commit 16031aae965083ec7a33db0380a8d67ee5095bce Author: wehub-resource-sync Date: Mon Jul 13 12:26:24 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..1800bc2 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,44 @@ +# see https://docs.codecov.io/docs/codecov-yaml +# Validation check: +# $ curl --data-binary @.codecov.yml https://codecov.io/validate + +# https://docs.codecov.io/docs/codecovyml-reference +codecov: + bot: "codecov-io" + strict_yaml_branch: "yaml-config" + require_ci_to_pass: yes + notify: + # after_n_builds: 2 + wait_for_ci: yes + +coverage: + precision: 0 # 2 = xx.xx%, 0 = xx% + round: nearest # how coverage is rounded: down/up/nearest + range: 40...100 # custom range of coverage colors from red -> yellow -> green + status: + # https://codecov.readme.io/v1.0/docs/commit-status + project: + default: + informational: true + target: 95% # specify the target coverage for each commit status + threshold: 10% # allow this little decrease on project + # https://github.com/codecov/support/wiki/Filtering-Branches + # branches: master + if_ci_failed: error + # https://github.com/codecov/support/wiki/Patch-Status + patch: + default: + informational: true + target: 95% # enforce same coverage target for new/changed code as the project + threshold: 10% # allow only a small decrease on patch coverage + changes: false + +# https://docs.codecov.com/docs/github-checks#disabling-github-checks-patch-annotations +github_checks: + annotations: false + +comment: + layout: header, diff + require_changes: false + behavior: default # update if exists else create new + # branches: * diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..76755f0 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +# These owners will be the default owners for everything in the repo. +# They will be requested for review when someone opens a pull request. +* @SkalskiP @Borda @probicheaux @isaacrob + +# supervise the core model modules +/rfdetr @probicheaux @isaacrob @Matvezy + +# supervise the training modules +/src/rfdetr/training/ @Borda @probicheaux @isaacrob +/tests/training/ @Borda @probicheaux @isaacrob diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..7968623 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,497 @@ +# Contributing to RF-DETR + +Thank you for helping to advance RF-DETR! Your participation is invaluable in evolving our platform—whether you’re squashing bugs, refining documentation, or rolling out new features. Every contribution pushes the project forward. + +## Table of Contents + +1. [How to Contribute](#how-to-contribute) +2. [Project Structure](#project-structure) +3. [Development Environment Setup](#development-environment-setup) +4. [Test-Driven Development](#test-driven-development) +5. [Code Quality and Linting](#code-quality-and-linting) +6. [Deprecation Policy](#deprecation-policy) +7. [Building Documentation](#building-documentation) +8. [CLA Signing](#cla-signing) +9. [Google-Style Docstrings and Mandatory Type Hints](#google-style-docstrings-and-mandatory-type-hints) +10. [Reporting Bugs](#reporting-bugs) +11. [Adding a New Model](#adding-a-new-model) +12. [Security Considerations](#security-considerations) +13. [License](#license) + +## How to Contribute + +Your contributions can be in many forms—whether it’s enhancing existing features, improving documentation, resolving bugs, or proposing new ideas. Here’s a high-level overview to get you started: + +1. [Fork the Repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo): Click the “Fork” button on our GitHub page to create your own copy. +2. [Clone Locally](https://docs.github.com/en/enterprise-server@3.11/repositories/creating-and-managing-repositories/cloning-a-repository): Download your fork to your local development environment. +3. [Create a Branch](https://docs.github.com/en/desktop/making-changes-in-a-branch/managing-branches-in-github-desktop): Use a descriptive name with appropriate prefix: + ```bash + # Branch naming convention: {type}/{issue_number}-name_or_description + git checkout -b fix/123-authentication_bug + git checkout -b feat/678-add_export_support + git checkout -b docs/update_readme + ``` + **Prefixes:** `fix/` (bug fixes), `feat/` (new features), `docs/` (documentation), `refactor/`, `test/`, `chore/` +4. Develop Your Changes: Make your updates, ensuring your commit messages clearly describe your modifications. +5. [Commit and Push](https://docs.github.com/en/desktop/making-changes-in-a-branch/committing-and-reviewing-changes-to-your-project-in-github-desktop): Run: + ```bash + git add . + git commit -m "A brief description of your changes" + git push -u origin your-descriptive-name + ``` +6. [Open a Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request): Submit your pull request against the main development branch. Please detail your changes and link any related issues. + +Before merging, check that all tests pass and that your changes adhere to our development and documentation standards. + +## Project Structure + +Understanding the project structure will help you navigate the codebase and make contributions effectively. + +``` +rf-detr/ +├── .github/ # GitHub configuration +│ ├── workflows/ # CI/CD pipelines (tests, builds, docs deployment) +│ ├── CONTRIBUTING.md # This file - contribution guidelines +│ ├── copilot-instructions.md # GitHub Copilot-specific guidance +│ └── ISSUE_TEMPLATE/ # Issue templates +├── docs/ # Documentation source (MkDocs) +│ ├── *.md # Documentation pages +│ └── assets/ # Images and other assets +├── src/rfdetr/ # Main package source code +│ ├── __init__.py # Package entry point +│ └── ... # Other modules (models, datasets, utils, etc.) +├── tests/ # Test suite +│ ├── test_*.py # Test files +│ └── conftest.py # Pytest configuration and fixtures +├── pyproject.toml # Project metadata, dependencies, tool configurations +├── mkdocs.yaml # Documentation configuration +├── .pre-commit-config.yaml # Pre-commit hooks configuration +├── README.md # Project overview and quick start +├── LICENSE # Apache 2.0 license +└── AGENTS.md # AI agent-specific technical documentation +``` + +**Key Directories:** + +- **`src/rfdetr/`** - All source code for the RF-DETR package + + - Contains models, datasets, training logic, deployment utilities, and more + - Internal organization may change as the project evolves + +- **`tests/`** - Comprehensive test suite + + - Unit tests, integration tests, and end-to-end tests + - Use `@pytest.mark.gpu` for GPU-dependent tests + +- **`docs/`** - Documentation source files + + - Written in Markdown, built with MkDocs + - Published to https://rfdetr.roboflow.com + +- **`.github/`** - GitHub-specific configuration + + - CI/CD workflows define automated testing and deployment + - Contributing guidelines and issue templates + +**Important Configuration Files:** + +- **`pyproject.toml`** - Single source of truth for: + + - Project metadata and dependencies + - Tool configurations (ruff, pytest, coverage, etc.) + - Build system configuration + +- **`.pre-commit-config.yaml`** - Defines pre-commit hooks for code quality + +- **`mkdocs.yaml`** - Documentation site configuration + +> [!TIP] +> When contributing, focus on the relevant directory for your change: +> +> - Bug fixes/features → `src/rfdetr/` and `tests/` +> - Documentation → `docs/` +> - CI/build issues → `.github/workflows/` or config files + +## Development Environment Setup + +RF-DETR uses **`uv`** as the package manager for dependency management. Ensure you have Python >=3.10 installed (supports 3.10, 3.11, 3.12, 3.13). + +### Installing uv + +```bash +pip install uv +``` + +### Setting Up Your Development Environment + +```bash +# Clone your fork +git clone https://github.com/YOUR_USERNAME/rf-detr.git +cd rf-detr + +# Install all development dependencies +uv sync --all-groups + +# Or install specific dependency groups +uv sync --group tests # Testing dependencies only +uv sync --group docs # Documentation dependencies only +uv sync --group build # Build tools only +``` + +**Important:** Always run `uv sync` after pulling changes to ensure your dependencies are up to date. + +### Running Tests + +> **CI Workflows as Source of Truth:** See `.github/workflows/ci-tests-cpu.yml` and `.github/workflows/ci-tests-gpu.yml` for the exact commands used in continuous integration. + +```bash +# Run CPU tests (default for local development; mirrors CI) +uv run --no-sync pytest src/ tests/ -n 2 -m "not gpu" --ignore=tests/run_smoke_all_models.py --cov=rfdetr --cov-report=xml --timeout=240 --durations=50 + +# Run GPU tests (requires GPU; mirrors CI) +uv run --no-sync pytest tests/ -m gpu -n 3 --reruns 1 --only-rerun "OutOfMemoryError" --cov=rfdetr --cov-report=xml --timeout=600 --durations=20 +``` + +**Development vs. PR Requirements:** + +- **During development:** Tests may fail as you work through TDD cycle (write failing test → implement → fix) +- **Before opening PR:** Your final commit MUST have all tests passing +- **Before each commit:** Run `pre-commit run --all-files` to ensure code quality + +### Building the Package + +```bash +# Build source and wheel distributions +uv build + +# Validate the build +uv run twine check --strict dist/* +``` + +## Test-Driven Development + +We follow test-driven development practices to ensure code quality and prevent regressions. + +### For Bug Fixes + +1. **Write a test that replicates the issue** - The test should fail initially, demonstrating the bug +2. **Commit the failing test** (optional during development, but commit message should note "WIP" or "test for issue #XXX") +3. **Implement the fix** - Make the minimal change needed to make the test pass +4. **Verify all tests pass** - Ensure your fix doesn't break existing functionality +5. **Commit the fix** - This commit MUST have all tests passing before opening PR + +**Note:** It's acceptable to have failing tests in intermediate commits during development. However, your **final commit before opening a PR must have all tests passing**. This aligns with test-driven development: first create a failing test that proves the bug exists, then fix it. + +### For New Features + +1. **Write tests covering all major use cases** - Think about edge cases, invalid inputs, and expected behaviors +2. **Implement the feature** - Build the feature to satisfy the test requirements +3. **Refactor if needed** - Clean up the implementation while keeping tests green + +### Test Organization + +**Use test classes to group related tests:** + +```python +import pytest + + +class TestModelInference: + def test_single_image_inference(self): + # Test code + pass + + def test_batch_inference(self): + # Test code + pass +``` + +**Use `pytest.mark.parametrize` to extend test cases:** + +```python +import pytest + + +@pytest.mark.parametrize( + "model_variant", + [ + pytest.param("nano", id="nano"), + pytest.param("small", id="small"), + pytest.param("medium", id="medium"), + ], +) +def test_model_loading(model_variant): + # Test code that runs for each model variant + pass +``` + +**Avoid multiple validation cases in a single test:** + +Do not write tests that loop through multiple cases internally. Instead, use `@pytest.mark.parametrize` so each case runs as a separate test: + +```python +import pytest +from rfdetr.assets.model_weights import ModelWeights + + +# BAD: Multiple cases in one test - all assertions must pass for test to pass +def test_all_models_have_valid_urls(): + for model in ModelWeights: + assert model.url.startswith("http") # Hard to identify which model failed + + +# GOOD: Parametrized - each model is a separate test case +@pytest.mark.parametrize("model", list(ModelWeights), ids=[m.filename for m in ModelWeights]) +def test_all_models_have_valid_urls(model): + assert model.url.startswith("http") # Clear which model failed +``` + +Benefits of parametrization: + +- Each case runs as an independent test (failures are isolated) +- Test IDs clearly identify which case failed +- Easier to debug and maintain +- Better test reporting in CI + +**Mark GPU-required or computationally heavy tests:** + +```python +import pytest + + +@pytest.mark.gpu # Use this marker for GPU-dependent or heavy tests (e.g., training) +def test_model_training(): + # Training test code + pass +``` + +Tests marked with `@pytest.mark.gpu` are excluded from CPU CI workflows and run separately on GPU infrastructure. + +### CI Testing + +> [!NOTE] +> **CI Workflows (Source of Truth):** See `.github/workflows/ci-tests-cpu.yml` and `.github/workflows/ci-tests-gpu.yml` for exact commands. + +Our continuous integration tests run on: + +- **Operating Systems:** Ubuntu, Windows, macOS +- **Python Versions:** 3.10, 3.11, 3.12, 3.13 +- **CPU Workflow:** `pytest -m "not gpu"` - Runs on all OS/Python combinations +- **GPU Workflow:** `pytest -m gpu` - Runs separately on GPU infrastructure + +This ensures your changes work across all supported platforms and Python versions. + +**Key GitHub Actions workflow files** (in `.github/workflows/`): + +- **ci-tests-cpu.yml** — CPU tests across Ubuntu/Windows/macOS × Python 3.10–3.13 +- **ci-tests-gpu.yml** — GPU-dependent tests +- **build-package.yml** — Build and validate distributions (`uv build` + `twine check`) +- **ci-build-docs.yml** — Documentation build validation +- **publish-docs.yml** — Deploy docs to GitHub Pages on release + +**Concurrency:** PRs cancel in-progress runs on new pushes. + +### Running Tests + +```bash +# Run tests with parallel execution (recommended) +uv run --no-sync pytest src/ tests/ -n 2 -m "not gpu" --ignore=tests/run_smoke_all_models.py --timeout=240 --durations=50 + +# Run a specific test file +uv run --no-sync pytest tests/models/test_model.py + +# Run a specific test +uv run --no-sync pytest tests/models/test_model.py::test_model_loading +``` + +## Code Quality and Linting + +All code must pass linting and formatting checks before being merged. We use **pre-commit hooks** to automate this process. + +> [!TIP] +> Pre-commit hooks will auto-format many issues. If pre-commit fails, review the changes it made and re-stage the files. + +### Setting Up Pre-commit + +```bash +# Install pre-commit +pip install pre-commit + +# Install the git hooks +pre-commit install + +# Run manually on all files +pre-commit run --all-files +``` + +**Configuration:** See `.pre-commit-config.yaml` for all hooks and `pyproject.toml` for tool-specific settings (e.g., `[tool.ruff]`). + +## Deprecation Policy + +RF-DETR uses [pyDeprecate](https://github.com/Borda/pyDeprecate) to emit structured deprecation warnings. Use `@deprecated` for functions and methods, `@deprecated_class` for classes. The importable package name is `deprecate` (not `pyDeprecate`); refer to its docs for advanced usage. + +```python +from deprecate import deprecated + + +@deprecated(target=new_fn, deprecated_in="1.7.0", remove_in="1.9.0") +def old_fn(*args, **kwargs): ... +``` + +**Rules:** + +- All version strings must be full semver: `1.7.0`, not `1.7`. +- Minimum window: a symbol deprecated in `X.Y.0` cannot be removed before `X.(Y+2).0` (two minor releases). +- Every new deprecation needs an entry in `docs/getting-started/migration.md` under a `### Deprecated (removal in vX.Z.0)` subsection. + +**Removal checklist** (when `remove_in` version arrives): + +1. Delete the deprecated symbol, class, or shim file. +2. Remove any remaining `@deprecated` / `@deprecated_class` decorators. +3. Add a breaking-change entry to `docs/getting-started/migration.md`. +4. Search for lingering imports of the removed symbol and update them. +5. Verify `pre-commit run --all-files` passes and tests are green. + +## Building Documentation + +RF-DETR's documentation is built with [MkDocs](https://www.mkdocs.org/) and the [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) theme. API reference pages are auto-generated from docstrings using [mkdocstrings](https://mkdocstrings.github.io/). + +> [!NOTE] +> Building the full documentation locally requires the `plus` extra (`rfdetr[plus]`), which provides the XLarge and 2XLarge model pages. Without it, the build will fail on those reference pages. + +### Install Documentation Dependencies + +```bash +# Full docs build (matches CI — required for XLarge/2XLarge model pages) +uv pip install -e ".[plus]" --group docs + +# Minimal install (skip plus models — XLarge/2XLarge pages will error) +uv sync --group docs +``` + +### Serve Locally with Live Reload + +```bash +uv run mkdocs serve +``` + +Open [http://localhost:8000](http://localhost:8000) in your browser. The server watches for file changes and reloads automatically — no restart needed as you edit documentation. + +### Build Static Site + +```bash +# Build static documentation site to the site/ directory +uv run mkdocs build +``` + +**Note:** `mkdocs.yaml` uses custom YAML tags (`!!python/name`). The `check-yaml` pre-commit hook runs with `--unsafe` to allow this — do not remove that flag. + +### Documentation Structure + +``` +docs/ +├── index.md # Home page +├── learn/ # How-to guides and tutorials +│ ├── install.md +│ ├── run/ # Detection and segmentation guides +│ └── train/ # Training guides (parameters, augmentations, loggers, etc.) +├── reference/ # Auto-generated API reference (from docstrings) +├── tutorials/ +└── theme/ # Custom theme overrides +mkdocs.yaml # MkDocs configuration and navigation +``` + +> [!TIP] +> When adding a new documentation page, add it to the `nav` section in `mkdocs.yaml` so it appears in the site navigation. Pages that exist in `docs/` but are not listed in `nav` will not be included in the site. + +## CLA Signing + +In order to maintain the integrity of our project, every pull request must include a signed Contributor License Agreement (CLA). This confirms that your contributions are properly licensed under our Apache 2.0 License. After opening your pull request, simply add a comment stating: + +``` +I have read the CLA Document and I sign the CLA. +``` + +This step is essential before any merge can occur. + +## Google-Style Docstrings and Mandatory Type Hints + +For clarity and maintainability, any new functions or classes must include [Google-style docstrings](https://google.github.io/styleguide/pyguide.html) and use Python type hints. Type hints are mandatory in all function definitions, ensuring explicit parameter and return type declarations. + +> [!IMPORTANT] +> Type hints are in the function signature. **Do not duplicate types in docstrings** - describe the parameter's purpose instead. + +For example: + +```python +def sample_function(param1: int, param2: int = 10) -> bool: + """ + Provides a brief description of function behavior. + + Args: + param1: Explanation of the first parameter's purpose. + param2: Explanation of the second parameter, defaulting to 10. + + Returns: + True if the operation succeeds, otherwise False. + + Examples: + >>> sample_function(5, 10) + True + """ + return param1 == param2 +``` + +Following this pattern helps ensure consistency throughout the codebase. + +## Reporting Bugs + +Bug reports are vital for continued improvement. When reporting an issue, please include a clear, minimal reproducible example that demonstrates the problem. Detailed bug reports assist us in swiftly diagnosing and addressing issues. + +## Adding a New Model + +> [!IMPORTANT] +> Before implementing a new model, **discuss with maintainers first**. Project structure and patterns are subject to change. + +**General workflow:** + +1. **Open an issue** describing the proposed model and approach + - You may ask maintainers to confirm the expected evaluation protocol (dataset, metrics) before running full benchmarks +2. **Demonstrate improvement** versus reference models on a standard public dataset (e.g., COCO val2017) + - If the change is for an existing RF-DETR model, show a case where the new approach is Pareto optimal (e.g., better accuracy at similar or lower latency/model size) over the existing model + - If the change is adding a new functionality, show a case where the new approach is Pareto optimal over comparable third-party models (see the [README model table](../README.md) for reference baselines) + - Provide a script for us to reproduce your results +3. **Wait for maintainer feedback** on architecture and integration approach +4. **Follow test-driven development:** + - Write comprehensive tests for the new model + - Implement the model following approved approach + - Ensure all tests pass +5. **Add documentation** as directed by maintainers +6. **Submit PR** with reference to the discussion issue + +Maintainers will guide you on specific files to modify and patterns to follow based on current project architecture. + +## Security Considerations + +- **Write secure code:** Avoid injection vulnerabilities (XSS, SQL injection, command injection) +- **Validate inputs:** Especially for file paths, URLs, and user-provided data +- **No credentials:** Never commit API keys, tokens, or credentials to the repository +- **Follow OWASP best practices** for any user-facing or network-facing code + +## License + +By contributing to RF-DETR, you agree that your contributions will be licensed under the Apache 2.0 License as specified in our [LICENSE](/LICENSE) file. + +Thank you for your commitment to making RF-DETR better. We look forward to your pull requests and continued collaboration. Happy coding! + +### License Headers + +All Python files must start with the following header: + +```python +# ------------------------------------------------------------------------ +# RF-DETR +# Copyright (c) 2025 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +``` diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..3f0562c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,80 @@ +name: 🐞 Bug Report +# title: " " +description: "Problems with RF-DETR" +labels: [bug] +body: + - type: markdown + attributes: + value: | + Thank you for submitting an RF-DETR 🐞 Bug Report! + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please search the RF-DETR documentation and [issues](https://github.com/roboflow/rf-detr/issues) to see if a similar bug report already exists. + options: + - label: > + I have searched the RF-DETR issues and found no similar bug report. + required: true + + - type: textarea + attributes: + label: Bug + description: > + Please provide as much information as possible. Include full console output and error messages (including any traceback). Use Markdown to format text, code, and logs. Screenshots can help clarify visual issues. + placeholder: | + Include logs, tracebacks, screenshots, or any other relevant info to help us quickly diagnose and solve your problem. + validations: + required: true + + - type: textarea + attributes: + label: Environment + description: | + Try the latest version of RF-DETR (`pip install -U rf-detr`) before reporting a bug. If the issue persists, please provide details about your setup: + - RF-DETR version + - OS (e.g. Ubuntu 20.04, Windows 10) + - Python version + - PyTorch version + - CUDA/cuDNN version + - GPU/CPU hardware + placeholder: | + Example: + - RF-DETR: 0.1.0 + - OS: Ubuntu 20.04 + - Python: 3.9.0 + - PyTorch: 1.13.1 + - GPU: NVIDIA RTX 4080 + validations: + required: true + + - type: textarea + attributes: + label: Minimal Reproducible Example + description: > + Provide code or steps that are as minimal and self-contained as possible to reproduce the issue. This helps us diagnose problems more quickly. + placeholder: | + ``` + # Example code to reproduce your issue + model = ... + ... + ``` + validations: + required: true + + - type: textarea + attributes: + label: Additional + description: | + Anything else you would like to share? (Optional) + placeholder: | + Additional context, suggestions, or observations. + + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + description: > + (Optional) We encourage you to submit a [Pull Request](https://github.com/roboflow/rf-detr/pulls) (PR) to help improve RF-DETR, especially if you already have a fix or feature in mind. + options: + - label: Yes, I'd like to help by submitting a PR! diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..ff2546c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,49 @@ +name: 🤩 Feature Request +description: Suggest an idea for RF-DETR +# title: " " +labels: [enhancement] +body: + - type: markdown + attributes: + value: | + Thank you for submitting an RF-DETR Feature Request! + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please search the [issues](https://github.com/roboflow/rf-detr/issues) to see if a similar feature request already exists. + options: + - label: > + I have searched the RF-DETR [issues](https://github.com/roboflow/rf-detr/issues) and found no similar feature requests. + required: true + + - type: textarea + attributes: + label: Description + description: A short description of your feature. + placeholder: | + What new feature would you like to see in RF-DETR? + validations: + required: true + + - type: textarea + attributes: + label: Use case + description: | + Describe the use case of your feature request. It will help us understand and prioritize the feature request. + placeholder: | + How would this feature be used, and who would use it? + + - type: textarea + attributes: + label: Additional + description: Anything else you would like to share? + + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + description: > + (Optional) We encourage you to submit a [Pull Request](https://github.com/roboflow/supervision/pulls) (PR) to help improve Supervision for everyone, especially if you have a good understanding of how to implement a fix or feature. + options: + - label: Yes I'd like to help by submitting a PR! diff --git a/.github/LICENSE_HEADER.txt b/.github/LICENSE_HEADER.txt new file mode 100644 index 0000000..2bdea91 --- /dev/null +++ b/.github/LICENSE_HEADER.txt @@ -0,0 +1,5 @@ +------------------------------------------------------------------------ +RF-DETR +Copyright (c) 2025 Roboflow. All Rights Reserved. +Licensed under the Apache License, Version 2.0 [see LICENSE for details] +------------------------------------------------------------------------ diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..181e2b2 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,97 @@ +# Security Policy + +## Project Status + +RF-DETR is a **research project** under active development. While we strive for stability, the codebase may contain undiscovered vulnerabilities typical of research-grade software. + +## Supported Versions + +Security fixes are generally provided for the latest stable release. + +Fixes for older versions may be provided at the maintainers' discretion, depending on severity and feasibility. + +| Version | Support Status | +| -------------- | ------------------ | +| Latest release | :white_check_mark: | +| Older versions | Case-by-case | + +## Reporting a Vulnerability + +Please report security issues privately. + +**Do not** create a public GitHub issue for security vulnerabilities. + +Report to: **security@roboflow.com** + +Include (if available): + +- A clear description and impact +- Steps to reproduce / proof-of-concept +- Affected versions, environment details, and relevant logs + +We aim to acknowledge reports within a few days and will work with you on appropriate disclosure timelines. Response times may vary depending on severity and complexity. + +## Security Considerations for ML Projects + +### Model Weights and Checkpoints + +**Critical**: PyTorch checkpoint files (`.pt`, `.pth`) can execute arbitrary code when loaded because they are commonly pickle-based. + +- **Only load models from trusted sources** +- Prefer safer formats (e.g. `safetensors`) when available +- When possible, use safer loading options (e.g. `torch.load(..., weights_only=True)` where supported) + +**Note**: ONNX models (`.onnx`) are not pickle-based, but parsing/optimizer toolchains can still have security vulnerabilities. Treat untrusted files cautiously. + +**Resources**: + +- [PyTorch Security Best Practices](https://pytorch.org/docs/stable/security.html) +- [PyTorch CVE Database](https://github.com/pytorch/pytorch/security/advisories) + +### Dependency Security + +RF-DETR depends on the PyTorch ecosystem and other ML libraries: + +- Keep PyTorch, torchvision, and transformers updated +- Monitor security advisories for dependencies +- Use virtual environments to isolate installations +- Regularly update dependencies (for users): `pip install --upgrade rfdetr` + +### Data Processing + +- Validate and sanitize input data +- Be cautious when processing data from untrusted sources +- Consider resource limits when processing large batches + +### Training and Inference + +- Untrusted training data may contain adversarial examples +- Monitor resource usage during training to detect anomalies +- Consider using resource limits in production environments + +## Known Limitations + +- This is research software not hardened for production use +- The package has not undergone formal security auditing +- Custom CUDA kernels may have memory safety issues +- Limited input validation in some code paths + +## Best Practices + +1. **Run in isolated environments**: Use containers or virtual machines for production deployments +2. **Limit resource access**: Apply appropriate resource constraints (memory, GPU, CPU) +3. **Monitor for anomalies**: Track unusual behavior during training or inference +4. **Keep updated**: Regularly update to the latest version +5. **Review dependencies**: Understand the security posture of all dependencies + +## Security Updates + +Security patches will be announced via: + +- GitHub Security Advisories +- Release notes +- Project README + +If a vulnerability is deemed significant, we may request a CVE identifier to ensure proper tracking across the ecosystem. + +Subscribe to repository notifications to stay informed. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..0883c60 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,140 @@ +# RF-DETR Copilot Instructions + +> [!NOTE] +> This document is GitHub Copilot-specific guidance. For canonical contribution guidelines (test-driven development, code quality, docstrings, etc.), see [CONTRIBUTING.md](CONTRIBUTING.md). For detailed agent-specific context, see [AGENTS.md](../AGENTS.md). + +## Repository Overview + +RF-DETR is a real-time transformer architecture for object detection and instance segmentation. Built on DINOv2 vision transformer backbone with PyTorch. + +**Project Type:** Python ML library (computer vision) +**Python:** >=3.10 (3.10, 3.11, 3.12, 3.13) +**License:** Apache 2.0 (Plus models under PML 1.0) + +> [!TIP] +> +> - **Configuration:** See `pyproject.toml` for dependencies, build settings, and tool configurations. +> - **Contributing:** See `.github/CONTRIBUTING.md` for contribution guidelines, CLA, and coding standards. + +## Quick Start + +**Package Manager:** This project uses `uv` for all dependency management. + +```bash +# Development setup +uv sync --all-groups + +# Run tests (always before committing) +uv run --no-sync pytest src/ tests/ -n 2 -m "not gpu" --cov=rfdetr --cov-report=xml + +# Build package +uv build +``` + +> [!IMPORTANT] +> Run `uv sync` after pulling changes to update dependencies. + +## Code Quality + +**Linting & Formatting:** All code must pass pre-commit checks. See **[Code Quality and Linting](CONTRIBUTING.md#code-quality-and-linting)** in CONTRIBUTING.md for setup and details. + +```bash +pre-commit run --all-files +``` + +> **Configuration:** `.pre-commit-config.yaml` (hooks) and `[tool.ruff]` in `pyproject.toml` (Python linting) + +## Key Conventions + +> [!NOTE] +> Internal package organization (`src/rfdetr/`) is subject to change as this is an active research project. Explore the codebase to understand current module organization. + +**Imports:** + +- Always use direct imports: `from rfdetr.utilities.distributed import get_rank, is_main_process` +- Logger: `from rfdetr.utilities.logger import get_logger` (reads `LOG_LEVEL` env var) +- **Never use** `rfdetr.util.*` or `rfdetr.deploy.*` — deprecated shims scheduled for removal in v1.9.0 +- TQDM: `from tqdm.auto import tqdm` (NOT `from tqdm import tqdm`) + +## Testing & Development Workflow + +**Test-Driven Development:** Follow TDD practices - write tests first for bugs, comprehensive tests for features. See **[Test-Driven Development](CONTRIBUTING.md#test-driven-development)** in CONTRIBUTING.md for detailed guidelines. + +**Quick reference:** + +- Bug fixes: Write failing test → Fix → Verify all pass +- Features: Write comprehensive tests → Implement → Refactor +- Use test classes and `@pytest.mark.parametrize` for organization +- Mark GPU/heavy tests with `@pytest.mark.gpu` + +**Testing Requirements:** + +- ⚠️ During development: Tests may fail (TDD cycle is fine) +- ✅ Before PR: Final commit MUST have all tests passing +- ✅ Before commit: Run `pre-commit run --all-files` + +**CI/CD:** See `.github/workflows/` for source of truth. Tests run on Python 3.10-3.13 across Ubuntu, Windows, macOS. + +## Coding Standards + +**Type Hints & Docstrings:** MANDATORY for all functions/classes. See **[Google-Style Docstrings and Mandatory Type Hints](CONTRIBUTING.md#google-style-docstrings-and-mandatory-type-hints)** in CONTRIBUTING.md for examples. + +**Import Conventions:** + +```python +# Always use direct imports (NOT import ... as pattern) +from rfdetr.utilities.distributed import get_rank, is_main_process, save_on_master +from rfdetr.utilities.logger import get_logger + +# TQDM (for environment compatibility) +from tqdm.auto import tqdm # NOT from tqdm import tqdm +``` + +**Project-Specific Patterns:** + +- **Logging:** Use `logger.debug()` for detailed tensor/shape info (not `logger.info()`) +- **Segmentation models:** Return `pred_masks` as `torch.Tensor` or dict with keys `['spatial_features', 'query_features', 'bias']` +- **Checkpoint handling:** Always check file existence before operations +- **License headers:** All Python files require Apache 2.0 header (enforced by pre-commit) + +**Best Practices:** + +- Make minimal, surgical changes - avoid over-engineering +- Use existing patterns and libraries +- Write secure code - avoid injection vulnerabilities (XSS, SQL injection, command injection) +- Follow Python ML development best practices + +## Pre-Commit Checklist + +Before submitting changes: + +1. ✅ Run tests: `uv run --no-sync pytest src/ tests/ -n 2 -m "not gpu"` +2. ✅ Run pre-commit: `pre-commit run --all-files` +3. ✅ Verify new functions have type hints + docstrings +4. ✅ Review changes for minimal scope + +## Resources + +- **Docs:** https://rfdetr.roboflow.com +- **Contributing:** `.github/CONTRIBUTING.md` +- **Config:** `pyproject.toml`, `.pre-commit-config.yaml` +- **Issues:** https://github.com/roboflow/rf-detr/issues + +## Maintaining Agentic Documentation + +**If your contribution:** + +- Changes project structure or introduces new patterns +- Receives major feedback in PR review about conventions/patterns + +**Then update the relevant documents:** + +- This file (copilot-instructions.md) for high-level guidance +- AGENTS.md for detailed technical patterns +- CONTRIBUTING.md if it affects human contribution workflow + +This ensures future contributions stay consistent and reduces repeated feedback. + +--- + +**Note:** These instructions are GitHub Copilot-specific. When in doubt, refer to existing code patterns, contributing guidelines, and test files for examples. diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml new file mode 100644 index 0000000..20f3bcf --- /dev/null +++ b/.github/workflows/build-package.yml @@ -0,0 +1,41 @@ +name: Build Package + +permissions: + contents: read + +on: + workflow_call: + inputs: + python-version: + required: false + type: string + default: "3.10" + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: 🐍 Install uv and set Python version ${{ inputs.python-version }} + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: ${{ inputs.python-version }} + activate-environment: true + + - name: 🏗️ Build source and wheel distributions + run: | + uv pip install --group build + uv build + uv run --no-sync twine check --strict dist/* + + - name: 📤 Upload distribution artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dist + path: dist/ + + - name: Minimize uv cache + run: uv cache prune --ci diff --git a/.github/workflows/ci-build-docs.yml b/.github/workflows/ci-build-docs.yml new file mode 100644 index 0000000..38797b8 --- /dev/null +++ b/.github/workflows/ci-build-docs.yml @@ -0,0 +1,47 @@ +name: Docs/Test WorkFlow + +on: + pull_request: + branches: ["main", "release/*", "develop"] + push: + branches: ["main", "release/*", "develop"] + workflow_dispatch: {} + +# Restrict permissions by default +permissions: + contents: read # Required for checkout + checks: write # Required for test reporting + +jobs: + docs-build-test: + name: Test docs build + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + + - name: 🐍 Install uv and set Python + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: "3.10" + activate-environment: true + + - name: 🏗️ Install dependencies + timeout-minutes: 5 + run: uv pip install -e ".[plus]" --group docs + + - name: 🧪 Test Docs Build + run: uv run --no-sync mkdocs build --verbose + + - name: 📤 Upload docs artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docs-site + path: site/ + retention-days: 7 + + - name: Minimize uv cache + run: uv cache prune --ci diff --git a/.github/workflows/ci-deps-resolution.yml b/.github/workflows/ci-deps-resolution.yml new file mode 100644 index 0000000..570fff2 --- /dev/null +++ b/.github/workflows/ci-deps-resolution.yml @@ -0,0 +1,52 @@ +name: Dependency resolution + +on: + push: + branches: ["main", "release/*", "develop"] + paths: ["pyproject.toml"] + pull_request: + paths: ["pyproject.toml"] + +permissions: + contents: read + +defaults: + run: + shell: bash + +concurrency: + group: deps-resolution-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + tflite-resolution: + name: Resolve [tflite] extra — Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # tflite deps carry python_version>='3.12' and <'3.13' markers; onnx2tf pins numpy==1.26.4 + # exactly, which conflicts with ml-dtypes>=0.5.1 requiring numpy>=2.1.0 on 3.13. + # Add 3.13 and relax the <'3.13' upper bound when onnx2tf drops the exact numpy pin. + python-version: ["3.12"] + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: 🐍 Install uv and set Python version ${{ matrix.python-version }} + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: ${{ matrix.python-version }} + + - name: 🔍 Resolve [tflite] optional dependencies + run: | + uv pip compile pyproject.toml \ + --extra tflite \ + --python-version ${{ matrix.python-version }} \ + --no-header \ + --quiet + + - name: Minimize uv cache + continue-on-error: true + run: uv cache prune --ci diff --git a/.github/workflows/ci-integrations.yml b/.github/workflows/ci-integrations.yml new file mode 100644 index 0000000..5d336f9 --- /dev/null +++ b/.github/workflows/ci-integrations.yml @@ -0,0 +1,49 @@ +name: Smoke Tests + +on: + push: + branches: ["main", "release/*", "develop"] + pull_request: + branches: ["main", "release/*", "develop"] + +permissions: + contents: read + +env: + # UV_TORCH_BACKEND only works with 'uv pip' commands, not 'uv sync'. + # Used by 'uv pip install torch torchvision' step to install CPU-only PyTorch. + UV_TORCH_BACKEND: "cpu" + +concurrency: + group: model-smoke-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + try-all-models: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.10", "3.13"] + timeout-minutes: 15 + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: 🐍 Install uv and set Python version ${{ matrix.python-version }} + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: ${{ matrix.python-version }} + activate-environment: true + + - name: 🚀 Install Packages (plus extras) + timeout-minutes: 5 + # Install PyTorch CPU-only first (UV_TORCH_BACKEND=cpu works with 'uv pip') + run: uv pip install -e .[plus] + + - name: 🔎 Smoke-test model instantiation, downloads, and inference + run: python tests/run_smoke_all_models.py + + - name: Minimize uv cache + run: uv cache prune --ci diff --git a/.github/workflows/ci-tests-cpu.yml b/.github/workflows/ci-tests-cpu.yml new file mode 100644 index 0000000..13ea448 --- /dev/null +++ b/.github/workflows/ci-tests-cpu.yml @@ -0,0 +1,99 @@ +name: CPU tests Workflow + +on: + push: + branches: ["main", "release/*", "develop", "feat/*"] + pull_request: + +permissions: + contents: read +defaults: + run: + shell: bash + +env: + # UV_TORCH_BACKEND only works with 'uv pip' commands, not 'uv sync'. + # Used by 'uv pip install torch torchvision' step to install CPU-only PyTorch. + UV_TORCH_BACKEND: "cpu" + +concurrency: + group: pytest-test-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build-pkg: + uses: ./.github/workflows/build-package.yml + + run-tests: + name: Testing + # needs: build # todo: consider using this build package for testing + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10", "3.11", "3.12", "3.13"] + include: + - { os: "windows-latest", python-version: "3.10" } + - { os: "macos-latest", python-version: "3.10" } + - { os: "windows-latest", python-version: "3.13" } + - { os: "macos-latest", python-version: "3.13" } + runs-on: ${{ matrix.os }} + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: 🐍 Install uv and set Python version ${{ matrix.python-version }} + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: ${{ matrix.python-version }} + activate-environment: true + + - name: 🚀 Install Packages + timeout-minutes: 5 + # Install PyTorch CPU-only first (UV_TORCH_BACKEND=cpu works with 'uv pip') + run: uv pip install -e ".[train,cli,visual,kornia]" --group tests + + - name: 🧪 Run the Test + run: | + uv run --no-sync pytest src/ tests/ \ + -n 2 -m "not gpu and not coco17" \ + --ignore=tests/run_smoke_all_models.py \ + --cov=rfdetr --cov-report=xml \ + --timeout=420 \ + --durations=50 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: "coverage.xml" + flags: cpu,${{ runner.os }},py${{ matrix.python-version }} + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + + - name: Minimize uv cache + continue-on-error: true + run: uv cache prune --ci + + testing-guardian: + runs-on: ubuntu-latest + needs: run-tests + if: always() + steps: + - name: 📋 Display test result + run: echo "${{ needs.run-tests.result }}" + - name: ❌ Fail guardian on test failure + if: needs.run-tests.result == 'failure' + run: exit 1 + # Ensure that cancelled or skipped test runs still cause this guardian job to fail, + # using an explicit exit code instead of relying on timeout behavior. + - name: ⚠️ cancelled or skipped... + if: contains(fromJSON('["cancelled", "skipped"]'), needs.run-tests.result) + run: | + echo "run-tests job result is '${{ needs.run-tests.result }}'; failing explicitly." + exit 1 + - name: ✅ tests succeeded + if: needs.run-tests.result == 'success' + run: echo "All tests completed successfully in job 'run-tests'." diff --git a/.github/workflows/ci-tests-gpu.yml b/.github/workflows/ci-tests-gpu.yml new file mode 100644 index 0000000..44e7eed --- /dev/null +++ b/.github/workflows/ci-tests-gpu.yml @@ -0,0 +1,75 @@ +name: GPU tests Workflow + +on: + push: + branches: ["main", "release/*", "develop", "feat/*"] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read +defaults: + run: + shell: bash + +env: + PYTHON_VERSION: "3.12" + # UV_TORCH_BACKEND only works with 'uv pip' commands, not 'uv sync'. + # Set to "auto" for GPU workflow (auto-detects CUDA version). + UV_TORCH_BACKEND: "auto" + +concurrency: + group: pytest-gpu-test-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + run-gpu-tests: + name: Testing + if: github.event_name == 'push' || !github.event.pull_request.draft + timeout-minutes: 35 + runs-on: Roboflow-GPU-VM-Runner + steps: + - name: 🖥️ Print GPU information + run: nvidia-smi + - name: 🧰 Install build tools + run: | + sudo apt-get update + sudo apt-get install -y cmake + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: 🐍 Install uv and set Python version + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + activate-environment: true + + - name: 🚀 Install Packages + timeout-minutes: 5 + # Use uv pip install (not uv sync) to avoid universal lock resolution failing on + # extras that require Python>=3.12 (e.g. tflite/onnx2tf) when project supports >=3.10. + # UV_TORCH_BACKEND=auto (set above) works only with uv pip, not uv sync. + run: uv pip install -e ".[onnx,plus,train,visual]" --group tests --group ci-gpu-pin + + - name: 🧪 Run the Test + run: | + uv run --no-sync pytest tests/ \ + -m gpu \ + -n 3 \ + --reruns 1 --only-rerun "OutOfMemoryError" \ + --cov=rfdetr --cov-report=xml \ + --timeout=600 \ + --durations=20 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: "coverage.xml" + flags: gpu,${{ runner.os }},py${{ env.PYTHON_VERSION }} + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + + - name: Minimize uv cache + run: uv cache prune --ci diff --git a/.github/workflows/ci-typing.yml b/.github/workflows/ci-typing.yml new file mode 100644 index 0000000..0e7beb3 --- /dev/null +++ b/.github/workflows/ci-typing.yml @@ -0,0 +1,47 @@ +name: Mypy Type Check + +on: + push: + branches: ["main", "release/*", "develop"] + pull_request: + +permissions: + contents: read +defaults: + run: + shell: bash + +env: + # UV_TORCH_BACKEND only works with 'uv pip' commands, not 'uv sync'. + # Used by 'uv pip install torch torchvision' step to install CPU-only PyTorch. + UV_TORCH_BACKEND: "cpu" + +concurrency: + group: mypy-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + type-check: + name: Type Check + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: 🐍 Install uv and set Python version + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: "3.10" + activate-environment: true + + - name: 🚀 Install Packages + timeout-minutes: 5 + run: uv pip install -e ".[train,cli,visual]" --group typing + + - name: 🔍 Run mypy + run: uv run --no-sync mypy src/rfdetr/ --no-error-summary + + - name: Minimize uv cache + continue-on-error: true + run: uv cache prune --ci diff --git a/.github/workflows/pr-conflict-labeler.yml b/.github/workflows/pr-conflict-labeler.yml new file mode 100644 index 0000000..ab11e3b --- /dev/null +++ b/.github/workflows/pr-conflict-labeler.yml @@ -0,0 +1,26 @@ +name: PR Conflict Labeler + +on: + # So that PRs touching the same files as the push are updated + push: + branches: ["main", "release/*", "develop"] + # So that the `dirtyLabel` is removed if conflicts are resolved + # We recommend `pull_request_target` so that github secrets are available. + # In `pull_request` we wouldn't be able to change labels of fork PRs + pull_request_target: + types: [synchronize] + +permissions: + pull-requests: write + issues: write + +jobs: + labeling: + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - name: check if prs are dirty + uses: eps1lon/actions-label-merge-conflict@1df065ebe6e3310545d4f4c4e862e43bdca146f0 # v3 + with: + dirtyLabel: "has conflicts" + repoToken: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 0000000..71f5b29 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,208 @@ +# Build and deploy MkDocs documentation to GitHub Pages (https://rfdetr.roboflow.com/) +# using mike for versioned docs. +# +# Deploy matrix: +# push develop → mike deploy develop (rfdetr.roboflow.com/develop/) +# push release/latest → mike deploy latest (rfdetr.roboflow.com/latest/) +# release published (stable/post) → mike deploy (rfdetr.roboflow.com//; .postN stripped; RC/pre-release skipped) +# workflow_dispatch target=develop → same as push develop +# workflow_dispatch target=release/latest → same as push release/latest (→ latest alias) +# workflow_dispatch target= → mike deploy +# +# Every deploy also: +# - Copies robots.txt, llms.txt, llms-full.txt to gh-pages root +# - Regenerates sitemap.xml normalising versioned paths to /latest/ +# - Pings IndexNow (Bing) with canonical /latest/ URLs + +name: Build and Publish Docs + +on: + push: + branches: ["release/latest", "develop"] + release: + types: [published] + workflow_dispatch: + inputs: + target: + description: "Deploy target: 'develop', 'release/latest', or a version tag (e.g. '1.2.3'). Determines both the checkout ref and the mike version label." + required: true + default: "release/latest" + +# Ensure only one concurrent deployment +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref_name || github.event_name == 'workflow_dispatch' && inputs.target || github.event_name == 'release' && github.event.release.tag_name || github.ref_name }} + cancel-in-progress: true + +# Restrict permissions by default +permissions: + contents: write # Required for committing to gh-pages + pages: write # Required for deploying to Pages + pull-requests: write # Required for PR comments + +jobs: + doc-deploy: + name: Publish Docs + runs-on: ubuntu-latest + environment: + name: documentation-deployment + url: https://rfdetr.roboflow.com/ + timeout-minutes: 10 + steps: + - name: 🔍 Validate dispatch target + if: github.event_name == 'workflow_dispatch' + run: | + TARGET="${{ inputs.target }}" + if [ "$TARGET" = "develop" ] || [ "$TARGET" = "release/latest" ]; then + echo "✓ Target '$TARGET' is valid" + elif echo "$TARGET" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?$'; then + echo "✓ Target '$TARGET' is valid (version tag)" + else + echo "::error::Invalid target '$TARGET'. Must be 'develop', 'release/latest', or a version tag (e.g. '1.2.3' or '1.2.3.post1')" + exit 1 + fi + + - name: 📥 Checkout the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + ref: ${{ inputs.target || github.ref }} + + - name: 🐍 Install uv and set Python + uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + with: + python-version: "3.10" + activate-environment: true + + - name: 🏗️ Install dependencies + # Install PyTorch CPU-only first (UV_TORCH_BACKEND=cpu works with 'uv pip') + run: uv pip install -e ".[plus]" --group docs + + - name: ⚙️ Configure git for github-actions + run: | + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" + + - name: 🚀 Deploy Development Docs + if: github.event_name == 'push' && github.ref == 'refs/heads/develop' + env: + MKDOCS_GIT_COMMITTERS_APIKEY: ${{ secrets.GITHUB_TOKEN }} + run: uv run --no-sync mike deploy --push develop + + - name: 🚀 Deploy Latest Branch Docs + if: github.event_name == 'push' && github.ref == 'refs/heads/release/latest' + env: + MKDOCS_GIT_COMMITTERS_APIKEY: ${{ secrets.GITHUB_TOKEN }} + run: uv run --no-sync mike deploy --push --update-aliases latest + + - name: 🚀 Deploy Release Docs + if: github.event_name == 'release' && github.event.action == 'published' && !github.event.release.prerelease + env: + MKDOCS_GIT_COMMITTERS_APIKEY: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ github.event.release.tag_name }}" + DOC_VERSION="${TAG%.post*}" + uv run --no-sync mike deploy --push "$DOC_VERSION" + + - name: 🚀 Deploy Docs (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + env: + MKDOCS_GIT_COMMITTERS_APIKEY: ${{ secrets.GITHUB_TOKEN }} + run: | + TARGET="${{ inputs.target }}" + if [ "$TARGET" = "develop" ]; then + uv run --no-sync mike deploy --push develop + elif [ "$TARGET" = "release/latest" ]; then + uv run --no-sync mike deploy --push --update-aliases latest + else + DOC_VERSION="${TARGET%.post*}" + uv run --no-sync mike deploy --push "$DOC_VERSION" + fi + + - name: 🏗️ Build docs for artifact + run: uv run --no-sync mkdocs build + + - name: 📤 Upload docs artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docs-site + path: site/ + retention-days: 7 + + - name: Deploy root static files to gh-pages + if: >- + (github.event_name == 'push' && + (github.ref == 'refs/heads/develop' || + github.ref == 'refs/heads/release/latest')) || + (github.event_name == 'release' && + github.event.action == 'published' && + !github.event.release.prerelease) || + github.event_name == 'workflow_dispatch' + run: | + cp docs/root-static/robots.txt /tmp/robots.txt + cp docs/root-static/llms.txt /tmp/llms.txt + cp docs/root-static/llms-full.txt /tmp/llms-full.txt + cp docs/root-static/cace9ef287cb8d641db90d4295fcb1e1.txt /tmp/indexnow.txt + git fetch origin gh-pages:refs/remotes/origin/gh-pages + git checkout -B gh-pages origin/gh-pages + cp /tmp/robots.txt robots.txt + cp /tmp/llms.txt llms.txt + cp /tmp/llms-full.txt llms-full.txt + cp /tmp/indexnow.txt cace9ef287cb8d641db90d4295fcb1e1.txt + git add robots.txt llms.txt llms-full.txt cace9ef287cb8d641db90d4295fcb1e1.txt + # Normalize versioned paths (/X.Y.Z/) to /latest/ so sitemap URLs are + # not blocked by the User-agent: * Disallow rules in robots.txt. + # latest/ may be a mike alias redirect (no sitemap.xml) — fall back to highest versioned dir. + SITEMAP_SRC="" + if [ -f latest/sitemap.xml ]; then + SITEMAP_SRC="latest/sitemap.xml" + else + LATEST_VER=$(ls -d [0-9]* 2>/dev/null | grep -E '^[0-9]+\.[0-9]+' | sort -V | tail -1) + if [ -n "$LATEST_VER" ] && [ -f "$LATEST_VER/sitemap.xml" ]; then + SITEMAP_SRC="$LATEST_VER/sitemap.xml" + elif [ -f develop/sitemap.xml ]; then + SITEMAP_SRC="develop/sitemap.xml" + fi + fi + if [ -n "$SITEMAP_SRC" ]; then + sed 's|rfdetr\.roboflow\.com/[^/]*/|rfdetr.roboflow.com/latest/|g' "$SITEMAP_SRC" > sitemap.xml + git add sitemap.xml + fi + git diff --staged --quiet || git commit -m "chore: update root static files and sitemap" + git push origin gh-pages || (git pull --rebase origin gh-pages && git push origin gh-pages) + + # IndexNow key: cace9ef287cb8d641db90d4295fcb1e1 + # Key file served at https://rfdetr.roboflow.com/cace9ef287cb8d641db90d4295fcb1e1.txt + # (deployed via "Deploy root static files" step above). + # To rotate: generate new key, update docs/root-static/.txt, replace key string here. + - name: 📡 Notify IndexNow (Bing) + if: >- + (github.event_name == 'push' && + (github.ref == 'refs/heads/develop' || + github.ref == 'refs/heads/release/latest')) || + (github.event_name == 'release' && + github.event.action == 'published' && + !github.event.release.prerelease) || + github.event_name == 'workflow_dispatch' + run: | + curl -s -o /dev/null -w "%{http_code}" -X POST "https://api.indexnow.org/IndexNow" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d '{ + "host": "rfdetr.roboflow.com", + "key": "cace9ef287cb8d641db90d4295fcb1e1", + "keyLocation": "https://rfdetr.roboflow.com/cace9ef287cb8d641db90d4295fcb1e1.txt", + "urlList": [ + "https://rfdetr.roboflow.com/", + "https://rfdetr.roboflow.com/latest/", + "https://rfdetr.roboflow.com/latest/learn/install/", + "https://rfdetr.roboflow.com/latest/learn/pretrained/", + "https://rfdetr.roboflow.com/latest/learn/run/detection/", + "https://rfdetr.roboflow.com/latest/learn/run/segmentation/", + "https://rfdetr.roboflow.com/latest/learn/train/", + "https://rfdetr.roboflow.com/latest/learn/export/", + "https://rfdetr.roboflow.com/latest/learn/deploy/", + "https://rfdetr.roboflow.com/latest/learn/benchmarks/" + ] + }' || true + + - name: Minimize uv cache + run: uv cache prune --ci diff --git a/.github/workflows/publish-pre-release.yml b/.github/workflows/publish-pre-release.yml new file mode 100644 index 0000000..da0d956 --- /dev/null +++ b/.github/workflows/publish-pre-release.yml @@ -0,0 +1,51 @@ +name: Publish Pre-Releases to PyPI + +on: + push: + tags: + # Matches Semantic Versioning pre-release tags (e.g., 1.2.3a0, 1.2.3.a0, 1.2.3rc2, 1.2.3.rc2) + # Format: MAJOR.MINOR.PATCH[.|]a|b|rcN + - "[0-9]*.[0-9]*.[0-9]*a[0-9]*" + - "[0-9]*.[0-9]*.[0-9]*.a[0-9]*" + - "[0-9]*.[0-9]*.[0-9]*b[0-9]*" + - "[0-9]*.[0-9]*.[0-9]*.b[0-9]*" + - "[0-9]*.[0-9]*.[0-9]*rc[0-9]*" + - "[0-9]*.[0-9]*.[0-9]*.rc[0-9]*" + workflow_dispatch: + pull_request: + branches: ["main", "release/latest", "develop"] + paths: + - ".github/workflows/build-package.yml" + - ".github/workflows/publish-pre-release.yml" + +permissions: + contents: read + +jobs: + build-pkg: + uses: ./.github/workflows/build-package.yml + + publish-pre-release: + name: Publish Pre-release Package + needs: build-pkg + runs-on: ubuntu-latest + environment: + name: test + url: https://pypi.org/project/rfdetr/ + timeout-minutes: 10 + permissions: + id-token: write # Required for PyPI publishing + steps: + - name: 📥 Download distribution artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dist + path: dist/ + - name: Display distribution files + run: ls -lh dist/ + + - name: 🚀 Publish to PyPi + if: github.event_name != 'pull_request' + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + attestations: true diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 0000000..7df611a --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,49 @@ +name: Publish Releases to PyPI + +on: + release: + types: [published] + workflow_dispatch: + pull_request: + branches: ["main", "release/latest", "develop"] + paths: + - ".github/workflows/build-package.yml" + - ".github/workflows/publish-release.yml" + +jobs: + build-pkg: + uses: ./.github/workflows/build-package.yml + + publish-release: + name: Publish Release Package + needs: build-pkg + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/project/rfdetr/ + timeout-minutes: 10 + permissions: + id-token: write # Required for PyPI publishing + contents: write # Add this line + steps: + - name: 📥 Download distribution artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dist + path: dist/ + - name: Display distribution files + run: ls -lh dist/ + + - name: 🚀 Publish to PyPi + # Only publish non-pre-releases + if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + attestations: true + + - name: 📤 Upload assets to GitHub Release + if: github.event_name == 'release' + uses: AButler/upload-release-assets@3d6774fae0ed91407dc5ae29d576b166536d1777 # v3.0 + with: + files: "dist/*.whl;dist/*.tar.gz" + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a3062b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,219 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +docs/adr/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Jetbrain's IDEs +.idea/ + +# model artifacts +rf-detr* +output/* + +train_test.py + +# test artifacts +test_visualizations/ + +# datasets +data/ + +debugging/ + +# AI assistant +.claude/ + +# Local docs development (versions.json is managed by mike on gh-pages in production) +docs/versions.json + +# do not upload python version or notebooks +notebooks/ + +# tracking work in progress +.codex/logs +.plans/ +.reports/ +.temp/ +cache-gh/ +checkpoints/ +releases/ +tasks/ + +# local demo files +demo_*.jpg +demo_*.py +demo_*.ipynb + +# original notebooks before conversion from scripts +docs/cookbooks/*.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..482e430 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,98 @@ +ci: + autofix_prs: true + autoupdate_schedule: monthly + autofix_commit_msg: "fix(pre-commit): 🎨 auto format pre-commit hooks" + autoupdate_commit_msg: "chore(pre-commit): ⬆ pre_commit autoupdate" + skip: [mypy] + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: check-yaml + args: ["--unsafe"] + - id: check-executables-have-shebangs + - id: check-json + - id: check-toml + - id: check-case-conflict + - id: check-added-large-files + args: ["--maxkb=200", "--enforce-all"] + exclude: uv.lock + - id: detect-private-key + - id: pretty-format-json + args: ["--autofix", "--no-sort-keys", "--indent=4"] + exclude: \.ipynb$ + - id: end-of-file-fixer + - id: mixed-line-ending + + - repo: https://github.com/JoC0de/pre-commit-prettier + rev: v3.9.4 # using tag; previously pinned SHA when tags were not persistent + hooks: + - id: prettier + files: \.(ya?ml|toml)$ + # https://prettier.io/docs/en/options.html#print-width + args: ["--print-width=120"] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.20 + hooks: + - id: ruff-check + args: ["--exit-non-zero-on-fix"] + exclude: ^notebooks/.*\.py$ + - id: ruff-format + exclude: ^notebooks/.*\.py$ + + - repo: https://github.com/PyCQA/docformatter + rev: v1.7.8 + hooks: + - id: docformatter + language_version: python3.10 + additional_dependencies: [tomli] + args: ["--in-place"] + + - repo: local + hooks: + - id: mypy + name: mypy + # Keep the mypy requirement in pyproject.toml aligned when changing this hook. + entry: python -m mypy src/rfdetr/ --no-error-summary + language: system + pass_filenames: false + + - repo: https://github.com/executablebooks/mdformat + rev: 1.0.0 + hooks: + - id: mdformat + exclude: ^docs/reference/ + additional_dependencies: + - "mdformat-mkdocs[recommended]>=2.1.0" + - "mdformat-ruff" + args: ["--number"] + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.2 + hooks: + - id: codespell + # args: [--write-changes] + additional_dependencies: + - tomli + + #- repo: https://github.com/mwouts/jupytext + # rev: v1.16.4 + # hooks: + # - id: jupytext + # args: [--update, --to, notebook] + # files: ^notebooks/.*\.py$ + + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.6 + hooks: + - id: insert-license + files: \.py$ + exclude: ^notebooks/ + args: + - --license-filepath + - .github/LICENSE_HEADER.txt + - --comment-style + - "#" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d9151a1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,326 @@ +# RF-DETR - Agent Instructions + +This file provides detailed technical context for AI coding agents working with RF-DETR. + +**Canonical Sources:** + +- **Contribution Guidelines:** [CONTRIBUTING.md](.github/CONTRIBUTING.md) - The authoritative source for all contribution practices +- **Human Documentation:** [README.md](README.md) - Project overview and usage +- **Copilot Instructions:** [.github/copilot-instructions.md](.github/copilot-instructions.md) - GitHub Copilot-specific guidance + +This document supplements the contribution guidelines with detailed technical information for automated tooling. + +## Agent Responsibilities + +As an AI agent contributing to RF-DETR, you are responsible for: + +1. **Following test-driven development practices** + + - Write failing tests first for bug fixes + - Write comprehensive tests for new features + - Ensure final PR commit has all tests passing + +2. **Adhering to code quality standards** + + - Run `pre-commit run --all-files` before every commit + - Follow type hint and docstring requirements + - Prefer direct project imports; conventional third-party aliases are allowed + +3. **Maintaining agentic documentation** + + - Update `AGENTS.md` when architecture patterns or technical conventions change + - Update `.github/copilot-instructions.md` when high-level guidance changes + - Update `.github/CONTRIBUTING.md` when human workflow is affected + - Apply updates after receiving major feedback in PR reviews + +4. **Consulting maintainers before major changes** + + - Open an issue before adding new models or significant features + - Wait for approval on approach before implementing + +5. **Writing secure, minimal code** + + - Avoid over-engineering and unnecessary abstractions + - Write secure code (prevent injection vulnerabilities) + - Follow existing patterns in the codebase + +> [!NOTE] +> Keeping documentation current ensures consistency across agent contributions and reduces repeated feedback on the same issues. + +## Build & Development Environment + +> [!NOTE] +> **Canonical Reference:** See [Development Environment Setup](.github/CONTRIBUTING.md#development-environment-setup) in CONTRIBUTING.md for complete setup instructions. + +### Setup + +```bash +# Install uv (if not already installed) +pip install uv + +# Full development environment (always use this) +uv sync --all-groups +``` + +**Prerequisites:** Python >=3.10 (tested on 3.10-3.13) + +### Dependency Information + +See `pyproject.toml` for complete dependency specifications: + +- **Core:** PyTorch, torchvision, transformers, supervision, pydantic, pyDeprecate +- **Optional:** `[train]` (training, including peft and pycocotools), `[lora]` (LoRA fine-tuning), `[plus]` (Plus models), `[onnx]` (ONNX export), `[loggers]` (tensorboard, wandb, mlflow, clearml) +- **Development:** `tests`, `docs`, `build` groups + +**Important version constraints:** + +- PyTorch: >=2.2.0, \<3.0.0 +- Transformers: >=5.0.0, \<6.0.0 + +## Testing + +> [!NOTE] +> **Canonical Reference:** See [Test-Driven Development](.github/CONTRIBUTING.md#test-driven-development) in CONTRIBUTING.md for complete guidelines. +> +> **CI Workflows (Source of Truth):** See `.github/workflows/ci-tests-cpu.yml` and `.github/workflows/ci-tests-gpu.yml` for exact test commands used in CI. + +### Commands + +```bash +# CPU tests (default for local development; mirrors CI) +uv run --no-sync pytest src/ tests/ -n 1 -m "not gpu" --ignore=tests/run_smoke_all_models.py --cov=rfdetr --cov-report=xml --timeout=240 --durations=50 + +# GPU tests (requires GPU; mirrors CI) +uv run --no-sync pytest tests/ -m gpu -n 2 --reruns 1 --only-rerun "OutOfMemoryError" --cov=rfdetr --cov-report=xml --timeout=600 --durations=20 + +# Pre-commit checks (ALWAYS run before committing) +pre-commit run --all-files +``` + +### Testing Principles + +> [!IMPORTANT] +> **Testing Requirements:** +> +> - ⚠️ **During development:** Tests may fail as you work through TDD cycle +> - ✅ **Before opening PR:** Final commit MUST have all tests passing +> - ✅ **Before each commit:** Run `pre-commit run --all-files` + +**Test-Driven Development:** + +1. **Bug fixes:** Write failing test → Fix code → Verify all tests pass +2. **New features:** Write comprehensive tests → Implement feature → Refactor + +**Test Organization:** + +- Group related tests in classes +- Use `@pytest.mark.parametrize` with `pytest.param(..., id="name")` +- Mark GPU/heavy tests with `@pytest.mark.gpu` +- Avoid multiple validation cases in a single test - see [CONTRIBUTING.md](.github/CONTRIBUTING.md#avoid-multiple-validation-cases-in-a-single-test) for details + +**CI Information:** +See [CI Testing](.github/CONTRIBUTING.md#ci-testing) in CONTRIBUTING.md for details on OS/Python version matrix and workflow configurations. + +## Code Quality & Linting + +> [!NOTE] +> **Canonical Reference:** See [Code Quality and Linting](.github/CONTRIBUTING.md#code-quality-and-linting) in CONTRIBUTING.md for setup and details. + +### Command + +```bash +# Always run full pre-commit (not individual tools) +pre-commit run --all-files +``` + +> [!TIP] +> Pre-commit hooks will auto-format many issues. Review changes and re-stage files. + +**Configuration Files:** + +- `.pre-commit-config.yaml` - Pre-commit hooks (ruff, mdformat, prettier, codespell, license headers) +- `pyproject.toml` - Ruff linting rules (`[tool.ruff]` section) + +**License Header (required for all Python files):** + +```python +# ------------------------------------------------------------------------ +# RF-DETR +# Copyright (c) 2025 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +``` + +## Documentation + +### Building Docs + +```bash +# Full install (matches CI — required for XLarge/2XLarge model pages) +uv pip install -e ".[plus]" --group docs + +# Serve locally (live reload) +uv run mkdocs serve + +# Build static site +uv run mkdocs build +``` + +**Documentation Structure:** + +- **Source:** `docs/` directory (Markdown) +- **Config:** `mkdocs.yaml` (uses custom YAML tags: `!!python/name`) +- **Deployment:** GitHub Actions publishes to GitHub Pages + +**Note:** `mkdocs.yaml` is checked by the `check-yaml` pre-commit hook with `--unsafe` so custom YAML tags such as `!!python/name` are accepted. + +## Package Building + +```bash +# Install build dependencies +uv sync --group build + +# Build distributions +uv build + +# Validate build +uv run twine check --strict dist/* +``` + +**Build outputs:** + +- Source distribution: `dist/rfdetr-*.tar.gz` +- Wheel: `dist/rfdetr-*.whl` + +## Project Structure + +> [!NOTE] +> **Canonical Reference:** See [Project Structure](.github/CONTRIBUTING.md#project-structure) in CONTRIBUTING.md for complete project organization, directory descriptions, and configuration files. +> +> **Quick summary:** `src/rfdetr/` (source code), `tests/` (test suite), `docs/` (documentation), `.github/` (CI/CD), `pyproject.toml` (dependencies and config). +> +> Internal package organization within `src/rfdetr/` is subject to change as this is an active research and development project. + +## Architecture & Conventions + +### Key Patterns + +**Model Architecture:** + +- RFDETR wrappers: `self.model` is the model context returned by `get_model()` +- Underlying PyTorch module: `self.model.model` +- Segmentation models return `pred_masks` as `torch.Tensor` or dict with keys `['spatial_features', 'query_features', 'bias']` + +**Imports:** + +```python +# Prefer direct project imports. Standard aliases such as `numpy as np`, +# `torch.nn.functional as F`, and lazy module aliases are allowed when conventional. +from rfdetr.util.misc import get_rank, get_world_size, is_main_process, save_on_master +from rfdetr.util.logger import get_logger + +# Logger usage +logger = get_logger() # Default name: "rf-detr", reads LOG_LEVEL env var + +# TQDM (environment compatibility) +from tqdm.auto import tqdm # NOT: from tqdm import tqdm +``` + +**Plus Models (XLarge, 2XLarge):** + +- Requires separate `rfdetr_plus` package (PML 1.0 license) +- Import handled lazily via `__getattr__` in `src/rfdetr/platform/models.py` +- Raises `ImportError` if package not installed + +**Subprocess Usage:** + +```python +import subprocess + +result = subprocess.run( + ["command", "arg1", "arg2"], + check=True, # Raise CalledProcessError on failure + text=True, # Return stdout/stderr as strings + capture_output=True, +) +# Note: stderr is already a string, don't decode +``` + +**Logging:** + +- Use `logger.debug()` for detailed tensor/shape information (not `logger.info()`) +- Use `logger.info()` for high-level progress/status + +**Checkpoint Handling:** + +- Always check file existence before operations +- Prevents errors when training is interrupted + +### Type Hints & Docstrings + +> [!IMPORTANT] +> **Canonical Reference:** See [Google-Style Docstrings and Mandatory Type Hints](.github/CONTRIBUTING.md#google-style-docstrings-and-mandatory-type-hints) in CONTRIBUTING.md for complete requirements and examples. + +**Requirements:** + +- MANDATORY type hints for all function parameters and return types +- MANDATORY Google-style docstrings for all functions and classes +- **Do not duplicate types in docstrings** - types are in the function signature +- Target Python version: 3.10+ + +## Common Workflows + +### Making Changes + +1. **Setup:** `uv sync --all-groups` +2. **Before changes:** Run tests to establish baseline +3. **Development:** + - Make minimal, focused changes + - Follow existing patterns and conventions + - Add type hints and docstrings +4. **Testing:** + - Bug fixes: Write test first, then fix + - Features: Test all major use cases + - Run: `uv run --no-sync pytest src/ tests/ -n 2 -m "not gpu" --ignore=tests/run_smoke_all_models.py --timeout=240 --durations=50` +5. **Quality checks:** `pre-commit run --all-files` +6. **Build (if needed):** `uv build` +7. **Commit:** Pre-commit hooks run automatically + +### Adding New Model Variants + +> [!IMPORTANT] +> **Canonical Reference:** See [Adding a New Model](.github/CONTRIBUTING.md#adding-a-new-model) in CONTRIBUTING.md for detailed guidance. +> +> Always consult maintainers before implementing new models. + +### Security Considerations + +- **Write secure code:** Avoid injection vulnerabilities (XSS, SQL injection, command injection) +- **Validate inputs:** Especially for file paths, URLs, and user-provided data +- **No credentials:** Never commit API keys, tokens, or credentials +- **Follow OWASP best practices** + +## CI/CD Workflows + +GitHub Actions workflows in `.github/workflows/`: + +- **ci-tests-cpu.yml:** CPU tests across OS/Python versions +- **ci-tests-gpu.yml:** GPU-dependent tests +- **build-package.yml:** Build and validate distributions +- **ci-build-docs.yml:** Documentation builds +- **publish-docs.yml:** Deploy docs to GitHub Pages + +**Concurrency:** PRs cancel in-progress runs on new pushes + +## Additional Resources + +- **Documentation:** https://rfdetr.roboflow.com +- **Repository:** https://github.com/roboflow/rf-detr +- **Issues:** https://github.com/roboflow/rf-detr/issues +- **Discord:** https://discord.gg/GbfgXGJ8Bk +- **Contributing:** `.github/CONTRIBUTING.md` +- **Copilot Instructions:** `.github/copilot-instructions.md` + +--- + +**Note:** This file is designed for AI coding agents. For human-readable project information, see README.md. For contribution guidelines, see CONTRIBUTING.md. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a8a7f7f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,423 @@ +# Changelog + +All notable changes to RF-DETR are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.8.3] — 2026-06-27 + +### Added + +- `trust_checkpoint: bool = False` keyword-only parameter on `RFDETR.from_checkpoint()` to opt into full pickle deserialization for trusted checkpoint files. +- `optimize_for_inference(inplace=True)` — new keyword-only argument on `RFDETR.optimize_for_inference()`; skips the deep-copy of the base model for memory-constrained inference-only deployments (~0.5× model-weight peak memory reduction). Requires `compile=False`. After inplace optimization, `export()` raises `RuntimeError` and `remove_optimized_model()` issues a `UserWarning` and returns cleanly instead of silently clearing state. New `RFDETR.is_optimized_inplace` property returns `True` after a successful inplace optimization. ([#1089](https://github.com/roboflow/rf-detr/pull/1089)) +- `CocoKeypointSchema.keypoint_flip_pairs` and `YoloKeypointSchema.keypoint_flip_pairs` fields — horizontal-flip swap pairs inferred automatically from keypoint names (left/right naming convention) for COCO schemas, and from `flip_idx` permutation for YOLO schemas. Auto-populated by `infer_coco_keypoint_schema` and `infer_yolo_keypoint_schema` respectively. ([#1164](https://github.com/roboflow/rf-detr/pull/1164)) +- `infer_coco_keypoint_schema` and `infer_yolo_keypoint_schema` re-exported from `rfdetr.datasets` (previously only accessible from `rfdetr.datasets._keypoint_schema`). ([#1164](https://github.com/roboflow/rf-detr/pull/1164)) + +### Changed + +- `RFDETR.from_checkpoint()` now uses safe deserialization by default (`weights_only=True`). Checkpoints containing custom Python objects beyond `argparse.Namespace` or `types.SimpleNamespace` must pass `trust_checkpoint=True`. Previously, full pickle deserialization was always used. +- Horizontal flip detection in `AlbumentationsWrapper` now uses Albumentations `ReplayCompose` replay metadata instead of heuristic bbox-center mirroring; eliminates false positives on non-flip transforms that shift box centers. Falls back to `alb.Compose` with a `UserWarning` when `albumentations <1.3` is detected. ([#1164](https://github.com/roboflow/rf-detr/pull/1164)) +- Keypoint schema inference now supports native COCO format (`dataset_file="coco"`) in addition to `"roboflow"` and `"yolo"`. ([#1164](https://github.com/roboflow/rf-detr/pull/1164)) +- `_keypoint_schema_cache` key changed from `dataset_dir` (string) to `(dataset_file, dataset_dir)` tuple to prevent cross-format cache collisions when the same directory is used with different dataset formats. ([#1164](https://github.com/roboflow/rf-detr/pull/1164)) + +### Fixed + +- Predicted bounding boxes are now clamped to image bounds `[0, width] × [0, height]` in `PostProcess._postprocess_boxes()`; model regression is unbounded and could previously produce negative or out-of-frame coordinates. `scale_fct` is also cast to `boxes.dtype` before multiplication to prevent dtype mismatch when boxes are `float16`. ([#1168](https://github.com/roboflow/rf-detr/pull/1168)) +- `SegmentationTrainConfig.cls_loss_coef` default corrected from `5.0` to `1.0` to restore the pre-v1.7 effective classification loss weight. The `5.0` value was present in `SegmentationTrainConfig` since v1.6 but was dead code until the v1.7 TrainConfig ownership migration activated it, silently over-penalising classification relative to mask losses during segmentation fine-tuning. To reproduce pre-fix behaviour, pass `cls_loss_coef=5.0` explicitly. ([#1165](https://github.com/roboflow/rf-detr/pull/1165)) +- `KeypointTrainConfig.keypoint_nll_loss_coef` default restored to `1.0` to align with the other keypoint loss terms (`keypoint_l1_loss_coef`, `keypoint_findable_loss_coef`, `keypoint_visible_loss_coef`). The previous default of `0.5` was set to dampen OKS@75 oscillation but under-weighted the NLL loss relative to other terms in practice. ([#1165](https://github.com/roboflow/rf-detr/pull/1165)) + +--- + +## [1.8.2] — 2026-06-25 + +### Added + +- YOLO pose keypoint dataset support: load Ultralytics YOLO pose datasets (`.yaml` with `kpt_shape`) directly for keypoint fine-tuning. Schema is inferred automatically via `infer_yolo_keypoint_schema`. ([#1156](https://github.com/roboflow/rf-detr/pull/1156)) +- `is_bg_first_schema`, `to_active_first`, `to_bg_first`, `schemas_semantically_equal` utilities in `rfdetr.utilities.keypoints` (and re-exported from `rfdetr.utilities`) for schema-aware keypoint processing. ([#1160](https://github.com/roboflow/rf-detr/pull/1160)) +- `amp_dtype` field on `TrainConfig` (`"auto"` / `"bf16"` / `"fp16"`): pin the mixed-precision autocast dtype instead of relying on device-capability auto-detection. `"auto"` (default) preserves the historical behaviour — `bf16-mixed` on Ampere+ CUDA, `16-mixed` otherwise. Invalid values degrade gracefully to `"auto"` with a `UserWarning`. ([#1143](https://github.com/roboflow/rf-detr/pull/1143)) +- Instance segmentation fine-tuning cookbook (`docs/cookbooks/fine-tune_segmentation.ipynb`) — end-to-end walkthrough using `RFDETRSegSmall` across seven diverse segmentation datasets. ([#1159](https://github.com/roboflow/rf-detr/pull/1159)) +- Inference latency benchmark cookbook (`docs/cookbooks/inference-latency-benchmark.ipynb`) — benchmarks CPU/GPU throughput across model sizes with reproducible measurement methodology. ([#1152](https://github.com/roboflow/rf-detr/pull/1152)) + +### Changed + +- Default `num_keypoints_per_class` in `RFDETRKeypointPreviewConfig` changed from `[0, 17]` (background-first) to `[17]` (active-first). Legacy bg-first checkpoints auto-align on load via `_kp_active_mask`. ([#1160](https://github.com/roboflow/rf-detr/pull/1160)) + +### Fixed + +- `RFDETR.from_checkpoint()` now correctly infers `num_classes` and `num_keypoints_per_class` from checkpoint weights (`class_embed.weight.shape[0] - 1` and `_kp_active_mask` respectively). Previously, `num_classes` was read as `shape[0]` (i.e. `num_classes + 1` including the background class), causing `load_state_dict` shape mismatches or a silent extra output class on every load. `BestModelCallback._serialize_model_config` is also fixed to persist the correct foreground-only `num_classes`. ([#1158](https://github.com/roboflow/rf-detr/pull/1158)) +- `HungarianMatcher.forward()` now uses the configured `focal_alpha` in the focal classification matching cost. Previously the value was hardcoded to `0.25`, silently ignoring any non-default `focal_alpha` passed to the constructor or `build_matcher`. This misaligned the bipartite matching cost with the focal classification loss in `criterion.py`, which correctly used `self.focal_alpha`. ([#1147](https://github.com/roboflow/rf-detr/pull/1147)) +- `spatial_shapes` in `Transformer.forward()` is now built from symbolic `Shape` ops (`torch.stack` of per-level `torch._shape_as_tensor` slices) instead of `torch.empty` + in-place index assignment. The previous pattern emitted a `ScatterND` feeding a shape tensor (`level_start_index`), which TensorRT rejected with "IScatterLayer cannot be used to compute a shape tensor". This fix is required to export any RF-DETR model to a TensorRT engine. ([#1155](https://github.com/roboflow/rf-detr/pull/1155)) +- Keypoint model inference now returns the correct `class_name` field in predictions. ([#1151](https://github.com/roboflow/rf-detr/pull/1151)) +- `predict()` now re-asserts eval mode before each call for unoptimized models, preventing silent train-mode inference after the first prediction. ([#1146](https://github.com/roboflow/rf-detr/pull/1146)) +- TFLite inference preprocessing and mask decoder aligned with PyTorch `predict()` behaviour. ([#1131](https://github.com/roboflow/rf-detr/pull/1131)) +- Python version mismatch in optional-dependency version overrides resolved. ([#1137](https://github.com/roboflow/rf-detr/pull/1137)) + +--- + +## [1.8.1] — 2026-06-19 + +### Changed + +- Config path parameters (e.g. `dataset_dir`, `output_dir`, `pretrain_weights`) now accept `pathlib.Path` objects in addition to strings. Paths are coerced to `str` automatically via the `expand_paths` validator. No API changes required; existing string usage unaffected. ([#1124](https://github.com/roboflow/rf-detr/pull/1124)) +- Keypoint training now disables horizontal flip augmentation until keypoint flip-pair swapping is implemented. Previously, flipping was applied without reordering keypoint pairs, producing incorrect labels. ([#1122](https://github.com/roboflow/rf-detr/pull/1122)) +- Training metric plots improved with optional seaborn error bands, AP@0.75 metric grouping, and custom AP metric group configuration. ([#1122](https://github.com/roboflow/rf-detr/pull/1122)) + +### Fixed + +- Keypoint encoder in eval mode now routes all queries through head 0 instead of splitting across group heads. Previously, `group_detr = len(self.enc_out_keypoint_embed)` caused the encoder keypoint path to split `num_queries` queries across all group heads in eval; now uses `if self.training else 1` guard. ([#1135](https://github.com/roboflow/rf-detr/pull/1135)) +- `config.use_return_dict` (deprecated in `transformers`) replaced with `config.return_dict` in DINOv2 windowed attention backbone. ([#1135](https://github.com/roboflow/rf-detr/pull/1135)) +- Epoch metric tables now display correctly when a Rich progress bar callback is active. Tables are printed through the progress bar's owned Rich console, preventing cursor conflicts with active live displays. ([#1128](https://github.com/roboflow/rf-detr/pull/1128)) +- Keypoint fine-tuning checkpoint selection stabilised with smoothed (EMA) best-metric comparison to avoid spurious checkpoint switches on noisy OKS metrics. Smoothing state is correctly restored on training resume. ([#1122](https://github.com/roboflow/rf-detr/pull/1122)) +- Group DETR train-time metric evaluation now evaluates only the primary query group, preventing crashes on non-tensor mask outputs from auxiliary decoder layers. ([#1122](https://github.com/roboflow/rf-detr/pull/1122)) +- `_detect_horizontal_flip` in Albumentations transform pipeline now uses `len(bboxes) == 0` instead of `not bboxes` to correctly handle Albumentations 2.x where bboxes is a NumPy array (falsy even when non-empty). ([#1126](https://github.com/roboflow/rf-detr/pull/1126)) +- TensorBoard logger is now disabled gracefully when `tensorboard` is installed alongside a NumPy-2.0-incompatible `tensorflow`. Training degrades to CSV-only logging with a clear warning instead of crashing inside `_log_hyperparams`. ([#1123](https://github.com/roboflow/rf-detr/pull/1123)) + +--- + +## [1.8.0] — 2026-06-13 + +### Added + +- `RFDETRKeypointPreview` — keypoint detection model variant with GroupPose-style head, covariance-based uncertainty (precision-Cholesky parameterization), and COCO keypoint AP evaluation. Public config classes: `KeypointTrainConfig`, `RFDETRKeypointPreviewConfig` (from `rfdetr.config`). Utility: `precision_cholesky_to_pixel_covariance` (from `rfdetr.utilities`). Schema helpers `infer_coco_keypoint_schema`, `CocoKeypointSchema`, `active_keypoint_counts` accessible via `rfdetr.datasets._keypoint_schema`. ([#1099](https://github.com/roboflow/rf-detr/pull/1099)) +- `RFDETR.export_for_roboflow(output_dir)` — writes a Roboflow upload bundle (`weights.pt` + `class_names.txt`) without a network call; extracted from `deploy_to_roboflow`, which now delegates to it. ([#1086](https://github.com/roboflow/rf-detr/pull/1086)) +- Keypoint fine-tuning cookbook (`docs/cookbooks/fine-tune_keypoints.ipynb`) — end-to-end walkthrough: dataset download, schema inference, `KeypointTrainConfig`, training metrics, and inference with covariance uncertainty. ([#1104](https://github.com/roboflow/rf-detr/pull/1104)) +- `MetricKeypointOKS` — reusable OKS metric facade over `CocoEvaluator`, exported from `rfdetr.evaluation`. Supports arbitrary keypoint counts, per-category OKS sigma values, DDP-safe evaluation with first-rank-wins deduplication, and an `OKSKey` enum (`mAP`, `mAP@50`, `mAP@75`, `mAR`) for standardised metric keys. ([#1107](https://github.com/roboflow/rf-detr/pull/1107)) + +### Changed + +- DDP strategy now enables `find_unused_parameters=True` for all detection, keypoint, and segmentation models when running under `strategy='ddp'` or `strategy='auto'` with a distributed launcher. Previously only enabled for segmentation. Opt out via `trainer_kwargs={"strategy": DDPStrategy(find_unused_parameters=False)}`. ([#1094](https://github.com/roboflow/rf-detr/pull/1094)) +- `rfdetr.datasets.aug_config` module renamed to `rfdetr.datasets.aug_configs` (plural). Direct imports from `rfdetr.datasets.aug_config` must be updated to `rfdetr.datasets.aug_configs`; the augmentation preset constants (`AUG_AGGRESSIVE`, etc.) are unchanged. ([#1103](https://github.com/roboflow/rf-detr/pull/1103)) + +### Removed + +- `RFDETR.export(simplify=..., force=...)` — both kwargs removed from the signature. Deprecated since v1.6.0 with `remove_in="1.8.0"`; both were no-ops during the deprecation window. Callers passing these args must remove them before upgrading. ([#1102](https://github.com/roboflow/rf-detr/pull/1102)) + +### Fixed + +- `RFDETR.from_checkpoint()` no longer treats `num_classes` loaded from the checkpoint as a user-supplied override. Previously, fine-tuning a checkpoint model on a dataset with a different class count was silently refused — the head refused to re-initialise and trained against the stale class count. An explicit `num_classes` kwarg from the caller still wins over both the checkpoint value and the dataset. ([#1106](https://github.com/roboflow/rf-detr/pull/1106)) +- Scale jitter restored in non-square training crop. `RandomCrop` in the `option_b` branch replaced with `RandomSizedCrop`, restoring the scale-augmentation behaviour lost during the Albumentations migration. ([#1088](https://github.com/roboflow/rf-detr/pull/1088)) +- Multi-GPU validation deadlock in COCO mAP synchronization prevented. `_merge_metric_state_across_ranks` now safe across zero-batch ranks. ([#1085](https://github.com/roboflow/rf-detr/pull/1085)) +- `import rfdetr` no longer fails on NumPy 2.x when a transitive dependency references the removed `np.complex_` alias. ([#1064](https://github.com/roboflow/rf-detr/pull/1064)) +- `rfdetr_plus` module availability check corrected; false-positive hit when the package was partially installed. ([#1083](https://github.com/roboflow/rf-detr/pull/1083)) +- Fixed spurious "Keypoint class-logit boost has N classes but detection head has M" warning on custom (non-Roboflow) keypoint datasets: `_align_num_classes_from_dataset` now zero-pads `num_keypoints_per_class` when auto-adjusting `num_classes` beyond the schema length ([#1113](https://github.com/roboflow/rf-detr/pull/1113)) +- Loss scaling corrected for keypoint training under gradient accumulation (`accumulate_grad_batches > 1`). Keypoint models now use manual optimization to normalize losses by the accumulated box count across the effective batch; detection and segmentation remain on Lightning's automatic-optimization path. Optimizer-step scheduling, LR warmup/decay, and epoch-boundary flushing are correctly handled in both paths. ([#1117](https://github.com/roboflow/rf-detr/pull/1117)) +- Device auto-detection now verifies accelerator runtime availability before selecting a device (PyTorch ≥ 2.4: `torch.accelerator.current_accelerator`; older builds: `torch.cuda.is_available()`). Previously, a machine with CUDA headers but no GPU driver could be assigned a CUDA device that fails at first use. ([#1111](https://github.com/roboflow/rf-detr/pull/1111)) +- `RFDETR.from_checkpoint()` and related APIs now honor an explicit `num_classes` argument even when its value equals the model default. Previously, passing `num_classes=N` where `N` is the default (e.g. 80 for COCO) was silently treated as unset, causing fine-tuning on a different class count to be refused. ([#1109](https://github.com/roboflow/rf-detr/pull/1109)) +- `RFDETR.from_checkpoint()` correctly infers the model variant from the checkpoint filename when `pretrain_weights` is absent or unset-like (empty string, `None`, whitespace). Previously, starter-like checkpoints without an explicit `pretrain_weights` entry raised an error or silently loaded the wrong model class. ([#1065](https://github.com/roboflow/rf-detr/pull/1065)) + +--- + +## [1.7.0] — 2026-04-29 + +### Added + +- `augmentation_backend` field on `TrainConfig` (`"cpu"` / `"auto"` / `"gpu"`): opt-in GPU-side augmentation via [Kornia](https://kornia.readthedocs.io) applied in `RFDETRDataModule.on_after_batch_transfer` after the batch is resident on the GPU. CPU path is unchanged and remains the default. Install with `pip install 'rfdetr[kornia]'`. Supports detection and segmentation (see below). ([#1003](https://github.com/roboflow/rf-detr/pull/1003)) +- Kornia GPU augmentation now supports instance segmentation: images, boxes, and per-instance masks are augmented in sync on the GPU. New public helper `collate_masks` packs `[N_i, H, W]` boolean masks into a `[B, N_max, H, W]` float32 tensor for Kornia; `build_kornia_pipeline` gains a `with_masks: bool = False` parameter; `unpack_boxes` gains an optional `masks_aug` tensor that re-binarises and filters masks in sync with boxes. Previously `augmentation_backend="gpu"/"auto"` was silently ignored for segmentation models; now it works identically to detection. **Note**: the mask buffer is `[B, N_max, H, W]` float32 — approximately 500 MB at `B=8, N_max=50, H=W=560`; use `augmentation_backend="cpu"` on cards with limited VRAM. ([#1003](https://github.com/roboflow/rf-detr/pull/1003), closes [#997](https://github.com/roboflow/rf-detr/issues/997)) +- `BuilderArgs` — a `@runtime_checkable` `typing.Protocol` documenting the minimum attribute set consumed by `build_model()`, `build_backbone()`, `build_transformer()`, and `build_criterion_and_postprocessors()`. Enables static type-checker support for custom builder integrations. Exported from `rfdetr.models`. ([#841](https://github.com/roboflow/rf-detr/pull/841)) +- `build_model_from_config(model_config, train_config=None, defaults=MODEL_DEFAULTS)` — config-native alternative to `build_model(build_namespace(mc, tc))`; accepts Pydantic config objects directly and constructs the internal namespace automatically. Exported from `rfdetr.models`. ([#845](https://github.com/roboflow/rf-detr/pull/845)) +- `build_criterion_from_config(model_config, train_config, defaults=MODEL_DEFAULTS)` — config-native alternative to `build_criterion_and_postprocessors(build_namespace(mc, tc))`; returns a `(SetCriterion, PostProcess)` tuple. Exported from `rfdetr.models`. ([#845](https://github.com/roboflow/rf-detr/pull/845)) +- `ModelDefaults` dataclass — exposes the 35 hardcoded architectural constants previously buried inside `build_namespace()`. Pass a `dataclasses.replace(MODEL_DEFAULTS, ...)` override to the new config-native builders to customise individual constants. **Note:** fields may be promoted to `ModelConfig`/`TrainConfig` in future phases. Exported from `rfdetr.models`. ([#845](https://github.com/roboflow/rf-detr/pull/845)) +- `MODEL_DEFAULTS` — the canonical `ModelDefaults` singleton with production defaults. Exported from `rfdetr.models`. ([#845](https://github.com/roboflow/rf-detr/pull/845)) +- `RFDETR.predict(include_source_image=...)` — opt-out flag (default `True`) to skip storing the source image in `detections.metadata["source_image"]`; set to `False` to reduce memory use when the image is not needed for annotation. ([#912](https://github.com/roboflow/rf-detr/pull/912)) +- `model_name` is now stored in checkpoint files during training so that `RFDETR.from_checkpoint()` can resolve the correct model class directly from the checkpoint, without requiring the caller to know or pass a class hint. `strip_checkpoint()` preserves this key. Backward-compatible: checkpoints without `model_name` continue to resolve via `pretrain_weights` filename matching. ([#895](https://github.com/roboflow/rf-detr/pull/895)) +- `rfdetr_version` is now stored in checkpoint files during training for provenance tracking and compatibility hints. `strip_checkpoint()` preserves this key. The key is omitted gracefully when the package version cannot be resolved (e.g. editable install without metadata). Backward-compatible: checkpoints without `rfdetr_version` continue to load normally. ([#918](https://github.com/roboflow/rf-detr/pull/918)) +- `notes` parameter on `RFDETR.train()` and `RFDETR.export()` — embed arbitrary JSON-serialisable provenance metadata (labeller, date, class names, etc.) into best-model `.pth` checkpoints (under `checkpoint["args"]["notes"]`) and ONNX files (under the `"rfdetr_notes"` metadata property). String values are stored verbatim; all other types are JSON-encoded. ([#1025](https://github.com/roboflow/rf-detr/pull/1025), closes [#1021](https://github.com/roboflow/rf-detr/issues/1021)) +- `RF_HOME` environment variable controls where pretrained model weights are cached (default: `~/.roboflow/models`). Bare filenames passed as `pretrain_weights` (e.g. `"rf-detr-base.pth"`) are now resolved relative to this directory; paths with a directory component are used as-is with parent directories created automatically. ([#130](https://github.com/roboflow/rf-detr/pull/130)) +- Grayscale and multispectral imagery support: RF-DETR models now accept inputs with any number of channels (not just 3). The pretrained DINOv2 patch-embedding weights are automatically adapted to the specified channel count at model construction time — no additional dependencies required. ([#180](https://github.com/roboflow/rf-detr/pull/180), closes [#75](https://github.com/roboflow/rf-detr/issues/75)) +- Training configuration is now saved to `training_config.json` in the output directory after training completes. The file captures the full `TrainConfig`, `ModelConfig`, effective training parameters, class names, and number of classes — useful for reproducibility and debugging predictions from older checkpoints. ([#194](https://github.com/roboflow/rf-detr/pull/194)) +- `dinov2_registers_windowed_small` backbone is now available as a config option in `ModelConfig.encoder`. ([#236](https://github.com/roboflow/rf-detr/pull/236)) +- `rfdetr.from_checkpoint(path)` — new top-level convenience function that loads a checkpoint and infers the correct model subclass automatically, without requiring the caller to specify a class. Equivalent to `RFDETR.from_checkpoint(path)` but importable directly from the `rfdetr` package. ([#664](https://github.com/roboflow/rf-detr/pull/664)) +- ONNX export filenames now include the model variant name (e.g. `rfdetr-medium.onnx`) instead of the generic `inference_model.onnx`. Exporting multiple variants to the same directory no longer overwrites previous exports. ([#910](https://github.com/roboflow/rf-detr/pull/910)) +- Background images (images without a matching label file) are now included in YOLO detection datasets as empty-detection samples instead of being silently dropped. Both detection and segmentation paths now use `_LazyYoloDetectionDataset` for consistent behaviour. ([#915](https://github.com/roboflow/rf-detr/pull/915)) +- TFLite export via `model.export(format="tflite")`. Converts through ONNX using `onnx2tf`; FP32 and FP16 outputs are always produced, INT8 quantization is available with a calibration image directory: `model.export(format="tflite", quantization="int8", calibration_data="path/to/images/")`. Requires `pip install 'rfdetr[onnx,tflite]'`. ([#920](https://github.com/roboflow/rf-detr/pull/920)) +- PyTorch Lightning `.ckpt` files are now accepted as `pretrain_weights`. Keys are automatically normalized from PTL format (`state_dict` with `model.`-prefixed keys, `hyper_parameters` → `args`) so that `load_pretrain_weights`, class-name extraction, and compatibility checks work without manual conversion. ([#951](https://github.com/roboflow/rf-detr/pull/951)) +- `skip_best_epochs` parameter for `RFDETR.train()` and `TrainConfig`: the first N epochs are excluded from best-checkpoint selection and early-stopping comparison, preventing strong pretrained weights or resumed checkpoints from locking in a suboptimal early score. ([#1000](https://github.com/roboflow/rf-detr/pull/1000), closes [#789](https://github.com/roboflow/rf-detr/issues/789)) +- TFLite inference now decodes segmentation mask outputs into `sv.Detections.mask`. Mask logits are upsampled to the source image size using Pillow bilinear resampling and thresholded at zero, matching the behaviour of `PostProcess.forward`. The mask tensor is detected by output name (`"masks"` substring) with a rank-4 shape fallback. ([#1053](https://github.com/roboflow/rf-detr/pull/1053)) +- `PretrainWeightsCompatibilityWarning` — new warning class emitted when a `ModelConfig` override (e.g. custom `encoder`, `num_queries`, or `num_feature_levels`) risks breaking pretrained weight loading. Importable as `from rfdetr.config import PretrainWeightsCompatibilityWarning` for targeted filtering. ([#1017](https://github.com/roboflow/rf-detr/pull/1017)) + +### Changed + +- `peft` is no longer installed as part of the default `rfdetr` package. It has moved to the `[lora]` and `[train]` optional extras. If you use LoRA fine-tuning, install with `pip install 'rfdetr[lora]'`. ([#838](https://github.com/roboflow/rf-detr/pull/838)) +- Native RLE annotation support in the COCO segmentation pipeline: `convert_coco_poly_to_mask` now explicitly detects and decodes both compressed (string counts) and uncompressed (int-list counts) RLE formats alongside existing polygon support. Malformed annotations now raise instead of being silently swallowed. ([#897](https://github.com/roboflow/rf-detr/pull/897)) +- Pinned PyTorch Lightning to exclude known-compromised versions. ([#1020](https://github.com/roboflow/rf-detr/pull/1020)) + +### Deprecated + +- `build_namespace(model_config, train_config)` — no longer used internally and deprecated in this release; use `build_model_from_config`, `build_criterion_from_config`, or `_namespace_from_configs` directly. It will be removed in v1.9 and currently emits a `DeprecationWarning` on use. ([#845](https://github.com/roboflow/rf-detr/pull/845)) +- `load_pretrain_weights(nn_model, model_config, train_config)` — the `train_config` positional argument is deprecated and will be removed in v1.9; it is no longer used internally. Omit it: `load_pretrain_weights(nn_model, model_config)`. Passing a non-`None` value emits a `DeprecationWarning`. ([#845](https://github.com/roboflow/rf-detr/pull/845)) +- `TrainConfig.group_detr` (architecture decision → `ModelConfig`), `TrainConfig.ia_bce_loss` (loss type tied to architecture family → `ModelConfig`), `TrainConfig.segmentation_head` (architecture flag → `ModelConfig`), `TrainConfig.num_select` (postprocessor count → `ModelConfig`; `SegmentationTrainConfig` users: remove the `num_select` override — the model config value is always used), `ModelConfig.cls_loss_coef` (training hyperparameter → `TrainConfig`) — each now emits `DeprecationWarning` when set on the wrong config object and will be **removed** in v1.9. ([#841](https://github.com/roboflow/rf-detr/pull/841)) +- `RFDETRBase` — use `RFDETRNano`, `RFDETRSmall`, `RFDETRMedium`, or `RFDETRLarge` instead. Emits `FutureWarning` on instantiation; scheduled for removal in v2.0. ([#900](https://github.com/roboflow/rf-detr/pull/900)) +- `RFDETRSegPreview` — use `RFDETRSegNano`, `RFDETRSegSmall`, `RFDETRSegMedium`, or `RFDETRSegLarge` instead. Emits `FutureWarning` on instantiation; scheduled for removal in v2.0. ([#900](https://github.com/roboflow/rf-detr/pull/900)) +- `rfdetr.util` and `rfdetr.deploy` sub-modules are deprecated and will be removed in v1.9. A `__getattr__` hook on the `rfdetr` package now emits a clear `ImportError` with migration guidance when these legacy paths are accessed. ([#839](https://github.com/roboflow/rf-detr/pull/839)) + +### Fixed + +- Fixed TFLite export (`format="tflite"`) producing detection scores that collapse to ~0.02 (vs ~0.62 from ONNX) on real inputs. Root cause is a long-standing onnx2tf bug ([PINTO0309/onnx2tf#274](https://github.com/PINTO0309/onnx2tf/issues/274)) where the `GridSample` lowering diverges numerically from ONNX while onnx2tf's own validator silently passes. RF-DETR's deformable cross-attention uses `F.grid_sample` once per decoder layer; drift compounds and is amplified by the attention softmax. The converter now detects onnx2tf's `GridSample → pseudo-GridSample` replacement kwarg at runtime (introspecting `convert()` via `inspect.signature`) and passes it as `True`; a warning is logged when the kwarg is absent. ([#1041](https://github.com/roboflow/rf-detr/pull/1041)) + +- `WindowedDinov2WithRegistersEmbeddings.forward()` now raises `ValueError` (instead of silently failing under `-O`) when input spatial dimensions are not divisible by `patch_size * num_windows`, with a clear message identifying the divisor and actual shape. ([#167](https://github.com/roboflow/rf-detr/pull/167)) + +- Fixed `_namespace.py`: `num_select` in the builder namespace now always reads from `ModelConfig`, eliminating a regression where `TrainConfig.num_select` (default 300) silently overrode model-specific values of 100–200 for segmentation variants (`RFDETRSegNano`, `RFDETRSegSmall`, `RFDETRSegMedium`, `RFDETRSegLarge`, `RFDETRSegPreview`). Post-processing now uses the correct top-k count for each model. ([#841](https://github.com/roboflow/rf-detr/pull/841)) + +- Fixed `models/weights.py`: `load_pretrain_weights` now correctly auto-aligns the model head when the checkpoint has fewer classes than the configured default, preventing a silent mismatch when `num_classes` was not explicitly set by the caller. ([#845](https://github.com/roboflow/rf-detr/pull/845)) + +- Fixed `models/weights.py`: `load_pretrain_weights` now slices `refpoint_embed.weight` and `query_feat.weight` per-group when reshaping checkpoint queries, instead of taking a flat `tensor[: num_queries * group_detr]` slice. The flat slice silently scrambled groups 1+ when `num_queries` decreased and `group_detr > 1`; inference (which only reads group 0) was unaffected, but training-resume corrupted query embeddings for groups 1 onward. ([#1019](https://github.com/roboflow/rf-detr/pull/1019)) + +- Fixed YOLO segmentation training on large datasets hitting OS out-of-memory: `supervision.DetectionDataset.from_yolo(force_masks=True)` was eager-rasterising H×W boolean masks for every image at dataset construction time (≈1 GB/1 000 images at 1024 px). A new `_LazyYoloDetectionDataset` stores polygon coordinates only and defers dense mask rasterisation to `__getitem__`, keeping RAM proportional to annotation count rather than (N × H × W). ([#851](https://github.com/roboflow/rf-detr/pull/851)) + +- Fixed ONNX/TRT dynamic batch inference: `gen_encoder_output_proposals` and `Transformer.forward` extracted the batch size as a Python int and passed it to `torch.full`, `.view(N_, ...)`, `.expand(N_, ...)`, and `.repeat(bs, ...)`, causing the ONNX tracer to bake the training batch size (e.g. 8) as a compile-time constant. TRT engines built with `--minShapes` smaller than the trace batch would fail at inference with `Reshape: reshaping failed`. All six call sites are now replaced with ONNX-symbolic equivalents (`zeros_like`, `-1` reshapes, `expand(memory.shape[0], ...)`), keeping the batch dimension fully dynamic. ([#950](https://github.com/roboflow/rf-detr/pull/950), closes [#949](https://github.com/roboflow/rf-detr/issues/949)) + +- Fixed training failure when `square_resize_div_64=False`: the non-square resize pipeline (`SmallestMaxSize` + `LongestMaxSize`) did not guarantee output dimensions divisible by `patch_size * num_windows`, causing `WindowedDinov2WithRegistersEmbeddings.forward` to raise `ValueError`. A `PadIfNeeded` step (with `pad_height_divisor` and `pad_width_divisor` set to `patch_size * num_windows`) is now appended after the resize pair in both the train and val/test pipelines. ([#991](https://github.com/roboflow/rf-detr/pull/991), closes [#983](https://github.com/roboflow/rf-detr/issues/983)) + +- Fixed non-square batch padding correctness: batch-level `block_size` rounding is now applied in the DataLoader collator (`nested_tensor_from_tensor_list` via `make_collate_fn`) in addition to the transform-level `PadIfNeeded`, ensuring divisibility by `patch_size * num_windows` survives any `Compose` reordering and applies uniformly to custom evaluation harnesses. ([#992](https://github.com/roboflow/rf-detr/pull/992)) + +- Fixed `RFDETRModelModule.on_load_checkpoint` crashing with `RuntimeError` when resuming training from a checkpoint saved at a different image resolution: DINOv2 positional embeddings in the checkpoint are now bicubic-interpolated to match `model_config.positional_encoding_size` before PyTorch Lightning applies the state dict. ([#1002](https://github.com/roboflow/rf-detr/pull/1002), closes [#998](https://github.com/roboflow/rf-detr/issues/998)) + +- Fixed `RFDETRLarge` initialization showing two conflicting `ValueError`s (for `patch_size=14` and `patch_size=16`) when the deprecated-config fallback retry also fails. The fallback now re-raises the original error without chained context, so users see a single deterministic message. ([#975](https://github.com/roboflow/rf-detr/pull/975)) + +- Fixed `RFDETRModelModule.__init__` crashing with `RuntimeError: size mismatch for backbone.0.encoder.encoder.embeddings.position_embeddings` when training segmentation models at a custom resolution (e.g. `RFDETRSegLarge(resolution=1008).train(...)`). The training entry path now delegates to the canonical `load_pretrain_weights` helper, which bicubic-interpolates the DINOv2 positional embeddings before `load_state_dict`. ([#1040](https://github.com/roboflow/rf-detr/pull/1040), closes [#1038](https://github.com/roboflow/rf-detr/issues/1038), [#1023](https://github.com/roboflow/rf-detr/issues/1023)) + +- Fixed TFLite detection scores collapsing for all queries (scores ~0.02 vs ~0.62 from ONNX) when `GridSample` was used as an onnx2tf pseudo-operator. The `GridSample` ONNX node is now rewritten to `Gather`-based integer-index arithmetic before conversion, eliminating all numerical drift from attention position sampling. This supersedes the pseudo-`GridSample` runtime-kwarg approach added in [#1041](https://github.com/roboflow/rf-detr/pull/1041). ([#1054](https://github.com/roboflow/rf-detr/pull/1054)) + +- Fixed `class_name` lookup for pretrained COCO models: COCO category IDs are sparse (1–90 with gaps for 80 classes), so flat 0-based indexing returned the wrong name (e.g. `class_id=18` ("dog") incorrectly returned `class_names[18]` instead of `class_names[16]`). Detection now uses a `coco_id → class_name` mapping built from the canonical `COCO_CLASSES` list so every COCO category resolves to its correct label. Fine-tuned models continue to use direct 0-based indexing unchanged. ([#1051](https://github.com/roboflow/rf-detr/pull/1051)) + +--- + +## [1.6.5] — 2026-04-22 + +### Breaking Changes + +- `predict()` now stores the source image in `detections.metadata["source_image"]` instead of `detections.data["source_image"]`. supervision indexes every value in `data` by the detection mask; `source_image` is per-image, not per-detection, so boolean/integer indexing raised `IndexError`. Moving it to `metadata` (passed through unchanged) fixes the issue. Update any code that reads `detections.data["source_image"]`. ([#972](https://github.com/roboflow/rf-detr/pull/972), [#968](https://github.com/roboflow/rf-detr/issues/968)) + +### Fixed + +- Fixed segmentation training crash on T4 and P100 GPUs: cuDNN engine selection fails for depthwise convolution backward on some CUDA stacks (Kaggle, Colab). A custom `autograd.Function` now disables cuDNN in both forward and backward passes. ([#967](https://github.com/roboflow/rf-detr/pull/967)) +- Fixed `ema_segm_mAP_50_95` and `ema_segm_mAP_50` being computed from the base (non-EMA) metric accumulator instead of the EMA accumulator, producing misleading validation scores for segmentation models. ([#980](https://github.com/roboflow/rf-detr/pull/980)) +- Fixed `BestModelCallback` losing the best EMA score on training resume because `_best_ema` was not persisted in `state_dict()`. ([#973](https://github.com/roboflow/rf-detr/pull/973)) +- Fixed `positional_encoding_size` not updating when `resolution` is set at construction time (e.g. `RFDETRLarge(resolution=640)`), causing shape mismatches during forward. A model validator now auto-syncs PE size. ([#956](https://github.com/roboflow/rf-detr/pull/956)) +- Fixed pretrained weight loading crash with custom resolution: DINOv2 positional embeddings are now bicubic-interpolated to match the target grid before `load_state_dict`. ([#964](https://github.com/roboflow/rf-detr/pull/964)) +- Fixed `validate_checkpoint_compatibility` producing a cryptic `RuntimeError` on `patch_size` mismatch when checkpoint lacks explicit `args.patch_size`. The function now infers `patch_size` from the DINOv2 projection weight shape and raises a descriptive `ValueError`. ([#971](https://github.com/roboflow/rf-detr/pull/971)) +- Fixed `predict()` storing `detections.data["source_shape"]` as a Python `tuple`, which caused `TypeError` whenever `sv.Detections` was iterated. The value is now an `np.ndarray` of shape `(N, 2)` and dtype `int64`. ([#966](https://github.com/roboflow/rf-detr/pull/966), [#963](https://github.com/roboflow/rf-detr/issues/963)) +- Fixed `predict()` emitting a misleading "class_id out of range" warning for the background/no-object class (class index `num_classes`). Background-class detections now map `data["class_name"]` to `"__background__"` without any warning. ([#970](https://github.com/roboflow/rf-detr/issues/970)) + +## [1.6.4] — 2026-04-10 + +### Changed + +- `predict()` now includes `class_name` in `detections.data`, mapping each detection's 0-indexed class ID to its human-readable name. ([#914](https://github.com/roboflow/rf-detr/pull/914)) + +### Fixed + +- Fixed segmentation multi-GPU DDP training crash: `build_trainer()` now wraps `strategy="ddp"` with `DDPStrategy(find_unused_parameters=True)` when `segmentation_head=True`. The segmentation head's `sparse_forward()` leaves parameters unused on some forward steps; plain `"ddp"` raised `RuntimeError: It looks like your LightningModule has parameters that were not used in producing the loss`. Non-segmentation DDP and other strategies are unchanged. ([#942](https://github.com/roboflow/rf-detr/pull/942), [#947](https://github.com/roboflow/rf-detr/pull/947)) +- Fixed fused AdamW crash under FP32 multi-GPU training: `configure_optimizers()` and `clip_gradients()` now gate fused AdamW on the trainer's actual precision (requiring a BF16 variant) rather than GPU capability alone. On Ampere+ hardware `torch.cuda.is_bf16_supported()` is always `True`, so the old code enabled fused AdamW even with `precision="32-true"`, causing `RuntimeError: params, grads, exp_avgs, and exp_avg_sqs must have same dtype, device, and layout` from DDP gradient bucket view stride mismatches. ([#942](https://github.com/roboflow/rf-detr/pull/942), [#947](https://github.com/roboflow/rf-detr/pull/947)) +- Fixed multi-GPU DDP training crashing in Jupyter notebooks and Kaggle: replaced fork-based `ddp_notebook` strategy with a spawn-based DDP strategy that avoids OpenMP thread pool corruption after `fork()`. ([#928](https://github.com/roboflow/rf-detr/pull/928)) +- Fixed `RFDETR.train(resolution=...)` being silently ignored — the kwarg is now applied to `model_config` before training begins, with validation that the value is divisible by `patch_size * num_windows`. ([#933](https://github.com/roboflow/rf-detr/pull/933)) +- Fixed `save_dataset_grids` being silently a no-op — `DatasetGridSaver` is now wired into the training loop, saving sample grids to `{output_dir}/dataset_grids/` when enabled. Grid save failures are caught without interrupting training. ([#946](https://github.com/roboflow/rf-detr/pull/946)) +- Fixed partial gradient-accumulation windows at the tail of training epochs: the training dataset is now padded to an exact multiple of `effective_batch_size * world_size`, ensuring every optimizer step uses a full gradient window. Workaround for [pytorch-lightning#19987](https://github.com/Lightning-AI/pytorch-lightning/issues/19987). ([#937](https://github.com/roboflow/rf-detr/pull/937)) +- Fixed `torch.export.export` failing on the transformer decoder by threading `spatial_shapes_hw` through all decoder layers. ([#936](https://github.com/roboflow/rf-detr/pull/936)) +- `download_pretrain_weights()` no longer overwrites fine-tuned checkpoints that share a filename with a registry model (e.g. `rf-detr-nano.pth`). Previously, an MD5 mismatch would fall through to `_download_file()` and silently replace the user's weights with the original COCO checkpoint. The function now returns early whenever the file exists and `redownload=False`, regardless of MD5 status — a warning is emitted when the hash differs. Pass `redownload=True` to force a fresh download. ([#935](https://github.com/roboflow/rf-detr/pull/935)) + +## [1.6.3] — 2026-04-02 + +### Changed + +- `predict()` now stores the original image and its shape on returned `sv.Detections` objects — `detections.data["source_image"]` (NumPy array) and `detections.data["source_shape"]` (NumPy array of shape `(N, 2)` where each row is `[height, width]`) let you annotate results without loading the image separately. ([#892](https://github.com/roboflow/rf-detr/pull/892)) +- `RFDETR.train()` auto-detects `num_classes` from the dataset directory when not explicitly set, reinitializing the detection head to the correct class count automatically. A warning is emitted when the configured value differs from the dataset count. ([#893](https://github.com/roboflow/rf-detr/pull/893)) +- `optimize_for_inference()` now accepts dtype as a string name (e.g. `"float16"`) in addition to a `torch.dtype` object; invalid dtype inputs uniformly raise `TypeError`. ([#899](https://github.com/roboflow/rf-detr/pull/899)) + +### Fixed + +- Fixed `models/lwdetr.py`: `reinitialize_detection_head` now replaces `nn.Linear` modules instead of mutating `.data` tensors in-place, ensuring `out_features` metadata stays consistent with the actual weight shape. This prevents ONNX export and `torch.jit.trace` from emitting stale (pre-fine-tuning) class counts for fine-tuned models. ([#904](https://github.com/roboflow/rf-detr/pull/904)) +- Fixed `RFDETR.optimize_for_inference()` leaking a CUDA context on multi-GPU setups: the deep-copy, export, and JIT-trace steps now run inside `torch.cuda.device(device)` to pin the context to the correct device. ([#899](https://github.com/roboflow/rf-detr/pull/899)) +- Fixed `optimize_for_inference()` leaving inconsistent state on failure: prior optimized state is now reset and flags are committed only after a successful build/trace; temp download files use unique per-process paths to avoid parallel worker collisions. +- Fixed `deploy_to_roboflow` failing with `FileNotFoundError` after PyTorch Lightning migration: `class_names.txt` is now written to the upload directory and `args.class_names` is populated before saving the checkpoint. ([#890](https://github.com/roboflow/rf-detr/pull/890)) + +## [1.6.2] — 2026-03-27 + +### Added + +- `RFDETR.predict(shape=...)` — optional `(height, width)` tuple overrides the default square inference resolution; useful when matching a non-square ONNX export. Both dimensions must be positive integers divisible by `patch_size × num_windows` as determined by the model configuration. ([#866](https://github.com/roboflow/rf-detr/pull/866)) + +### Changed + +- `ModelConfig.device` and `RFDETR.train(device=...)` now accept `torch.device` objects and indexed device strings such as `"cuda:0"`. Values are normalized to canonical torch-style strings. `RFDETR.train()` warns when an unmapped device type is passed to PyTorch Lightning auto-detection. ([#872](https://github.com/roboflow/rf-detr/pull/872)) + +### Fixed + +- Fixed ONNX export ignoring an explicit `patch_size` argument: `export()` and `predict()` now resolve `patch_size` from `model_config` by default, validate it strictly (positive integer, not bool), and enforce that `(H, W)` dimensions are divisible by `patch_size × num_windows`. ([#876](https://github.com/roboflow/rf-detr/pull/876)) +- Fixed ONNX export for models with dynamic batch dimensions — replaced `H_.expand(N_)` with `torch.full` for Python-int spatial dims to eliminate tracer failures. ([#871](https://github.com/roboflow/rf-detr/pull/871)) + +## [1.6.1] — 2026-03-25 + +### Deprecated + +- `RFDETR.export(..., simplify=..., force=...)` — both arguments are now no-ops and emit a `DeprecationWarning`. RF-DETR no longer runs ONNX simplification automatically; remove these arguments from your calls. They will be removed in v1.8. ([#861](https://github.com/roboflow/rf-detr/pull/861)) + +### Fixed + +- Fixed `RFDETR.train()`: a missing `rfdetr[train]` install (e.g. plain `pip install rfdetr` in Colab) now raises an `ImportError` with an actionable message — `pip install "rfdetr[train,loggers]"` — instead of a raw `ModuleNotFoundError` with no install hint. ([#858](https://github.com/roboflow/rf-detr/pull/858)) +- Fixed `AUG_AGGRESSIVE` preset: `translate_percent` was `(0.1, 0.1)` — a degenerate range that forced Albumentations `Affine` to always translate right/down by exactly 10%. Corrected to `(-0.1, 0.1)` for symmetric bidirectional translation. ([#863](https://github.com/roboflow/rf-detr/pull/863)) +- Fixed PTL training path: `latest.ckpt` and per-interval checkpoints (`checkpoint_interval_N.ckpt`) are now properly written and restored on resume. ([#847](https://github.com/roboflow/rf-detr/pull/847)) +- Fixed `BestModelCallback` and checkpoint monitor raising `MisconfigurationException` on non-eval epochs when `eval_interval > 1` — monitor key absence is now handled gracefully. ([#848](https://github.com/roboflow/rf-detr/pull/848)) +- Fixed `protobuf` version constraint in the `loggers` extra to guard against TensorBoard descriptor crash (`TypeError: Descriptors cannot be created directly`) with protobuf ≥ 4. ([#846](https://github.com/roboflow/rf-detr/pull/846)) +- Fixed duplicate `ModelCheckpoint` state keys when `checkpoint_interval=1`; `last.ckpt` is omitted in that configuration to avoid collision. ([#859](https://github.com/roboflow/rf-detr/pull/859)) + +## [1.6.0] — 2026-03-20 + +### Added + +- PyTorch Lightning training building blocks: `RFDETRModelModule`, `RFDETRDataModule`, `build_trainer()`, and individual callbacks (`RFDETREMACallback`, `COCOEvalCallback`, `BestModelCallback`, `DropPathCallback`, `MetricsPlotCallback`) — all standard PTL components, swap/subclass/extend any piece. Level 3: `rfdetr fit --config` CLI with zero Python required. ([#757](https://github.com/roboflow/rf-detr/pull/757), [#794](https://github.com/roboflow/rf-detr/pull/794)) +- Multi-GPU DDP via `model.train()`: `strategy`, `devices`, and `num_nodes` added to `TrainConfig`; single-GPU behaviour unchanged when omitted. ([#808](https://github.com/roboflow/rf-detr/pull/808)) +- `batch_size='auto'`: CUDA memory probe finds the largest safe micro-batch size, then recommends `grad_accum_steps` to reach a configurable effective batch target (default 16 via `auto_batch_target_effective`). ([#814](https://github.com/roboflow/rf-detr/pull/814)) +- `ModelContext` promoted from `_ModelContext` to a public, exported API — inspect `class_names`, `num_classes`, and related metadata via `model.context` after training. ([#835](https://github.com/roboflow/rf-detr/pull/835)) +- `backbone_lora` and `freeze_encoder` added as first-class fields in `ModelConfig`. ([#829](https://github.com/roboflow/rf-detr/pull/829)) +- `generate_coco_dataset(with_segmentation=True)` produces COCO polygon annotations alongside bounding boxes for segmentation fine-tuning with synthetic data. ([#781](https://github.com/roboflow/rf-detr/pull/781)) +- `set_attn_implementation("eager" | "sdpa")` on the DINOv2 backbone — switch attention implementation at runtime. ([#760](https://github.com/roboflow/rf-detr/pull/760)) +- `eval_max_dets`, `eval_interval`, and `log_per_class_metrics` added to `TrainConfig`. +- `python -m rfdetr` entry point alongside the `rfdetr` console script. +- `py.typed` marker — RF-DETR is now PEP 561–compliant. + +### Changed + +- **Breaking:** Minimum `transformers` version bumped to `>=5.1.0,<6.0.0`. The DINOv2 windowed-attention backbone now uses the transformers v5 API (`BackboneMixin._init_transformers_backbone()`, removed `head_mask` plumbing). Projects still on transformers v4 must pin `rfdetr<1.6.0`. ([#760](https://github.com/roboflow/rf-detr/pull/760)) +- **Breaking:** PyPI install extras renamed — `rfdetr[metrics]` → `rfdetr[loggers]`, `rfdetr[onnxexport]` → `rfdetr[onnx]`. +- `draw_synthetic_shape` now returns `Tuple[np.ndarray, List[float]]` instead of `np.ndarray`. The second element is a flat COCO-style polygon list `[x1, y1, x2, y2, …]`. Any caller that previously did `img = draw_synthetic_shape(...)` must be updated to `img, polygon = draw_synthetic_shape(...)`. ([#781](https://github.com/roboflow/rf-detr/pull/781)) +- Albumentations version constraint broadened to `>=1.4.24,<3.0.0`; `RandomSizedCrop` configs using `height`/`width` kwargs are automatically adapted to the 2.x `size=(height, width)` API. ([#786](https://github.com/roboflow/rf-detr/pull/786)) +- Current learning rate is now shown in the training progress bar alongside loss. ([#809](https://github.com/roboflow/rf-detr/pull/809)) +- `supervision`, `pytorch_lightning`, and other heavy dependencies are now imported lazily (on first use) rather than at module load, reducing cold-import time in inference-only environments. ([#801](https://github.com/roboflow/rf-detr/pull/801)) + +### Deprecated + +- `rfdetr.deploy.*` — redirects to `rfdetr.export.*` with a `DeprecationWarning`. Migrate before v1.7. +- `rfdetr.util.*` — redirects to `rfdetr.utilities.*` with a `DeprecationWarning`. Migrate before v1.7. + +### Fixed + +- Raised a descriptive `ValueError` instead of a cryptic `RuntimeError` / tensor-size mismatch when a checkpoint is incompatible with the current model architecture — covers `segmentation_head` mismatch and `patch_size` mismatch. ([#810](https://github.com/roboflow/rf-detr/pull/810)) +- Fixed `class_names` not reflecting dataset labels on `model.predict()` after training — class names are now synced from the dataset so inference always uses the correct label list. ([#816](https://github.com/roboflow/rf-detr/pull/816)) +- Fixed detection head reinitialization overwriting fine-tuned weights when loading a checkpoint with fewer classes than the model default. The second `reinitialize_detection_head` call now fires only in the backbone-pretrain scenario. ([#815](https://github.com/roboflow/rf-detr/pull/815), [#509](https://github.com/roboflow/rf-detr/pull/509)) +- Fixed `grid_sample` and bicubic interpolation silently falling back to CPU on MPS (Apple Silicon) — both now run natively on the MPS device. ([#821](https://github.com/roboflow/rf-detr/pull/821)) +- Fixed `early_stopping=False` in `TrainConfig` being silently ignored — the setting now propagates correctly. ([#835](https://github.com/roboflow/rf-detr/pull/835)) +- Fixed `AttributeError` crash in `update_drop_path` when the DINOv2 backbone layer structure does not match any known pattern. +- Added warning when `drop_path_rate > 0.0` is configured with a non-windowed DINOv2 backbone, where drop-path is silently ignored. +- Fixed `ValueError: matrix entries are not finite` in `HungarianMatcher` when the cost matrix contains NaN or Inf — non-finite entries are replaced with a finite sentinel before `linear_sum_assignment`, warning emitted at most once per matcher instance. ([#787](https://github.com/roboflow/rf-detr/pull/787)) +- Fixed YOLO dataset validation rejecting `data.yml` — both `.yaml` and `.yml` are now accepted. ([#777](https://github.com/roboflow/rf-detr/pull/777)) +- Silently dropped degenerate bounding boxes (zero width or height) before Albumentations validation instead of raising `ValueError`. ([#825](https://github.com/roboflow/rf-detr/pull/825)) + +--- + +## [1.5.2] — 2026-03-04 + +### Added + +- Added peak GPU memory (`max_mem` in MB) to training and evaluation progress bars on CUDA; omitted on CPU and MPS. ([#773](https://github.com/roboflow/rf-detr/pull/773)) + +### Fixed + +- Fixed `aug_config` being silently ignored when training on YOLO-format datasets — `build_roboflow_from_yolo` never forwarded the value, so transforms always fell back to the default. ([#774](https://github.com/roboflow/rf-detr/pull/774)) +- Fixed segmentation evaluation metrics not being written to `results_mask.json` during validation and test runs. ([#772](https://github.com/roboflow/rf-detr/pull/772)) +- Fixed `AttributeError` crash in `update_drop_path` when the DINOv2 backbone layer structure does not match any known pattern — `_get_backbone_encoder_layers` now returns `None` for unrecognised architectures. ([#762](https://github.com/roboflow/rf-detr/pull/762)) +- Fixed `drop_path_rate` not being forwarded to the DINOv2 model configuration; stochastic depth was never applied even when explicitly set. Added a warning when `drop_path_rate > 0.0` is used with a non-windowed backbone. ([#762](https://github.com/roboflow/rf-detr/pull/762)) +- Fixed incorrect COCO hierarchy filtering that excluded parent categories from the class list. ([#759](https://github.com/roboflow/rf-detr/pull/759)) +- Fixed evaluation metric corruption on 1-indexed Roboflow datasets caused by a flawed contiguity check in `_should_use_raw_category_ids`. ([#755](https://github.com/roboflow/rf-detr/pull/755)) + +## [1.5.1] — 2026-02-27 + +### Added + +- Added support for nested Albumentations containers (`OneOf`, `Sequential`) inside `aug_config`. ([#752](https://github.com/roboflow/rf-detr/pull/752)) + +### Changed + +- Migrated dataset transform pipeline to torchvision-native `Compose`, `ToImage`, and `ToDtype`; `Normalize` now defaults to ImageNet mean/std. ([#745](https://github.com/roboflow/rf-detr/pull/745)) + +### Fixed + +- Fixed `RFDETRMedium` missing from the public API — `__all__` contained a duplicate `RFDETRSmall` entry. ([#748](https://github.com/roboflow/rf-detr/pull/748)) +- Fixed `AR50_90` reporting an incorrect value in `MetricsMLFlowSink` due to a wrong COCO evaluation index. ([#735](https://github.com/roboflow/rf-detr/pull/735)) +- Fixed supercategory filtering in `_load_classes` for COCO datasets with flat or mixed supercategory structures. ([#744](https://github.com/roboflow/rf-detr/pull/744)) +- Fixed crash in geometric transforms when a sample contained zero-area or empty masks. ([#727](https://github.com/roboflow/rf-detr/pull/727)) +- Fixed segmentation training on Colab — `DepthwiseConvBlock` now disables cuDNN for depthwise separable convolutions. ([#728](https://github.com/roboflow/rf-detr/pull/728)) +- Pinned `onnxsim<0.6.0` to prevent `pip install` from hanging indefinitely. ([#749](https://github.com/roboflow/rf-detr/pull/749)) + +## [1.5.0] — 2026-02-23 + +### Added + +- Added custom training augmentations via `aug_config` in `model.train()` — accepts a dict of Albumentations transforms, a built-in preset (`AUG_CONSERVATIVE`, `AUG_AGGRESSIVE`, `AUG_AERIAL`, `AUG_INDUSTRIAL`), or `{}` to disable. Bounding boxes and segmentation masks are transformed automatically. ([#263](https://github.com/roboflow/rf-detr/pull/263), [#702](https://github.com/roboflow/rf-detr/pull/702)) +- Added `save_dataset_grids=True` in `TrainConfig` to write 3×3 JPEG grids of augmented samples to `output_dir` before training begins. ([#153](https://github.com/roboflow/rf-detr/pull/153)) +- Added ClearML logger: set `clearml=True` in `TrainConfig` to stream per-epoch metrics to ClearML. ([#520](https://github.com/roboflow/rf-detr/pull/520)) +- Added MLflow logger: set `mlflow=True` in `TrainConfig` to log runs and metrics to MLflow with custom tracking URI support. ([#109](https://github.com/roboflow/rf-detr/pull/109)) +- Added live progress bar for training and validation with structured per-epoch logs. ([#204](https://github.com/roboflow/rf-detr/pull/204)) +- Added `device` field to `TrainConfig` for explicit device selection. ([#687](https://github.com/roboflow/rf-detr/pull/687)) +- `ModelConfig` now raises an error on unknown parameters, preventing silent misconfiguration. ([#196](https://github.com/roboflow/rf-detr/pull/196)) + +### Changed + +- Deprecated `OPEN_SOURCE_MODELS` constant in favour of `ModelWeights` enum. ([#696](https://github.com/roboflow/rf-detr/pull/696)) +- Added MD5 checksum validation for pretrained weight downloads. ([#679](https://github.com/roboflow/rf-detr/pull/679)) + +### Fixed + +- Fixed Albumentations bool-mask crash during segmentation training. ([#706](https://github.com/roboflow/rf-detr/pull/706)) +- Fixed `UnboundLocalError` when resuming training from a completed checkpoint. ([#707](https://github.com/roboflow/rf-detr/pull/707)) +- Prevented corruption of `checkpoint_best_total.pth` via atomic checkpoint stripping. ([#708](https://github.com/roboflow/rf-detr/pull/708)) +- Fixed PyTorch 2.9+ compatibility issue with CUDA capability detection. ([#686](https://github.com/roboflow/rf-detr/pull/686)) +- Fixed dtype mismatch error when `use_position_supervised_loss=True`. ([#447](https://github.com/roboflow/rf-detr/pull/447)) +- Fixed inconsistent return values from `build_model`. ([#519](https://github.com/roboflow/rf-detr/pull/519)) +- Fixed `positional_encoding_size` type annotation (`bool` → `int`). ([#524](https://github.com/roboflow/rf-detr/pull/524)) +- Fixed ONNX export `output_names` to include masks when exporting segmentation models. ([#402](https://github.com/roboflow/rf-detr/pull/402)) +- Fixed `num_select` not being updated correctly during segmentation model fine-tuning. ([#399](https://github.com/roboflow/rf-detr/pull/399)) +- Fixed `np.argwhere` → `np.argmax` misuse. ([#536](https://github.com/roboflow/rf-detr/pull/536)) +- Fixed COCO sparse category ID remapping for non-contiguous or offset category IDs. ([#712](https://github.com/roboflow/rf-detr/pull/712)) +- Fixed segmentation mask filtering when using aggressive augmentations. ([#717](https://github.com/roboflow/rf-detr/pull/717)) + +--- + +## [1.4.3] — 2026-02-16 + +### Changed + +- Pretrained weight downloads now validate against an MD5 checksum to detect corrupted files. ([#679](https://github.com/roboflow/rf-detr/pull/679)) + +### Fixed + +- Fixed `deploy_to_roboflow` failing for segmentation model exports. ([#578](https://github.com/roboflow/rf-detr/pull/578)) +- Fixed missing `info` key in COCO export format. ([#681](https://github.com/roboflow/rf-detr/pull/681)) + +## [1.4.2] — 2026-02-12 + +### Added + +- Added `generate_coco_dataset()` utility for generating synthetic COCO-format datasets with configurable class counts, split ratios, and bounding box annotations. ([#617](https://github.com/roboflow/rf-detr/pull/617)) +- Added `run_test=False` to `TrainConfig` — skip test-split evaluation when your dataset has no test set. ([#628](https://github.com/roboflow/rf-detr/pull/628)) + +### Changed + +- `model.predict()` now accepts image URLs directly — no need to download images before inference. ([#629](https://github.com/roboflow/rf-detr/pull/629)) +- Plus models (`RFDETRXLarge`, `RFDETR2XLarge`) are now distributed as a separate `rfdetr_plus` package under the Roboflow Model License. ([#645](https://github.com/roboflow/rf-detr/pull/645)) + +### Fixed + +- Fixed segmentation ONNX export failure. ([#626](https://github.com/roboflow/rf-detr/pull/626)) + +## [1.4.1] — 2026-01-30 + +### Added + +- Added native YOLO dataset format support alongside COCO. ([#74](https://github.com/roboflow/rf-detr/pull/74)) +- Added `--print-freq` CLI argument to control training log frequency. ([#603](https://github.com/roboflow/rf-detr/pull/603)) + +### Changed + +- Pinned `transformers` to `<5.0.0` to prevent incompatibility with the transformers v5 API. ([#599](https://github.com/roboflow/rf-detr/pull/599)) + +### Fixed + +- Fixed class count mismatch in `train_from_config` for Roboflow-uploaded datasets. ([#588](https://github.com/roboflow/rf-detr/pull/588)) +- Improved `num_classes` mismatch warning messages to be actionable rather than misleading. ([#261](https://github.com/roboflow/rf-detr/pull/261)) +- Fixed CLI crash when specifying the `device` argument. ([#246](https://github.com/roboflow/rf-detr/pull/246)) + +## [1.4.0] — 2026-01-22 + +Headline release introducing new pre-trained model sizes — L, XL, and 2XL for object detection, and the full N/S/M/L/XL/2XL range for instance segmentation. Also added YOLO format training support, simplified the dependency footprint by removing several heavy packages (`cython`, `fairscale`, `timm`, `einops`, and others), and fixed per-class precision/recall/F1 computation. Drops Python 3.9 support. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..602a331 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,28 @@ +cff-version: 1.2.0 +title: "RF-DETR" +message: >- + If you use this software, please cite it using the + metadata from this file. +type: software +authors: + - family-names: Robinson + given-names: Isaac + email: isaac@roboflow.com + affiliation: Roboflow + - family-names: Robicheaux + given-names: Peter + email: peter@roboflow.com + affiliation: Roboflow + - family-names: Popov + given-names: Matvei + email: matvei@roboflow.com + affiliation: Roboflow +repository-code: 'https://github.com/roboflow/rf-detr' +abstract: 'A state-of-the-art, real-time object detection model developed by Roboflow.' +date-released: 2025-03-20 +keywords: + - object detection + - computer vision + - rf-detr + - detr +license: "Apache-2.0" diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2eefdd7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +# Claude Code Project Instructions + + + +@AGENTS.md diff --git a/CNAME b/CNAME new file mode 100644 index 0000000..f39fba4 --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +rfdetr.roboflow.com diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..56db37c --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Roboflow, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5c9d525 --- /dev/null +++ b/README.md @@ -0,0 +1,338 @@ +# RF-DETR: Real-Time SOTA Object Detection, Instance Segmentation, and Keypoint Detection + +
+ +[![version](https://badge.fury.io/py/rfdetr.svg)](https://badge.fury.io/py/rfdetr) +[![downloads](https://img.shields.io/pypi/dm/rfdetr)](https://pypistats.org/packages/rfdetr) +[![codecov](https://codecov.io/gh/roboflow/rf-detr/graph/badge.svg?token=K8V4ARR3XV)](https://codecov.io/gh/roboflow/rf-detr) +[![python-version](https://img.shields.io/pypi/pyversions/rfdetr)](https://badge.fury.io/py/rfdetr) +[![license](https://img.shields.io/badge/license-Apache%202.0-blue)](https://github.com/roboflow/rf-detr/blob/main/LICENSE) + +[![arXiv](https://img.shields.io/badge/arXiv-2511.09554-b31b1b.svg)](https://arxiv.org/abs/2511.09554) +[![hf space](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/SkalskiP/RF-DETR) +[![colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/how-to-finetune-rf-detr-on-detection-dataset.ipynb) +[![roboflow](https://raw.githubusercontent.com/roboflow-ai/notebooks/main/assets/badges/roboflow-blogpost.svg)](https://blog.roboflow.com/rf-detr) +[![discord](https://img.shields.io/discord/1159501506232451173?logo=discord&label=discord&labelColor=fff&color=5865f2&link=https%3A%2F%2Fdiscord.gg%2FGbfgXGJ8Bk)](https://discord.gg/GbfgXGJ8Bk) + + +roboflow%2Frf-detr | Trendshift + + +
+ +--- + +RF-DETR is a real-time transformer architecture for object detection, instance segmentation, and keypoint detection (preview) developed by Roboflow. Built on a DINOv2 vision transformer backbone, RF-DETR delivers state-of-the-art accuracy and latency trade-offs on [Microsoft COCO](https://cocodataset.org/#home) and [RF100-VL](https://github.com/roboflow/rf100-vl). + +RF-DETR uses a DINOv2 vision transformer backbone and supports object detection, instance segmentation, and keypoint detection (preview) in a single, consistent API. The open-source `rfdetr` package and Apache-designated models are released under Apache 2.0, while Plus components (`rfdetr_plus`, including RF-DETR-XL/2XL detection models) are licensed under PML 1.0. + +The published RF-DETR sizes were created with neural architecture search (NAS) — and the same NAS method is now available on the [Roboflow platform](https://app.roboflow.com/), so you can discover the best architecture for your own dataset. Learn more in the [NAS docs](https://docs.roboflow.com/train/neural-architecture-search). + +https://github.com/user-attachments/assets/add23fd1-266f-4538-8809-d7dd5767e8e6 + +## Install + +To install RF-DETR, install the `rfdetr` package in a [**Python>=3.10**](https://www.python.org/) environment with `pip`. + +```bash +pip install rfdetr +``` + +
+Install from source + +
+ +By installing RF-DETR from source, you can explore the most recent features and enhancements that have not yet been officially released. **Please note that these updates are still in development and may not be as stable as the latest published release.** + +```bash +pip install https://github.com/roboflow/rf-detr/archive/refs/heads/develop.zip +``` + +
+ +## Benchmarks + +RF-DETR achieves state-of-the-art results in both object detection and instance segmentation, with benchmarks reported on Microsoft COCO and RF100-VL (RF100-VL for detection only). The charts and tables below compare RF-DETR against other top real-time models across accuracy and latency for detection and segmentation. All latency numbers were measured on an NVIDIA T4 using TensorRT, FP16, and batch size 1. For full benchmarking methodology and reproducibility details, see [roboflow/sab](https://github.com/roboflow/single_artifact_benchmarking). + +### Detection + +rf_detr_1-4_latency_accuracy_object_detection + +
+See object detection benchmark numbers + +
+ +| Architecture | COCO AP50 | COCO AP50:95 | RF100VL AP50 | RF100VL AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :-----------: | :------------------: | :---------------------: | :---------------------: | :------------------------: | :----------: | :--------: | :--------: | :--------: | +| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384x384 | Apache 2.0 | +| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512x512 | Apache 2.0 | +| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576x576 | Apache 2.0 | +| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704x704 | Apache 2.0 | +| RF-DETR-XL △ | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700x700 | PML 1.0 | +| RF-DETR-2XL △ | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880x880 | PML 1.0 | +| YOLO11-N | 52.0 | 37.4 | 81.4 | 55.3 | 2.5 | 2.6 | 640x640 | AGPL-3.0 | +| YOLO11-S | 59.7 | 44.4 | 82.3 | 56.2 | 3.2 | 9.4 | 640x640 | AGPL-3.0 | +| YOLO11-M | 64.1 | 48.6 | 82.5 | 56.5 | 5.1 | 20.1 | 640x640 | AGPL-3.0 | +| YOLO11-L | 64.9 | 49.9 | 82.2 | 56.5 | 6.5 | 25.3 | 640x640 | AGPL-3.0 | +| YOLO11-X | 66.1 | 50.9 | 81.7 | 56.2 | 10.5 | 56.9 | 640x640 | AGPL-3.0 | +| YOLO26-N | 55.8 | 40.3 | 76.7 | 52.0 | 1.7 | 2.6 | 640x640 | AGPL-3.0 | +| YOLO26-S | 64.3 | 47.7 | 82.7 | 57.0 | 2.6 | 9.4 | 640x640 | AGPL-3.0 | +| YOLO26-M | 69.7 | 52.5 | 84.4 | 58.7 | 4.4 | 20.1 | 640x640 | AGPL-3.0 | +| YOLO26-L | 71.1 | 54.1 | 85.0 | 59.3 | 5.7 | 25.3 | 640x640 | AGPL-3.0 | +| YOLO26-X | 74.0 | 56.9 | 85.6 | 60.0 | 9.6 | 56.9 | 640x640 | AGPL-3.0 | +| LW-DETR-T | 60.7 | 42.9 | 84.7 | 57.1 | 1.9 | 12.1 | 640x640 | Apache 2.0 | +| LW-DETR-S | 66.8 | 48.0 | 85.0 | 57.4 | 2.6 | 14.6 | 640x640 | Apache 2.0 | +| LW-DETR-M | 72.0 | 52.6 | 86.8 | 59.8 | 4.4 | 28.2 | 640x640 | Apache 2.0 | +| LW-DETR-L | 74.6 | 56.1 | 87.4 | 61.5 | 6.9 | 46.8 | 640x640 | Apache 2.0 | +| LW-DETR-X | 76.9 | 58.3 | 87.9 | 62.1 | 13.0 | 118.0 | 640x640 | Apache 2.0 | +| D-FINE-N | 60.2 | 42.7 | 84.4 | 58.2 | 2.1 | 3.8 | 640x640 | Apache 2.0 | +| D-FINE-S | 67.6 | 50.6 | 85.3 | 60.3 | 3.5 | 10.2 | 640x640 | Apache 2.0 | +| D-FINE-M | 72.6 | 55.0 | 85.5 | 60.6 | 5.4 | 19.2 | 640x640 | Apache 2.0 | +| D-FINE-L | 74.9 | 57.2 | 86.4 | 61.6 | 7.5 | 31.0 | 640x640 | Apache 2.0 | +| D-FINE-X | 76.8 | 59.3 | 86.9 | 62.2 | 11.5 | 62.0 | 640x640 | Apache 2.0 | + +
+ +### Segmentation + +rf_detr_1-4_latency_accuracy_instance_segmentation + +
+See instance segmentation benchmark numbers + +
+ +| Architecture | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :-------------: | :------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: | +| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312x312 | Apache 2.0 | +| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384x384 | Apache 2.0 | +| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432x432 | Apache 2.0 | +| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504x504 | Apache 2.0 | +| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624x624 | Apache 2.0 | +| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768x768 | Apache 2.0 | +| YOLOv8-N-Seg | 45.6 | 28.3 | 3.5 | 3.4 | 640x640 | AGPL-3.0 | +| YOLOv8-S-Seg | 53.8 | 34.0 | 4.2 | 11.8 | 640x640 | AGPL-3.0 | +| YOLOv8-M-Seg | 58.2 | 37.3 | 7.0 | 27.3 | 640x640 | AGPL-3.0 | +| YOLOv8-L-Seg | 60.5 | 39.0 | 9.7 | 46.0 | 640x640 | AGPL-3.0 | +| YOLOv8-XL-Seg | 61.3 | 39.5 | 14.0 | 71.8 | 640x640 | AGPL-3.0 | +| YOLOv11-N-Seg | 47.8 | 30.0 | 3.6 | 2.9 | 640x640 | AGPL-3.0 | +| YOLOv11-S-Seg | 55.4 | 35.0 | 4.6 | 10.1 | 640x640 | AGPL-3.0 | +| YOLOv11-M-Seg | 60.0 | 38.5 | 6.9 | 22.4 | 640x640 | AGPL-3.0 | +| YOLOv11-L-Seg | 61.5 | 39.5 | 8.3 | 27.6 | 640x640 | AGPL-3.0 | +| YOLOv11-XL-Seg | 62.4 | 40.1 | 13.7 | 62.1 | 640x640 | AGPL-3.0 | +| YOLO26-N-Seg | 54.3 | 34.7 | 2.31 | 2.7 | 640x640 | AGPL-3.0 | +| YOLO26-S-Seg | 62.4 | 40.2 | 3.47 | 10.4 | 640x640 | AGPL-3.0 | +| YOLO26-M-Seg | 67.8 | 44.0 | 6.32 | 23.6 | 640x640 | AGPL-3.0 | +| YOLO26-L-Seg | 69.8 | 45.5 | 7.58 | 28.0 | 640x640 | AGPL-3.0 | +| YOLO26-X-Seg | 71.6 | 46.8 | 12.92 | 62.8 | 640x640 | AGPL-3.0 | + +
+ +### Keypoints + +RF-DETR Keypoint mAP vs latency chart comparing against YOLO26-pose and YOLO11-pose on MS COCO + +
+See keypoint detection benchmark numbers + +
+ +| Architecture | COCO AP50:95 | Latency (ms) | License | +| :------------------------: | :---------------------: | :----------: | :--------: | +| RF-DETR Keypoint (Preview) | 71.8 | 9.7 | Apache 2.0 | +| YOLO11-pose N | 48.9 | 3.2 | AGPL-3.0 | +| YOLO11-pose S | 57.5 | 3.4 | AGPL-3.0 | +| YOLO11-pose M | 64.2 | 5.2 | AGPL-3.0 | +| YOLO11-pose L | 65.2 | 6.6 | AGPL-3.0 | +| YOLO11-pose X | 68.6 | 10.6 | AGPL-3.0 | +| YOLO26-pose N | 55.9 | 1.9 | AGPL-3.0 | +| YOLO26-pose S | 62.0 | 2.7 | AGPL-3.0 | +| YOLO26-pose M | 68.0 | 4.6 | AGPL-3.0 | +| YOLO26-pose L | 69.2 | 5.9 | AGPL-3.0 | +| YOLO26-pose X | 71.0 | 9.8 | AGPL-3.0 | + +
+ +> Keypoint benchmarks report AP50:95 (OKS-based); this is the standard COCO keypoint comparison metric. + +## Run Models + +### Detection + +RF-DETR provides multiple model sizes, ranging from Nano to 2XLarge. To use a different model size, replace the class name in the code snippet below with another class from the table. + +```python +import supervision as sv +from rfdetr import RFDETRMedium +from rfdetr.assets.coco_classes import COCO_CLASSES + +model = RFDETRMedium() + +detections = model.predict("https://media.roboflow.com/dog.jpg", threshold=0.5) + +labels = [f"{COCO_CLASSES[class_id]}" for class_id in detections.class_id] + +annotated_image = sv.BoxAnnotator().annotate(detections.metadata["source_image"], detections) +annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) +``` + +> **Note:** `COCO_CLASSES` works for COCO-pretrained models. For fine-tuned models, use `detections.data["class_name"]` instead — it resolves class names from the checkpoint and works for both COCO and custom datasets. + +
+Run RF-DETR with Inference + +
+ +You can also run RF-DETR models using the Inference library. To switch model size, select the appropriate inference package alias from the table below. + +```python +import requests +import supervision as sv +from PIL import Image +from inference import get_model + +model = get_model("rfdetr-medium") + +image = Image.open(requests.get("https://media.roboflow.com/dog.jpg", stream=True).raw) +predictions = model.infer(image, confidence=0.5)[0] +detections = sv.Detections.from_inference(predictions) + +annotated_image = sv.BoxAnnotator().annotate(image, detections) +annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections) +``` + +
+ +| Size | RF-DETR package class | Inference package alias | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :--: | :-------------------: | :---------------------- | :------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: | +| N | `RFDETRNano` | `rfdetr-nano` | 67.6 | 48.4 | 2.3 | 30.5 | 384x384 | Apache 2.0 | +| S | `RFDETRSmall` | `rfdetr-small` | 72.1 | 53.0 | 3.5 | 32.1 | 512x512 | Apache 2.0 | +| M | `RFDETRMedium` | `rfdetr-medium` | 73.6 | 54.7 | 4.4 | 33.7 | 576x576 | Apache 2.0 | +| L | `RFDETRLarge` | `rfdetr-large` | 75.1 | 56.5 | 6.8 | 33.9 | 704x704 | Apache 2.0 | +| XL | `RFDETRXLarge` △ | `rfdetr-xlarge` | 77.4 | 58.6 | 11.5 | 126.4 | 700x700 | PML 1.0 | +| 2XL | `RFDETR2XLarge` △ | `rfdetr-2xlarge` | 78.5 | 60.1 | 17.2 | 126.9 | 880x880 | PML 1.0 | + +> △ Requires the `rfdetr_plus` extension: `pip install rfdetr[plus]`. See [License](#license) for details. + +### Segmentation + +RF-DETR supports instance segmentation with model sizes from Nano to 2XLarge. To use a different model size, replace the class name in the code snippet below with another class from the table. + +```python +import supervision as sv +from rfdetr import RFDETRSegMedium +from rfdetr.assets.coco_classes import COCO_CLASSES + +model = RFDETRSegMedium() + +detections = model.predict("https://media.roboflow.com/dog.jpg", threshold=0.5) + +labels = [f"{COCO_CLASSES[class_id]}" for class_id in detections.class_id] + +annotated_image = sv.MaskAnnotator().annotate(detections.metadata["source_image"], detections) +annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) +``` + +
+Run RF-DETR-Seg with Inference + +
+ +You can also run RF-DETR-Seg models using the Inference library. To switch model size, select the appropriate inference package alias from the table below. + +```python +import requests +import supervision as sv +from PIL import Image +from inference import get_model + +model = get_model("rfdetr-seg-medium") + +image = Image.open(requests.get("https://media.roboflow.com/dog.jpg", stream=True).raw) +predictions = model.infer(image, confidence=0.5)[0] +detections = sv.Detections.from_inference(predictions) + +annotated_image = sv.MaskAnnotator().annotate(image, detections) +annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections) +``` + +
+ +| Size | RF-DETR package class | Inference package alias | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :--: | :-------------------: | :---------------------- | :------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: | +| N | `RFDETRSegNano` | `rfdetr-seg-nano` | 63.0 | 40.3 | 3.4 | 33.6 | 312x312 | Apache 2.0 | +| S | `RFDETRSegSmall` | `rfdetr-seg-small` | 66.2 | 43.1 | 4.4 | 33.7 | 384x384 | Apache 2.0 | +| M | `RFDETRSegMedium` | `rfdetr-seg-medium` | 68.4 | 45.3 | 5.9 | 35.7 | 432x432 | Apache 2.0 | +| L | `RFDETRSegLarge` | `rfdetr-seg-large` | 70.5 | 47.1 | 8.8 | 36.2 | 504x504 | Apache 2.0 | +| XL | `RFDETRSegXLarge` | `rfdetr-seg-xlarge` | 72.2 | 48.8 | 13.5 | 38.1 | 624x624 | Apache 2.0 | +| 2XL | `RFDETRSeg2XLarge` | `rfdetr-seg-2xlarge` | 73.1 | 49.9 | 21.8 | 38.6 | 768x768 | Apache 2.0 | + +### Keypoints + +RF-DETR supports keypoint detection (preview) with `RFDETRKeypointPreview`, pretrained on COCO person keypoints. + +```python +from rfdetr import RFDETRKeypointPreview + +model = RFDETRKeypointPreview() +key_points = model.predict("image.jpg", threshold=0.5) +``` + +| Size | RF-DETR package class | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :----------------: | :---------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: | +| Keypoint (Preview) | `RFDETRKeypointPreview` | 71.8 | 9.7 | 126.4 | 576x576 | Apache 2.0 | + +### Train Models + +RF-DETR supports training for object detection, instance segmentation, and keypoint detection (preview). You can train models in [Google Colab](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/how-to-finetune-rf-detr-on-detection-dataset.ipynb) or directly on the Roboflow platform. Below you will find a step-by-step video fine-tuning tutorial. + +[![rf-detr-tutorial-banner](https://github.com/user-attachments/assets/555a45c3-96e8-4d8a-ad29-f23403c8edfd)](https://youtu.be/-OvpdLAElFA) + +## Documentation + +Visit our [documentation website](https://rfdetr.roboflow.com) to learn more about how to use RF-DETR. + +## License + +Licensing is split by component: + +- The open-source `rfdetr` package and Apache-designated model weights are licensed under Apache License 2.0. See [`LICENSE`](LICENSE). +- Plus components, including the `rfdetr_plus` extension and RF-DETR-XL / RF-DETR-2XL detection models, are licensed under PML 1.0. + +## Acknowledgements + +Our work is built upon [LW-DETR](https://arxiv.org/pdf/2406.03459), [DINOv2](https://arxiv.org/pdf/2304.07193), and [Deformable DETR](https://arxiv.org/pdf/2010.04159). Thanks to their authors for their excellent work! + +## Citation + +If you find our work helpful for your research, please consider citing the following BibTeX entry. + +```bibtex +@inproceedings{robinson2026rfdetr, + title = {RF-DETR: Real-Time Detection Transformer}, + author = {Robinson, Isaac and Robicheaux, Peter and Popov, Matvei and Ramanan, Deva and Peri, Neehar}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2026}, + url = {https://arxiv.org/abs/2511.09554} +} +``` + +## Contribute + +We welcome and appreciate all contributions! If you notice any issues or bugs, have questions, or would like to suggest new features, please [open an issue](https://github.com/roboflow/rf-detr/issues/new) or pull request. By sharing your ideas and improvements, you help make RF-DETR better for everyone. + +

+ + + + + + + + + + + +

diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..bfd8ffb --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`roboflow/rf-detr` +- 原始仓库：https://github.com/roboflow/rf-detr +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/configs/rfdetr_base.yaml b/configs/rfdetr_base.yaml new file mode 100644 index 0000000..5302ac7 --- /dev/null +++ b/configs/rfdetr_base.yaml @@ -0,0 +1,21 @@ +# RF-DETR Base — example training configuration (resolution 560, patch 14). +# Usage: +# rfdetr fit --config configs/rfdetr_base.yaml +# rfdetr fit --config configs/rfdetr_base.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRBaseConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.TrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_base + epochs: 100 + batch_size: 4 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_large.yaml b/configs/rfdetr_large.yaml new file mode 100644 index 0000000..3fcbea0 --- /dev/null +++ b/configs/rfdetr_large.yaml @@ -0,0 +1,22 @@ +# RF-DETR Large — example training configuration (resolution 704, patch 16). +# Usage: +# rfdetr fit --config configs/rfdetr_large.yaml +# rfdetr fit --config configs/rfdetr_large.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRLargeConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.TrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_large + epochs: 100 + batch_size: 2 + grad_accum_steps: 8 # effective batch size 16 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_medium.yaml b/configs/rfdetr_medium.yaml new file mode 100644 index 0000000..0f5a1eb --- /dev/null +++ b/configs/rfdetr_medium.yaml @@ -0,0 +1,21 @@ +# RF-DETR Medium — example training configuration (resolution 576, patch 16). +# Usage: +# rfdetr fit --config configs/rfdetr_medium.yaml +# rfdetr fit --config configs/rfdetr_medium.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRMediumConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.TrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_medium + epochs: 100 + batch_size: 4 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_nano.yaml b/configs/rfdetr_nano.yaml new file mode 100644 index 0000000..97d98f0 --- /dev/null +++ b/configs/rfdetr_nano.yaml @@ -0,0 +1,21 @@ +# RF-DETR Nano — example training configuration (resolution 384, patch 16). +# Usage: +# rfdetr fit --config configs/rfdetr_nano.yaml +# rfdetr fit --config configs/rfdetr_nano.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRNanoConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.TrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_nano + epochs: 100 + batch_size: 8 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_seg_2xlarge.yaml b/configs/rfdetr_seg_2xlarge.yaml new file mode 100644 index 0000000..6ee64b4 --- /dev/null +++ b/configs/rfdetr_seg_2xlarge.yaml @@ -0,0 +1,22 @@ +# RF-DETR Seg 2XLarge — example segmentation training configuration (resolution 768, patch 12). +# Usage: +# rfdetr fit --config configs/rfdetr_seg_2xlarge.yaml +# rfdetr fit --config configs/rfdetr_seg_2xlarge.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSeg2XLargeConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.SegmentationTrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_seg_2xlarge + epochs: 100 + batch_size: 1 + grad_accum_steps: 16 # effective batch size 16 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_seg_large.yaml b/configs/rfdetr_seg_large.yaml new file mode 100644 index 0000000..ce2d04d --- /dev/null +++ b/configs/rfdetr_seg_large.yaml @@ -0,0 +1,22 @@ +# RF-DETR Seg Large — example segmentation training configuration (resolution 504, patch 12). +# Usage: +# rfdetr fit --config configs/rfdetr_seg_large.yaml +# rfdetr fit --config configs/rfdetr_seg_large.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSegLargeConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.SegmentationTrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_seg_large + epochs: 100 + batch_size: 2 + grad_accum_steps: 8 # effective batch size 16 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_seg_medium.yaml b/configs/rfdetr_seg_medium.yaml new file mode 100644 index 0000000..c295e6b --- /dev/null +++ b/configs/rfdetr_seg_medium.yaml @@ -0,0 +1,21 @@ +# RF-DETR Seg Medium — example segmentation training configuration (resolution 432, patch 12). +# Usage: +# rfdetr fit --config configs/rfdetr_seg_medium.yaml +# rfdetr fit --config configs/rfdetr_seg_medium.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSegMediumConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.SegmentationTrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_seg_medium + epochs: 100 + batch_size: 3 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_seg_nano.yaml b/configs/rfdetr_seg_nano.yaml new file mode 100644 index 0000000..a3b1fe7 --- /dev/null +++ b/configs/rfdetr_seg_nano.yaml @@ -0,0 +1,21 @@ +# RF-DETR Seg Nano — example segmentation training configuration (resolution 312, patch 12). +# Usage: +# rfdetr fit --config configs/rfdetr_seg_nano.yaml +# rfdetr fit --config configs/rfdetr_seg_nano.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSegNanoConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.SegmentationTrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_seg_nano + epochs: 100 + batch_size: 4 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_seg_small.yaml b/configs/rfdetr_seg_small.yaml new file mode 100644 index 0000000..f69b1ca --- /dev/null +++ b/configs/rfdetr_seg_small.yaml @@ -0,0 +1,21 @@ +# RF-DETR Seg Small — example segmentation training configuration (resolution 384, patch 12). +# Usage: +# rfdetr fit --config configs/rfdetr_seg_small.yaml +# rfdetr fit --config configs/rfdetr_seg_small.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSegSmallConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.SegmentationTrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_seg_small + epochs: 100 + batch_size: 4 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_seg_xlarge.yaml b/configs/rfdetr_seg_xlarge.yaml new file mode 100644 index 0000000..a20737f --- /dev/null +++ b/configs/rfdetr_seg_xlarge.yaml @@ -0,0 +1,22 @@ +# RF-DETR Seg XLarge — example segmentation training configuration (resolution 624, patch 12). +# Usage: +# rfdetr fit --config configs/rfdetr_seg_xlarge.yaml +# rfdetr fit --config configs/rfdetr_seg_xlarge.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSegXLargeConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.SegmentationTrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_seg_xlarge + epochs: 100 + batch_size: 2 + grad_accum_steps: 8 # effective batch size 16 + num_workers: 4 + tensorboard: true diff --git a/configs/rfdetr_small.yaml b/configs/rfdetr_small.yaml new file mode 100644 index 0000000..b2f8131 --- /dev/null +++ b/configs/rfdetr_small.yaml @@ -0,0 +1,21 @@ +# RF-DETR Small — example training configuration (resolution 512, patch 16). +# Usage: +# rfdetr fit --config configs/rfdetr_small.yaml +# rfdetr fit --config configs/rfdetr_small.yaml \ +# --model.train_config.init_args.dataset_dir /data/my_dataset \ +# --trainer.devices 4 + +model: + model_config: + class_path: rfdetr.config.RFDETRSmallConfig + init_args: + num_classes: 80 # set to your dataset class count + train_config: + class_path: rfdetr.config.TrainConfig + init_args: + dataset_dir: /data/coco # required: path to your COCO-format dataset + output_dir: output/rfdetr_small + epochs: 100 + batch_size: 6 + num_workers: 4 + tensorboard: true diff --git a/docs/CNAME b/docs/CNAME new file mode 100644 index 0000000..f39fba4 --- /dev/null +++ b/docs/CNAME @@ -0,0 +1 @@ +rfdetr.roboflow.com diff --git a/docs/assets/keypoints/bridge-1.jpg b/docs/assets/keypoints/bridge-1.jpg new file mode 100644 index 0000000..738b066 Binary files /dev/null and b/docs/assets/keypoints/bridge-1.jpg differ diff --git a/docs/assets/keypoints/bridge-2.jpg b/docs/assets/keypoints/bridge-2.jpg new file mode 100644 index 0000000..b53c55c Binary files /dev/null and b/docs/assets/keypoints/bridge-2.jpg differ diff --git a/docs/assets/keypoints/kp-map-latency.png b/docs/assets/keypoints/kp-map-latency.png new file mode 100644 index 0000000..853e40e Binary files /dev/null and b/docs/assets/keypoints/kp-map-latency.png differ diff --git a/docs/assets/roboflow-logo.svg b/docs/assets/roboflow-logo.svg new file mode 100644 index 0000000..1725e11 --- /dev/null +++ b/docs/assets/roboflow-logo.svg @@ -0,0 +1,29 @@ + + + + + + + + + + + + diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..180125d --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,19 @@ +--- +title: Changelog +description: RF-DETR release history and changelog. +hide: + - navigation +--- + +# Changelog + +RF-DETR release notes are maintained on [GitHub Releases](https://github.com/roboflow/rf-detr/releases). +Use the release feed to review versioned package changes, migration notes, and model updates. + +- [Install the latest PyPI package](https://pypi.org/project/rfdetr/) +- [Migration guide](getting-started/migration.md) — upgrade steps between major versions +- [Cookbooks](cookbooks.md) — runnable notebooks for training, fine-tuning, export, and deployment + +--- + +--8<-- "CHANGELOG.md" diff --git a/docs/cookbooks.md b/docs/cookbooks.md new file mode 100644 index 0000000..ec4d2dd --- /dev/null +++ b/docs/cookbooks.md @@ -0,0 +1,7 @@ +--- +template: notebooks.html +description: Practical RF-DETR cookbooks — runnable notebooks covering training, fine-tuning, export, and deployment workflows. +hide: + - navigation + - toc +--- diff --git a/docs/cookbooks/NOTES.md b/docs/cookbooks/NOTES.md new file mode 100644 index 0000000..fb19687 --- /dev/null +++ b/docs/cookbooks/NOTES.md @@ -0,0 +1,58 @@ +# Notebooks + +Each `.ipynb` file here is rendered as a page under `/cookbooks/` in the docs site. + +Cards on the cookbooks landing page are driven by [`cards.yaml`](cards.yaml). The MkDocs hook +`docs/hooks/cookbooks_cards.py` loads that file and exposes it to `docs/theme/notebooks.html`, +which renders each entry as a card via a Jinja loop. + +## Converting a jupytext `.py` to `.ipynb` + +Cookbook source files live as jupytext percent-format `.py` scripts (e.g. `fine-tune_keypoints.py`) inside `docs/cookbooks/`. Each script requires at minimum a **docs render copy**; some also have a `notebooks/` copy for users who want to run it directly. Regenerate every existing copy after each edit: + +```bash +# Docs render copy (served by mkdocs-jupyter at /cookbooks/) — always required +jupytext --to notebook fine-tune_keypoints.py --output docs/cookbooks/fine-tune_keypoints.ipynb + +# Runnable copy in notebooks/ — only for notebooks explicitly placed there +jupytext --to notebook fine-tune_keypoints.py --output notebooks/fine-tune_keypoints.ipynb +``` + +New notebooks default to the docs-only copy. Add a `notebooks/` copy only when there is an explicit need (e.g. a runnable starter notebook shipped with the repo). Omit `--execute` — notebooks are rendered statically by `mkdocs-jupyter` with `execute: false`, so pre-run outputs in the `.ipynb` are displayed as-is. + +If jupytext is not installed: `pip install jupytext` (or `uv add jupytext --dev`). + +## Adding a notebook + +1. Add the `.ipynb` file here, named after its content (e.g. `custom-augmentations.ipynb`, `onnx-export.ipynb`). +2. Add a new entry to `docs/cookbooks/cards.yaml` under the `cards:` list: + + + +```yaml + - href: content-slug/ + name: Short Title + labels: [LABEL1, LABEL2] + version: vX.Y.0 + author: GitHubUsername + description: One sentence describing what the notebook demonstrates. +``` + +Available labels (reuse these to keep tags standardised): `TRAINING`, `AUGMENTATION`, `EXPORT`, `TFLITE`, `PYTORCH LIGHTNING`, `INFERENCE`, `SEGMENTATION`, `DEPLOY`. +Current tag colours are assigned dynamically by the docs UI, so they may change if cards or labels are added or reordered. + +## Removing a notebook + +1. Delete the `.ipynb` file. +2. Remove the matching entry (the `- href: content-slug/` block) from `docs/cookbooks/cards.yaml`. + +## Current notebooks + +| File | Card title | Version | +| ----------------------------------- | ----------------------------------------------- | ------- | +| `custom-augmentations.ipynb` | Custom Augmentations and Live Training Progress | v1.5.0 | +| `fine-tune_detection.ipynb` | Fine-Tune RF-DETR Object Detection | v1.8.0 | +| `fine-tune_keypoints.ipynb` | Fine-Tune RF-DETR Keypoint Detection | v1.8.0 | +| `fine-tune_segmentation.ipynb` | Fine-Tune RF-DETR Instance Segmentation | v1.8.2 | +| `inference-latency-benchmark.ipynb` | Inference Latency Benchmark | v1.8.2 | +| `pytorch-lightning.ipynb` | Training with PyTorch Lightning | v1.6.0 | diff --git a/docs/cookbooks/cards.yaml b/docs/cookbooks/cards.yaml new file mode 100644 index 0000000..415edb8 --- /dev/null +++ b/docs/cookbooks/cards.yaml @@ -0,0 +1,37 @@ +cards: + - href: fine-tune_detection/ + name: "Fine-Tune RF-DETR Object Detection" + labels: [TRAINING, PYTORCH LIGHTNING, INFERENCE] + version: v1.8.2 + author: Borda + description: "Fine-tune RF-DETR object detection on any COCO detection dataset from Roboflow Universe — covers PPE safety, traffic, sports, infrastructure, and marine-wildlife use cases." + - href: inference-latency-benchmark/ + name: "Inference Latency Benchmark" + labels: [INFERENCE, ONNX, GPU] + version: v1.8.1 + author: Borda + description: "Benchmark detection, segmentation, and keypoint models across FP32, FP16+JIT, and ONNX Runtime — measures latency and FPS on GPU using CUDA events." + - href: fine-tune_segmentation/ + name: "Fine-Tune RF-DETR Instance Segmentation" + labels: [TRAINING, PYTORCH LIGHTNING, SEGMENTATION, INFERENCE] + version: v1.8.1 + author: Borda + description: "Fine-tune RF-DETR instance segmentation on any COCO segmentation dataset from Roboflow Universe — covers training, mask loss metrics, and inference with mask overlays." + - href: fine-tune_keypoints/ + name: "Fine-Tune RF-DETR Keypoint Detection" + labels: [TRAINING, PYTORCH LIGHTNING, INFERENCE] + version: v1.8.0 + author: Borda + description: "Fine-tune RF-DETR keypoint detection on any COCO keypoint dataset from Roboflow Universe — covers training, metrics, and inference with uncertainty ellipses." + - href: custom-augmentations/ + name: "Custom Augmentations and Live Training Progress" + labels: [TRAINING, AUGMENTATION] + version: v1.5.0 + author: Borda + description: "Add custom Albumentations augmentations and monitor training metrics in real time." + - href: pytorch-lightning/ + name: "Training with PyTorch Lightning" + labels: [TRAINING, PYTORCH LIGHTNING] + version: v1.6.0 + author: Borda + description: "Train RF-DETR with PyTorch Lightning — callbacks, checkpointing, and the LightningModule API." diff --git a/docs/cookbooks/custom-augmentations.ipynb b/docs/cookbooks/custom-augmentations.ipynb new file mode 100644 index 0000000..6857902 --- /dev/null +++ b/docs/cookbooks/custom-augmentations.ipynb @@ -0,0 +1,417 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0f72b166", + "metadata": {}, + "source": [ + "# Custom Augmentations and Live Training Progress\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow/rf-detr/blob/develop/docs/cookbooks/custom-augmentations.ipynb)\n", + "\n", + "**RF-DETR** is a real-time object detection model that combines the accuracy of\n", + "transformer-based detectors with inference speeds suitable for production.\n", + "1.5.0 brings two headline additions:\n", + "\n", + "1. **Custom training augmentations via Albumentations** — a flexible system that\n", + " lets you control exactly how images are transformed during training, with bounding\n", + " boxes and segmentation masks kept in sync automatically. Four ready-made presets\n", + " cover the most common scenarios out of the box.\n", + "2. **Live progress bars and structured epoch logs** — per-epoch rich / tqdm progress\n", + " so you can monitor batch-level metrics without parsing raw log output.\n", + "\n", + "You will learn how to:\n", + "- Explore and compare the four built-in augmentation presets\n", + "- Visually inspect augmented samples *before* committing to a full training run\n", + "- Define a fully custom augmentation pipeline\n", + "- Train RF-DETR with your chosen augmentation config\n", + "- Run inference with the trained model" + ] + }, + { + "cell_type": "markdown", + "id": "49ad10f9", + "metadata": {}, + "source": [ + "## 1. Install RF-DETR 1.5.0\n", + "\n", + "`rfdetr` includes the augmentation system and progress-bar support introduced in\n", + "this release. `supervision` handles visualization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c1df817", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q rfdetr==1.5.0" + ] + }, + { + "cell_type": "markdown", + "id": "075cf0b4", + "metadata": {}, + "source": [ + "## 2. Check GPU availability\n", + "\n", + "RF-DETR trains on GPU when one is available and falls back to CPU otherwise.\n", + "The cell below detects your device and prints VRAM size — a useful sanity check\n", + "before choosing batch size." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d005aa12", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import torch\n", + "\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"Using device: {device}\")\n", + "if device == \"cuda\":\n", + " print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n", + " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n", + "\n", + "# Scale data-loader workers with available CPUs so the GPU is kept fed.\n", + "num_workers = max(os.cpu_count() or 0, 2)\n", + "print(f\"Data loader workers: {num_workers}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c926124e", + "metadata": {}, + "source": [ + "## 3. Download COCO 2017\n", + "\n", + "We use the official train/val splits — no manual splitting needed.\n", + "\n", + "| Split | Images | Size |\n", + "|---|---|---|\n", + "| `train2017` | ~118 000 | ~18 GB |\n", + "| `val2017` | 5 000 | ~1 GB |\n", + "| annotations | — | ~241 MB |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3cc746d", + "metadata": {}, + "outputs": [], + "source": [ + "!wget -q --show-progress http://images.cocodataset.org/zips/train2017.zip -O train2017.zip\n", + "!wget -q --show-progress http://images.cocodataset.org/zips/val2017.zip -O val2017.zip\n", + "!wget -q --show-progress http://images.cocodataset.org/annotations/annotations_trainval2017.zip -O annotations.zip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "985d1e18", + "metadata": {}, + "outputs": [], + "source": [ + "!unzip -q train2017.zip\n", + "!unzip -q val2017.zip\n", + "!unzip -q annotations.zip" + ] + }, + { + "cell_type": "markdown", + "id": "27b7ae27", + "metadata": {}, + "source": [ + "### Set up the dataset directory structure\n", + "\n", + "`model.train()` expects:\n", + "```\n", + "dataset/\n", + " train/ _annotations.coco.json + images\n", + " valid/ _annotations.coco.json + images\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da25896a", + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir -p coco_demo\n", + "!mv train2017 coco_demo/train\n", + "!mv val2017 coco_demo/valid\n", + "!cp annotations/instances_train2017.json coco_demo/train/_annotations.coco.json\n", + "!cp annotations/instances_val2017.json coco_demo/valid/_annotations.coco.json" + ] + }, + { + "cell_type": "markdown", + "id": "69343c86", + "metadata": {}, + "source": [ + "## 4. Explore the built-in augmentation presets\n", + "\n", + "RF-DETR ships four ready-made presets tuned for common use cases. Each preset\n", + "is a plain Python dict mapping Albumentations transform names to their constructor\n", + "kwargs (including `p`, the probability of applying the transform). You can\n", + "inspect, merge, or extend them like any other dict.\n", + "\n", + "| Preset | When to use |\n", + "|---|---|\n", + "| `AUG_CONSERVATIVE` | Small datasets (< 500 images) — gentle transforms to avoid overfitting |\n", + "| `AUG_AGGRESSIVE` | Large datasets (2 000+ images) — stronger augmentations for better generalisation |\n", + "| `AUG_AERIAL` | Satellite and overhead imagery — rotation-invariant transforms |\n", + "| `AUG_INDUSTRIAL` | Manufacturing and inspection data — handles structured textures |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "409e006a", + "metadata": {}, + "outputs": [], + "source": [ + "import rfdetr.datasets.aug_configs as aug_config\n", + "from rfdetr.datasets.aug_configs import AUG_AGGRESSIVE\n", + "\n", + "for name in (\"AUG_CONSERVATIVE\", \"AUG_AGGRESSIVE\", \"AUG_AERIAL\", \"AUG_INDUSTRIAL\"):\n", + " preset = getattr(aug_config, name)\n", + " print(f\"\\n{name}:\")\n", + " for transform, params in preset.items():\n", + " print(f\" {transform}: {params}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d10837d5", + "metadata": {}, + "source": [ + "## 5. Inspect augmented samples before training\n", + "\n", + "Setting `save_dataset_grids=True` writes 3×3 JPEG grids of augmented training\n", + "and validation images to `output_dir` *before any weight updates occur*. Use\n", + "this to visually sanity-check your pipeline in seconds — catching problems like\n", + "flipped labels or extreme colour shifts before committing to a full training run.\n", + "\n", + "Saved files:\n", + "```\n", + "output/\n", + " train_batch0_grid.jpg train_batch1_grid.jpg train_batch2_grid.jpg\n", + " val_batch0_grid.jpg val_batch1_grid.jpg val_batch2_grid.jpg\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7797651", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from rfdetr import RFDETRNano # smallest & fastest model — ideal for demos\n", + "\n", + "DATASET_DIR = \"coco_demo\"\n", + "OUTPUT_DIR = \"output\"\n", + "os.makedirs(OUTPUT_DIR, exist_ok=True)\n", + "\n", + "model = RFDETRNano()\n", + "model.train(\n", + " dataset_dir=str(DATASET_DIR),\n", + " epochs=1, # one epoch is enough to generate the grids\n", + " batch_size=12,\n", + " aug_config=AUG_AGGRESSIVE,\n", + " save_dataset_grids=True,\n", + " output_dir=OUTPUT_DIR,\n", + " device=device,\n", + " num_workers=num_workers,\n", + " run_test=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "7b7cb782", + "metadata": {}, + "source": [ + "### Display the saved grids inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28df5b59", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "%matplotlib inline\n", + "\n", + "import matplotlib.image as mpimg\n", + "import matplotlib.pyplot as plt\n", + "\n", + "grids = sorted(Path(OUTPUT_DIR).glob(\"*_grid.jpg\"))\n", + "if grids:\n", + " fig, axes = plt.subplots(len(grids), 1, figsize=(6, 6 * len(grids)))\n", + " if len(grids) == 1:\n", + " axes = [axes]\n", + " for ax, grid_path in zip(axes, grids):\n", + " ax.imshow(mpimg.imread(grid_path))\n", + " ax.set_title(grid_path.name)\n", + " ax.axis(\"off\")\n", + " plt.tight_layout()\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "d7f10179", + "metadata": {}, + "source": [ + "## 6. Choose your augmentation config\n", + "\n", + "Pick one of the four options below. Option A (a built-in preset) is the fastest\n", + "way to get started. Options B–D give increasing levels of control. The selected\n", + "`AUG_CONFIG` is passed straight to `model.train()` in the next section." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f62b76de", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Option A: built-in preset ---\n", + "AUG_CONFIG = AUG_AGGRESSIVE\n", + "\n", + "# --- Option B: extend a preset ---\n", + "# AUG_CONFIG = {**AUG_AGGRESSIVE, \"VerticalFlip\": {\"p\": 0.3}}\n", + "\n", + "# --- Option C: fully custom ---\n", + "# AUG_CONFIG = {\n", + "# \"HorizontalFlip\": {\"p\": 0.5},\n", + "# \"Rotate\": {\"limit\": 15, \"p\": 0.3},\n", + "# \"RandomBrightnessContrast\": {\"brightness_limit\": 0.2, \"contrast_limit\": 0.2, \"p\": 0.4},\n", + "# \"GaussianBlur\": {\"blur_limit\": 3, \"p\": 0.2},\n", + "# }\n", + "\n", + "# --- Option D: no augmentations ---\n", + "# AUG_CONFIG = {}\n", + "\n", + "print(\"Selected augmentation config:\")\n", + "for transform, params in AUG_CONFIG.items():\n", + " print(f\" {transform}: {params}\")" + ] + }, + { + "cell_type": "markdown", + "id": "5a5ff7b9", + "metadata": {}, + "source": [ + "## 7. Train with augmentations\n", + "\n", + "Run a full training pass with the augmentation config chosen above. The\n", + "`progress_bar=\"rich\"` setting activates per-epoch live progress bars — a batch\n", + "counter, loss, and learning rate are updated in real time. Structured metric\n", + "tables are printed at the end of each epoch.\n", + "\n", + "`aug_config` accepts any dict of Albumentations transforms, including the\n", + "`AUG_CONFIG` defined above. Passing `{}` disables all augmentations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79d29fd9", + "metadata": {}, + "outputs": [], + "source": [ + "model = RFDETRNano()\n", + "model.train(\n", + " dataset_dir=str(DATASET_DIR),\n", + " epochs=2,\n", + " batch_size=24,\n", + " aug_config=AUG_CONFIG,\n", + " output_dir=OUTPUT_DIR,\n", + " device=device,\n", + " num_workers=num_workers,\n", + " progress_bar=\"rich\",\n", + " run_test=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "beccaf40", + "metadata": {}, + "source": [ + "## 8. Run inference\n", + "\n", + "Load a sample image and run `model.predict()`. The method returns a\n", + "`supervision.Detections` object, which makes it straightforward to draw\n", + "bounding boxes and labels with the `supervision` annotators." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d79c7612", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import supervision as sv\n", + "from PIL import Image\n", + "\n", + "from rfdetr.util.coco_classes import COCO_CLASSES\n", + "\n", + "image_url = \"https://media.roboflow.com/dog.jpg\"\n", + "image = Image.open(requests.get(image_url, stream=True).raw)\n", + "\n", + "detections = model.predict(image, threshold=0.5)\n", + "\n", + "labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]\n", + "annotated = sv.BoxAnnotator().annotate(image.copy(), detections)\n", + "annotated = sv.LabelAnnotator().annotate(annotated, detections, labels)\n", + "sv.plot_image(annotated)" + ] + }, + { + "cell_type": "markdown", + "id": "d69f8215", + "metadata": {}, + "source": [ + "## 9. Next steps\n", + "\n", + "You have seen the complete 1.5.0 augmentation workflow — from exploring presets\n", + "through live inspection to a full training run. From here:\n", + "\n", + "- [Augmentation docs](https://rfdetr.roboflow.com/1.5.0/learn/train/augmentations/) — full transform reference and custom pipeline guide\n", + "- [Advanced training options](https://rfdetr.roboflow.com/1.5.0/learn/train/advanced/) — EMA, gradient accumulation, learning rate schedules\n", + "- [Logger integrations (ClearML, MLflow, W&B)](https://rfdetr.roboflow.com/1.5.0/learn/train/loggers/) — experiment tracking\n", + "- [Export your model](https://rfdetr.roboflow.com/1.5.0/learn/export/) — ONNX, TensorRT, CoreML\n", + "- [RF-DETR 1.6.0 notebook](https://colab.research.google.com/github/roboflow/rf-detr/blob/develop/notebooks/release-demo_1-6.ipynb) — PyTorch Lightning building blocks" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "formats": "ipynb,py", + "main_language": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/cookbooks/fine-tune_detection.ipynb b/docs/cookbooks/fine-tune_detection.ipynb new file mode 100644 index 0000000..3513527 --- /dev/null +++ b/docs/cookbooks/fine-tune_detection.ipynb @@ -0,0 +1,944 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "920e5414", + "metadata": {}, + "source": [ + "# RF-DETR object detection fine-tuning on Roboflow Universe datasets\n", + "\n", + "Select one dataset with `DATASET_KEY`, then run the same fine-tuning, plotting, checkpoint loading, and inference cells.\n", + "\n", + "## What is object detection?\n", + "\n", + "Object detection answers two questions simultaneously: *what* objects are in an image and *where* they are.\n", + "Each detected object gets a **bounding box** (four coordinates) and a **class label** (e.g. \"car\", \"person\").\n", + "This is the most common computer vision task — it powers traffic monitoring, safety compliance, retail analytics,\n", + "medical imaging, environmental surveys, and virtually every real-time vision application.\n", + "\n", + "## Why fine-tune instead of train from scratch?\n", + "\n", + "RF-DETR ships pre-trained on the COCO dataset (80 common object classes). Fine-tuning reuses the backbone's\n", + "rich visual representations — edges, textures, object parts — learned from millions of images.\n", + "Starting from these weights instead of random initialisation typically:\n", + "- **Reduces training time** from days to minutes or hours\n", + "- **Requires far fewer labelled images** (hundreds instead of tens of thousands)\n", + "- **Achieves higher accuracy** than training from scratch on small datasets\n", + "\n", + "The process is straightforward: swap out the final classification head for your number of classes,\n", + "then continue training with a small learning rate so the pre-trained features are preserved.\n", + "\n", + "## Dataset overview\n", + "\n", + "Five datasets covering different domains are pre-configured. Set `DATASET_KEY` to one of:\n", + "\n", + "| Key | Dataset | Domain | Classes |\n", + "|-----|---------|--------|---------|\n", + "| `\"hard_hat_ppes\"` | Hard Hat Worker Safety | Construction / PPE | hat, person, vest |\n", + "| `\"road_damage\"` | Pavement Distress Detection | Infrastructure Inspection | pot_hole, cracking, ravelling, ... |\n", + "| `\"traffic\"` | Traffic Detection | Autonomous Driving | car, bus, truck, motorbike, bike, traffic lights |\n", + "| `\"football\"` | Football Player Detection | Sports Analytics | player, ball, goalkeeper, referee |\n", + "| `\"brackish_underwater\"` | Brackish Underwater | Marine Wildlife | fish, crab, jellyfish, shrimp, starfish, small_fish |\n", + "\n", + "Every subsequent cell (fine-tuning, plotting, checkpoint loading, inference) adapts automatically.\n", + "By the end you will have a fine-tuned model, training curves, and annotated inference images with bounding boxes." + ] + }, + { + "cell_type": "markdown", + "id": "b6eb7cd5", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install `rfdetr` with the `train` and `visual` extras. The `train` extra pulls in the training loop dependencies\n", + "(PyTorch Lightning, COCO evaluation tools, and data augmentation libraries), while `visual` adds the visualisation\n", + "helpers used later in the notebook. The `roboflow` package handles dataset download; `pandas` and `seaborn` are\n", + "needed for the metrics table and optional plot styling. If you hit an `ImportError` after running this cell,\n", + "the most likely cause is a stale in-memory import — restart the Python runtime once and re-run from the top.\n", + "\n", + "**GPU recommended.** Fine-tuning 50 epochs on a ~1,500-image dataset takes roughly 10–25 minutes on a modern GPU\n", + "(RTX 3090, A10, T4). CPU-only is possible for a quick test but will be much slower." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e79facfd", + "metadata": { + "title": "[bash]" + }, + "outputs": [], + "source": "!pip install -q \"rfdetr[train,visual]>=1.8.0\" roboflow pandas seaborn" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62aff634", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Shared RF-DETR object detection fine-tuning demo for several Roboflow COCO detection exports.\"\"\"\n", + "\n", + "import json\n", + "import os\n", + "from pathlib import Path\n", + "from typing import Any\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import display\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib.figure import Figure\n", + "from PIL import Image\n", + "from roboflow import Roboflow\n", + "\n", + "from rfdetr import RFDETRSmall\n", + "from rfdetr.config import TrainConfig\n", + "from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer\n", + "from rfdetr.training.callbacks.best_model import BestModelCallback\n", + "from rfdetr.utilities.reproducibility import seed_all\n", + "from rfdetr.visualize.training import plot_loss_metrics, plot_map_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9c78daa", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "PROJECT_ROOT = Path(__file__).resolve().parent if \"__file__\" in globals() else Path.cwd()\n", + "DATASETS_DIR = PROJECT_ROOT / \"datasets\"\n", + "\n", + "# Resolution must be divisible by patch_size × num_windows (32 for RFDETRSmall: patch_size=16, num_windows=2).\n", + "# Higher resolution improves detection of small objects at the cost of GPU memory and training time.\n", + "# Practical range: 384–640. The default 512 suits most fine-tuning runs.\n", + "DATASETS: dict[str, dict[str, Any]] = {\n", + " \"hard_hat_ppes\": {\n", + " \"name\": \"Hard Hat Worker Safety\",\n", + " \"workspace\": \"safety-first\",\n", + " \"project\": \"hard-hat-worker-safety-equipments\",\n", + " \"version\": 9,\n", + " \"source_url\": \"https://universe.roboflow.com/safety-first/hard-hat-worker-safety-equipments\",\n", + " \"output_name\": \"det_hard_hat_ppes\",\n", + " # 512 — outdoor construction scenes; workers appear at medium scale so the\n", + " # default resolution is sufficient to distinguish helmet vs no-helmet.\n", + " \"resolution\": 512,\n", + " },\n", + " \"road_damage\": {\n", + " \"name\": \"Pavement Distress Detection\",\n", + " \"workspace\": \"pavement-distresses\",\n", + " \"project\": \"distress-detection\",\n", + " \"version\": 2,\n", + " \"source_url\": \"https://universe.roboflow.com/pavement-distresses/distress-detection\",\n", + " \"output_name\": \"det_road_damage\",\n", + " # 544 — road-surface images captured from dashcams and drones; fine cracks\n", + " # and potholes are small relative to the frame, so the extra resolution\n", + " # helps the model distinguish closely spaced defect categories.\n", + " \"resolution\": 544,\n", + " },\n", + " \"traffic\": {\n", + " \"name\": \"Traffic Detection\",\n", + " \"workspace\": \"redlightrunningdection\",\n", + " \"project\": \"traffic-detection-sutq6\",\n", + " \"version\": 36,\n", + " \"source_url\": \"https://universe.roboflow.com/redlightrunningdection/traffic-detection-sutq6\",\n", + " \"output_name\": \"det_traffic\",\n", + " # 512 — street-level traffic scenes; vehicles span a wide size range\n", + " # (large buses to distant motorbikes); 512 gives a good speed/accuracy trade-off.\n", + " \"resolution\": 512,\n", + " },\n", + " \"football\": {\n", + " \"name\": \"Football Player Detection\",\n", + " \"workspace\": \"football-gozni\",\n", + " \"project\": \"football-player-detection-bfswn\",\n", + " \"version\": 1,\n", + " \"source_url\": \"https://universe.roboflow.com/football-gozni/football-player-detection-bfswn\",\n", + " \"output_name\": \"det_football\",\n", + " # 544 — broadcast and stadium-angle shots; the ball is a tiny object relative\n", + " # to the field; the extra resolution meaningfully reduces false negatives.\n", + " \"resolution\": 544,\n", + " },\n", + " \"brackish_underwater\": {\n", + " \"name\": \"Brackish Underwater\",\n", + " \"workspace\": \"brad-dwyer\",\n", + " \"project\": \"brackish-underwater\",\n", + " \"version\": 2,\n", + " \"source_url\": \"https://universe.roboflow.com/brad-dwyer/brackish-underwater\",\n", + " \"output_name\": \"det_brackish_underwater\",\n", + " # 512 — sonar and underwater camera frames; marine creatures appear at\n", + " # medium-to-large scale; the default resolution captures enough detail.\n", + " \"resolution\": 512,\n", + " },\n", + "}\n", + "\n", + "DATASET_KEY = \"hard_hat_ppes\"\n", + "DATASET_INFO = DATASETS[DATASET_KEY]\n", + "\n", + "OUTPUT_DIR = PROJECT_ROOT / \"output\" / str(DATASET_INFO[\"output_name\"])\n", + "METRICS_CSV = OUTPUT_DIR / \"metrics.csv\"\n", + "VALIDATION_METRICS_JSON = OUTPUT_DIR / \"validation_metrics.json\"\n", + "FINAL_CHECKPOINT_PATH = OUTPUT_DIR / \"checkpoint_final_demo.pth\"\n", + "\n", + "RESOLUTION = int(DATASET_INFO[\"resolution\"])\n", + "\n", + "SEED = 7\n", + "EPOCHS = 50\n", + "BATCH_SIZE = 8\n", + "GRAD_ACCUM_STEPS = 2\n", + "NUM_WORKERS = 8\n", + "LR = 1e-4\n", + "LR_ENCODER = 1e-4\n", + "SAMPLE_PREVIEW_COUNT = 6\n", + "SAMPLE_PREVIEW_COLUMNS = 3\n", + "SAMPLE_PREVIEW_FIGURE_SIZE: tuple[float, float] = (15.0, 10.0)\n", + "INFERENCE_COUNT = 6\n", + "INFERENCE_COLUMNS = 3\n", + "INFERENCE_THRESHOLD = 0.3\n", + "PLOT_LOSS_LOG_SCALE = False\n", + "\n", + "print(f\"dataset_key={DATASET_KEY}\")\n", + "print(f\"dataset={DATASET_INFO['name']}\")\n", + "print(f\"source_url={DATASET_INFO['source_url']}\")\n", + "print(f\"resolution={RESOLUTION}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cdce84bd", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## Notebook display\n", + "\n", + "This helper registers the `%matplotlib inline` magic so figures render directly beneath each cell when you run the\n", + "notebook in Jupyter or Google Colab. If you are running this file as a plain Python script the function detects the\n", + "absence of an IPython kernel and exits silently, so it is always safe to call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "993a411e", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _enable_notebook_inline_matplotlib() -> None:\n", + " \"\"\"Enable inline matplotlib figures when running in IPython.\"\"\"\n", + " get_ipython_func = globals().get(\"get_ipython\")\n", + " if not callable(get_ipython_func):\n", + " return\n", + " ipython = get_ipython_func()\n", + " if ipython is not None:\n", + " ipython.run_line_magic(\"matplotlib\", \"inline\")\n", + " ipython.run_line_magic(\"config\", \"InlineBackend.close_figures = True\")\n", + "\n", + "\n", + "_enable_notebook_inline_matplotlib()" + ] + }, + { + "cell_type": "markdown", + "id": "9c7f944e", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 1 - Download dataset\n", + "\n", + "Roboflow exports object detection datasets in **COCO format**: a JSON file containing image metadata and per-image\n", + "annotations, each with a `bbox` field (x, y, width, height in pixels) and a `category_id` referencing the class.\n", + "The download cell fetches the exact dataset version listed in `DATASETS` and places it under\n", + "`datasets//`. You need a Roboflow API key — get one for free at `app.roboflow.com/settings/api`\n", + "and set it as the `ROBOFLOW_API_KEY` environment variable (or as a Colab secret with the same name).\n", + "\n", + "The download is **idempotent**: if the target directory already exists Roboflow skips the network transfer,\n", + "so re-running this cell after a successful download is fast.\n", + "\n", + "> **Tip:** All five datasets in this notebook are open-access on Roboflow Universe. You can browse them in your\n", + "> browser to explore image samples and annotation statistics before downloading." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20fea837", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "try:\n", + " from google.colab import userdata\n", + "\n", + " try:\n", + " ROBOFLOW_API_KEY = userdata.get(\"ROBOFLOW_API_KEY\") or \"\"\n", + " except Exception:\n", + " ROBOFLOW_API_KEY = \"\"\n", + "except ImportError:\n", + " ROBOFLOW_API_KEY = \"\"\n", + "\n", + "if not ROBOFLOW_API_KEY:\n", + " ROBOFLOW_API_KEY = os.environ.get(\"ROBOFLOW_API_KEY\", \"\")\n", + "if not ROBOFLOW_API_KEY:\n", + " raise RuntimeError(\n", + " \"ROBOFLOW_API_KEY not found. \"\n", + " \"In Colab: add it via Secrets (key icon). \"\n", + " \"Locally: set the environment variable before running.\"\n", + " )\n", + "\n", + "rf = Roboflow(api_key=ROBOFLOW_API_KEY)\n", + "dataset = (\n", + " rf.workspace(str(DATASET_INFO[\"workspace\"]))\n", + " .project(str(DATASET_INFO[\"project\"]))\n", + " .version(int(DATASET_INFO[\"version\"]))\n", + " .download(\"coco\", location=str(DATASETS_DIR / DATASET_KEY))\n", + ")\n", + "DATASET_DIR = Path(dataset.location)\n", + "TRAIN_ANNOTATIONS = DATASET_DIR / \"train\" / \"_annotations.coco.json\"\n", + "print(f\"dataset_dir={DATASET_DIR}\")" + ] + }, + { + "cell_type": "markdown", + "id": "6516f6ee", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 2 - Infer class names\n", + "\n", + "The COCO annotation file defines a `categories` list mapping integer IDs to human-readable names. RF-DETR uses\n", + "**0-based class indices** internally, so we sort the categories by their original `id` field and read names in order.\n", + "The resulting `CLASS_NAMES` list maps each index back to a label, which is used during inference visualisation.\n", + "\n", + "If your own dataset has a single category, this list will have one element — that is expected and valid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ca32aa1", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "with TRAIN_ANNOTATIONS.open() as _f:\n", + " _coco = json.load(_f)\n", + "\n", + "CLASS_NAMES: list[str] = [cat[\"name\"] for cat in sorted(_coco[\"categories\"], key=lambda c: c[\"id\"])]\n", + "NUM_CLASSES = len(CLASS_NAMES)\n", + "\n", + "print(f\"class_names={CLASS_NAMES}\")\n", + "print(f\"num_classes={NUM_CLASSES}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3e39e02e", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_save_final_checkpoint` writes a self-contained `.pth` file that bundles model weights with the full training and\n", + "model config. This is separate from the PTL checkpoint because it can be loaded with a single `from_checkpoint`\n", + "call on any machine, without reconstructing the original config." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89450ec9", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _save_final_checkpoint(\n", + " module: RFDETRModelModule,\n", + " trainer: Any,\n", + " train_config: TrainConfig,\n", + " model_config: Any,\n", + " output_path: Path,\n", + ") -> Path:\n", + " \"\"\"Save a final RF-DETR .pth checkpoint that can be loaded with ``from_checkpoint``.\"\"\"\n", + " raw_model: Any = getattr(module.model, \"_orig_mod\", module.model)\n", + " output_path.parent.mkdir(parents=True, exist_ok=True)\n", + " torch.save(\n", + " BestModelCallback._build_checkpoint_payload(\n", + " raw_model.state_dict(),\n", + " train_config.model_dump(),\n", + " trainer,\n", + " model_name=\"RFDETRSmall\",\n", + " model_config_dict=model_config.model_dump(),\n", + " ),\n", + " output_path,\n", + " )\n", + " return output_path" + ] + }, + { + "cell_type": "markdown", + "id": "e0a0ad62", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_detection_grid_figure` arranges multiple annotated images into a fixed-column matplotlib grid.\n", + "Bounding boxes are drawn with `sv.BoxAnnotator` and labels with `sv.LabelAnnotator`, showing the class name and\n", + "confidence score for each detection. Using a grid keeps the visual summary compact and makes it easy to compare\n", + "predictions across images at a glance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efae67bb", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _detection_grid_figure(\n", + " items: list[tuple[str, np.ndarray, sv.Detections]],\n", + " columns: int,\n", + " class_names: list[str],\n", + ") -> Figure:\n", + " \"\"\"Render bounding box detections in a fixed-column subplot grid.\"\"\"\n", + " if columns <= 0:\n", + " raise ValueError(f\"columns must be positive, got {columns}.\")\n", + "\n", + " box_annotator = sv.BoxAnnotator(thickness=2)\n", + " label_annotator = sv.LabelAnnotator(text_scale=0.5, text_thickness=1, text_padding=4)\n", + "\n", + " rows = max(1, (len(items) + columns - 1) // columns)\n", + " figure, axes = plt.subplots(rows, columns, figsize=(5 * columns, 5 * rows))\n", + " axes_array = np.asarray(axes, dtype=object).reshape(-1)\n", + " for axis in axes_array:\n", + " axis.axis(\"off\")\n", + "\n", + " for axis, (title, image, detections) in zip(axes_array, items, strict=False):\n", + " scene = image.copy()\n", + " scene = box_annotator.annotate(scene=scene, detections=detections)\n", + " if len(detections) > 0 and detections.class_id is not None:\n", + " conf_list = (\n", + " detections.confidence.tolist() if detections.confidence is not None else [None] * len(detections)\n", + " )\n", + " labels = [\n", + " f\"{class_names[cid] if cid < len(class_names) else cid}\" + (f\" {conf:.2f}\" if conf is not None else \"\")\n", + " for cid, conf in zip(detections.class_id.tolist(), conf_list)\n", + " ]\n", + " scene = label_annotator.annotate(scene=scene, detections=detections, labels=labels)\n", + " axis.imshow(scene)\n", + " axis.set_title(title, fontsize=10)\n", + " axis.axis(\"off\")\n", + "\n", + " figure.tight_layout()\n", + " return figure" + ] + }, + { + "cell_type": "markdown", + "id": "38359554", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 3 - Configure training\n", + "\n", + "`TrainConfig` centralises every hyperparameter the training loop needs. The most important fields to understand\n", + "before customising them for your own dataset:\n", + "\n", + "**Learning rates** — `lr` and `lr_encoder` are both set to `1e-4` for fine-tuning. The `lr` controls the\n", + "detection head and decoder; `lr_encoder` controls the vision backbone (DINOv2). For fine-tuning, both are usually\n", + "equal. If you notice the backbone overfitting early — validation loss rising while training loss keeps falling —\n", + "halve `lr_encoder` to protect the pre-trained features while the head continues to adapt.\n", + "\n", + "**Batch size and gradient accumulation** — `batch_size=8` with `grad_accum_steps=2` gives an effective batch size\n", + "of 16 without requiring extra GPU memory. Effective batch size affects how smoothly gradients are estimated each\n", + "update; too small (< 8) leads to noisy updates; too large (> 64) may over-smooth and slow convergence.\n", + "\n", + "**EMA** — `use_ema=False` keeps this demo fast. For a production run you can enable EMA\n", + "(`use_ema=True`) to maintain an exponential moving average of the weights, which typically adds 0.5–1.0 mAP points\n", + "on small datasets by reducing the impact of noisy late-epoch updates.\n", + "\n", + "**Multi-scale training** — `multi_scale=False` and `expanded_scales=False` disable the multi-resolution\n", + "augmentation used during pre-training. Turning them off shortens epoch time by up to 40 % and is fine for\n", + "most fine-tuning runs on small custom datasets. Re-enable if your validation mAP plateaus early.\n", + "\n", + "**Notes dict** — stored verbatim in the checkpoint file alongside the weights, giving you a lightweight experiment\n", + "log that travels with the model file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8c0bbcf", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "seed_all(SEED)\n", + "variant = RFDETRSmall( # type: ignore[no-untyped-call]\n", + " num_classes=NUM_CLASSES,\n", + " resolution=RESOLUTION,\n", + ")\n", + "variant.model_config.model_name = type(variant).__name__\n", + "\n", + "train_config = TrainConfig(\n", + " dataset_file=\"roboflow\",\n", + " dataset_dir=str(DATASET_DIR),\n", + " output_dir=str(OUTPUT_DIR),\n", + " epochs=EPOCHS,\n", + " batch_size=BATCH_SIZE,\n", + " grad_accum_steps=GRAD_ACCUM_STEPS,\n", + " num_workers=NUM_WORKERS,\n", + " lr=LR,\n", + " lr_encoder=LR_ENCODER,\n", + " use_ema=False,\n", + " run_test=False,\n", + " compute_train_metrics=True,\n", + " compute_val_loss=True,\n", + " multi_scale=False,\n", + " expanded_scales=False,\n", + " do_random_resize_via_padding=False,\n", + " tensorboard=False,\n", + " wandb=False,\n", + " mlflow=False,\n", + " clearml=False,\n", + " class_names=CLASS_NAMES,\n", + " notes={\n", + " \"demo\": f\"detection PTL fine-tune on Roboflow Universe {DATASET_INFO['project']}\",\n", + " \"source_url\": DATASET_INFO[\"source_url\"],\n", + " \"roboflow_workspace\": DATASET_INFO[\"workspace\"],\n", + " \"roboflow_project\": DATASET_INFO[\"project\"],\n", + " \"roboflow_version\": DATASET_INFO[\"version\"],\n", + " \"num_classes\": NUM_CLASSES,\n", + " \"class_names\": CLASS_NAMES,\n", + " },\n", + " progress_bar=\"tqdm\",\n", + ")\n", + "\n", + "datamodule = RFDETRDataModule(variant.model_config, train_config)\n", + "model = RFDETRModelModule(variant.model_config, train_config)\n", + "trainer = build_trainer(train_config, variant.model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "c1eaa77a", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 4 - Preview dataset inputs\n", + "\n", + "**Always look at your data before starting a long training run.** This cell renders a grid of annotated training\n", + "images drawn from the augmented pipeline — the same view the model will see during training.\n", + "\n", + "What to check here:\n", + "- **Box alignment** — do bounding boxes tightly surround the objects, or are they shifted or too large?\n", + "- **Class correctness** — are labels correct for each box? Mixed-up classes are one of the most common\n", + " annotation errors and are hard to recover from after training.\n", + "- **Augmentation sanity** — horizontal flips, colour jitter, and random crops should look plausible;\n", + " cropped boxes at image edges are normal.\n", + "- **Class imbalance** — if one class dominates the preview, the model will likely be biased toward it;\n", + " rebalancing or oversampling rare classes can help.\n", + "\n", + "Catching label noise here costs a few seconds; catching it after 50 epochs of training costs much more." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a42a8135", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "sample_figure = datamodule._show_samples(\n", + " SAMPLE_PREVIEW_COUNT,\n", + " split=\"train\",\n", + " columns=SAMPLE_PREVIEW_COLUMNS,\n", + " figure_size=SAMPLE_PREVIEW_FIGURE_SIZE,\n", + ")\n", + "display(sample_figure)\n", + "plt.close(sample_figure)" + ] + }, + { + "cell_type": "markdown", + "id": "58343884", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 5 - Fine-tune\n", + "\n", + "`trainer.fit` hands control to PyTorch Lightning, which drives the full training loop: forward pass, bipartite\n", + "matching loss (classification + bounding-box L1 + GIoU), gradient accumulation, weight updates,\n", + "learning-rate scheduling, and periodic validation with COCO bounding-box mAP.\n", + "\n", + "After each epoch the trainer appends a row to `metrics.csv` and, if validation mAP improves,\n", + "saves a new best checkpoint via `BestModelCallback`.\n", + "\n", + "**Typical training times on a single GPU** (RTX 3090 or similar):\n", + "- PPE / hard hat dataset (~2,600 images): ~20–30 min for 50 epochs\n", + "- Traffic dataset (~1,400 images): ~15–20 min for 50 epochs\n", + "- Underwater dataset (~12,000 images): ~60–90 min for 50 epochs\n", + "\n", + "To **resume an interrupted run**, set `train_config.resume` to the path of the last PTL checkpoint\n", + "(e.g. `OUTPUT_DIR / \"last.ckpt\"`) before calling this cell again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c78f17f", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "trainer.fit(model, datamodule=datamodule, ckpt_path=train_config.resume or None)\n", + "print(f\"output_dir={OUTPUT_DIR}\")" + ] + }, + { + "cell_type": "markdown", + "id": "92918976", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 6 - Save checkpoint/model\n", + "\n", + "PTL saves its own checkpoints during training (optimizer state, scheduler state, epoch counter), but those files\n", + "are not directly portable — they require the same class hierarchy to load. The `.pth` file written here is a\n", + "**self-contained RF-DETR checkpoint**: it bundles the model weights together with the full training and model\n", + "configs, including the class names. You can share the file with a colleague and they can run inference with a\n", + "single `RFDETRSmall.from_checkpoint(path)` call, with no need to reconstruct the original config.\n", + "\n", + "For full reproducibility, keep this checkpoint alongside the dataset version number printed in section 1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47156e82", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "final_checkpoint = _save_final_checkpoint(model, trainer, train_config, variant.model_config, FINAL_CHECKPOINT_PATH)\n", + "print(f\"saved_checkpoint={final_checkpoint}\")" + ] + }, + { + "cell_type": "markdown", + "id": "9dbf2232", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 7 - Validate metrics\n", + "\n", + "This cell runs a clean post-training validation: no augmentation, the best checkpoint loaded from\n", + "`checkpoint_best_total.pth` written by `BestModelCallback`, and results serialised to JSON for downstream\n", + "comparison or reporting.\n", + "\n", + "**Key metrics to inspect:**\n", + "\n", + "- **`bbox/map`** — the primary metric: standard COCO bounding-box mAP averaged across IoU thresholds 0.50–0.95\n", + " in steps of 0.05. A value of 0.50 means the model correctly localises and classifies 50 % of objects at strict\n", + " IoU thresholds. This is the most commonly reported number in object detection benchmarks.\n", + "- **`bbox/map_50`** — mAP at IoU ≥ 0.50 (a box is \"correct\" if it overlaps the ground-truth box by at least 50 %).\n", + " Rises fastest and is the clearest early signal of whether the model is learning at all.\n", + "- **`bbox/map_75`** — mAP at the stricter IoU ≥ 0.75 threshold. Reflects how precisely the model localises\n", + " objects, not just whether it finds them.\n", + "\n", + "A model with high `bbox/map_50` but low `bbox/map_75` finds objects reliably but draws imprecise boxes.\n", + "This usually improves with more training epochs or by increasing `RESOLUTION`.\n", + "\n", + "> **Note:** `checkpoint_best_total.pth` is written after the first completed validation epoch. If training was\n", + "> interrupted before any epoch finished, this file will not exist and the cell below will raise `FileNotFoundError`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b35e7f96", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "# checkpoint_best_total.pth stores plain model weights under key \"model\", not a full\n", + "# PTL checkpoint. Passing it as ckpt_path to trainer.validate() triggers a KeyError\n", + "# on \"validate_loop\" because PTL tries to restore loop state that isn't present.\n", + "# Load the weights directly and run validate without ckpt_path instead.\n", + "_ckpt = torch.load(OUTPUT_DIR / \"checkpoint_best_total.pth\", map_location=\"cpu\", weights_only=False)\n", + "model.model.load_state_dict(_ckpt[\"model\"], strict=True)\n", + "del _ckpt\n", + "validation_results = trainer.validate(model, datamodule=datamodule)\n", + "validation_metrics = {key: float(value) for key, value in validation_results[0].items()} if validation_results else {}\n", + "VALIDATION_METRICS_JSON.write_text(json.dumps(validation_metrics, indent=2, sort_keys=True), encoding=\"utf-8\")\n", + "print(f\"validation_metrics={validation_metrics}\")\n", + "print(f\"validation_metrics_json={VALIDATION_METRICS_JSON}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cbb8112e", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 8 - Plot CSVLogger metrics\n", + "\n", + "Reading loss and mAP curves is the fastest way to diagnose training problems.\n", + "\n", + "**Loss curves (first plot):**\n", + "- A healthy run shows both training loss and validation loss decreasing together.\n", + "- A **growing gap** between train and val loss is a classic overfitting signal — the model is memorising the\n", + " training set. Add augmentation (`multi_scale=True`, or custom Albumentations), reduce learning rate, or collect\n", + " more data.\n", + "- A **flat or rising loss from epoch 1** usually means the learning rate is too high. Try reducing `LR` to `5e-5`.\n", + "\n", + "**mAP curves (second plot):**\n", + "- `bbox_map_50` (solid line) rises fastest and is the clearest signal of learning.\n", + "- On small datasets (a few hundred images) you typically see rapid improvement in epochs 1–20 followed by\n", + " a plateau around epoch 30–50.\n", + "- If mAP is **still rising at epoch 50**, extend `EPOCHS` to 75 or 100.\n", + "- If mAP **never rises above 0.05**, check class names, annotation quality, and whether\n", + " `RESOLUTION` is appropriate for the objects' scale in your dataset.\n", + "\n", + "Set `PLOT_LOSS_LOG_SCALE = True` if the loss drops by an order of magnitude in the first few epochs and the later,\n", + "more meaningful portion of the curve gets compressed into a flat line." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "278892c2", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"metrics_csv={METRICS_CSV}\")\n", + "loss_figure = plot_loss_metrics(str(METRICS_CSV), loss_log_scale=PLOT_LOSS_LOG_SCALE)\n", + "display(loss_figure)\n", + "plt.close(loss_figure)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0f5f468", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "map_figure = plot_map_metrics(str(METRICS_CSV))\n", + "display(map_figure)\n", + "plt.close(map_figure)" + ] + }, + { + "cell_type": "markdown", + "id": "550cf8f7", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 9 - Load checkpoint/model\n", + "\n", + "`from_checkpoint` is the standard entry point for loading a saved RF-DETR model. It reads both the weights and the\n", + "stored config from the `.pth` file, so the reconstructed model has the correct number of classes and resolution\n", + "without you having to pass them explicitly.\n", + "\n", + "This cell also confirms that the save–load round trip works before you proceed to inference, so any file corruption\n", + "or version mismatch surfaces here rather than silently producing wrong predictions later.\n", + "\n", + "To deploy this model in a production pipeline, you only need to ship the `.pth` file and call `from_checkpoint`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd0a0623", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "loaded_model = RFDETRSmall.from_checkpoint(FINAL_CHECKPOINT_PATH)" + ] + }, + { + "cell_type": "markdown", + "id": "929edc54", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 10 - Select inference images\n", + "\n", + "This cell implements a fallback chain — test split first, then validation, then train — so the notebook always\n", + "finds images to run inference on even when the dataset has no dedicated test split.\n", + "\n", + "**Using test images gives an unbiased view of model performance** because those images were never seen during\n", + "training or used to select the best checkpoint. If your deployment images look different from the training set\n", + "(different lighting, camera angle, resolution), replace `inference_image_paths` with a list of `Path` objects\n", + "pointing to your own files to assess real-world performance before shipping the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc90e306", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "_IMAGE_EXTS = {\".jpg\", \".jpeg\", \".png\"}\n", + "\n", + "\n", + "def _find_images(directory: Path) -> list[Path]:\n", + " if not directory.is_dir():\n", + " return []\n", + " return sorted(p for p in directory.glob(\"**/*\") if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)\n", + "\n", + "\n", + "inference_image_paths = _find_images(DATASET_DIR / \"test\")\n", + "if not inference_image_paths:\n", + " inference_image_paths = _find_images(DATASET_DIR / \"valid\")\n", + "if not inference_image_paths:\n", + " inference_image_paths = _find_images(DATASET_DIR / \"train\")\n", + "if not inference_image_paths:\n", + " raise FileNotFoundError(f\"No images ({', '.join(sorted(_IMAGE_EXTS))}) found under {DATASET_DIR}\")\n", + "\n", + "inference_image_paths = inference_image_paths[:INFERENCE_COUNT]\n", + "print(f\"inference_images={[str(p) for p in inference_image_paths]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f49ac0a4", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 11 - Visualize inference\n", + "\n", + "`INFERENCE_THRESHOLD` is a **detection confidence threshold**: only detections whose bounding-box score exceeds\n", + "this value are shown. The default `0.3` is a reasonable starting point; adjust it based on what you see:\n", + "\n", + "- **Too many false positives** (boxes on background or wrong objects) → raise the threshold towards `0.5`–`0.7`.\n", + "- **Missing true objects** (objects in the image not detected) → lower the threshold towards `0.1`–`0.2`.\n", + "- **Zero detections on most images** → lower the threshold; the model may be well-calibrated to lower confidence\n", + " scores on a custom dataset than the COCO-pretrained default.\n", + "\n", + "The confidence score reflects how certain the model is about each box. On a well-fine-tuned model you will\n", + "typically see scores above 0.7 for clear, unoccluded objects of the target classes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4f7739a5", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "inference_grid_items: list[tuple[str, np.ndarray, sv.Detections]] = []\n", + "for image_path in inference_image_paths:\n", + " with Image.open(image_path) as image:\n", + " image_rgb = image.convert(\"RGB\")\n", + " detections = loaded_model.predict(image_rgb, threshold=INFERENCE_THRESHOLD)\n", + " if not isinstance(detections, sv.Detections):\n", + " raise RuntimeError(f\"Expected RFDETRSmall.predict() to return sv.Detections, got {type(detections)!r}.\")\n", + " inference_grid_items.append((image_path.name, np.array(image_rgb), detections))\n", + "\n", + "if inference_grid_items:\n", + " figure = _detection_grid_figure(inference_grid_items, columns=INFERENCE_COLUMNS, class_names=CLASS_NAMES)\n", + " display(figure)\n", + " plt.close(figure)" + ] + }, + { + "cell_type": "markdown", + "id": "40582975", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 12 - Detection summary\n", + "\n", + "Print per-image detection counts and confidence scores as a quick sanity check. This table is useful for:\n", + "\n", + "- **Spotting images with zero detections** — if many images produce no detections at your chosen threshold,\n", + " the model may be under-confident; try lowering `INFERENCE_THRESHOLD` first.\n", + "- **Checking average confidence** — a well-calibrated fine-tuned model typically reports average confidence\n", + " between 0.5 and 0.9 on in-distribution images. Scores consistently below 0.3 suggest the model is uncertain\n", + " and may benefit from more training epochs or additional labelled data.\n", + "- **Class distribution** — if certain classes never appear in inference results but were present in training data,\n", + " they may be under-represented in the training set. Consider adding more examples for those classes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa58d24e", + "metadata": {}, + "outputs": [], + "source": [ + "summary_rows = []\n", + "for image_path, (_, _, detections) in zip(inference_image_paths, inference_grid_items):\n", + " num_det = len(detections)\n", + " avg_conf = (\n", + " float(np.mean(detections.confidence)) if detections.confidence is not None and num_det > 0 else float(\"nan\")\n", + " )\n", + " class_counts: dict[str, int] = {}\n", + " if num_det > 0 and detections.class_id is not None:\n", + " for cid in detections.class_id.tolist():\n", + " label = CLASS_NAMES[cid] if cid < len(CLASS_NAMES) else str(cid)\n", + " class_counts[label] = class_counts.get(label, 0) + 1\n", + " summary_rows.append(\n", + " {\n", + " \"image\": image_path.name,\n", + " \"detections\": num_det,\n", + " \"avg_confidence\": round(avg_conf, 3),\n", + " \"classes\": \", \".join(f\"{cls}×{cnt}\" for cls, cnt in sorted(class_counts.items())),\n", + " }\n", + " )\n", + "\n", + "summary_df = pd.DataFrame(summary_rows)\n", + "display(summary_df)" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "title,-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/cookbooks/fine-tune_keypoints.ipynb b/docs/cookbooks/fine-tune_keypoints.ipynb new file mode 100644 index 0000000..dd099b9 --- /dev/null +++ b/docs/cookbooks/fine-tune_keypoints.ipynb @@ -0,0 +1,878 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7e465973", + "metadata": {}, + "source": [ + "# RF-DETR keypoint training demo on multiple Roboflow Universe datasets\n", + "\n", + "Select one dataset with `DATASET_KEY`, then run the same fine-tuning, plotting, checkpoint loading, and inference cells.\n", + "\n", + "RF-DETR extends bounding-box detection to predict structured keypoint skeletons — useful for human pose\n", + "estimation, sports field calibration, product inspection, and any task where *where* matters as much as *what*.\n", + "\n", + "**Datasets** — set `DATASET_KEY` to one of:\n", + "\n", + "| Key | Dataset |\n", + "|-----|---------|\n", + "| `\"dart\"` | Darts Detection |\n", + "| `\"human_pose\"` | Human Body Pose |\n", + "| `\"basketball_court\"` | Basketball Court Detection |\n", + "| `\"football_field\"` | Football Field Detection |\n", + "| `\"tennis_court\"` | Tennis Court Keypoint |\n", + "\n", + "Every subsequent cell (fine-tuning, plotting, checkpoint loading, inference) adapts automatically.\n", + "By the end you will have a fine-tuned model, training curves, and annotated inference images with\n", + "optional uncertainty ellipses." + ] + }, + { + "cell_type": "markdown", + "id": "6d361242", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install `rfdetr` with the `train` and `visual` extras. The `train` extra pulls in the training loop dependencies (PyTorch Lightning, COCO evaluation tools, and data augmentation libraries), while `visual` adds the visualisation helpers used later in the notebook. The `roboflow` package handles dataset download; `pandas` and `seaborn` are needed for the metrics table and optional plot styling. If you hit an `ImportError` after running this cell, the most likely cause is a stale in-memory import — restart the Python runtime once and re-run from the top." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca1b5461", + "metadata": { + "title": "[bash]" + }, + "outputs": [], + "source": [ + "!pip install -q \"rfdetr[train,visual]==1.8.2\" roboflow pandas seaborn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d89771c9", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Shared RF-DETR keypoint fine-tuning demo for several Roboflow COCO keypoint exports.\"\"\"\n", + "\n", + "import json\n", + "import os\n", + "from pathlib import Path\n", + "from typing import Any\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import display\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib.figure import Figure\n", + "from PIL import Image\n", + "from roboflow import Roboflow\n", + "\n", + "from rfdetr import RFDETRKeypointPreview\n", + "from rfdetr.config import KeypointTrainConfig\n", + "from rfdetr.datasets._keypoint_schema import infer_coco_keypoint_schema, infer_yolo_keypoint_schema\n", + "from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer\n", + "from rfdetr.training.callbacks.best_model import BestModelCallback\n", + "from rfdetr.utilities.reproducibility import seed_all\n", + "from rfdetr.visualize.keypoints import _key_points_for_display, _keypoint_prediction_records\n", + "from rfdetr.visualize.training import plot_loss_metrics, plot_map_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b7de4e2", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "PROJECT_ROOT = Path(__file__).resolve().parent if \"__file__\" in globals() else Path.cwd()\n", + "DATASETS_DIR = PROJECT_ROOT / \"datasets\"\n", + "\n", + "# Resolution must be divisible by the model patch_size (12 for RFDETRKeypointPreview).\n", + "# Larger resolutions improve fine-grained keypoint localisation at the cost of GPU memory\n", + "# and training time. Recommended choices per dataset are annotated below.\n", + "DATASETS: dict[str, dict[str, Any]] = {\n", + " \"dart\": {\n", + " \"name\": \"Darts Detection\",\n", + " \"workspace\": \"dartdetection-vfgjd\",\n", + " \"project\": \"darts_detection-yp7lt\",\n", + " \"version\": 5,\n", + " \"source_url\": \"https://universe.roboflow.com/dartdetection-vfgjd/darts_detection-yp7lt\",\n", + " \"output_name\": \"keypoint_darts_detection_demo\",\n", + " \"keypoint_flip_pairs\": [],\n", + " # 576 — close-up shots where the dart tip occupies a large fraction of the frame;\n", + " # default resolution provides sufficient detail for tip localisation.\n", + " \"resolution\": 576,\n", + " },\n", + " \"human_pose\": {\n", + " \"name\": \"Dataset Ridimensionato\",\n", + " \"workspace\": \"poseestimation-wzidb\",\n", + " \"project\": \"dataset-ridimensionato\",\n", + " \"version\": 65,\n", + " \"source_url\": \"https://universe.roboflow.com/poseestimation-wzidb/dataset-ridimensionato/dataset/65\",\n", + " \"output_name\": \"keypoint_dataset_ridimensionato_demo\",\n", + " \"format\": \"yolo\",\n", + " \"keypoint_flip_pairs\": [],\n", + " # 768 — body joints (wrists, ankles) require fine localisation across a range of\n", + " # person scales; a larger canvas reduces positional error for distant people.\n", + " \"resolution\": 768,\n", + " },\n", + " \"basketball_court\": {\n", + " \"name\": \"Basketball Court Detection 2\",\n", + " \"workspace\": \"roboflow-jvuqo\",\n", + " \"project\": \"basketball-court-detection-2\",\n", + " \"version\": 19,\n", + " \"source_url\": \"https://universe.roboflow.com/roboflow-jvuqo/basketball-court-detection-2\",\n", + " \"output_name\": \"keypoint_basketball_court_detection_demo\",\n", + " \"keypoint_flip_pairs\": [],\n", + " # 576 — court lines are coarse features that fill the frame; the default resolution\n", + " # is sufficient to locate corners and centre-circle keypoints accurately.\n", + " \"resolution\": 576,\n", + " },\n", + " \"football_field\": {\n", + " \"name\": \"Football Field Detection\",\n", + " \"workspace\": \"roboflow-jvuqo\",\n", + " \"project\": \"football-field-detection-f07vi\",\n", + " \"version\": 18,\n", + " \"source_url\": \"https://universe.roboflow.com/roboflow-jvuqo/football-field-detection-f07vi\",\n", + " \"output_name\": \"keypoint_football_field_detection_demo\",\n", + " \"keypoint_flip_pairs\": [],\n", + " # 768 — a full football pitch captured in a single frame compresses many line\n", + " # intersections into a small pixel area; the higher resolution separates\n", + " # closely spaced markings and reduces keypoint confusion near the centre circle.\n", + " \"resolution\": 768,\n", + " },\n", + " \"tennis_court\": {\n", + " \"name\": \"Tennis Court Keypoint\",\n", + " \"workspace\": \"shj-pk1wt\",\n", + " \"project\": \"tennis_court_keypoint\",\n", + " \"version\": 6,\n", + " \"source_url\": \"https://universe.roboflow.com/shj-pk1wt/tennis_court_keypoint\",\n", + " \"output_name\": \"keypoint_tennis_court_demo\",\n", + " \"format\": \"yolo\",\n", + " \"keypoint_flip_pairs\": [],\n", + " # 768 — a full court viewed from broadcast angles packs many line intersections\n", + " # into a small pixel area; the higher resolution reduces confusion between\n", + " # closely spaced baseline and service-box corners.\n", + " \"resolution\": 768,\n", + " },\n", + "}\n", + "\n", + "DATASET_KEY = \"dart\"\n", + "DATASET_INFO = DATASETS[DATASET_KEY]\n", + "DATASET_FORMAT = DATASET_INFO.get(\"format\", \"coco\") # \"coco\" or \"yolo\"\n", + "\n", + "OUTPUT_DIR = PROJECT_ROOT / \"output\" / str(DATASET_INFO[\"output_name\"])\n", + "METRICS_CSV = OUTPUT_DIR / \"metrics.csv\"\n", + "VALIDATION_METRICS_JSON = OUTPUT_DIR / \"validation_metrics.json\"\n", + "FINAL_CHECKPOINT_PATH = OUTPUT_DIR / \"checkpoint_final_demo.pth\"\n", + "\n", + "RESOLUTION = int(DATASET_INFO[\"resolution\"])\n", + "\n", + "SEED = 7\n", + "EPOCHS = 50\n", + "BATCH_SIZE = 8\n", + "GRAD_ACCUM_STEPS = 2\n", + "NUM_WORKERS = 8\n", + "LR = 1e-4\n", + "LR_ENCODER = 1e-4\n", + "SAMPLE_PREVIEW_COUNT = 6\n", + "SAMPLE_PREVIEW_COLUMNS = 3\n", + "SAMPLE_PREVIEW_FIGURE_SIZE: tuple[float, float] | None = None\n", + "INFERENCE_COUNT = 6\n", + "INFERENCE_COLUMNS = 3\n", + "INFERENCE_THRESHOLD = 0.25\n", + "KEYPOINT_THRESHOLD = 0.1\n", + "PLOT_LOSS_LOG_SCALE = False\n", + "DRAW_UNCERTAINTY_ELLIPSES = True\n", + "ELLIPSE_SIGMA = 1.0\n", + "MAX_ELLIPSE_RADIUS = 36.0\n", + "\n", + "print(f\"dataset_key={DATASET_KEY}\")\n", + "print(f\"dataset={DATASET_INFO['name']}\")\n", + "print(f\"dataset_format={DATASET_FORMAT}\")\n", + "print(f\"source_url={DATASET_INFO['source_url']}\")\n", + "print(f\"resolution={RESOLUTION}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3d1ca469", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## Notebook display\n", + "\n", + "This helper registers the `%matplotlib inline` magic so figures render directly beneath each cell when you run the notebook in Jupyter or Google Colab. If you are running this file as a plain Python script the function detects the absence of an IPython kernel and exits silently, so it is always safe to call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7920201", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _enable_notebook_inline_matplotlib() -> None:\n", + " \"\"\"Enable inline matplotlib figures when running in IPython.\"\"\"\n", + " get_ipython_func = globals().get(\"get_ipython\")\n", + " if not callable(get_ipython_func):\n", + " return\n", + " ipython = get_ipython_func()\n", + " if ipython is not None:\n", + " ipython.run_line_magic(\"matplotlib\", \"inline\")\n", + " ipython.run_line_magic(\"config\", \"InlineBackend.close_figures = True\")\n", + "\n", + "\n", + "_enable_notebook_inline_matplotlib()" + ] + }, + { + "cell_type": "markdown", + "id": "08de3f94", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 1 - Download dataset\n", + "\n", + "Roboflow exports datasets in COCO keypoint format: a JSON file where each annotation contains a flat `keypoints` array of `[x, y, visibility, x, y, visibility, ...]` triplets, one per defined skeleton joint. The download cell fetches the exact dataset version listed in `DATASETS` and places it under `datasets//`. You need a Roboflow API key — get one for free at app.roboflow.com/settings/api and set it as the `ROBOFLOW_API_KEY` environment variable (or as a Colab secret with the same name). The download is idempotent: if the target directory already exists Roboflow skips the network transfer, so re-running this cell after a successful download is fast." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2109a1e", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "try:\n", + " from google.colab import userdata\n", + "\n", + " try:\n", + " ROBOFLOW_API_KEY = userdata.get(\"ROBOFLOW_API_KEY\") or \"\"\n", + " except Exception:\n", + " ROBOFLOW_API_KEY = \"\"\n", + "except ImportError:\n", + " ROBOFLOW_API_KEY = \"\"\n", + "\n", + "if not ROBOFLOW_API_KEY:\n", + " ROBOFLOW_API_KEY = os.environ.get(\"ROBOFLOW_API_KEY\", \"\")\n", + "if not ROBOFLOW_API_KEY:\n", + " raise RuntimeError(\n", + " \"ROBOFLOW_API_KEY not found. \"\n", + " \"In Colab: add it via Secrets (key icon). \"\n", + " \"Locally: set the environment variable before running.\"\n", + " )\n", + "\n", + "rf = Roboflow(api_key=ROBOFLOW_API_KEY)\n", + "_rf_format = \"yolov8\" if DATASET_FORMAT == \"yolo\" else \"coco\"\n", + "dataset = (\n", + " rf.workspace(str(DATASET_INFO[\"workspace\"]))\n", + " .project(str(DATASET_INFO[\"project\"]))\n", + " .version(int(DATASET_INFO[\"version\"]))\n", + " .download(_rf_format, location=str(DATASETS_DIR / DATASET_KEY))\n", + ")\n", + "DATASET_DIR = Path(dataset.location)\n", + "print(f\"dataset_dir={DATASET_DIR}\")\n", + "print(f\"rf_format={_rf_format}\")" + ] + }, + { + "cell_type": "markdown", + "id": "2f59cc9e", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 2 - Infer keypoint schema\n", + "\n", + "`infer_coco_keypoint_schema` reads the training annotation JSON and extracts three things the model needs: the class names, the number of keypoints per class, and the OKS sigmas. OKS stands for Object Keypoint Similarity — it is the keypoint analogue of IoU and ranges from 0 to 1. Each sigma is a per-keypoint scale factor that controls how strictly localisation is penalised during evaluation: a smaller sigma (e.g. 0.025 for a wrist) means the model must predict that joint more precisely to score well, while a larger sigma (e.g. 0.107 for a hip) is more forgiving. `VALIDATE_KEYPOINT_METRICS` will be `False` when your dataset has categories with different keypoint counts, because the standard COCO OKS evaluator assumes a uniform skeleton across all instances; bounding-box mAP is always computed regardless." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3111b71a", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "if DATASET_FORMAT == \"yolo\":\n", + " schema = infer_yolo_keypoint_schema(DATASET_DIR / \"data.yaml\")\n", + " _flip_idx = schema.flip_idx\n", + " _pairs: list[int] = []\n", + " _seen: set[int] = set()\n", + " for _i, _j in enumerate(_flip_idx):\n", + " if _i in _seen or _j in _seen or _i == _j:\n", + " _seen.add(_i)\n", + " continue\n", + " if _j < len(_flip_idx) and _flip_idx[_j] == _i:\n", + " _pairs.extend([_i, _j])\n", + " _seen.update({_i, _j})\n", + " KEYPOINT_FLIP_PAIRS = _pairs\n", + "else:\n", + " TRAIN_ANNOTATIONS = DATASET_DIR / \"train\" / \"_annotations.coco.json\"\n", + " schema = infer_coco_keypoint_schema(TRAIN_ANNOTATIONS) # type: ignore[assignment]\n", + " KEYPOINT_FLIP_PAIRS = list(DATASET_INFO.get(\"keypoint_flip_pairs\", []))\n", + "\n", + "CLASS_NAMES = schema.class_names\n", + "NUM_KEYPOINTS_PER_CLASS = schema.num_keypoints_per_class\n", + "NUM_CLASSES = len(CLASS_NAMES)\n", + "KEYPOINT_OKS_SIGMAS = schema.keypoint_oks_sigmas\n", + "ACTIVE_KEYPOINT_COUNTS = [count for count in NUM_KEYPOINTS_PER_CLASS if count > 0]\n", + "VALIDATE_KEYPOINT_METRICS = len(set(ACTIVE_KEYPOINT_COUNTS)) <= 1\n", + "\n", + "print(f\"class_names={CLASS_NAMES}\")\n", + "print(f\"num_keypoints_per_class={NUM_KEYPOINTS_PER_CLASS}\")\n", + "print(\"bbox_validation=True\")\n", + "print(f\"keypoint_oks_validation={VALIDATE_KEYPOINT_METRICS}\")\n", + "if VALIDATE_KEYPOINT_METRICS:\n", + " print(f\"keypoint_oks_sigmas={len(KEYPOINT_OKS_SIGMAS)} values\")\n", + "else:\n", + " print(\"keypoint_oks_validation=skipped because COCO OKS requires one keypoint count across categories\")" + ] + }, + { + "cell_type": "markdown", + "id": "4c9b5a6b", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_save_final_checkpoint` writes a self-contained `.pth` file that bundles model weights with the full training and model config. This is separate from the PTL checkpoint because it can be loaded with a single `from_checkpoint` call on any machine, without reconstructing the original config." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0bd03c1", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _save_final_checkpoint(\n", + " module: RFDETRModelModule,\n", + " trainer: Any,\n", + " train_config: KeypointTrainConfig,\n", + " model_config: Any,\n", + " output_path: Path,\n", + ") -> Path:\n", + " \"\"\"Save a final RF-DETR .pth checkpoint that can be loaded with ``from_checkpoint``.\"\"\"\n", + " raw_model: Any = getattr(module.model, \"_orig_mod\", module.model)\n", + " output_path.parent.mkdir(parents=True, exist_ok=True)\n", + " torch.save(\n", + " BestModelCallback._build_checkpoint_payload(\n", + " raw_model.state_dict(),\n", + " train_config.model_dump(),\n", + " trainer,\n", + " model_name=\"RFDETRKeypointPreview\",\n", + " model_config_dict=model_config.model_dump(),\n", + " ),\n", + " output_path,\n", + " )\n", + " return output_path" + ] + }, + { + "cell_type": "markdown", + "id": "f516409d", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_annotated_keypoint_scene` renders keypoint dots and, when covariance data is available, uncertainty ellipses on top of an image. Drawing ellipses here rather than in the grid helper keeps the annotation logic in one place and makes it easy to toggle `DRAW_UNCERTAINTY_ELLIPSES` without touching the layout code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80f4317b", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _annotated_keypoint_scene(image: np.ndarray, key_points: sv.KeyPoints) -> np.ndarray:\n", + " \"\"\"Return an image with visible vertices and optional uncertainty ellipses.\"\"\"\n", + " scene = image.copy()\n", + " if DRAW_UNCERTAINTY_ELLIPSES and \"covariance\" in key_points.data:\n", + " scene = sv.VertexEllipseAnnotator(\n", + " sigma=ELLIPSE_SIGMA,\n", + " color=sv.Color.ROBOFLOW,\n", + " max_axis=MAX_ELLIPSE_RADIUS,\n", + " ).annotate(scene=scene, key_points=key_points)\n", + "\n", + " return sv.VertexAnnotator(radius=3).annotate(scene=scene, key_points=key_points)" + ] + }, + { + "cell_type": "markdown", + "id": "80f50cd8", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_keypoint_grid_figure` arranges multiple annotated images into a fixed-column matplotlib grid. Using a grid rather than individual figures keeps the visual summary compact and makes it easy to compare predictions across images at a glance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51734ce1", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _keypoint_grid_figure(items: list[tuple[str, np.ndarray, sv.KeyPoints]], columns: int) -> Figure:\n", + " \"\"\"Render keypoint annotations in a fixed-column subplot grid.\"\"\"\n", + " if columns <= 0:\n", + " raise ValueError(f\"columns must be positive, got {columns}.\")\n", + "\n", + " rows = max(1, (len(items) + columns - 1) // columns)\n", + " figure, axes = plt.subplots(rows, columns, figsize=(5 * columns, 5 * rows))\n", + " axes_array = np.asarray(axes, dtype=object).reshape(-1)\n", + " for axis in axes_array:\n", + " axis.axis(\"off\")\n", + "\n", + " for axis, (title, image, key_points) in zip(axes_array, items, strict=False):\n", + " axis.imshow(_annotated_keypoint_scene(image, key_points))\n", + " axis.set_title(title, fontsize=10)\n", + " axis.axis(\"off\")\n", + "\n", + " figure.tight_layout()\n", + " return figure" + ] + }, + { + "cell_type": "markdown", + "id": "2e548c8c", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_display_keypoint_records` prints per-image keypoint predictions as a pandas table. Fields that are constant across all keypoints in an image (such as the filename or detection score) are printed once as a header line rather than repeated in every row, keeping the output readable for images with many joints." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "757c151a", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def _display_keypoint_records(records: list[dict[str, Any]]) -> None:\n", + " \"\"\"Display keypoint rows while printing per-image constant fields once.\"\"\"\n", + " if not records:\n", + " print(\"keypoint_rows=[]\")\n", + " return\n", + "\n", + " keypoint_columns = {\"detection_index\", \"keypoint_index\", \"x\", \"y\", \"keypoint_confidence\"}\n", + " records_frame = pd.DataFrame(records)\n", + " for image_name, image_frame in records_frame.groupby(\"image\", dropna=False, sort=False):\n", + " constant_columns = [\n", + " column\n", + " for column in image_frame.columns\n", + " if column not in keypoint_columns and image_frame[column].nunique(dropna=False) == 1\n", + " ]\n", + " if constant_columns:\n", + " constants = image_frame.iloc[0][constant_columns]\n", + " print(\", \".join(f\"{column}={constants[column]}\" for column in constant_columns))\n", + " elif image_name is not None:\n", + " print(f\"image={image_name}\")\n", + "\n", + " display(image_frame.drop(columns=constant_columns).reset_index(drop=True))" + ] + }, + { + "cell_type": "markdown", + "id": "105aefc9", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 3 - Configure training\n", + "\n", + "`KeypointTrainConfig` centralises every hyperparameter the training loop needs. A few fields are worth understanding before you customise them for your own dataset. `lr` and `lr_encoder` are usually set to the same value for fine-tuning; if you notice the backbone overfitting early you can halve `lr_encoder` to protect the pre-trained features while the detection head continues to adapt. `grad_accum_steps=2` means gradients are accumulated over two mini-batches before a weight update, which effectively doubles the batch size without requiring extra GPU memory — useful on smaller GPUs. `use_ema=False` keeps the demo fast; for a production run you can set this to `True` to maintain an exponential moving average of weights, which often improves final accuracy by a few tenths of a mAP point. `multi_scale=False` and `expanded_scales=False` disable the multi-resolution augmentation that RF-DETR uses during pre-training; turning them off shortens epoch time significantly and is fine for most fine-tuning runs. The `notes` dict is stored in the checkpoint alongside the weights, giving you a lightweight experiment log that travels with the model file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe060e29", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "seed_all(SEED)\n", + "variant = RFDETRKeypointPreview( # type: ignore[no-untyped-call]\n", + " num_classes=NUM_CLASSES,\n", + " num_keypoints_per_class=NUM_KEYPOINTS_PER_CLASS,\n", + " resolution=RESOLUTION,\n", + ")\n", + "variant.model_config.model_name = type(variant).__name__\n", + "\n", + "train_config = KeypointTrainConfig(\n", + " dataset_file=\"yolo\" if DATASET_FORMAT == \"yolo\" else \"roboflow\",\n", + " dataset_dir=str(DATASET_DIR),\n", + " output_dir=str(OUTPUT_DIR),\n", + " epochs=EPOCHS,\n", + " batch_size=BATCH_SIZE,\n", + " grad_accum_steps=GRAD_ACCUM_STEPS,\n", + " num_workers=NUM_WORKERS,\n", + " lr=LR,\n", + " lr_encoder=LR_ENCODER,\n", + " use_ema=False,\n", + " run_test=False,\n", + " compute_train_metrics=True,\n", + " compute_val_loss=True,\n", + " multi_scale=False,\n", + " expanded_scales=False,\n", + " do_random_resize_via_padding=False,\n", + " tensorboard=False,\n", + " wandb=False,\n", + " mlflow=False,\n", + " clearml=False,\n", + " class_names=CLASS_NAMES,\n", + " keypoint_flip_pairs=KEYPOINT_FLIP_PAIRS,\n", + " keypoint_oks_sigmas=KEYPOINT_OKS_SIGMAS if VALIDATE_KEYPOINT_METRICS else None,\n", + " notes={\n", + " \"demo\": f\"keypoint-preview PTL fine-tune on Roboflow Universe {DATASET_INFO['project']}\",\n", + " \"source_url\": DATASET_INFO[\"source_url\"],\n", + " \"roboflow_workspace\": DATASET_INFO[\"workspace\"],\n", + " \"roboflow_project\": DATASET_INFO[\"project\"],\n", + " \"roboflow_version\": DATASET_INFO[\"version\"],\n", + " \"num_keypoints_per_class\": NUM_KEYPOINTS_PER_CLASS,\n", + " \"keypoint_flip_pairs\": KEYPOINT_FLIP_PAIRS,\n", + " \"bbox_validation_enabled\": True,\n", + " \"keypoint_oks_validation_enabled\": VALIDATE_KEYPOINT_METRICS,\n", + " \"keypoint_oks_sigmas\": KEYPOINT_OKS_SIGMAS if VALIDATE_KEYPOINT_METRICS else None,\n", + " },\n", + " progress_bar=\"tqdm\",\n", + ")\n", + "\n", + "datamodule = RFDETRDataModule(variant.model_config, train_config)\n", + "model = RFDETRModelModule(variant.model_config, train_config)\n", + "trainer = build_trainer(train_config, variant.model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4cdfa696", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 4 - Preview dataset inputs\n", + "\n", + "Always look at your data before you start a long training run. This cell renders a grid of annotated training images so you can confirm that keypoint skeletons are aligned with objects, that flips and colour jitter look reasonable, and that there are no systematic labelling errors such as swapped left/right joints or keypoints sitting outside their bounding box. Catching label noise here costs a few seconds; catching it after 50 epochs of training costs much more. If the preview shows empty images or missing annotations, the most common cause is a mismatch between the image filenames in the COCO JSON and the files on disk — check that the download completed fully." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df126db6", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "sample_figure = datamodule._show_samples(\n", + " SAMPLE_PREVIEW_COUNT,\n", + " split=\"train\",\n", + " columns=SAMPLE_PREVIEW_COLUMNS,\n", + " figure_size=SAMPLE_PREVIEW_FIGURE_SIZE,\n", + ")\n", + "display(sample_figure)\n", + "plt.close(sample_figure)" + ] + }, + { + "cell_type": "markdown", + "id": "76be07a3", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 5 - Fine-tune\n", + "\n", + "`trainer.fit` hands control to PyTorch Lightning, which drives the full training loop: forward pass, loss computation, gradient accumulation, weight updates, learning-rate scheduling, and periodic validation. After each epoch the trainer appends a row to `metrics.csv` and, if validation mAP improves, saves a new best checkpoint via `BestModelCallback`. On a single consumer GPU (RTX 3090 or similar) 50 epochs on the dart dataset takes roughly 15–30 minutes depending on `BATCH_SIZE` and `NUM_WORKERS`. If you need to resume an interrupted run, set `train_config.resume` to the path of the last PTL checkpoint before calling this cell again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71d28853", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "trainer.fit(model, datamodule=datamodule, ckpt_path=train_config.resume or None)\n", + "print(f\"output_dir={OUTPUT_DIR}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1e72b80", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 6 - Save checkpoint/model\n", + "\n", + "PTL saves its own checkpoints during training (optimizer state, scheduler state, epoch counter), but those files are not directly portable — they require the same class hierarchy to load. The `.pth` file written here is a self-contained RF-DETR checkpoint: it bundles the model weights together with the full training and model configs, including the keypoint schema and class names. This means you can share the file with a colleague and they can run inference with a single `RFDETRKeypointPreview.from_checkpoint(path)` call, with no need to reconstruct the original config. For full reproducibility, keep this checkpoint alongside the dataset version number printed in section 1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a052495", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "final_checkpoint = _save_final_checkpoint(model, trainer, train_config, variant.model_config, FINAL_CHECKPOINT_PATH)\n", + "print(f\"saved_checkpoint_model={final_checkpoint}\")" + ] + }, + { + "cell_type": "markdown", + "id": "1482fa79", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 7 - Validate metrics\n", + "\n", + "The validation pass that runs at the end of every training epoch uses the best weights seen so far and applies augmentation. This cell runs a clean post-training validation: no augmentation, the best checkpoint loaded from `checkpoint_best_total.pth` written by `BestModelCallback`, and results serialised to JSON for downstream comparison. Two metrics are most important to inspect here. `bbox/map` is the standard COCO bounding-box mAP at IoU 0.50:0.95 — it tells you how reliably the detector finds and bounds each object. `keypoint/oks` is the OKS-based mAP — it measures how precisely the model places each joint within the detected bounding boxes. A model can have a high `bbox/map` but a low `keypoint/oks` if it finds objects well but struggles to localise their joints; addressing that usually means more labelled data or tighter OKS sigmas.\n", + "\n", + "**Note:** `checkpoint_best_total.pth` is written after the first completed validation epoch. If training was interrupted before any epoch finished, this file will not exist and the cell below will raise `FileNotFoundError`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7aa26516", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "# checkpoint_best_total.pth stores plain model weights under key \"model\", not a full\n", + "# PTL checkpoint. Passing it as ckpt_path to trainer.validate() triggers a KeyError\n", + "# on \"validate_loop\" because PTL tries to restore loop state that isn't present.\n", + "# Load the weights directly and run validate without ckpt_path instead.\n", + "_ckpt = torch.load(OUTPUT_DIR / \"checkpoint_best_total.pth\", map_location=\"cpu\", weights_only=False)\n", + "model.model.load_state_dict(_ckpt[\"model\"], strict=True)\n", + "del _ckpt\n", + "validation_results = trainer.validate(model, datamodule=datamodule)\n", + "validation_metrics = {key: float(value) for key, value in validation_results[0].items()} if validation_results else {}\n", + "if not VALIDATE_KEYPOINT_METRICS:\n", + " validation_metrics[\"keypoint_oks_skipped_mixed_keypoint_counts\"] = 1.0\n", + "VALIDATION_METRICS_JSON.write_text(json.dumps(validation_metrics, indent=2, sort_keys=True), encoding=\"utf-8\")\n", + "print(f\"validation_metrics={validation_metrics}\")\n", + "print(f\"validation_metrics_json={VALIDATION_METRICS_JSON}\")" + ] + }, + { + "cell_type": "markdown", + "id": "0aa1a943", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 8 - Plot CSVLogger metrics\n", + "\n", + "Reading loss curves is one of the fastest ways to diagnose training problems. Healthy runs show both training loss and validation loss decreasing together; a large and growing gap between the two is a classic overfitting signal — the model is memorising the training set rather than generalising. For the mAP curves, on small datasets (a few hundred images) you typically see rapid improvement in the first 10–20 epochs followed by a plateau around epoch 30–50; if mAP is still climbing at epoch 50, consider extending `EPOCHS`. Set `PLOT_LOSS_LOG_SCALE = True` if the loss drops by an order of magnitude in the first few epochs and the later, more meaningful portion of the curve gets compressed into a flat line at the bottom of the plot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37589b33", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"metrics_csv={METRICS_CSV}\")\n", + "loss_figure = plot_loss_metrics(str(METRICS_CSV), loss_log_scale=PLOT_LOSS_LOG_SCALE)\n", + "display(loss_figure)\n", + "plt.close(loss_figure)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5608045", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "map_figure = plot_map_metrics(str(METRICS_CSV))\n", + "display(map_figure)\n", + "plt.close(map_figure)" + ] + }, + { + "cell_type": "markdown", + "id": "690a200a", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 9 - Load checkpoint/model\n", + "\n", + "`from_checkpoint` is the standard entry point for loading a saved RF-DETR model. It reads both the weights and the stored config from the `.pth` file, so the reconstructed model has the correct number of classes and keypoints without you having to pass them explicitly. You can share this checkpoint file with teammates and they can run inference immediately — the keypoint schema, class names, and OKS sigmas are all embedded in the file alongside the weights. This cell also confirms that the save-load round trip works before you proceed to inference, so any corruption or version mismatch surfaces here rather than silently producing wrong predictions later." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8d75b0f", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "loaded_model = RFDETRKeypointPreview.from_checkpoint(FINAL_CHECKPOINT_PATH)" + ] + }, + { + "cell_type": "markdown", + "id": "7bee8d28", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 10 - Select inference images\n", + "\n", + "This cell implements a fallback chain — test split first, then validation, then train — so the notebook always finds images to run inference on even when the dataset has no dedicated test split. Using images from the test set gives you an unbiased view of model performance because those images were never seen during training or used to pick the best checkpoint. If you want to run inference on your own images, replace `inference_image_paths` with a list of `Path` objects pointing to your files before running the next cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5182f47", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "_IMAGE_EXTS = {\".jpg\", \".jpeg\", \".png\"}\n", + "\n", + "\n", + "def _find_images(directory: Path) -> list[Path]:\n", + " if not directory.is_dir():\n", + " return []\n", + " # Iterate through all subfolders to find images\n", + " return sorted(p for p in directory.glob(\"**/*\") if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)\n", + "\n", + "\n", + "validation_image_paths = _find_images(DATASET_DIR / \"test\")\n", + "if not validation_image_paths:\n", + " validation_image_paths = _find_images(DATASET_DIR / \"valid\")\n", + "if not validation_image_paths:\n", + " validation_image_paths = _find_images(DATASET_DIR / \"train\")\n", + "if not validation_image_paths:\n", + " raise FileNotFoundError(f\"No images ({', '.join(sorted(_IMAGE_EXTS))}) found under {DATASET_DIR}\")\n", + "\n", + "inference_image_paths = validation_image_paths[:INFERENCE_COUNT]\n", + "print(f\"inference_images={[str(path) for path in inference_image_paths]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "1c7654a9", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 11 - Visualize inference\n", + "\n", + "Two thresholds control what you see here. `INFERENCE_THRESHOLD` is a detection confidence threshold: only detections whose bounding-box score exceeds this value are shown. Raise it to suppress false positives; lower it to surface low-confidence detections. `KEYPOINT_THRESHOLD` operates within each accepted detection: individual keypoints whose confidence falls below this value are treated as not visible and omitted from the overlay. The uncertainty ellipses (drawn when `DRAW_UNCERTAINTY_ELLIPSES = True`) visualise the predicted covariance for each keypoint — a larger ellipse means the model is less certain about that joint's exact location." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e520b84c", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "inference_grid_items = []\n", + "keypoint_rows = []\n", + "for image_path in inference_image_paths:\n", + " with Image.open(image_path) as image:\n", + " image = image.convert(\"RGB\")\n", + " key_points_raw = loaded_model.predict(image, threshold=INFERENCE_THRESHOLD)\n", + " if not isinstance(key_points_raw, sv.KeyPoints):\n", + " raise RuntimeError(\n", + " f\"Expected RFDETRKeypointPreview.predict() to return sv.KeyPoints, got {type(key_points_raw)!r}.\"\n", + " )\n", + "\n", + " key_points = _key_points_for_display(key_points_raw, keypoint_threshold=KEYPOINT_THRESHOLD)\n", + " rows = _keypoint_prediction_records(key_points, image=image_path, keypoint_threshold=KEYPOINT_THRESHOLD)\n", + " inference_grid_items.append((image_path.name, np.array(image), key_points))\n", + " keypoint_rows.extend(rows)\n", + "\n", + "if inference_grid_items:\n", + " figure = _keypoint_grid_figure(inference_grid_items, columns=INFERENCE_COLUMNS)\n", + " display(figure)\n", + " plt.close(figure)" + ] + }, + { + "cell_type": "markdown", + "id": "93df94eb", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 12 - Keypoint prediction table\n", + "\n", + "The table below gives you the raw coordinates and per-keypoint confidence scores for every detection in the grid above. Fields that are constant across all keypoints in an image (filename, detection score) are printed once as a header line rather than repeated in every row. This format is easy to copy into a spreadsheet or feed into downstream analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de9b2707", + "metadata": {}, + "outputs": [], + "source": [ + "_display_keypoint_records(keypoint_rows)" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "title,-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/cookbooks/fine-tune_segmentation.ipynb b/docs/cookbooks/fine-tune_segmentation.ipynb new file mode 100644 index 0000000..a58df5f --- /dev/null +++ b/docs/cookbooks/fine-tune_segmentation.ipynb @@ -0,0 +1,846 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e6fd4da4", + "metadata": {}, + "source": [ + "# RF-DETR instance segmentation fine-tuning on Roboflow Universe datasets\n", + "\n", + "Select one dataset with `DATASET_KEY`, then run the same fine-tuning, plotting, checkpoint loading, and inference cells.\n", + "\n", + "RF-DETR extends bounding-box detection with a sparse segmentation head that predicts per-object binary masks — useful\n", + "for facade analysis, infrastructure damage inspection, vehicle part assessment, dental imaging, litter mapping, and\n", + "any task where *what shape* matters as much as *what class*.\n", + "\n", + "**Datasets** — set `DATASET_KEY` to one of:\n", + "\n", + "| Key | Dataset | Classes |\n", + "|-----|---------|---------|\n", + "| `\"building_facade\"` | Building Facade Segmentation | facade, window, street, vegetation, car, fence, ... |\n", + "| `\"spalling_rebar\"` | Spalling and Exposed Rebar | exposed_rebar, spalling |\n", + "| `\"car_damage_parts\"` | Car Part Damage Segmentation | bumper, wheel, door, headlight, dent, scratch, ... |\n", + "| `\"teeth_numbering\"` | Teeth Numbering Segmentation | tooth IDs 11-48 |\n", + "| `\"taco_trash\"` | TACO Trash | bottle, can, carton, cup, lid, straw, wrapper, ... |\n", + "\n", + "Every subsequent cell (fine-tuning, plotting, checkpoint loading, inference) adapts automatically.\n", + "By the end you will have a fine-tuned model, training curves, and annotated inference images with segmentation masks." + ] + }, + { + "cell_type": "markdown", + "id": "6eaacf22", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install `rfdetr` with the `train` and `visual` extras. The `train` extra pulls in the training loop dependencies\n", + "(PyTorch Lightning, COCO evaluation tools, and data augmentation libraries), while `visual` adds the visualisation\n", + "helpers used later in the notebook. The `roboflow` package handles dataset download; `pandas` and `seaborn` are needed\n", + "for the metrics table and optional plot styling. If you hit an `ImportError` after running this cell, the most likely\n", + "cause is a stale in-memory import — restart the Python runtime once and re-run from the top." + ] + }, + { + "cell_type": "code", + "id": "64736ca1", + "metadata": {}, + "source": "!pip install -q \"rfdetr[train,visual]>=1.8.2\" roboflow pandas seaborn", + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b4797e9", + "metadata": { + "title": "[bash]" + }, + "outputs": [], + "source": [ + "\"\"\"Shared RF-DETR instance segmentation fine-tuning demo for several Roboflow COCO segmentation exports.\"\"\"\n", + "\n", + "import json\n", + "import os\n", + "from pathlib import Path\n", + "from typing import Any\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import display\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib.figure import Figure\n", + "from PIL import Image\n", + "from roboflow import Roboflow\n", + "\n", + "from rfdetr import RFDETRSegSmall\n", + "from rfdetr.config import SegmentationTrainConfig\n", + "from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer\n", + "from rfdetr.training.callbacks.best_model import BestModelCallback\n", + "from rfdetr.utilities.reproducibility import seed_all\n", + "from rfdetr.visualize.training import plot_loss_metrics, plot_map_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99f1f539", + "metadata": {}, + "outputs": [], + "source": [ + "PROJECT_ROOT = Path(__file__).resolve().parent if \"__file__\" in globals() else Path.cwd()\n", + "DATASETS_DIR = PROJECT_ROOT / \"datasets\"\n", + "\n", + "# Resolution must be divisible by patch_size × num_windows (24 for RFDETRSegSmall).\n", + "# Larger resolutions capture fine-grained mask boundaries at the cost of GPU memory and time.\n", + "DATASETS: dict[str, dict[str, Any]] = {\n", + " \"building_facade\": {\n", + " \"name\": \"Building Facade Segmentation\",\n", + " \"workspace\": \"building-facade\",\n", + " \"project\": \"building-facade-segmentation-instance\",\n", + " \"version\": 4,\n", + " \"source_url\": \"https://universe.roboflow.com/building-facade/building-facade-segmentation-instance\",\n", + " \"output_name\": \"seg_building_facade\",\n", + " # 432 — structured street scenes with medium-to-large facade regions plus\n", + " # smaller windows, cars, vegetation, and infrastructure boundaries.\n", + " \"resolution\": 432,\n", + " },\n", + " \"spalling_rebar\": {\n", + " \"name\": \"Spalling and Exposed Rebar\",\n", + " \"workspace\": \"uni-of-birmingham\",\n", + " \"project\": \"spalling-and-exposed-rebar\",\n", + " \"version\": 7,\n", + " \"source_url\": \"https://universe.roboflow.com/uni-of-birmingham/spalling-and-exposed-rebar\",\n", + " \"output_name\": \"seg_spalling_rebar\",\n", + " # 432 — civil-infrastructure damage masks with irregular thin regions and\n", + " # exposed-rebar details that stress boundary quality.\n", + " \"resolution\": 432,\n", + " },\n", + " \"car_damage_parts\": {\n", + " \"name\": \"Car Part Damage Segmentation\",\n", + " \"workspace\": \"car-damaged-detection-e66m0\",\n", + " \"project\": \"car-part-detection-with-damage-part\",\n", + " \"version\": 1,\n", + " \"source_url\": \"https://universe.roboflow.com/car-damaged-detection-e66m0/car-part-detection-with-damage-part\",\n", + " \"output_name\": \"seg_car_damage_parts\",\n", + " # 432 — fine-grained automotive part masks with many adjacent classes and\n", + " # damage regions, useful for checking class-specific mask boundaries.\n", + " \"resolution\": 432,\n", + " },\n", + " \"teeth_numbering\": {\n", + " \"name\": \"Teeth Numbering Segmentation\",\n", + " \"workspace\": \"godento2\",\n", + " \"project\": \"teeth-seg-3537-iaky1\",\n", + " \"version\": 1,\n", + " \"source_url\": \"https://universe.roboflow.com/godento2/teeth-seg-3537-iaky1\",\n", + " \"output_name\": \"seg_teeth_numbering\",\n", + " # 432 — dental images with many repeated small instances whose labels are\n", + " # tied to position, stressing class confusion and small-mask quality.\n", + " \"resolution\": 432,\n", + " },\n", + " \"taco_trash\": {\n", + " \"name\": \"TACO Trash\",\n", + " \"workspace\": \"mohamed-traore-2ekkp\",\n", + " \"project\": \"taco-trash-annotations-in-context\",\n", + " \"version\": 13,\n", + " \"source_url\": \"https://universe.roboflow.com/mohamed-traore-2ekkp/taco-trash-annotations-in-context\",\n", + " \"output_name\": \"seg_taco_trash\",\n", + " # 432 — long-tail litter segmentation with many small, cluttered object\n", + " # categories. Use as a harder stress test after the simpler datasets.\n", + " \"resolution\": 432,\n", + " },\n", + "}\n", + "\n", + "DATASET_KEY = \"building_facade\"\n", + "DATASET_INFO = DATASETS[DATASET_KEY]\n", + "\n", + "OUTPUT_DIR = PROJECT_ROOT / \"output\" / str(DATASET_INFO[\"output_name\"])\n", + "METRICS_CSV = OUTPUT_DIR / \"metrics.csv\"\n", + "VALIDATION_METRICS_JSON = OUTPUT_DIR / \"validation_metrics.json\"\n", + "FINAL_CHECKPOINT_PATH = OUTPUT_DIR / \"checkpoint_final_demo.pth\"\n", + "\n", + "RESOLUTION = int(DATASET_INFO[\"resolution\"])\n", + "\n", + "SEED = 7\n", + "EPOCHS = 50\n", + "BATCH_SIZE = 8\n", + "GRAD_ACCUM_STEPS = 2\n", + "NUM_WORKERS = 8\n", + "LR = 1e-4\n", + "LR_ENCODER = 1e-4\n", + "SAMPLE_PREVIEW_COUNT = 6\n", + "SAMPLE_PREVIEW_COLUMNS = 3\n", + "SAMPLE_PREVIEW_FIGURE_SIZE: tuple[float, float] = (15.0, 10.0)\n", + "INFERENCE_COUNT = 6\n", + "INFERENCE_COLUMNS = 3\n", + "INFERENCE_THRESHOLD = 0.3\n", + "PLOT_LOSS_LOG_SCALE = False\n", + "\n", + "print(f\"dataset_key={DATASET_KEY}\")\n", + "print(f\"dataset={DATASET_INFO['name']}\")\n", + "print(f\"source_url={DATASET_INFO['source_url']}\")\n", + "print(f\"resolution={RESOLUTION}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b60971f9", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## Notebook display\n", + "\n", + "This helper registers the `%matplotlib inline` magic so figures render directly beneath each cell when you run the\n", + "notebook in Jupyter or Google Colab. If you are running this file as a plain Python script the function detects the\n", + "absence of an IPython kernel and exits silently, so it is always safe to call." + ] + }, + { + "cell_type": "code", + "id": "1c40c67d", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "def _enable_notebook_inline_matplotlib() -> None:\n", + " \"\"\"Enable inline matplotlib figures when running in IPython.\"\"\"\n", + " get_ipython_func = globals().get(\"get_ipython\")\n", + " if not callable(get_ipython_func):\n", + " return\n", + " ipython = get_ipython_func()\n", + " if ipython is not None:\n", + " ipython.run_line_magic(\"matplotlib\", \"inline\")\n", + " ipython.run_line_magic(\"config\", \"InlineBackend.close_figures = True\")\n", + "\n", + "\n", + "_enable_notebook_inline_matplotlib()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "7f99c55d", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 1 - Download dataset\n", + "\n", + "Roboflow exports instance segmentation datasets in COCO format: a JSON file where each annotation contains a\n", + "`segmentation` field with polygon coordinates that trace the precise object boundary. The download cell fetches the\n", + "exact dataset version listed in `DATASETS` and places it under `datasets//`. You need a Roboflow API key\n", + "— get one for free at app.roboflow.com/settings/api and set it as the `ROBOFLOW_API_KEY` environment variable (or as\n", + "a Colab secret with the same name). The download is idempotent: if the target directory already exists Roboflow skips\n", + "the network transfer, so re-running this cell after a successful download is fast." + ] + }, + { + "cell_type": "code", + "id": "ab8fdeb6", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "try:\n", + " from google.colab import userdata\n", + "\n", + " try:\n", + " ROBOFLOW_API_KEY = userdata.get(\"ROBOFLOW_API_KEY\") or \"\"\n", + " except Exception:\n", + " ROBOFLOW_API_KEY = \"\"\n", + "except ImportError:\n", + " ROBOFLOW_API_KEY = \"\"\n", + "\n", + "if not ROBOFLOW_API_KEY:\n", + " ROBOFLOW_API_KEY = os.environ.get(\"ROBOFLOW_API_KEY\", \"\")\n", + "if not ROBOFLOW_API_KEY:\n", + " raise RuntimeError(\n", + " \"ROBOFLOW_API_KEY not found. \"\n", + " \"In Colab: add it via Secrets (key icon). \"\n", + " \"Locally: set the environment variable before running.\"\n", + " )\n", + "\n", + "rf = Roboflow(api_key=ROBOFLOW_API_KEY)\n", + "dataset = (\n", + " rf.workspace(str(DATASET_INFO[\"workspace\"]))\n", + " .project(str(DATASET_INFO[\"project\"]))\n", + " .version(int(DATASET_INFO[\"version\"]))\n", + " .download(\"coco\", location=str(DATASETS_DIR / DATASET_KEY))\n", + ")\n", + "DATASET_DIR = Path(dataset.location)\n", + "TRAIN_ANNOTATIONS = DATASET_DIR / \"train\" / \"_annotations.coco.json\"\n", + "print(f\"dataset_dir={DATASET_DIR}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "e38db150", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 2 - Infer class names\n", + "\n", + "Read the class names directly from the COCO annotation file. RF-DETR uses 0-based class indices internally; the\n", + "`class_names` list maps each index to a human-readable label so the inference visualisation shows category names\n", + "rather than numbers. If your dataset has a single category the list will have one element — that is expected." + ] + }, + { + "cell_type": "code", + "id": "2c6b8dc4", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "with TRAIN_ANNOTATIONS.open() as _f:\n", + " _coco = json.load(_f)\n", + "\n", + "CLASS_NAMES: list[str] = [cat[\"name\"] for cat in sorted(_coco[\"categories\"], key=lambda c: c[\"id\"])]\n", + "NUM_CLASSES = len(CLASS_NAMES)\n", + "\n", + "print(f\"class_names={CLASS_NAMES}\")\n", + "print(f\"num_classes={NUM_CLASSES}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "945822fa", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_save_final_checkpoint` writes a self-contained `.pth` file that bundles model weights with the full training and\n", + "model config. This is separate from the PTL checkpoint because it can be loaded with a single `from_checkpoint` call\n", + "on any machine, without reconstructing the original config." + ] + }, + { + "cell_type": "code", + "id": "ed36df66", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "def _save_final_checkpoint(\n", + " module: RFDETRModelModule,\n", + " trainer: Any,\n", + " train_config: SegmentationTrainConfig,\n", + " model_config: Any,\n", + " output_path: Path,\n", + ") -> Path:\n", + " \"\"\"Save a final RF-DETR .pth checkpoint that can be loaded with ``from_checkpoint``.\"\"\"\n", + " raw_model: Any = getattr(module.model, \"_orig_mod\", module.model)\n", + " output_path.parent.mkdir(parents=True, exist_ok=True)\n", + " torch.save(\n", + " BestModelCallback._build_checkpoint_payload(\n", + " raw_model.state_dict(),\n", + " train_config.model_dump(),\n", + " trainer,\n", + " model_name=\"RFDETRSegSmall\",\n", + " model_config_dict=model_config.model_dump(),\n", + " ),\n", + " output_path,\n", + " )\n", + " return output_path" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "7000d9a4", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "`_segmentation_grid_figure` arranges multiple annotated images into a fixed-column matplotlib grid.\n", + "Masks are drawn with `sv.MaskAnnotator` (translucent colour fill) and bounding boxes with\n", + "`sv.BoxAnnotator`. Labels show the class name and confidence score." + ] + }, + { + "cell_type": "code", + "id": "477cd2f6", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "def _segmentation_grid_figure(\n", + " items: list[tuple[str, np.ndarray, sv.Detections]],\n", + " columns: int,\n", + " class_names: list[str],\n", + ") -> Figure:\n", + " \"\"\"Render segmentation masks and bounding boxes in a fixed-column subplot grid.\"\"\"\n", + " if columns <= 0:\n", + " raise ValueError(f\"columns must be positive, got {columns}.\")\n", + "\n", + " mask_annotator = sv.MaskAnnotator()\n", + " box_annotator = sv.BoxAnnotator(thickness=2)\n", + " label_annotator = sv.LabelAnnotator(text_scale=0.5, text_thickness=1, text_padding=4)\n", + "\n", + " rows = max(1, (len(items) + columns - 1) // columns)\n", + " figure, axes = plt.subplots(rows, columns, figsize=(5 * columns, 5 * rows))\n", + " axes_array = np.asarray(axes, dtype=object).reshape(-1)\n", + " for axis in axes_array:\n", + " axis.axis(\"off\")\n", + "\n", + " for axis, (title, image, detections) in zip(axes_array, items, strict=False):\n", + " scene = image.copy()\n", + " if detections.mask is not None and len(detections) > 0:\n", + " scene = mask_annotator.annotate(scene=scene, detections=detections)\n", + " scene = box_annotator.annotate(scene=scene, detections=detections)\n", + " if len(detections) > 0 and detections.class_id is not None:\n", + " conf_list = (\n", + " detections.confidence.tolist() if detections.confidence is not None else [None] * len(detections)\n", + " )\n", + " labels = [\n", + " f\"{class_names[cid] if cid < len(class_names) else cid}\" + (f\" {conf:.2f}\" if conf is not None else \"\")\n", + " for cid, conf in zip(detections.class_id.tolist(), conf_list)\n", + " ]\n", + " scene = label_annotator.annotate(scene=scene, detections=detections, labels=labels)\n", + " axis.imshow(scene)\n", + " axis.set_title(title, fontsize=10)\n", + " axis.axis(\"off\")\n", + "\n", + " figure.tight_layout()\n", + " return figure" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "8e53a5a5", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 3 - Configure training\n", + "\n", + "`SegmentationTrainConfig` centralises every hyperparameter the training loop needs, including mask-specific loss\n", + "coefficients (`mask_ce_loss_coef` and `mask_dice_loss_coef`). The defaults (5.0 each) work well for most datasets;\n", + "if the segmentation head overfits early you can reduce them towards 2.0 while keeping `cls_loss_coef=5.0`.\n", + "\n", + "`lr` and `lr_encoder` control the head and backbone learning rates respectively. For fine-tuning both are usually\n", + "set to the same value; if you notice the backbone overfitting early, halve `lr_encoder` to protect the pre-trained\n", + "features. `grad_accum_steps=2` doubles the effective batch size without extra GPU memory.\n", + "\n", + "`multi_scale=False` and `expanded_scales=False` disable multi-resolution augmentation to shorten epoch time during\n", + "fine-tuning — they rarely help on small custom datasets. The `notes` dict is stored alongside the weights in the\n", + "checkpoint, giving you a lightweight experiment log that travels with the model file." + ] + }, + { + "cell_type": "code", + "id": "3e71bc58", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "seed_all(SEED)\n", + "variant = RFDETRSegSmall( # type: ignore[no-untyped-call]\n", + " num_classes=NUM_CLASSES,\n", + " resolution=RESOLUTION,\n", + ")\n", + "variant.model_config.model_name = type(variant).__name__\n", + "\n", + "train_config = SegmentationTrainConfig(\n", + " dataset_file=\"roboflow\",\n", + " dataset_dir=str(DATASET_DIR),\n", + " output_dir=str(OUTPUT_DIR),\n", + " epochs=EPOCHS,\n", + " batch_size=BATCH_SIZE,\n", + " grad_accum_steps=GRAD_ACCUM_STEPS,\n", + " num_workers=NUM_WORKERS,\n", + " lr=LR,\n", + " lr_encoder=LR_ENCODER,\n", + " use_ema=False,\n", + " run_test=False,\n", + " compute_train_metrics=True,\n", + " compute_val_loss=True,\n", + " multi_scale=False,\n", + " expanded_scales=False,\n", + " do_random_resize_via_padding=False,\n", + " tensorboard=False,\n", + " wandb=False,\n", + " mlflow=False,\n", + " clearml=False,\n", + " class_names=CLASS_NAMES,\n", + " notes={\n", + " \"demo\": f\"segmentation PTL fine-tune on Roboflow Universe {DATASET_INFO['project']}\",\n", + " \"source_url\": DATASET_INFO[\"source_url\"],\n", + " \"roboflow_workspace\": DATASET_INFO[\"workspace\"],\n", + " \"roboflow_project\": DATASET_INFO[\"project\"],\n", + " \"roboflow_version\": DATASET_INFO[\"version\"],\n", + " \"num_classes\": NUM_CLASSES,\n", + " \"class_names\": CLASS_NAMES,\n", + " },\n", + " progress_bar=\"tqdm\",\n", + ")\n", + "\n", + "datamodule = RFDETRDataModule(variant.model_config, train_config)\n", + "model = RFDETRModelModule(variant.model_config, train_config)\n", + "trainer = build_trainer(train_config, variant.model_config)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "5f0ccbd3", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 4 - Preview dataset inputs\n", + "\n", + "Always look at your data before you start a long training run. This cell renders a grid of annotated training images\n", + "so you can confirm that segmentation masks align with objects, colour jitter and flips look reasonable, and there are\n", + "no systematic labelling errors such as masks placed on the wrong object or masks that bleed outside the object\n", + "boundary. Catching label noise here costs a few seconds; catching it after 50 epochs costs much more." + ] + }, + { + "cell_type": "code", + "id": "71e1a7db", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "sample_figure = datamodule._show_samples(\n", + " SAMPLE_PREVIEW_COUNT,\n", + " split=\"train\",\n", + " columns=SAMPLE_PREVIEW_COLUMNS,\n", + " figure_size=SAMPLE_PREVIEW_FIGURE_SIZE,\n", + ")\n", + "display(sample_figure)\n", + "plt.close(sample_figure)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "993b1cd1", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 5 - Fine-tune\n", + "\n", + "`trainer.fit` hands control to PyTorch Lightning, which drives the full training loop: forward pass, loss\n", + "computation (cross-entropy + Dice on sampled mask points), gradient accumulation, weight updates,\n", + "learning-rate scheduling, and periodic validation with COCO mask mAP. After each epoch the trainer appends a row to\n", + "`metrics.csv` and, if validation mAP improves, saves a new best checkpoint via `BestModelCallback`. On a single\n", + "consumer GPU (RTX 3090 or similar) 50 epochs on the crack dataset takes roughly 20–35 minutes depending on\n", + "`BATCH_SIZE` and `NUM_WORKERS`." + ] + }, + { + "cell_type": "code", + "id": "0564b2c6", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "trainer.fit(model, datamodule=datamodule, ckpt_path=train_config.resume or None)\n", + "print(f\"output_dir={OUTPUT_DIR}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "c746e44d", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 6 - Save checkpoint/model\n", + "\n", + "PTL saves its own checkpoints during training (optimizer state, scheduler state, epoch counter), but those files are\n", + "not directly portable — they require the same class hierarchy to load. The `.pth` file written here is a\n", + "self-contained RF-DETR checkpoint: it bundles the model weights together with the full training and model configs,\n", + "including the class names. You can share the file with a colleague and they can run inference with a single\n", + "`RFDETRSegSmall.from_checkpoint(path)` call. For full reproducibility, keep this checkpoint alongside the dataset\n", + "version number printed in section 1." + ] + }, + { + "cell_type": "code", + "id": "684059b3", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "final_checkpoint = _save_final_checkpoint(model, trainer, train_config, variant.model_config, FINAL_CHECKPOINT_PATH)\n", + "print(f\"saved_checkpoint={final_checkpoint}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "dadba8e3", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 7 - Validate metrics\n", + "\n", + "This cell runs a clean post-training validation with no augmentation, using the best checkpoint loaded from\n", + "`checkpoint_best_total.pth` written by `BestModelCallback`, and serialises results to JSON. Two metrics are most\n", + "important to inspect. `bbox/map` is the standard COCO bounding-box mAP at IoU 0.50:0.95. `segm/map` (when logged)\n", + "is the mask mAP — it measures how precisely the predicted binary masks overlap with the annotated polygons. A model\n", + "can have high `bbox/map` but low `segm/map` if the detector locates objects well but the mask boundaries are coarse;\n", + "this usually improves with more training epochs or by increasing `resolution`.\n", + "\n", + "**Note:** `checkpoint_best_total.pth` is written after the first completed validation epoch. If training was\n", + "interrupted before any epoch finished, this file will not exist and the cell below will raise `FileNotFoundError`." + ] + }, + { + "cell_type": "code", + "id": "ea69ee92", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "# checkpoint_best_total.pth stores plain model weights under key \"model\", not a full\n", + "# PTL checkpoint. Passing it as ckpt_path to trainer.validate() triggers a KeyError\n", + "# on \"validate_loop\" because PTL tries to restore loop state that isn't present.\n", + "# Load the weights directly and run validate without ckpt_path instead.\n", + "_ckpt = torch.load(OUTPUT_DIR / \"checkpoint_best_total.pth\", map_location=\"cpu\", weights_only=False)\n", + "model.model.load_state_dict(_ckpt[\"model\"], strict=True)\n", + "del _ckpt\n", + "validation_results = trainer.validate(model, datamodule=datamodule)\n", + "validation_metrics = {key: float(value) for key, value in validation_results[0].items()} if validation_results else {}\n", + "VALIDATION_METRICS_JSON.write_text(json.dumps(validation_metrics, indent=2, sort_keys=True), encoding=\"utf-8\")\n", + "print(f\"validation_metrics={validation_metrics}\")\n", + "print(f\"validation_metrics_json={VALIDATION_METRICS_JSON}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "e9b7a1cb", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 8 - Plot CSVLogger metrics\n", + "\n", + "Reading loss curves is one of the fastest ways to diagnose training problems. Healthy segmentation runs show the\n", + "combined detection + mask loss decreasing together for train and val. A loss that rises or plateaus from epoch 1 is\n", + "usually a sign that the learning rate is too high or the dataset is too small for the chosen `BATCH_SIZE`. For the\n", + "mAP curves, `segm mAP 50` (mask IoU ≥ 0.50) rises fastest and is the clearest signal of whether the model is\n", + "learning to segment objects; `segm mAP 50:95` (averaged across stricter thresholds) follows more slowly.\n", + "Set `PLOT_LOSS_LOG_SCALE = True` if the loss drops by an order of magnitude in the first few epochs and the later,\n", + "more meaningful portion of the curve gets compressed into a flat line." + ] + }, + { + "cell_type": "code", + "id": "abab244f", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "print(f\"metrics_csv={METRICS_CSV}\")\n", + "loss_figure = plot_loss_metrics(str(METRICS_CSV), loss_log_scale=PLOT_LOSS_LOG_SCALE)\n", + "display(loss_figure)\n", + "plt.close(loss_figure)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c93477c9", + "metadata": {}, + "outputs": [], + "source": [ + "map_figure = plot_map_metrics(str(METRICS_CSV))\n", + "display(map_figure)\n", + "plt.close(map_figure)" + ] + }, + { + "cell_type": "markdown", + "id": "e46c2424", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 9 - Load checkpoint/model\n", + "\n", + "`from_checkpoint` is the standard entry point for loading a saved RF-DETR model. It reads both the weights and the\n", + "stored config from the `.pth` file, so the reconstructed model has the correct number of classes and resolution\n", + "without you having to pass them explicitly. You can share this file with teammates and they can run inference\n", + "immediately — the class names and architecture are all embedded alongside the weights." + ] + }, + { + "cell_type": "code", + "id": "58b00247", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": "loaded_model = RFDETRSegSmall.from_checkpoint(FINAL_CHECKPOINT_PATH)", + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "d2252a7a", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 10 - Select inference images\n", + "\n", + "This cell implements a fallback chain — test split first, then validation, then train — so the notebook always finds\n", + "images to run inference on even when the dataset has no dedicated test split. Using images from the test set gives\n", + "you an unbiased view of model performance because those images were never seen during training or used to pick the\n", + "best checkpoint. Replace `inference_image_paths` with a list of `Path` objects pointing to your own files to run\n", + "inference on custom images." + ] + }, + { + "cell_type": "code", + "id": "f03e8dcc", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "_IMAGE_EXTS = {\".jpg\", \".jpeg\", \".png\"}\n", + "\n", + "\n", + "def _find_images(directory: Path) -> list[Path]:\n", + " if not directory.is_dir():\n", + " return []\n", + " return sorted(p for p in directory.glob(\"**/*\") if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)\n", + "\n", + "\n", + "validation_image_paths = _find_images(DATASET_DIR / \"test\")\n", + "if not validation_image_paths:\n", + " validation_image_paths = _find_images(DATASET_DIR / \"valid\")\n", + "if not validation_image_paths:\n", + " validation_image_paths = _find_images(DATASET_DIR / \"train\")\n", + "if not validation_image_paths:\n", + " raise FileNotFoundError(f\"No images ({', '.join(sorted(_IMAGE_EXTS))}) found under {DATASET_DIR}\")\n", + "\n", + "inference_image_paths = validation_image_paths[:INFERENCE_COUNT]\n", + "print(f\"inference_images={[str(p) for p in inference_image_paths]}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "74116bbe", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 11 - Visualize inference\n", + "\n", + "`INFERENCE_THRESHOLD` is a detection confidence threshold: only detections whose bounding-box score exceeds this\n", + "value are shown. Raise it to suppress false positives; lower it to surface low-confidence detections. Each accepted\n", + "detection includes a binary mask resized to the original image dimensions — `sv.MaskAnnotator` renders it as a\n", + "translucent colour overlay so you can see both the mask shape and the image beneath it.\n", + "\n", + "If the mask boundaries look jagged, consider increasing `RESOLUTION` (in multiples of 24) and retraining; higher\n", + "resolution gives the segmentation head more spatial detail to work with." + ] + }, + { + "cell_type": "code", + "id": "f3a68bfe", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "inference_grid_items: list[tuple[str, np.ndarray, sv.Detections]] = []\n", + "for image_path in inference_image_paths:\n", + " with Image.open(image_path) as image:\n", + " image_rgb = image.convert(\"RGB\")\n", + " detections = loaded_model.predict(image_rgb, threshold=INFERENCE_THRESHOLD)\n", + " if not isinstance(detections, sv.Detections):\n", + " raise RuntimeError(f\"Expected RFDETRSegSmall.predict() to return sv.Detections, got {type(detections)!r}.\")\n", + " inference_grid_items.append((image_path.name, np.array(image_rgb), detections))\n", + "\n", + "if inference_grid_items:\n", + " figure = _segmentation_grid_figure(inference_grid_items, columns=INFERENCE_COLUMNS, class_names=CLASS_NAMES)\n", + " display(figure)\n", + " plt.close(figure)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "63027bc2", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 12 - Detection summary\n", + "\n", + "Print per-image detection counts and confidence scores as a quick sanity check. If you see zero detections on most\n", + "images, try lowering `INFERENCE_THRESHOLD` (towards 0.1) — the model may be calibrated to lower confidence scores\n", + "on a custom dataset than the COCO-pretrained default. If you see too many false positives, raise the threshold or\n", + "add more negative examples to your training set." + ] + }, + { + "cell_type": "code", + "id": "d723ad36", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "summary_rows = []\n", + "for image_path, (_, _, detections) in zip(inference_image_paths, inference_grid_items):\n", + " num_det = len(detections)\n", + " has_masks = detections.mask is not None and detections.mask.shape[0] > 0\n", + " avg_conf = (\n", + " float(np.mean(detections.confidence)) if detections.confidence is not None and num_det > 0 else float(\"nan\")\n", + " )\n", + " summary_rows.append(\n", + " {\n", + " \"image\": image_path.name,\n", + " \"detections\": num_det,\n", + " \"has_masks\": has_masks,\n", + " \"avg_confidence\": round(avg_conf, 3),\n", + " }\n", + " )\n", + "\n", + "summary_df = pd.DataFrame(summary_rows)\n", + "display(summary_df)" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "title,-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/cookbooks/inference-latency-benchmark.ipynb b/docs/cookbooks/inference-latency-benchmark.ipynb new file mode 100644 index 0000000..59eb9cc --- /dev/null +++ b/docs/cookbooks/inference-latency-benchmark.ipynb @@ -0,0 +1,375 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a5720dea", + "metadata": {}, + "outputs": [], + "source": [ + "# ------------------------------------------------------------------------\n", + "# RF-DETR\n", + "# Copyright (c) 2025 Roboflow. All Rights Reserved.\n", + "# Licensed under the Apache License, Version 2.0 [see LICENSE for details]\n", + "# ------------------------------------------------------------------------" + ] + }, + { + "cell_type": "markdown", + "id": "66c62889", + "metadata": {}, + "source": [ + "# RF-DETR Inference Latency Benchmark\n", + "\n", + "Measures inference latency for three RF-DETR families across three configs:\n", + "\n", + "| Config | Description |\n", + "|--------|-------------|\n", + "| **FP32** | `predict()` — unoptimized baseline |\n", + "| **FP16+JIT** | `optimize_for_inference(dtype=torch.float16)` |\n", + "| **ONNX** | exported `.onnx` via `onnxruntime-gpu` |" + ] + }, + { + "cell_type": "markdown", + "id": "d1568b82", + "metadata": {}, + "source": [ + "## 1. Install\n", + "\n", + "We need `onnxruntime-gpu` built against CUDA 12 — the default PyPI wheel targets CUDA 11.8 and silently\n", + "falls back to CPU on modern GPUs. The Microsoft CUDA-12 package index ships the correct build.\n", + "\n", + "> **Colab**: after running this cell, go to **Runtime → Restart session**, then run from the next cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a116013", + "metadata": { + "title": "[bash]" + }, + "outputs": [], + "source": [ + "!pip uninstall -y onnxruntime onnxruntime-gpu\n", + "!pip install -q \"rfdetr[onnx]\" pillow pandas\n", + "!pip install -q onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/" + ] + }, + { + "cell_type": "markdown", + "id": "1ca0838b", + "metadata": {}, + "source": [ + "## 2. Config\n", + "\n", + "`WARMUP_RUNS` discards the first N inferences — GPU kernels are JIT-compiled on first use, so early timings\n", + "are outliers. `MEASURE_RUNS` then collects the steady-state distribution. 20 + 100 is a reasonable balance\n", + "between statistical stability and total wall-clock time per model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e47b983", + "metadata": {}, + "outputs": [], + "source": [ + "from collections.abc import Callable\n", + "from pathlib import Path\n", + "from typing import Any, NamedTuple\n", + "\n", + "import numpy as np\n", + "import torch\n", + "from PIL import Image\n", + "\n", + "WARMUP_RUNS = 20\n", + "MEASURE_RUNS = 100\n", + "EXPORT_DIR = Path(\"benchmark_output\")\n", + "EXPORT_DIR.mkdir(exist_ok=True)\n", + "\n", + "if not torch.cuda.is_available():\n", + " raise RuntimeError(\"This benchmark requires a CUDA GPU.\")\n", + "print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n", + "\n", + "import onnxruntime as ort\n", + "\n", + "_ort_providers = ort.get_available_providers()\n", + "print(f\"ORT {ort.__version__}, providers: {_ort_providers}\")\n", + "if \"CUDAExecutionProvider\" not in _ort_providers:\n", + " raise RuntimeError(\n", + " f\"onnxruntime-gpu with CUDA support required. Available providers: {_ort_providers}. \"\n", + " \"Fix: reinstall from the CUDA-12 index (see install cell) then restart runtime.\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "2ccdc87e", + "metadata": {}, + "source": [ + "## 3. Sample images\n", + "\n", + "Latency depends on resolution, not pixel content, so synthetic noise images are equivalent to real photos\n", + "for benchmarking purposes. Using a fixed seed makes results reproducible across runs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06a16bb0", + "metadata": {}, + "outputs": [], + "source": [ + "rng = np.random.default_rng(42)\n", + "images: list[Image.Image] = [Image.fromarray(rng.integers(0, 256, (640, 640, 3), dtype=np.uint8)) for _ in range(10)]\n", + "print(f\"Generated {len(images)} synthetic 640×640 RGB images\")" + ] + }, + { + "cell_type": "markdown", + "id": "db0d3d7c", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 4. Latency helpers\n", + "\n", + "GPU kernels execute asynchronously — `time.perf_counter()` returns before the GPU finishes, giving\n", + "misleadingly low numbers. CUDA events are inserted directly into the GPU command stream and timestamped\n", + "on the device, so `elapsed_time()` measures actual kernel execution. `torch.cuda.synchronize()` after\n", + "each run flushes the stream and ensures the event fires before we read it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f928319", + "metadata": {}, + "outputs": [], + "source": [ + "class BenchmarkResult(NamedTuple):\n", + " \"\"\"Single benchmark measurement.\"\"\"\n", + "\n", + " label: str\n", + " mean_ms: float\n", + " std_ms: float\n", + "\n", + " @property\n", + " def fps(self) -> float:\n", + " \"\"\"Frames per second.\"\"\"\n", + " return 1000.0 / self.mean_ms\n", + "\n", + "\n", + "def measure_latency_gpu(\n", + " fn: Callable[[], object],\n", + " warmup: int = WARMUP_RUNS,\n", + " runs: int = MEASURE_RUNS,\n", + ") -> tuple[float, float]:\n", + " \"\"\"Return (mean_ms, std_ms) using CUDA events.\"\"\"\n", + " for _ in range(warmup):\n", + " fn()\n", + " torch.cuda.synchronize()\n", + " start = torch.cuda.Event(enable_timing=True)\n", + " end = torch.cuda.Event(enable_timing=True)\n", + " timings: list[float] = []\n", + " for _ in range(runs):\n", + " start.record()\n", + " fn()\n", + " end.record()\n", + " torch.cuda.synchronize()\n", + " timings.append(start.elapsed_time(end))\n", + " arr = np.array(timings)\n", + " return float(arr.mean()), float(arr.std())\n", + "\n", + "\n", + "_measure = measure_latency_gpu" + ] + }, + { + "cell_type": "markdown", + "id": "8f7ee4f5", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 5. Per-config benchmark functions\n", + "\n", + "Three inference paths are compared:\n", + "\n", + "- **FP32** — `predict()` as shipped. Full 32-bit arithmetic on GPU. Baseline.\n", + "- **FP16+JIT** — `optimize_for_inference(dtype=torch.float16)` fuses layers with `torch.jit.script` and\n", + " halves the arithmetic precision. Typically 1.5–2× faster than FP32 with negligible accuracy loss on\n", + " modern tensor cores.\n", + "- **ONNX** — the model is exported to the Open Neural Network Exchange format and run through ONNX Runtime,\n", + " bypassing PyTorch entirely. ORT applies its own graph optimisations and can use the TensorRT execution\n", + " provider for additional speedup. Benchmarked separately on CPU and GPU to show the provider impact." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "533f037b", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "from rfdetr.export._onnx.inference import _onnx_runtime\n", + "\n", + "\n", + "def _predict_fp32(model: Any, image: Image.Image) -> BenchmarkResult:\n", + " \"\"\"Baseline FP32 predict() latency.\"\"\"\n", + " mean, std = _measure(lambda: model.predict(image))\n", + " return BenchmarkResult(\"predict() FP32\", mean, std)\n", + "\n", + "\n", + "def _predict_fp16(model: Any, image: Image.Image) -> BenchmarkResult:\n", + " \"\"\"FP16+JIT latency — applies and removes optimize_for_inference.\"\"\"\n", + " model.optimize_for_inference(dtype=torch.float16)\n", + " mean, std = _measure(lambda: model.predict(image))\n", + " model.remove_optimized_model()\n", + " return BenchmarkResult(\"predict() FP16+JIT\", mean, std)\n", + "\n", + "\n", + "def _export_onnx(model: Any, export_dir: Path) -> Path:\n", + " \"\"\"Export model to ONNX and return the path.\"\"\"\n", + " return Path(model.export(output_dir=str(export_dir)))" + ] + }, + { + "cell_type": "markdown", + "id": "2f6bc92a", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "## 6. Model benchmark runner\n", + "\n", + "Each model is loaded fresh, exported once, and then each inference config is timed independently.\n", + "The ONNX path reuses the same exported file for both CPU and CUDA providers so export cost is not\n", + "counted in latency. CUDA ONNX is skipped gracefully when no GPU is available; missing `onnxruntime-gpu`\n", + "raises immediately so misconfigured environments are caught early." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8afd179a", + "metadata": {}, + "outputs": [], + "source": [ + "def run_model_benchmark(\n", + " model_cls: type,\n", + " model_name: str,\n", + " images: list[Image.Image],\n", + ") -> list[BenchmarkResult]:\n", + " \"\"\"Run FP32 / FP16 / ONNX benchmarks for one model and print results.\"\"\"\n", + " print(f\"\\n{'=' * 62}\")\n", + " print(f\" {model_name}\")\n", + " print(\"=\" * 62)\n", + "\n", + " model: Any = model_cls()\n", + " export_dir = EXPORT_DIR / model_name.split()[0]\n", + " export_dir.mkdir(exist_ok=True)\n", + " image = images[0]\n", + "\n", + " fp32 = _predict_fp32(model, image)\n", + " fp16 = _predict_fp16(model, image)\n", + " results: list[BenchmarkResult] = [fp32, fp16]\n", + "\n", + " onnx_path = _export_onnx(model, export_dir)\n", + " for providers in ([\"CPUExecutionProvider\"], [\"CUDAExecutionProvider\", \"CPUExecutionProvider\"]):\n", + " if providers[0] == \"CUDAExecutionProvider\" and not torch.cuda.is_available():\n", + " print(\" ⚠ ONNX (CUDA) skipped — no CUDA GPU\")\n", + " continue\n", + " mean_ms, std_ms, label = _onnx_runtime(onnx_path, image, providers, WARMUP_RUNS, MEASURE_RUNS)\n", + " results.append(BenchmarkResult(f\"ONNX ({label})\", mean_ms, std_ms))\n", + "\n", + " for r in results:\n", + " print(f\" {r.label:<30} {r.mean_ms:6.2f} ms ± {r.std_ms:5.2f} ({r.fps:6.1f} FPS)\")\n", + "\n", + " onnx_results = [r for r in results if r.label.startswith(\"ONNX\")]\n", + " speedups = [f\"FP16 {fp32.mean_ms / fp16.mean_ms:.1f}×\"]\n", + " if onnx_results:\n", + " speedups.append(f\"ONNX {fp32.mean_ms / onnx_results[0].mean_ms:.1f}×\")\n", + " print(f\" Speedup vs FP32: {' | '.join(speedups)}\")\n", + "\n", + " return results" + ] + }, + { + "cell_type": "markdown", + "id": "e7ccd41c", + "metadata": {}, + "source": [ + "## 7. Benchmark loop — detection · segmentation · keypoint\n", + "\n", + "Three model families are benchmarked to show how task complexity affects latency. Detection (`RFDETRMedium`)\n", + "outputs boxes only; segmentation (`RFDETRSegSmall`) additionally predicts per-object masks, which adds\n", + "decoder cost; keypoint (`RFDETRKeypointPreview`) predicts skeleton joints and is typically the lightest\n", + "of the three at smaller resolutions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ab7ffbc", + "metadata": {}, + "outputs": [], + "source": [ + "from rfdetr import RFDETRKeypointPreview, RFDETRMedium, RFDETRSegSmall\n", + "\n", + "MODELS: list[tuple[type, str]] = [\n", + " (RFDETRMedium, \"RFDETRMedium — detection\"),\n", + " (RFDETRSegSmall, \"RFDETRSegSmall — segmentation\"),\n", + " (RFDETRKeypointPreview, \"RFDETRKeypointPreview — keypoint\"),\n", + "]\n", + "\n", + "all_results: dict[str, list[BenchmarkResult]] = {}\n", + "for _model_cls, _model_name in MODELS:\n", + " all_results[_model_name] = run_model_benchmark(_model_cls, _model_name, images)" + ] + }, + { + "cell_type": "markdown", + "id": "ebbba970", + "metadata": {}, + "source": [ + "## 8. Summary\n", + "\n", + "The table shows FPS (frames per second) for each model × config combination. Higher is better.\n", + "Compare columns to see which model fits your latency budget; compare rows to choose the right\n", + "inference backend for your deployment target (Python server, edge CPU, or ONNX Runtime service)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d8eddd9", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "summary = {\n", + " model_name.split()[0]: {r.label: round(r.fps, 1) for r in results} for model_name, results in all_results.items()\n", + "}\n", + "df = pd.DataFrame(summary)\n", + "df.index.name = \"Config \\\\ Model\"\n", + "print(df.to_string())\n", + "print(f\"\\nFPS — {MEASURE_RUNS} timed + {WARMUP_RUNS} warmup runs, batch 1, GPU: {torch.cuda.get_device_name(0)}.\")" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "title,-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/cookbooks/pytorch-lightning.ipynb b/docs/cookbooks/pytorch-lightning.ipynb new file mode 100644 index 0000000..ff528d9 --- /dev/null +++ b/docs/cookbooks/pytorch-lightning.ipynb @@ -0,0 +1,451 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1777c5f9", + "metadata": {}, + "source": [ + "# Training with PyTorch Lightning\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow/rf-detr/blob/develop/docs/cookbooks/pytorch-lightning.ipynb)\n", + "\n", + "**RF-DETR** is a real-time object detection model that combines the accuracy of\n", + "transformer-based detectors with inference speeds suitable for production.\n", + "The training stack is built on **PyTorch Lightning** — giving you\n", + "composable building blocks you can adopt incrementally, without changing your\n", + "existing code.\n", + "\n", + "| Building block | Role |\n", + "|---|---|\n", + "| `RFDETRModelModule` | `LightningModule` — model, loss, optimizer, scheduler |\n", + "| `RFDETRDataModule` | `LightningDataModule` — datasets and dataloaders |\n", + "| `build_trainer()` | Factory that assembles a `Trainer` with all RF-DETR callbacks |\n", + "\n", + "**Key design principle:** start simple, then pick up building blocks without losing\n", + "your trained weights.\n", + "\n", + "- **Phase 1** — `model.train()` one-liner (`EPOCHS_PHASE_1` epochs)\n", + "- **Phase 2** — swap in the PTL components and continue for `EPOCHS_PHASE_2` more\n", + " epochs from the same checkpoint, same output folder — no conversion required\n", + "- **End** — full training curve, single-image inference, and batch inference with\n", + " `trainer.predict()`" + ] + }, + { + "cell_type": "markdown", + "id": "a750078f", + "metadata": {}, + "source": [ + "## 1. Install RF-DETR 1.6.0\n", + "\n", + "`rfdetr[train,loggers]` pulls in PyTorch Lightning, torchmetrics, and the full\n", + "callback stack. `roboflow` downloads the demo dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4688278", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q rfdetr[train,loggers]==1.6.0 roboflow" + ] + }, + { + "cell_type": "markdown", + "id": "77fab4e5", + "metadata": {}, + "source": [ + "## 2. Config\n", + "\n", + "All notebook-level knobs in one place. Adjust `EPOCHS_PHASE_*` and `BATCH_SIZE`\n", + "to match your hardware — every downstream cell reads from these variables.\n", + "\n", + "`num_workers` is set to `os.cpu_count()` inside a Jupyter/Colab kernel where\n", + "process forking is safe, and to `0` when running as a plain Python script.\n", + "On macOS and Windows, spawn-based multiprocessing would otherwise re-import\n", + "this module as `__main__` and retrigger training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "174445e1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "DATASET_DIR = os.environ.get(\"DATASET_DIR\", \"\")\n", + "OUTPUT_DIR = \"output\"\n", + "EPOCHS_PHASE_1 = 20\n", + "EPOCHS_PHASE_2 = 10\n", + "BATCH_SIZE = 12\n", + "THRESHOLD = 0.3\n", + "\n", + "os.makedirs(OUTPUT_DIR, exist_ok=True)\n", + "\n", + "try:\n", + " from IPython import get_ipython\n", + "\n", + " _in_notebook = get_ipython() is not None\n", + "except Exception:\n", + " _in_notebook = False\n", + "\n", + "# Outside a notebook kernel, macOS/Windows spawn-based multiprocessing will\n", + "# re-import this script as __main__, triggering training again.\n", + "# Use 0 workers in that case; inside a kernel the usual forking rules apply safely.\n", + "num_workers = (os.cpu_count() or 0) if _in_notebook else 0" + ] + }, + { + "cell_type": "markdown", + "id": "3a43c723", + "metadata": {}, + "source": [ + "## 3. Dataset\n", + "\n", + "[Aquarium Combined](https://universe.roboflow.com/brad-dwyer/aquarium-combined)\n", + "— 638 images across 7 classes (`fish`, `jellyfish`, `penguin`, `puffin`,\n", + "`shark`, `starfish`, `stingray`). It is small enough to complete a demo run\n", + "in a few minutes yet diverse enough to produce meaningful detection results.\n", + "\n", + "Set `ROBOFLOW_API_KEY` as a Colab secret (Secrets panel, key icon) or as an\n", + "environment variable before running this cell. The dataset is downloaded in\n", + "COCO format, which RF-DETR reads natively." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34478ada", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from roboflow import Roboflow\n", + "\n", + "try:\n", + " from google.colab import userdata # type: ignore[import]\n", + "\n", + " API_KEY = userdata.get(\"ROBOFLOW_API_KEY\")\n", + "except Exception:\n", + " API_KEY = os.environ[\"ROBOFLOW_API_KEY\"]\n", + "\n", + "rf = Roboflow(api_key=API_KEY)\n", + "dataset = rf.workspace(\"brad-dwyer\").project(\"aquarium-combined\").version(1).download(\"coco\", location=\"datasets\")\n", + "DATASET_DIR = dataset.location\n", + "\n", + "with open(Path(DATASET_DIR) / \"train\" / \"_annotations.coco.json\") as f:\n", + " _ann = json.load(f)\n", + "\n", + "CLASS_NAMES = [c[\"name\"] for c in sorted(_ann[\"categories\"], key=lambda c: c[\"id\"])]\n", + "NUM_CLASSES = len(CLASS_NAMES)\n", + "print(f\"Dataset : {DATASET_DIR}\")\n", + "print(f\"Classes : {NUM_CLASSES} — {CLASS_NAMES}\")\n", + "\n", + "with open(Path(DATASET_DIR) / \"valid\" / \"_annotations.coco.json\") as f:\n", + " _val_ann = json.load(f)\n", + "\n", + "val_images_dir = Path(DATASET_DIR) / \"valid\"\n", + "val_image_files = [img[\"file_name\"] for img in _val_ann[\"images\"]]" + ] + }, + { + "cell_type": "markdown", + "id": "44bbcce0", + "metadata": {}, + "source": [ + "## 4. Phase 1 — `model.train()` one-liner\n", + "\n", + "The same high-level API that has been in RF-DETR since v1.0 — nothing here\n", + "changes from previous releases. If you have existing training scripts, they\n", + "keep working without modification.\n", + "\n", + "`pretrain_weights=\"rf-detr-medium.pth\"` downloads COCO-pretrained backbone\n", + "weights automatically on first run and caches them locally. `use_ema=True`\n", + "maintains an exponential moving average of the weights to stabilise validation\n", + "metrics. `run_test=False` skips the final test-set evaluation to keep Phase 1\n", + "fast; Phase 2 turns it back on.\n", + "\n", + "After this cell completes, `OUTPUT_DIR/checkpoint_best_total.pth` holds the\n", + "best weights seen so far — the starting point for Phase 2." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf7b136e", + "metadata": {}, + "outputs": [], + "source": [ + "from rfdetr import RFDETRMedium\n", + "\n", + "model = RFDETRMedium(num_classes=NUM_CLASSES, pretrain_weights=\"rf-detr-medium.pth\")\n", + "model.train(\n", + " dataset_dir=DATASET_DIR,\n", + " epochs=EPOCHS_PHASE_1,\n", + " batch_size=BATCH_SIZE,\n", + " grad_accum_steps=4,\n", + " lr=1e-4,\n", + " num_workers=num_workers,\n", + " output_dir=OUTPUT_DIR,\n", + " use_ema=True,\n", + " run_test=False,\n", + " progress_bar=\"rich\",\n", + " tensorboard=True,\n", + " seed=42,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "178dd044", + "metadata": {}, + "source": [ + "## 5. Phase 2 — PTL building blocks\n", + "\n", + "Pick up the three PTL components and call `trainer.fit()` pointing at the Phase 1\n", + "checkpoint. No weight conversion is needed — `RFDETRModelModule.on_load_checkpoint`\n", + "detects the `.pth` format and remaps keys automatically.\n", + "\n", + "A lower learning rate (`5e-5`) is used because the model is already partially\n", + "converged. `epochs=EPOCHS_PHASE_1 + EPOCHS_PHASE_2` sets the *absolute* epoch\n", + "ceiling; because the loaded checkpoint records the last completed epoch, PTL\n", + "runs exactly `EPOCHS_PHASE_2` additional epochs before stopping. The same\n", + "`OUTPUT_DIR` is reused so checkpoints and metrics all land in one place." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4526ab0", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from rfdetr import RFDETRDataModule, RFDETRModelModule, build_trainer\n", + "from rfdetr.config import RFDETRMediumConfig, TrainConfig\n", + "\n", + "# Read Phase 1 metrics before Phase 2 overwrites the CSV.\n", + "df1 = pd.read_csv(f\"{OUTPUT_DIR}/metrics.csv\")\n", + "\n", + "model_config = RFDETRMediumConfig(\n", + " num_classes=NUM_CLASSES,\n", + " pretrain_weights=\"rf-detr-medium.pth\",\n", + ")\n", + "\n", + "# epochs = EPOCHS_1 + EPOCHS_2 so PTL (which resumes the epoch counter from the\n", + "# checkpoint) runs exactly EPOCHS_2 additional epochs before reaching max_epochs.\n", + "train_config = TrainConfig(\n", + " dataset_dir=DATASET_DIR,\n", + " epochs=EPOCHS_PHASE_1 + EPOCHS_PHASE_2,\n", + " batch_size=BATCH_SIZE,\n", + " grad_accum_steps=4,\n", + " lr=5e-5,\n", + " num_workers=num_workers,\n", + " output_dir=OUTPUT_DIR,\n", + " use_ema=True,\n", + " run_test=True,\n", + " progress_bar=\"tqdm\",\n", + " tensorboard=True,\n", + " seed=42,\n", + ")\n", + "\n", + "module = RFDETRModelModule(model_config=model_config, train_config=train_config)\n", + "datamodule = RFDETRDataModule(model_config=model_config, train_config=train_config)\n", + "trainer = build_trainer(train_config, model_config)\n", + "\n", + "# Resume directly from the Phase 1 .pth — no conversion needed.\n", + "trainer.fit(module, datamodule, ckpt_path=f\"{OUTPUT_DIR}/checkpoint_best_total.pth\")" + ] + }, + { + "cell_type": "markdown", + "id": "344147ee", + "metadata": {}, + "source": [ + "## 6. Training curve\n", + "\n", + "Phase 1 and Phase 2 each emit their own `metrics.csv` (Phase 2 overwrites Phase 1's\n", + "file when it starts). We captured the Phase 1 copy before fitting so we can\n", + "concatenate both DataFrames and plot a single continuous curve across all epochs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "751e8277", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "from IPython.display import display\n", + "from PIL import Image\n", + "\n", + "from rfdetr.visualize.training import plot_metrics\n", + "\n", + "df2 = pd.read_csv(f\"{OUTPUT_DIR}/metrics.csv\")\n", + "\n", + "combined_csv = f\"{OUTPUT_DIR}/metrics_combined.csv\"\n", + "pd.concat([df1, df2], ignore_index=True).to_csv(combined_csv, index=False)\n", + "print(f\"Combined CSV: {combined_csv} ({len(df1) + len(df2)} rows)\")\n", + "\n", + "fig = plot_metrics(combined_csv)\n", + "display(fig)\n", + "print(f\"Saved: {OUTPUT_DIR}/metrics_plot.png\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3f29ab1", + "metadata": {}, + "source": [ + "## 7. Single-image inference — `model.predict()`\n", + "\n", + "`RFDETRMedium` can be instantiated directly from a checkpoint path for inference\n", + "— no training config needed. `model.predict()` accepts a PIL `Image`, runs\n", + "preprocessing, the forward pass, and postprocessing internally, and returns a\n", + "`supervision.Detections` object that is ready to annotate and display." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90e06abb", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "model = RFDETRMedium(pretrain_weights=f\"{OUTPUT_DIR}/checkpoint_best_total.pth\", num_classes=NUM_CLASSES)\n", + "\n", + "image = Image.open(val_images_dir / val_image_files[0])\n", + "detections = model.predict(image, threshold=THRESHOLD)\n", + "\n", + "annotated = sv.BoxAnnotator().annotate(image.copy(), detections)\n", + "annotated = sv.LabelAnnotator().annotate(annotated, detections, labels=[CLASS_NAMES[c] for c in detections.class_id])\n", + "\n", + "plt.figure(figsize=(10, 7))\n", + "plt.imshow(np.array(annotated))\n", + "plt.axis(\"off\")\n", + "plt.show()\n", + "\n", + "print(f\"Detected {len(detections)} object(s)\")" + ] + }, + { + "cell_type": "markdown", + "id": "32baa2d8", + "metadata": {}, + "source": [ + "## 8. Batch inference — `trainer.predict()`\n", + "\n", + "Instead of calling `model.predict()` one image at a time, `trainer.predict()`\n", + "streams the entire validation set through the model in batches and collects\n", + "all results — useful for dataset-level evaluation or offline export pipelines.\n", + "\n", + "**What happens under the hood:**\n", + "\n", + "1. PTL calls `datamodule.setup(\"predict\")` — this builds `_dataset_val` if it\n", + " does not exist yet. Because `trainer.fit()` already ran above, the dataset\n", + " is already in memory and `setup` is a no-op.\n", + "2. PTL calls `datamodule.predict_dataloader()` — this returns the *validation*\n", + " dataset wrapped in a `SequentialSampler` (no shuffle, no augmentation),\n", + " identical to `val_dataloader`.\n", + "3. For each batch, `RFDETRModelModule.predict_step()` runs a forward pass under\n", + " `torch.no_grad()` and returns a list of `{\"scores\", \"labels\", \"boxes\"}`\n", + " dicts — one dict per image in the batch.\n", + "4. `trainer.predict()` collects all batch results into a\n", + " `List[List[dict]]` (outer = batches, inner = images).\n", + "\n", + "Flatten, apply a confidence threshold, and wrap in `sv.Detections`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d7543ff", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import itertools\n", + "\n", + "# Returns List[List[dict]] — outer: batches, inner: one dict per image\n", + "# Each dict has keys: \"scores\" (N,), \"labels\" (N,), \"boxes\" (N, 4) — all tensors\n", + "all_preds = trainer.predict(module, datamodule)\n", + "\n", + "# Flatten the batch dimension → one result dict per validation image\n", + "flat_preds = [img_result for batch in all_preds for img_result in batch]\n", + "print(f\"Ran predict on {len(flat_preds)} validation images\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed942d37", + "metadata": {}, + "outputs": [], + "source": [ + "# Build sv.Detections from raw tensors and visualise the first four images\n", + "annotated_images = []\n", + "for img_file, result in itertools.islice(zip(val_image_files, flat_preds), 4):\n", + " keep = result[\"scores\"] > THRESHOLD\n", + " detections = sv.Detections(\n", + " xyxy=result[\"boxes\"][keep].cpu().float().numpy(),\n", + " confidence=result[\"scores\"][keep].cpu().float().numpy(),\n", + " class_id=result[\"labels\"][keep].cpu().long().numpy(),\n", + " )\n", + " image = Image.open(val_images_dir / img_file)\n", + " annotated = sv.BoxAnnotator().annotate(image.copy(), detections)\n", + " annotated = sv.LabelAnnotator().annotate(\n", + " annotated, detections, labels=[CLASS_NAMES[c] for c in detections.class_id]\n", + " )\n", + " annotated_images.append(np.array(annotated))\n", + " print(f\" {img_file}: {len(detections)} detection(s)\")\n", + "\n", + "fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n", + "for ax, img in zip(axes.flat, annotated_images):\n", + " ax.imshow(img)\n", + " ax.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3387e060", + "metadata": {}, + "source": [ + "## 9. Next steps\n", + "\n", + "You have now seen the complete 1.6.0 stack — from a one-liner `model.train()`\n", + "through composable PTL components to batch inference. From here:\n", + "\n", + "- [PyTorch Lightning training docs](https://rfdetr.roboflow.com/1.6.0/reference/training/) — custom callbacks, multi-GPU, mixed precision\n", + "- [Advanced training options](https://rfdetr.roboflow.com/1.6.0/learn/train/advanced/) — augmentations, EMA, learning rate schedules\n", + "- [Logger integrations (ClearML, MLflow, W&B)](https://rfdetr.roboflow.com/1.6.0/learn/train/loggers/) — experiment tracking\n", + "- [Export your model](https://rfdetr.roboflow.com/1.6.0/learn/export/) — ONNX, TensorRT, CoreML" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "formats": "ipynb,py", + "main_language": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md new file mode 100644 index 0000000..6521c1d --- /dev/null +++ b/docs/getting-started/install.md @@ -0,0 +1,117 @@ +--- +description: Install RF-DETR via pip, uv, or from source. Set up a development environment for contributing to Roboflow's real-time detection transformer. +--- + +# Installation + +Welcome to RF-DETR! This guide will help you install and set up RF-DETR for your projects. Whether you're a developer looking to contribute or an end-user ready to start using RF-DETR, we've got you covered. + +## Installation Methods + +RF-DETR supports several installation methods. Choose the option which best fits your workflow. + +!!! example "Installation" + + === "pip (recommended)" + + The easiest way to install RF-DETR is using `pip`. This method is recommended for most users. + + ```bash + pip install rfdetr + ``` + + === "uv" + + If you are using `uv`, you can install RF-DETR using the following command: + + ```bash + uv pip install rfdetr + ``` + + For `uv` projects, you can also use: + + ```bash + uv add rfdetr + ``` + + === "Source Archive" + + To install the latest development version of RF-DETR from source without cloning the full repository, run the command below. + + ```bash + pip install https://github.com/roboflow/rf-detr/archive/refs/heads/develop.zip + ``` + +## Dev Environment + +If you plan to contribute to RF-DETR or modify the codebase locally, set up a local development environment using the steps below. + +!!! example "Development Setup" + + === "virtualenv" + + ```bash + # Clone the repository and navigate to the root directory + git clone --depth 1 -b develop https://github.com/roboflow/rf-detr.git + cd rf-detr + + # Set up a Python virtual environment with a specific Python version (e.g., 3.10) + python3.10 -m venv venv + + # Activate the virtual environment + source venv/bin/activate + + # Upgrade pip + pip install --upgrade pip + + # Install the package in development mode + pip install -e "." + ``` + + === "uv" + + ```bash + # Clone the repository and navigate to the root directory + git clone --depth 1 -b develop https://github.com/roboflow/rf-detr.git + cd rf-detr + + # Pin Python version (optional but recommended) + uv python pin 3.11 + + # Sync environment (creates .venv, installs pinned Python, and installs dependencies) + uv sync + + # Install the package in development mode with all extras + uv pip install -e . --all-extras + ``` + +## Optional Extras + +RF-DETR provides several optional extras for additional functionality: + +| Extra | Install command | Purpose | +| --------- | ----------------------------------- | --------------------------------------------------------------- | +| `train` | `pip install "rfdetr[train]"` | Training dependencies (PyTorch Lightning, albumentations, etc.) | +| `loggers` | `pip install "rfdetr[loggers]"` | Experiment tracking (TensorBoard, W&B, MLflow, ClearML) | +| `onnx` | `pip install "rfdetr[onnx]"` | ONNX export | +| `tflite` | `pip install "rfdetr[onnx,tflite]"` | TFLite export (Python 3.12 only) | +| `trt` | `pip install "rfdetr[trt]"` | TensorRT inference (pycuda, onnxruntime-gpu, tensorrt) | +| `kornia` | `pip install "rfdetr[kornia]"` | GPU-accelerated augmentations via Kornia | +| `lora` | `pip install "rfdetr[lora]"` | LoRA fine-tuning with PEFT | +| `visual` | `pip install "rfdetr[visual]"` | Visualization utilities (matplotlib, pandas, seaborn) | +| `cli` | `pip install "rfdetr[cli]"` | CLI with typed argument parsing (jsonargparse) | +| `plus` | `pip install "rfdetr[plus]"` | XLarge and 2XLarge detection models (PML 1.0 license) | + +## Additional Notes + +- Ensure you have Python 3.10 or higher installed. +- For development, it is recommended to use a virtual environment to avoid conflicts with other packages. +- If you encounter any issues during installation, refer to the [troubleshooting](#troubleshooting) section or open an issue on the [GitHub repository](https://github.com/roboflow/rf-detr). + +## Troubleshooting + +If you encounter any issues during installation, here are some common solutions: + +- **Permission Issues**: Use `pip install --user rfdetr` to install the package for your user only. +- **Dependency Conflicts**: Use a virtual environment to isolate the installation. +- **Python Version**: Ensure you are using Python 3.10 or higher. diff --git a/docs/getting-started/migration.md b/docs/getting-started/migration.md new file mode 100644 index 0000000..b8d619a --- /dev/null +++ b/docs/getting-started/migration.md @@ -0,0 +1,404 @@ +--- +description: Per-version migration guide for RF-DETR. Covers breaking changes and deprecated APIs for each release series. +--- + +# Migration Guide + +Read each section between your current version and your target — every section covers +only the delta between two adjacent releases. + +``` +1.4.x → 1.5 → 1.6 → 1.7 → 1.8 +``` + +You can apply all changes in one go; working through sections one release at a time +and verifying between each step is optional but makes failures easier to isolate. +Deprecated APIs emit a `DeprecationWarning` until the version marked for removal. +See the [Changelog](../changelog.md) for the full list of changes in each release. + +--- + +## Upgrade 1.8 → 1.9 + +### Planned for Removal in v1.9 + +The following APIs were deprecated in earlier releases and will be removed in v1.9. They still work in the current release (v1.8.x) but emit `DeprecationWarning`. Update your code before upgrading. + +!!! warning "Planned for removal: `rfdetr.util.*` and `rfdetr.deploy.*` import paths" + + Deprecated since v1.6. Use the canonical replacements listed in the [Upgrade 1.5 → 1.6](#upgrade-15--16) section. + + ```python + # These imports still work in v1.8 but emit DeprecationWarning; update before v1.9 + from rfdetr.util.coco_classes import COCO_CLASSES # → rfdetr.assets.coco_classes + from rfdetr.util.misc import get_rank # → rfdetr.utilities + from rfdetr.deploy import export_onnx # → rfdetr.export.main + ``` + +!!! warning "Planned for removal: `build_namespace(model_config, train_config)`" + + Deprecated since v1.7. Use `build_model_from_config` and `build_criterion_from_config` instead. + +!!! warning "Planned for removal: `load_pretrain_weights(nn_model, model_config, train_config)` with `train_config`" + + Deprecated since v1.7. Drop the `train_config` positional argument. + +!!! warning "Planned for removal: `start_epoch` kwarg in `train()`" + + Deprecated since v1.7. PyTorch Lightning resumes automatically via `resume=`. + +!!! warning "Planned for removal: `do_benchmark` kwarg in `train()`" + + Deprecated since v1.7. Use the `rfdetr.export.benchmark` module instead. + +!!! warning "Planned for removal: `callbacks` dict kwarg in `train()`" + + Deprecated since v1.7. Pass PTL `Callback` objects directly via the Lightning API instead. + +!!! warning "Planned for removal: misplaced config fields" + + The following `TrainConfig` and `ModelConfig` fields moved to their correct config class in v1.7 and the deprecated compatibility shims will be removed in v1.9. Update any direct references: + + | Field | Removed from | Use in | + | ------------------- | ------------- | ------------- | + | `group_detr` | `TrainConfig` | `ModelConfig` | + | `ia_bce_loss` | `TrainConfig` | `ModelConfig` | + | `segmentation_head` | `TrainConfig` | `ModelConfig` | + | `num_select` | `TrainConfig` | `ModelConfig` | + | `cls_loss_coef` | `ModelConfig` | `TrainConfig` | + +--- + +## Upgrade 1.7 → 1.8 + +### Breaking changes + +!!! note "Breaking in v1.8.2: default keypoint schema changed to active-first `[17]`" + + New checkpoints created from v1.8.2 onwards use `class_id=0` for person. Legacy `[0, 17]` checkpoints + are still supported — RF-DETR auto-detects the schema from the checkpoint at load time. + + If your post-processing code offsets class IDs by 1 (common for background-first models), update it: + + ```python + # Before (background-first [0, 17]: person was at class_id=1) + class_name = "person" if detection.class_id == 1 else "other" + + # After (active-first [17]: person is at class_id=0) + class_name = "person" if detection.class_id == 0 else "other" + ``` + + Use `detection.data["class_name"]` for schema-agnostic name resolution. + +!!! warning "Breaking: `rfdetr.datasets.aug_config` renamed to `rfdetr.datasets.aug_configs`" + + The augmentation config module was renamed (singular → plural). If you import from it directly: + + ```python + # Before + from rfdetr.datasets.aug_config import AUG_AGGRESSIVE + + # After + from rfdetr.datasets.aug_configs import AUG_AGGRESSIVE + ``` + + All preset constants (`AUG_AGGRESSIVE`, `AUG_CONSERVATIVE`, etc.) are unchanged. + +!!! warning "Breaking: `supervision>=0.29.0` now required" + + Required for `sv.KeyPoints` support. `pip install rfdetr==1.8.0` pulls this automatically. + If another dependency pins `supervision<0.29.0`, resolve the conflict manually. + +!!! warning "Breaking: `pyDeprecate` constraint narrowed to `>=0.9,<0.10`" + + Was `>=0.6,<0.8`. If another package pins an older version, resolve with: + + ```bash + pip install "rfdetr==1.8.0" "pyDeprecate>=0.9,<0.10" + ``` + +--- + +## Upgrade 1.6 → 1.7 + +### Breaking changes + +!!! warning "Breaking: `peft` removed from the default install" + + LoRA fine-tuning now requires the `lora` extra. If you use LoRA adapters during + training, update your install command. + + ```bash + # Before + pip install rfdetr + + # After + pip install 'rfdetr[lora]' + ``` + +!!! warning "Breaking: `predict()` stores source image in `detections.metadata`" + + **`predict()` stores the source image in `detections.metadata`, not `detections.data`.** + + ```python + # Before (1.6.4 and earlier) + source = detections.data["source_image"] + + # After + source = detections.metadata["source_image"] + ``` + +!!! warning "Breaking: `pyDeprecate` constraint changed to `>=0.6,<0.8`" + + Was `>=0.3,<0.6`. If another package pins an older version, resolve with: + + ```bash + pip install "rfdetr==1.7.0" "pyDeprecate>=0.6,<0.8" + ``` + +### Deprecated in v1.7 → Remove in v1.9 + +!!! note "Deprecated: `build_namespace()` split into two functions" + + **`build_namespace(model_config, train_config)`** — use `build_model_from_config` or + `build_criterion_from_config` instead. + + ```python + # Before (deprecated) + from rfdetr.models import build_namespace + + ns = build_namespace(model_config, train_config) + + # After + from rfdetr.models import build_model_from_config, build_criterion_from_config + + model = build_model_from_config(model_config) + criterion = build_criterion_from_config(model_config, train_config) + ``` + +!!! note "Deprecated: `load_pretrain_weights()` no longer takes `train_config`" + + **`load_pretrain_weights(nn_model, model_config, train_config)`** — drop the + `train_config` positional argument. + + ```python + # Before (deprecated) + from rfdetr.models import load_pretrain_weights + + load_pretrain_weights(nn_model, model_config, train_config) + + # After + from rfdetr.models import load_pretrain_weights + + load_pretrain_weights(nn_model, model_config) + ``` + +!!! note "Deprecated: config fields moved between `ModelConfig` and `TrainConfig`" + + **Config fields placed in the wrong config object.** Move them as shown: + + | Field | Was in | Move to | + | ------------------- | ------------- | ------------- | + | `group_detr` | `TrainConfig` | `ModelConfig` | + | `ia_bce_loss` | `TrainConfig` | `ModelConfig` | + | `segmentation_head` | `TrainConfig` | `ModelConfig` | + | `num_select` | `TrainConfig` | `ModelConfig` | + | `cls_loss_coef` | `ModelConfig` | `TrainConfig` | + + ```python + # Before (deprecated) + train_config = TrainConfig(group_detr=13, cls_loss_coef=2.0) + + # After + model_config = ModelConfig(group_detr=13) + train_config = TrainConfig(cls_loss_coef=2.0) + ``` + +### Deprecated in v1.7 → Remove in v2.0 + +!!! note "Deprecated: `RFDETRBase` replaced by size-specific classes" + + **`RFDETRBase`** defaulted to the small variant and is replaced by size-specific + classes. Choose the variant that matches your previous model size. If you used + `RFDETRBase()` without arguments, switch to `RFDETRSmall()`. + + ```python + # Before (deprecated) + from rfdetr import RFDETRBase + + model = RFDETRBase() + + # After — pick one + from rfdetr import RFDETRNano, RFDETRSmall, RFDETRMedium, RFDETRLarge + + model = RFDETRSmall() + ``` + +!!! note "Deprecated: `RFDETRSegPreview` replaced by size-specific segmentation classes" + + **`RFDETRSegPreview`** defaulted to the small variant and is replaced by size-specific + segmentation classes. If you used `RFDETRSegPreview()` without arguments, switch to + `RFDETRSegSmall()`. + + ```python + # Before (deprecated) + from rfdetr import RFDETRSegPreview + + model = RFDETRSegPreview() + + # After — pick one + from rfdetr import RFDETRSegNano, RFDETRSegSmall, RFDETRSegMedium, RFDETRSegLarge + + model = RFDETRSegSmall() + ``` + +--- + +## Upgrade 1.5 → 1.6 + +### Breaking changes + +!!! warning "Breaking: `transformers` minimum version raised to `>=5.1.0`" + + **`transformers` minimum version raised to `>=5.1.0,<6.0.0`.** + + Projects pinned to `transformers<5.0.0` must upgrade. If upgrading is not possible, + pin `rfdetr<1.6.0`. + + ```bash + pip install 'transformers>=5.1.0,<6.0.0' + ``` + +!!! warning "Breaking: PyPI extras renamed" + + **PyPI extras renamed.** + + Update your `pip install` commands and `requirements*.txt` files. + + | Old extra | New extra | + | -------------------- | ----------------- | + | `rfdetr[metrics]` | `rfdetr[loggers]` | + | `rfdetr[onnxexport]` | `rfdetr[onnx]` | + + ```bash + # Before + pip install 'rfdetr[metrics]' + pip install 'rfdetr[onnxexport]' + + # After + pip install 'rfdetr[loggers]' + pip install 'rfdetr[onnx]' + ``` + +!!! warning "Breaking: `draw_synthetic_shape()` now returns a tuple" + + **`draw_synthetic_shape()` now returns `(image, polygon)` instead of `image`.** + + Update every call site that unpacks only the image. + + ```python + # Before + img = draw_synthetic_shape(...) + + # After + img, polygon = draw_synthetic_shape(...) + ``` + +### Deprecated in v1.6 → Removed in v1.8 + +!!! note "Deprecated: `simplify` and `force` arguments in `RFDETR.export()`" + + **`RFDETR.export(..., simplify=..., force=...)`** — both arguments are no-ops. + Remove them from your calls. + + ```python + # Before (deprecated) + model.export("model.onnx", simplify=True, force=True) + + # After + model.export("model.onnx") + ``` + +### Deprecated in v1.6 → Remove in v1.9 + +!!! note "Deprecated: `rfdetr.util.*` and `rfdetr.deploy.*` import paths" + + Backward-compatibility shims are still active but emit `DeprecationWarning` on import. + Replace with the canonical paths listed in the table below. + + | Deprecated module | Canonical replacement | + | --------------------------------- | ---------------------------------- | + | `rfdetr.util.coco_classes` | `rfdetr.assets.coco_classes` | + | `rfdetr.util.misc` | `rfdetr.utilities` | + | `rfdetr.util.logger` | `rfdetr.utilities.logger` | + | `rfdetr.util.box_ops` | `rfdetr.utilities.box_ops` | + | `rfdetr.util.files` | `rfdetr.utilities.files` | + | `rfdetr.util.package` | `rfdetr.utilities.package` | + | `rfdetr.util.get_param_dicts` | `rfdetr.training.param_groups` | + | `rfdetr.util.drop_scheduler` | `rfdetr.training.drop_schedule` | + | `rfdetr.util.visualize` | `rfdetr.visualize.data` | + | `rfdetr.deploy` | `rfdetr.export` | + | `rfdetr.models.segmentation_head` | `rfdetr.models.heads.segmentation` | + + **Examples:** + + ```python + # Before (deprecated) + from rfdetr.util.coco_classes import COCO_CLASSES + from rfdetr.util.misc import get_rank, get_world_size, is_main_process, save_on_master + from rfdetr.util.logger import get_logger + from rfdetr.util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou + from rfdetr.util.get_param_dicts import get_param_dict + from rfdetr.util.drop_scheduler import drop_scheduler + from rfdetr.util.visualize import save_gt_predictions_visualization + from rfdetr.deploy import export_onnx + from rfdetr.models.segmentation_head import SegmentationHead + + # After + from rfdetr.assets.coco_classes import COCO_CLASSES + from rfdetr.utilities.distributed import get_rank, get_world_size, is_main_process, save_on_master + from rfdetr.utilities.logger import get_logger + from rfdetr.utilities.box_ops import box_cxcywh_to_xyxy, generalized_box_iou + from rfdetr.training.param_groups import get_param_dict + from rfdetr.training.drop_schedule import drop_scheduler + from rfdetr.visualize.data import save_gt_predictions_visualization + from rfdetr.export.main import export_onnx + from rfdetr.models.heads.segmentation import SegmentationHead + ``` + +--- + +## Upgrade 1.4 → 1.5 + +### Breaking changes + +!!! warning "Breaking: `ModelConfig` rejects unknown keyword arguments" + + **`ModelConfig` now raises `ValidationError` on unknown keyword arguments.** + + Previously, unrecognised fields were silently ignored. Remove or rename any + unrecognised keys you pass to `ModelConfig(...)`. + + ```python + # Before — silently accepted + config = ModelConfig(unknown_field=True) + + # Now raises ValidationError — remove the unknown key + config = ModelConfig() + ``` + +### Deprecated in v1.5 → Removed in v1.7 + +!!! note "Deprecated: `OPEN_SOURCE_MODELS` replaced by `ModelWeights` enum" + + **`OPEN_SOURCE_MODELS` constant** — use the `ModelWeights` enum instead. A + `DeprecationWarning` is emitted on access. See the + [API reference](../reference/rfdetr.md) for available enum values. + + ```python + # Before (deprecated) + from rfdetr import OPEN_SOURCE_MODELS + + # After + from rfdetr.assets.model_weights import ModelWeights + ``` diff --git a/docs/hooks/cookbooks_cards.py b/docs/hooks/cookbooks_cards.py new file mode 100644 index 0000000..388a085 --- /dev/null +++ b/docs/hooks/cookbooks_cards.py @@ -0,0 +1,72 @@ +# ------------------------------------------------------------------------ +# RF-DETR +# Copyright (c) 2025 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +"""MkDocs hook for exposing cookbook card data to documentation templates.""" + +from pathlib import Path +from typing import Any + +import yaml # type: ignore[import-untyped] # yaml stubs not in docs group + + +def _load_cards(cards_path: Path) -> list[dict[str, Any]]: + """Load cookbook card definitions from a YAML file. + + Reads the YAML file and returns the ``cards`` list, which is consumed by the + cookbook landing-page template. Centralising card data in YAML keeps content + decoupled from presentation and avoids repeated edits to the HTML template. + + Args: + cards_path: Path to the cookbook cards YAML file. + + Returns: + Ordered list of card dictionaries from the YAML ``cards`` key. + + Raises: + FileNotFoundError: If the YAML file does not exist. + RuntimeError: If the YAML payload does not expose a list under ``cards``. + yaml.YAMLError: If the YAML file cannot be parsed. + + Example: + >>> cards = _load_cards(Path("docs/cookbooks/cards.yaml")) + >>> isinstance(cards, list) + True + """ + with cards_path.open("r", encoding="utf-8") as cards_file: + payload = yaml.safe_load(cards_file) + cards = (payload or {}).get("cards") + if not isinstance(cards, list): + msg = f"Missing list 'cards' in {cards_path}" + raise RuntimeError(msg) + return cards + + +def on_config(config: dict[str, Any]) -> dict[str, Any]: + """Expose cookbook card data to MkDocs templates. + + Adds ``config.extra.cookbooks_cards`` so the cookbooks landing-page template + can render cards from a single YAML source of truth instead of hardcoded + HTML blocks. + + Args: + config: MkDocs configuration object. + + Returns: + Updated MkDocs configuration object. + + Raises: + FileNotFoundError: If the cookbook cards YAML file is missing. + RuntimeError: If the YAML file does not expose a ``cards`` list. + + Example: + >>> cfg = {"extra": {}} + >>> updated = on_config(cfg) + >>> isinstance(updated["extra"]["cookbooks_cards"], list) + True + """ + cards_path = Path(__file__).resolve().parents[2] / "docs" / "cookbooks" / "cards.yaml" + extra = config.setdefault("extra", {}) + extra["cookbooks_cards"] = _load_cards(cards_path) + return config diff --git a/docs/hooks/package_version.py b/docs/hooks/package_version.py new file mode 100644 index 0000000..7efeb0c --- /dev/null +++ b/docs/hooks/package_version.py @@ -0,0 +1,97 @@ +# ------------------------------------------------------------------------ +# RF-DETR +# Copyright (c) 2025 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +"""MkDocs hooks for exposing package metadata to documentation templates.""" + +import sys +from pathlib import Path +from typing import Any + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli + + +def _load_pyproject(pyproject_path: Path) -> dict[str, Any]: + """Load project metadata from a pyproject file. + + Reads the TOML file in binary mode, matching the parser API. The returned + dictionary is used by MkDocs hooks to avoid duplicating package metadata. + + Args: + pyproject_path: Path to the repository's pyproject file. + + Returns: + Parsed pyproject data. + + Raises: + FileNotFoundError: If the pyproject file does not exist. + tomllib.TOMLDecodeError: If the pyproject file is invalid TOML on Python 3.11+. + tomli.TOMLDecodeError: If the pyproject file is invalid TOML on Python 3.10. + + Example: + >>> data = _load_pyproject(Path("pyproject.toml")) + >>> isinstance(data["project"]["version"], str) + True + """ + with pyproject_path.open("rb") as pyproject_file: + if sys.version_info >= (3, 11): + return tomllib.load(pyproject_file) + return tomli.load(pyproject_file) + + +def _read_project_version(pyproject_path: Path) -> str: + """Read the package version from pyproject project metadata. + + Validates that the version exists and is a string before exposing it to the + documentation templates. + + Args: + pyproject_path: Path to the repository's pyproject file. + + Returns: + Package version from the ``[project]`` table. + + Raises: + RuntimeError: If ``[project].version`` is missing or not a string. + + Example: + >>> _read_project_version(Path("pyproject.toml")) + '1.6.0' + """ + pyproject = _load_pyproject(pyproject_path) + version = pyproject.get("project", {}).get("version") + if not isinstance(version, str): + msg = f"Missing string [project].version in {pyproject_path}" + raise RuntimeError(msg) + return version + + +def on_config(config: dict[str, Any]) -> dict[str, Any]: + """Expose the package version to MkDocs templates. + + Adds ``config.extra.software_version`` so schema.org JSON-LD can stay in + sync with the package metadata in ``pyproject.toml``. + + Args: + config: MkDocs configuration object. + + Returns: + Updated MkDocs configuration object. + + Raises: + RuntimeError: If package metadata cannot provide a valid version. + + Example: + >>> cfg = {"extra": {}} + >>> updated = on_config(cfg) + >>> updated["extra"]["software_version"] + '1.6.0' + """ + pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml" + extra = config.setdefault("extra", {}) + extra["software_version"] = _read_project_version(pyproject_path) + return config diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..bcddb8c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,196 @@ +--- +description: RF-DETR is a real-time transformer for object detection, instance segmentation, and keypoint detection (preview) by Roboflow. DINOv2 backbone, SOTA on COCO (60.1 AP50:95). Apache 2.0. +hide: + - navigation +--- + +# RF-DETR: Real-Time SOTA Object Detection, Instance Segmentation, and Keypoint Detection + +RF-DETR is a real-time transformer architecture for object detection, instance segmentation, and keypoint detection (preview) developed by Roboflow. Built on a DINOv2 vision transformer backbone, RF-DETR achieves state-of-the-art accuracy–latency trade-offs: RF-DETR-L reaches 56.5 AP50:95 on COCO at 6.8 ms (NVIDIA T4, TensorRT FP16), and RF-DETR-2XL achieves 60.1 AP50:95 — the first real-time model to exceed 60 AP on COCO. Accepted at [ICLR 2026](https://arxiv.org/abs/2511.09554). + +RF-DETR uses a DINOv2 vision transformer backbone and supports object detection, instance segmentation, and keypoint detection (preview) in a single, consistent API. Core models (Nano through Large) and all code are released under the Apache 2.0 license; XL and 2XLarge detection models require `rfdetr[plus]` and are provided under PML 1.0. + +Developed by Isaac Robinson, Peter Robicheaux, Matvei Popov, Deva Ramanan (CMU), and Neehar Peri (CMU) at [Roboflow](https://roboflow.com). If you use RF-DETR in your research, please cite: + +```bibtex +@inproceedings{robinson2026rfdetr, + title = {RF-DETR: Real-Time Detection Transformer}, + author = {Robinson, Isaac and Robicheaux, Peter and Popov, Matvei and Ramanan, Deva and Peri, Neehar}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2026}, + url = {https://arxiv.org/abs/2511.09554} +} +``` + +## Install + +You can install and use `rfdetr` in a [**Python>=3.10**](https://www.python.org/) environment. For detailed installation instructions, including installing from source, and setting up a local development environment, check out our [install](getting-started/install.md) page. + +!!! example "Installation" + + version + python-version + license + downloads + + === "pip" + + ```bash + pip install rfdetr + ``` + + === "uv" + + ```bash + uv pip install rfdetr + ``` + + For uv projects: + + ```bash + uv add rfdetr + ``` + +## Quickstart + +
+ +- **Run Detection Models** + + --- + + Load and run pre-trained RF-DETR detection models. + + [:octicons-arrow-right-24: Tutorial](learn/run/detection.md) + +- **Run Segmentation Models** + + --- + + Load and run pre-trained RF-DETR-Seg segmentation models. + + [:octicons-arrow-right-24: Tutorial](learn/run/segmentation.md) + +- **Train Models** + + --- + + Learn how to fine-tune RF-DETR models for detection and segmentation. + + [:octicons-arrow-right-24: Tutorial](learn/train/index.md) + +
+ +## Tutorials + +
+ +- **Train RF-DETR on a Custom Dataset. Video** + + --- + + ![Train RF-DETR on a Custom Dataset](https://i.ytimg.com/vi/-OvpdLAElFA/maxresdefault.jpg){ width="1280" height="720" loading="lazy" } + + End to end walkthrough of training RF-DETR on a custom dataset. + + [:octicons-arrow-right-24: Watch the video](https://www.youtube.com/watch?v=-OvpdLAElFA) + +- **Deploy RF-DETR to NVIDIA Jetson. Article** + + --- + + ![Deploy RF-DETR to NVIDIA Jetson](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/06/inst-3-.png){ width="1000" height="563" loading="lazy" } + + Instructions for deploying RF-DETR on NVIDIA Jetson with Roboflow Inference. + + [:octicons-arrow-right-24: Read the tutorial](https://blog.roboflow.com/how-to-deploy-rf-detr-to-an-nvidia-jetson/) + +- **Train and Deploy RF-DETR with Roboflow** + + --- + + ![Train and Deploy RF-DETR with Roboflow](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/03/img-blog-nycerebro-2.png){ width="1000" height="563" loading="lazy" } + + Cloud training and hardware deployment workflow using Roboflow. + + [:octicons-arrow-right-24: Read the tutorial](https://blog.roboflow.com/train-and-deploy-rf-detr-models-with-roboflow/) + +
+ +## Benchmarks + +RF-DETR achieves the best accuracy–latency trade-off among real-time object detection and instance segmentation models. It also provides keypoint detection (preview) on COCO person keypoints. For detailed benchmark tables and methodology, check out our [benchmarks](learn/benchmarks.md) page. + +### Detection + +Pareto front — detection accuracy vs latency: RF-DETR-2XL achieves 78.5 COCO AP50 (60.1 AP50:95) at 17.2 ms; RF-DETR-L achieves 75.1 AP50 at 6.8 ms, outperforming YOLO11x at comparable latency + +| Architecture | COCO AP50 | COCO AP50:95 | RF100VL AP50 | RF100VL AP50:95 | Latency (ms) | Params (M) | Resolution | +| ------------ | -------------------- | ----------------------- | ----------------------- | -------------------------- | ------------ | ---------- | ---------- | +| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384×384 | +| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512×512 | +| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576×576 | +| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704×704 | +| RF-DETR-XL | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700×700 | +| RF-DETR-2XL | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880×880 | + +### Segmentation + +Pareto front — segmentation accuracy vs latency: RF-DETR-Seg-2XL achieves 73.1 COCO AP50 (49.9 AP50:95) at 21.8 ms; RF-DETR-Seg-L achieves 70.5 AP50 at 8.8 ms + +| Architecture | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | +| --------------- | -------------------- | ----------------------- | ------------ | ---------- | ---------- | +| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312×312 | +| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384×384 | +| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432×432 | +| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504×504 | +| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624×624 | +| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768×768 | + +### Keypoints + +RF-DETR Keypoint mAP vs latency chart comparing against YOLO26-pose and YOLO11-pose on MS COCO + +| Architecture | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | +| -------------------------- | ----------------------- | ------------ | ---------- | ---------- | +| RF-DETR Keypoint (Preview) | 71.8 | 9.7 | 126.4 | 576×576 | + +> Keypoint benchmarks report AP50:95 (OKS-based); this is the standard COCO keypoint comparison metric. +> For the full competitor comparison (YOLO11-pose, YOLO26-pose), see the [Benchmarks](learn/benchmarks.md#keypoints) page. + +## Frequently Asked Questions + +**What is RF-DETR?** +RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy–latency trade-offs on COCO (60.1 AP50:95 for RF-DETR-2XL) and RF100-VL. + +**How does RF-DETR compare to YOLOv11?** +RF-DETR-L achieves 56.5 AP50:95 on COCO at 6.8 ms latency on an NVIDIA T4, outperforming YOLOv11x (50.9 AP) at lower latency. The DINOv2 backbone gives RF-DETR stronger performance on domain-shift benchmarks such as RF100-VL. + +**What GPU is required to train RF-DETR?** +A CUDA-capable GPU with at least 8 GB VRAM (e.g., NVIDIA RTX 3060, T4, A10) is recommended for fine-tuning. Smaller models (RF-DETR-N and RF-DETR-S) can fit in 6 GB VRAM with reduced batch size. CPU inference is supported for evaluation. + +**Which dataset formats does RF-DETR support?** +RF-DETR supports COCO JSON and YOLO-format datasets (with `dataset_file: "yolo"`). Roboflow datasets export directly to both formats. Detection and segmentation datasets use the same format — the model variant determines the task. + +**Can RF-DETR run in real time?** +Yes. RF-DETR-N runs at 2.3 ms per frame on a T4 GPU (TensorRT FP16, batch 1), and RF-DETR-L at 6.8 ms — both well within real-time thresholds. ONNX and TFLite exports are available for edge deployment. + +**What is the difference between RF-DETR detection and segmentation models?** +Detection models (e.g., `RFDETRLarge`) output bounding boxes. Segmentation models (e.g., `RFDETRSegLarge`) additionally output instance masks. Both share the same backbone and training API; segmentation adds a mask head and requires COCO-format segmentation annotations. + +**Does RF-DETR support keypoint detection?** +RF-DETR Keypoint (Preview) detects 17 body keypoints per person on COCO, achieving 71.8 AP50:95 at 9.7 ms on NVIDIA T4. It is available in the `rfdetr` package as `RFDETRKeypointPreview`. See [Run Keypoint Models](learn/run/keypoints.md) for usage. + +**Is RF-DETR open source?** +Yes. Core models (Nano through Large) and all training/inference code are released under the Apache 2.0 license. XLarge and 2XLarge models require the `rfdetr[plus]` package (PML 1.0 license). + +**How do I fine-tune RF-DETR on a custom dataset?** +Instantiate a model and call `model.train(...)` with your dataset directory in COCO JSON or YOLO format. Example: `model = RFDETRLarge(); model.train(dataset_dir='./dataset', epochs=50, batch_size=4)`. The model downloads pretrained weights automatically and saves best checkpoints automatically (use `resume=` to continue from one). + +**How do I export RF-DETR to ONNX or TensorRT?** +Call `model.export(format="onnx")` after training or loading a checkpoint. ONNX export works on CPU and produces a single `.onnx` file compatible with ONNX Runtime and OpenCV DNN. For TensorRT deployment, first export to ONNX and then convert the `.onnx` model with TensorRT tooling or helpers such as `trtexec`; this requires TensorRT and a CUDA GPU. + +**Which RF-DETR model size should I use?** +RF-DETR-Nano (2.3 ms, 67.6 AP50 on COCO) is best for edge and real-time applications. RF-DETR-Large (6.8 ms, 56.5 AP50:95) offers the best accuracy–latency trade-off for server deployment. RF-DETR-2XLarge (17.2 ms, 60.1 AP50:95) maximizes accuracy when latency allows. + +> **Checkpoint note:** Current `RFDETRLarge` defaults to `rf-detr-large-2026.pth`. The older `rf-detr-large.pth` checkpoint is a legacy Large release kept for backward compatibility and has been superseded by the current release. diff --git a/docs/javascripts/cookbooks-card.js b/docs/javascripts/cookbooks-card.js new file mode 100644 index 0000000..5b6ebda --- /dev/null +++ b/docs/javascripts/cookbooks-card.js @@ -0,0 +1,154 @@ +document.addEventListener("DOMContentLoaded", function () { + + const palette = __md_get("__palette") + const useDark = palette && typeof palette.color === "object" && palette.color.scheme === "slate" + const theme = useDark ? "dark-theme" : "light-default"; + + const colorList = [ + "#22c55e", + "#14b8a6", + "#ef4444", + "#eab308", + "#8b5cf6", + "#f97316", + "#3b82f6", + ] + + const logoSrc = (document.querySelector('link[rel="icon"]') || {}).href || ''; + const authorCache = {}; + + const repoCards = document.querySelectorAll(".repo-card"); + const labelsAll = Array + .from(repoCards) + .flatMap((element) => (element.getAttribute('data-labels') || '').split(',')) + .map(label => label.trim()) + .filter(label => label !== ''); + const uniqueLabels = [...new Set(labelsAll)]; + + const labelToColor = uniqueLabels.reduce((map, label, index) => { + map[label] = colorList[index % colorList.length]; + return map; + }, {}); + + + async function renderCard(element, elementIndex) { + const name = element.getAttribute('data-name') || ''; + const description = element.getAttribute('data-description') || ''; + const labels = element.getAttribute('data-labels') || ''; + const version = element.getAttribute('data-version') || ''; + const authors = element.getAttribute('data-author') || ''; + + const labelHTML = labels ? labels.split(',').filter(label => label !== '').map((label, index) => { + const color = labelToColor[label.trim()]; + return ` + + ${label.trim()} + + `; + }).join(' ') : ''; + + const authorArray = authors ? authors.split(',').filter(a => a.trim()) : []; + const authorDataArray = await Promise.all(authorArray.map(async (author) => { + const login = author.trim(); + if (authorCache[login]) return authorCache[login]; + try { + const response = await fetch(`https://api.github.com/users/${login}`); + if (!response.ok) return { login, avatar_url: `https://github.com/${login}.png` }; + const data = await response.json(); + authorCache[login] = data; + return data; + } catch { + return { login, avatar_url: `https://github.com/${login}.png` }; + } + })); + + let authorAvatarsHTML = authorDataArray.map((authorData, index) => { + const marginLeft = index === 0 ? '0' : '-10px'; + return ` +
+ + ${authorData.login}'s avatar + +
+ `; + }).join(''); + + let authorNamesHTML = authorDataArray.map( + authorData => ` + + + ${authorData.login} + + ` + ).join(', '); + + let authorsHTML = ` +
+ ${authorAvatarsHTML} +
${authorNamesHTML}
+
+ `; + + const rawHTML = ` +
+
+ + ${name} + +
+ ${description ? `

${description}

` : ''} + ${authorsHTML} +
+
+ +   + ${version} +
+
+ ${labelHTML} +
+
+
+ `; + + element.innerHTML = DOMPurify.sanitize(rawHTML); + + element.querySelectorAll('.author-name').forEach(nameEl => { + nameEl.addEventListener('mouseenter', function () { + const login = this.getAttribute('data-login'); + element.querySelector(`.author-container[data-login="${login}"]`).classList.add('hover'); + }); + + nameEl.addEventListener('mouseleave', function () { + const login = this.getAttribute('data-login'); + element.querySelector(`.author-container[data-login="${login}"]`).classList.remove('hover'); + }); + }); + } + repoCards.forEach((element, index) => { + renderCard(element, index); + }); +}) diff --git a/docs/learn/benchmarks.md b/docs/learn/benchmarks.md new file mode 100644 index 0000000..27070ab --- /dev/null +++ b/docs/learn/benchmarks.md @@ -0,0 +1,112 @@ +--- +description: RF-DETR benchmark results on COCO and RF100-VL for detection, segmentation, and keypoint detection. Compare accuracy and latency against YOLO, D-FINE, and LW-DETR. +--- + +# Benchmarks + +!!! tip "Key Takeaways" + + - RF-DETR-2XL achieves 60.1 AP50:95 on COCO detection at 17.2 ms latency (T4, TensorRT FP16) + - RF-DETR-L outperforms YOLOv11x (56.5 vs 50.9 AP50:95) at lower latency (6.8 vs 10.5 ms) + - Segmentation models range from 40.3 AP (Nano, 3.4 ms) to 49.9 AP (2XL, 21.8 ms) on COCO + - All latency measured on NVIDIA T4 with TensorRT 10.4, CUDA 12.4, FP16, batch size 1 + - RF100-VL results demonstrate strong domain-shift generalization across 100 diverse datasets + - RF-DETR Keypoint (Preview) achieves 71.8 AP50:95 on COCO person keypoints at 9.7 ms (T4, TensorRT FP16) + +This page reports RF-DETR benchmark results for object detection, instance segmentation, and keypoint detection on Microsoft COCO and RF100-VL (detection only). All benchmark numbers and plots match the latest released checkpoints and tables shown below. Latency values are measured on an NVIDIA T4 with TensorRT in FP16 at batch size 1. For full methodology details and architectural context, see the RF-DETR paper. + +## Methodology + +Accuracy is reported using standard COCO metrics computed with pycocotools. For object detection, we report COCO AP50 and COCO AP50:95, and the same metrics are also reported for RF100-VL. COCO results are evaluated on the validation split, following common practice in detector benchmarking. RF100-VL results are averaged across all 100 datasets to reflect performance under diverse real-world data distributions. + +Latency is measured as single-image inference latency rather than sustained throughput. All latency numbers are obtained on an NVIDIA T4 GPU using TensorRT 10.4 and CUDA 12.4 with FP16 inference and batch size 1. To reduce variance caused by GPU power throttling and thermal effects, a 200 ms buffer is inserted between consecutive forward passes. This procedure improves reproducibility of latency measurements but is not intended to measure maximum throughput. + +Accuracy and latency are always measured using the same model artifact and the same numerical precision. This avoids reporting FP32 accuracy together with FP16 latency, which can lead to misleading comparisons because naive FP16 conversion can significantly degrade accuracy for some models. + +!!! info "Metric definitions" + + **AP50**: Detection accuracy at IoU threshold >= 0.50. + **AP50:95**: Mean accuracy averaged over IoU thresholds 0.50 to 0.95 (step 0.05) — the primary COCO metric. + Latency measured on NVIDIA T4, TensorRT 10.4, CUDA 12.4, FP16, batch size 1, + with 200 ms thermal buffer between passes to reduce GPU thermal variance. + +## Detection + +rf_detr_1-4_latency_accuracy_object_detection + +| Architecture | COCO AP50 | COCO AP50:95 | RF100VL AP50 | RF100VL AP50:95 | Latency (ms) | Params (M) | Resolution | +| :----------: | :------------------: | :---------------------: | :---------------------: | :------------------------: | :----------: | :--------: | :--------: | +| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384x384 | +| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512x512 | +| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576x576 | +| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704x704 | +| RF-DETR-XL | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700x700 | +| RF-DETR-2XL | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880x880 | +| YOLO11-N | 52.0 | 37.4 | 81.4 | 55.3 | 2.5 | 2.6 | 640x640 | +| YOLO11-S | 59.7 | 44.4 | 82.3 | 56.2 | 3.2 | 9.4 | 640x640 | +| YOLO11-M | 64.1 | 48.6 | 82.5 | 56.5 | 5.1 | 20.1 | 640x640 | +| YOLO11-L | 64.9 | 49.9 | 82.2 | 56.5 | 6.5 | 25.3 | 640x640 | +| YOLO11-X | 66.1 | 50.9 | 81.7 | 56.2 | 10.5 | 56.9 | 640x640 | +| YOLO26-N | 55.8 | 40.3 | 76.7 | 52.0 | 1.7 | 2.6 | 640x640 | +| YOLO26-S | 64.3 | 47.7 | 82.7 | 57.0 | 2.6 | 9.4 | 640x640 | +| YOLO26-M | 69.7 | 52.5 | 84.4 | 58.7 | 4.4 | 20.1 | 640x640 | +| YOLO26-L | 71.1 | 54.1 | 85.0 | 59.3 | 5.7 | 25.3 | 640x640 | +| YOLO26-X | 74.0 | 56.9 | 85.6 | 60.0 | 9.6 | 56.9 | 640x640 | +| LW-DETR-T | 60.7 | 42.9 | 84.7 | 57.1 | 1.9 | 12.1 | 640x640 | +| LW-DETR-S | 66.8 | 48.0 | 85.0 | 57.4 | 2.6 | 14.6 | 640x640 | +| LW-DETR-M | 72.0 | 52.6 | 86.8 | 59.8 | 4.4 | 28.2 | 640x640 | +| LW-DETR-L | 74.6 | 56.1 | 87.4 | 61.5 | 6.9 | 46.8 | 640x640 | +| LW-DETR-X | 76.9 | 58.3 | 87.9 | 62.1 | 13.0 | 118.0 | 640x640 | +| D-FINE-N | 60.2 | 42.7 | 84.4 | 58.2 | 2.1 | 3.8 | 640x640 | +| D-FINE-S | 67.6 | 50.6 | 85.3 | 60.3 | 3.5 | 10.2 | 640x640 | +| D-FINE-M | 72.6 | 55.0 | 85.5 | 60.6 | 5.4 | 19.2 | 640x640 | +| D-FINE-L | 74.9 | 57.2 | 86.4 | 61.6 | 7.5 | 31.0 | 640x640 | +| D-FINE-X | 76.8 | 59.3 | 86.9 | 62.2 | 11.5 | 62.0 | 640x640 | + +## Segmentation + +rf_detr_1-4_latency_accuracy_instance_segmentation + +| Architecture | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | +| :-------------: | :------------------: | :---------------------: | :----------: | :--------: | :--------: | +| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312x312 | +| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384x384 | +| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432x432 | +| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504x504 | +| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624x624 | +| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768x768 | +| YOLOv8-N-Seg | 45.6 | 28.3 | 3.5 | 3.4 | 640x640 | +| YOLOv8-S-Seg | 53.8 | 34.0 | 4.2 | 11.8 | 640x640 | +| YOLOv8-M-Seg | 58.2 | 37.3 | 7.0 | 27.3 | 640x640 | +| YOLOv8-L-Seg | 60.5 | 39.0 | 9.7 | 46.0 | 640x640 | +| YOLOv8-XL-Seg | 61.3 | 39.5 | 14.0 | 71.8 | 640x640 | +| YOLOv11-N-Seg | 47.8 | 30.0 | 3.6 | 2.9 | 640x640 | +| YOLOv11-S-Seg | 55.4 | 35.0 | 4.6 | 10.1 | 640x640 | +| YOLOv11-M-Seg | 60.0 | 38.5 | 6.9 | 22.4 | 640x640 | +| YOLOv11-L-Seg | 61.5 | 39.5 | 8.3 | 27.6 | 640x640 | +| YOLOv11-XL-Seg | 62.4 | 40.1 | 13.7 | 62.1 | 640x640 | +| YOLO26-N-Seg | 54.3 | 34.7 | 2.31 | 2.7 | 640x640 | +| YOLO26-S-Seg | 62.4 | 40.2 | 3.47 | 10.4 | 640x640 | +| YOLO26-M-Seg | 67.8 | 44.0 | 6.32 | 23.6 | 640x640 | +| YOLO26-L-Seg | 69.8 | 45.5 | 7.58 | 28.0 | 640x640 | +| YOLO26-X-Seg | 71.6 | 46.8 | 12.92 | 62.8 | 640x640 | + +## Keypoints + +RF-DETR Keypoint mAP vs latency chart comparing against YOLO26-pose and YOLO11-pose on MS COCO + +| Architecture | COCO AP50:95 | Latency (ms) | +| :------------------------: | :---------------------: | :----------: | +| RF-DETR Keypoint (Preview) | 71.8 | 9.7 | +| YOLO11-pose N | 48.9 | 3.2 | +| YOLO11-pose S | 57.5 | 3.4 | +| YOLO11-pose M | 64.2 | 5.2 | +| YOLO11-pose L | 65.2 | 6.6 | +| YOLO11-pose X | 68.6 | 10.6 | +| YOLO26-pose N | 55.9 | 1.9 | +| YOLO26-pose S | 62.0 | 2.7 | +| YOLO26-pose M | 68.0 | 4.6 | +| YOLO26-pose L | 69.2 | 5.9 | +| YOLO26-pose X | 71.0 | 9.8 | + +> Keypoint benchmarks report AP50:95 (OKS-based); this is the standard COCO keypoint comparison metric. diff --git a/docs/learn/deploy.md b/docs/learn/deploy.md new file mode 100644 index 0000000..5c83a74 --- /dev/null +++ b/docs/learn/deploy.md @@ -0,0 +1,115 @@ +--- +description: Deploy fine-tuned RF-DETR detection and segmentation models to Roboflow for cloud inference, edge hardware, and multi-step vision workflows. +--- + +# Deploy a Trained RF-DETR Model + +!!! tip "Key Takeaways" + + - Deploy fine-tuned RF-DETR models to Roboflow with a single `deploy_to_roboflow()` call + - Supports both detection and segmentation model deployment + - Run deployed models via Roboflow Inference on cloud, edge hardware, or NVIDIA Jetson + - Model weights are cached locally after the first inference run for fast subsequent predictions + +You can deploy a fine-tuned RF-DETR model to Roboflow. + +Deploying to Roboflow allows you to create multi-step computer vision applications that run both in the cloud and your own hardware. + +To deploy your model to Roboflow, run: + +=== "Object Detection" + + ```python + from rfdetr import RFDETRNano + + x = RFDETRNano(pretrain_weights="") + x.deploy_to_roboflow( + workspace="", + project_id="", + version=1, + api_key="", + ) + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegMedium + + x = RFDETRSegMedium(pretrain_weights="") + x.deploy_to_roboflow( + workspace="", + project_id="", + version=1, + api_key="", + ) + ``` + +Above, set your Roboflow Workspace ID, the ID of the project to which you want to upload your model, and your Roboflow API key. + +- [Learn how to find your Workspace and Project ID.](https://docs.roboflow.com/developer/authentication/workspace-and-project-ids) +- [Learn how to find your API key.](https://docs.roboflow.com/developer/authentication/find-your-roboflow-api-key) + +You can then run your model with Roboflow Inference: + +=== "Object Detection" + + ```python + import supervision as sv + from inference import get_model + from PIL import Image + from io import BytesIO + import requests + + url = "https://media.roboflow.com/dog.jpeg" + image = Image.open(BytesIO(requests.get(url).content)) + + model = get_model("rfdetr-large") # replace with your Roboflow model ID + + predictions = model.infer(image, confidence=0.5)[0] + + detections = sv.Detections.from_inference(predictions) + + labels = [prediction.class_name for prediction in predictions.predictions] + + annotated_image = image.copy() + annotated_image = sv.BoxAnnotator().annotate(annotated_image, detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) + + sv.plot_image(annotated_image) + ``` + +=== "Image Segmentation" + + ```python + import supervision as sv + from inference import get_model + from PIL import Image + from io import BytesIO + import requests + + url = "https://media.roboflow.com/dog.jpeg" + image = Image.open(BytesIO(requests.get(url).content)) + + model = get_model("rfdetr-seg-small") # replace with your Roboflow model ID + + predictions = model.infer(image, confidence=0.5)[0] + + detections = sv.Detections.from_inference(predictions) + + labels = [prediction.class_name for prediction in predictions.predictions] + + annotated_image = image.copy() + annotated_image = sv.MaskAnnotator().annotate(annotated_image, detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) + + sv.plot_image(annotated_image) + ``` + +Above, replace `rfdetr-large` with the your Roboflow model ID. You can find this ID from the "Models" list in your Roboflow dashboard: + +![](https://media.roboflow.com/rfdetr/models-list.png) + +When you first run this model, your model weights will be cached for local use with Inference. + +You will then see the results from your fine-tuned model. diff --git a/docs/learn/export.md b/docs/learn/export.md new file mode 100644 index 0000000..6f375b1 --- /dev/null +++ b/docs/learn/export.md @@ -0,0 +1,396 @@ +--- +description: Export RF-DETR models to ONNX, TensorRT, and TFLite (FP32/FP16/INT8) for high-performance inference on GPUs, mobile, and edge devices. +--- + +# Export RF-DETR Model + +!!! tip "Key Takeaways" + + - Export to ONNX for cross-platform inference with ONNX Runtime, OpenVINO, or TensorRT + - Export to TFLite (FP32, FP16, INT8) for mobile and edge deployment + - TensorRT conversion delivers lowest latency on NVIDIA GPUs (2.3 ms for Nano) + - INT8 quantization requires calibration data from your dataset for accurate results + - Custom input resolutions supported (must be divisible by `patch_size × num_windows`, which varies by model variant) + +RF-DETR supports exporting models to ONNX and TFLite formats, enabling deployment across a wide range of inference frameworks, edge devices, and hardware accelerators. + +## Installation + +Install the export dependencies you need: + +```bash +# ONNX export only +pip install "rfdetr[onnx]" + +# TFLite export (includes ONNX dependency) +pip install "rfdetr[onnx,tflite]" +``` + +## Basic Export + +Export your trained model to ONNX format: + +=== "Object Detection" + + ```python + from rfdetr import RFDETRMedium + + model = RFDETRMedium(pretrain_weights="") + + model.export() + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegMedium + + model = RFDETRSegMedium(pretrain_weights="") + + model.export() + ``` + +This command saves the ONNX model to the `output` directory by default. + +## Export Parameters + +The `export()` method accepts several parameters to customize the export process: + +| Parameter | Default | Description | +| ------------------ | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `output_dir` | `"output"` | Directory where the exported model will be saved. | +| `format` | `"onnx"` | Export format: `"onnx"` or `"tflite"`. | +| `quantization` | `None` | TFLite quantization mode: `None`/`"fp32"`, `"fp16"`, or `"int8"`. Only used when `format="tflite"`. | +| `calibration_data` | `None` | Calibration data for TFLite export. Image directory, `.npy` file path, NumPy array, or `None`. See [TFLite Export](#tflite-export). | +| `max_images` | `100` | Maximum number of images to load from a calibration directory for TFLite INT8 quantization. Ignored for other calibration data formats. | +| `infer_dir` | `None` | Optional directory of sample images for inference validation during export tracing. If not provided, a random dummy image is generated. | +| `backbone_only` | `False` | Export only the backbone feature extractor instead of the full model. | +| `opset_version` | `17` | ONNX opset version to use for export. Higher versions support more operations. | +| `verbose` | `True` | Whether to print verbose export information. | +| `shape` | `None` | Input shape as tuple `(height, width)`. Each dimension must be divisible by the selected model's block size (`patch_size * num_windows`). If not provided, uses the model's default resolution. | +| `batch_size` | `1` | Batch size for the exported model. | +| `dynamic_batch` | `False` | If `True`, export with a dynamic batch dimension so the ONNX model accepts variable batch sizes at runtime. | +| `patch_size` | `None` | Backbone patch size override. Defaults to the value from `model_config.patch_size`. Must match the instantiated model's patch size when provided. | +| `notes` | `None` | Optional user-defined metadata (string, dict, list, or any JSON-serialisable value) to embed in the exported ONNX model under the `"rfdetr_notes"` metadata property. | + +## Advanced Export Examples + +### Export with Custom Output Directory + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium(pretrain_weights="") + +model.export(output_dir="exports/my_model") +``` + +### Export with Custom Resolution + +Export the model with a specific input resolution. For example, `RFDETRMedium` expects dimensions divisible by `32` (`patch_size=16`, `num_windows=2`): + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium(pretrain_weights="") + +model.export(shape=(608, 608)) +``` + +### Export Backbone Only + +Export only the backbone feature extractor for use in custom pipelines: + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium(pretrain_weights="") + +model.export(backbone_only=True) +``` + +## Output Files + +After running the export, you will find the following files in your output directory: + +- `inference_model.onnx` - The exported ONNX model (or `backbone_model.onnx` if `backbone_only=True`) + +## Optional: Convert ONNX to TensorRT + +If you want lower latency on NVIDIA GPUs, you can convert the exported ONNX model to a TensorRT engine. + +> [!IMPORTANT] +> Run TensorRT conversion on the same machine and GPU family where you plan to deploy inference. + +### Prerequisites + +- Install TensorRT (`trtexec` must be available in your `PATH`) +- Export an ONNX model first (for example: `output/inference_model.onnx`) + +### Python API Conversion + +```python +from argparse import Namespace + +from rfdetr.export._tensorrt import trtexec + +args = Namespace( + verbose=True, + profile=False, + dry_run=False, +) + +trtexec("output/inference_model.onnx", args) +``` + +This produces `output/inference_model.engine`. If `profile=True`, it also writes an Nsight Systems report (`.nsys-rep`). + +## TFLite Export + +!!! warning "Experimental — Use with Caution" + + TFLite export is **experimental and work-in-progress**. The pipeline depends on + several upstream packages (`onnx2tf`, `ai_edge_litert`, `tflite-runtime`) that + have experienced breaking API changes and installation instabilities across + releases. You may encounter errors or unexpected results. + + **Known instabilities:** + + - `onnx2tf` output graph structure can change between minor versions, silently + altering output tensor layout and breaking downstream inference code. + - `ai_edge_litert` (Google's replacement for `tflite-runtime`) is still + stabilising its public API; version pinning is strongly recommended. + - INT8 quantization accuracy is sensitive to calibration data quality — poor + calibration causes silent precision loss with no error at export time. + - The ONNX → TF → TFLite conversion chain introduces numerical rounding that + may produce slightly different predictions from the original PyTorch model. + - Installation of the `[tflite]` extra may conflict with existing TensorFlow + or NumPy versions in your environment. + + **Recommendations:** + + - Pin your dependency versions (e.g. `onnx2tf==X.Y.Z`) and test before each upgrade. + - Validate exported `.tflite` files against a held-out evaluation set before deploying. + - Prefer ONNX export when your target runtime supports it — it is more stable and + better tested. + - If export fails, check the [open issues](https://github.com/roboflow/rf-detr/issues) + for known workarounds or report a new one with your environment details + (`pip freeze`, Python version, OS). + +Export your model to TFLite for deployment on mobile devices, microcontrollers, and edge hardware via TensorFlow Lite. The TFLite export pipeline converts ONNX → TensorFlow → TFLite using [onnx2tf](https://github.com/PINTO0309/onnx2tf). + +### Prerequisites + +```bash +pip install "rfdetr[onnx,tflite]" +``` + +### Basic TFLite Export (FP32) + +=== "Object Detection" + + ```python + from rfdetr import RFDETRSmall + + model = RFDETRSmall() + + model.export(format="tflite", output_dir="output") + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegNano + + model = RFDETRSegNano() + + model.export(format="tflite", output_dir="output") + ``` + +This produces both `output/inference_model_float32.tflite` and `output/inference_model_float16.tflite`. + +### INT8 Quantization with Calibration Data + +For INT8 quantization, provide representative images from your dataset as calibration data. This is **critical** for preserving model accuracy — without real calibration data, the quantizer uses random noise and accuracy will be poor. + +#### Option 1: Point to an Image Directory (Recommended) + +The simplest approach — just point `calibration_data` to a directory containing JPEG/PNG images. The converter automatically loads, resizes, and prepares the images: + +```python +from rfdetr import RFDETRNano + +model = RFDETRNano() +model.export( + format="tflite", + quantization="int8", + calibration_data="path/to/val2017/", # directory of images + output_dir="output", +) +``` + +The converter loads up to 100 images from the directory by default, resizes them to the model's input resolution, and uses them for both output validation and INT8 calibration. Supported formats: JPEG, PNG, BMP, WebP. + +You can control how many images are loaded with the `max_images` parameter: + +```python +model.export( + format="tflite", + quantization="int8", + calibration_data="path/to/val2017/", + max_images=200, # load up to 200 images (default: 100) + output_dir="output", +) +``` + +#### Option 2: NumPy `.npy` File + +Prepare calibration data as a NumPy array and save it to a `.npy` file: + +- Shape: `(N, H, W, 3)` — NHWC format with 3 color channels +- Data type: `float32` +- Value range: `[0, 1]` (divide by 255, but do **not** apply ImageNet normalization — the converter handles that automatically) +- Recommended: 20–100 representative images from your dataset + +```python +import numpy as np +from PIL import Image +from rfdetr import RFDETRSmall + +model = RFDETRSmall() +target_resolution = model.model_config.resolution + +# Load representative images from your dataset +images = [] +for path in image_paths[:50]: # 50 representative samples + img = Image.open(path).convert("RGB").resize((target_resolution, target_resolution)) + images.append(np.array(img, dtype=np.float32) / 255.0) + +calibration_data = np.stack(images) # shape: (50, H, W, 3) + +# Save to .npy for reuse +np.save("calibration_data.npy", calibration_data) + +# Export with INT8 quantization +model.export( + format="tflite", + quantization="int8", + calibration_data="calibration_data.npy", + output_dir="output", +) +``` + +#### Option 3: NumPy Array Directly + +You can also pass the NumPy array directly without saving to disk: + +```python +model.export( + format="tflite", + quantization="int8", + calibration_data=calibration_data, # np.ndarray + output_dir="output", +) +``` + +### FP16 Export + +FP16 models are always produced alongside FP32. You can explicitly request FP16 mode: + +```python +model.export(format="tflite", quantization="fp16", output_dir="output") +``` + +### TFLite Output Files + +The `onnx2tf` converter **always** produces both FP32 and FP16 TFLite files, regardless of the requested quantization mode. When `quantization="int8"` is specified, it additionally produces the INT8-quantized model. + +| File | Description | +| -------------------------------------- | --------------------------------------- | +| `inference_model_float32.tflite` | FP32 model (always produced) | +| `inference_model_float16.tflite` | FP16 model (always produced) | +| `inference_model_integer_quant.tflite` | INT8 model (when `quantization="int8"`) | + +!!! note + + Segmentation models produce TFLite files with three outputs: `dets` (bounding boxes), `labels` (class scores), and `masks` (per-instance segmentation masks). + +### TFLite Inference Example + +```python +import numpy as np +from PIL import Image + +# pip install tflite-runtime (or use tensorflow.lite) +import tflite_runtime.interpreter as tflite + +# Load model +interpreter = tflite.Interpreter(model_path="output/inference_model_float32.tflite") +interpreter.allocate_tensors() + +input_details = interpreter.get_input_details() +output_details = interpreter.get_output_details() + +# Prepare input — TFLite model expects NHWC, ImageNet-normalized +input_height, input_width = input_details[0]["shape"][1:3] +image = Image.open("image.jpg").convert("RGB").resize((input_width, input_height)) +image_array = np.array(image, dtype=np.float32) / 255.0 + +# Apply ImageNet normalization +mean = np.array([0.485, 0.456, 0.406]) +std = np.array([0.229, 0.224, 0.225]) +image_array = (image_array - mean) / std + +# Add batch dimension: (1, H, W, 3) +image_array = np.expand_dims(image_array, axis=0).astype(np.float32) + +# Run inference +interpreter.set_tensor(input_details[0]["index"], image_array) +interpreter.invoke() + +boxes = interpreter.get_tensor(output_details[0]["index"]) +labels = interpreter.get_tensor(output_details[1]["index"]) +``` + +## Using the Exported Model + +Once exported, you can use the ONNX model with various inference frameworks: + +### ONNX Runtime + +```python +import onnxruntime as ort +import numpy as np +from PIL import Image + +# Load the ONNX model +session = ort.InferenceSession("output/inference_model.onnx") + +# Prepare input image +input_height, input_width = session.get_inputs()[0].shape[2:4] +image = Image.open("image.jpg").convert("RGB") +image = image.resize((input_width, input_height)) # Resize to the exported model shape +image_array = np.array(image).astype(np.float32) / 255.0 + +# Normalize +mean = np.array([0.485, 0.456, 0.406]) +std = np.array([0.229, 0.224, 0.225]) +image_array = (image_array - mean) / std + +# Convert to NCHW format +image_array = np.transpose(image_array, (2, 0, 1)) +image_array = np.expand_dims(image_array, axis=0) + +# Run inference +outputs = session.run(None, {"input": image_array}) +boxes, labels = outputs +``` + +## Next Steps + +After exporting your model, you may want to: + +- [Deploy to Roboflow](deploy.md) for cloud-based inference and workflow integration +- Use the ONNX model with TensorRT for optimized GPU inference +- Deploy TFLite models on mobile/edge devices with TensorFlow Lite +- Integrate with edge deployment frameworks like ONNX Runtime or OpenVINO diff --git a/docs/learn/pretrained.md b/docs/learn/pretrained.md new file mode 100644 index 0000000..64f6966 --- /dev/null +++ b/docs/learn/pretrained.md @@ -0,0 +1,198 @@ +--- +description: Run pre-trained RF-DETR models (Nano to 2XLarge) on images, video, webcam, and RTSP streams. COCO-trained with real-time DINOv2 backbone. +--- + +You can run any of the four supported RF-DETR base models -- Nano, Small, Medium, Large -- with [Inference](https://github.com/roboflow/inference), an open source computer vision inference server. The base models are trained on the [Microsoft COCO dataset](https://universe.roboflow.com/microsoft/coco). XLarge and 2XLarge detection models are also available via `pip install rfdetr[plus]` and are provided under the PML 1.0 license. + +=== "Run on an Image" + + To run RF-DETR on an image, use the following code: + + ```python + import os + import supervision as sv + from inference import get_model + from PIL import Image + from io import BytesIO + import requests + + url = "https://media.roboflow.com/dog.jpeg" + image = Image.open(BytesIO(requests.get(url).content)) + + model = get_model("rfdetr-large") + + predictions = model.infer(image, confidence=0.5)[0] + + detections = sv.Detections.from_inference(predictions) + + labels = [prediction.class_name for prediction in predictions.predictions] + + annotated_image = image.copy() + annotated_image = sv.BoxAnnotator().annotate(annotated_image, detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) + + sv.plot_image(annotated_image) + ``` + + Above, replace the image URL with any image you want to use with the model. + + Here are the results from the code above: + +
+ ![](https://media.roboflow.com/rfdetr-docs/annotated_image_base.jpg){ width=300 } +
RF-DETR Base predictions
+
+ +=== "Run on a Video File" + + To run RF-DETR on a video file, use the following code: + + ```python + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + + def callback(frame, index): + detections = model.predict(frame[:, :, ::-1], threshold=0.5) + + labels = [ + f"{COCO_CLASSES[class_id]} {confidence:.2f}" + for class_id, confidence in zip(detections.class_id, detections.confidence) + ] + + annotated_frame = frame.copy() + annotated_frame = sv.BoxAnnotator().annotate(annotated_frame, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + return annotated_frame + + + sv.process_video( + source_path="", + target_path="", + callback=callback, + ) + ``` + + Above, set your `SOURCE_VIDEO_PATH` and `TARGET_VIDEO_PATH` to the directories of the video you want to process and where you want to save the results from inference, respectively. + +=== "Run on a Webcam Stream" + + To run RF-DETR on a webcam input, use the following code: + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + cap = cv2.VideoCapture(0) + while True: + success, frame = cap.read() + if not success: + break + + detections = model.predict(frame[:, :, ::-1], threshold=0.5) + + labels = [ + f"{COCO_CLASSES[class_id]} {confidence:.2f}" + for class_id, confidence in zip(detections.class_id, detections.confidence) + ] + + annotated_frame = frame.copy() + annotated_frame = sv.BoxAnnotator().annotate(annotated_frame, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("Webcam", annotated_frame) + + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + cap.release() + cv2.destroyAllWindows() + ``` + +=== "Run on an RTSP Stream" + + To run RF-DETR on an RTSP (Real Time Streaming Protocol) stream, use the following code: + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + cap = cv2.VideoCapture("") + while True: + success, frame = cap.read() + if not success: + break + + detections = model.predict(frame[:, :, ::-1], threshold=0.5) + + labels = [ + f"{COCO_CLASSES[class_id]} {confidence:.2f}" + for class_id, confidence in zip(detections.class_id, detections.confidence) + ] + + annotated_frame = frame.copy() + annotated_frame = sv.BoxAnnotator().annotate(annotated_frame, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RTSP Stream", annotated_frame) + + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + cap.release() + cv2.destroyAllWindows() + ``` + +You can change the RF-DETR model that the code snippet above uses. To do so, update `rfdetr-base` to any of the following values: + +- `rfdetr-nano` +- `rfdetr-small` +- `rfdetr-medium` +- `rfdetr-large` + +## Batch Inference + +You can provide `.predict()` with either a single image or a list of images. When multiple images are supplied, they are processed together in a single forward pass, resulting in a corresponding list of detections. + +```python +import io +import requests +import supervision as sv +from PIL import Image +from rfdetr import RFDETRMedium +from rfdetr.assets.coco_classes import COCO_CLASSES + +model = RFDETRMedium() + +urls = [ + "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", + "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", +] + +images = [Image.open(io.BytesIO(requests.get(url).content)) for url in urls] + +detections_list = model.predict(images, threshold=0.5) + +for image, detections in zip(images, detections_list): + labels = [ + f"{COCO_CLASSES[class_id]} {confidence:.2f}" + for class_id, confidence in zip(detections.class_id, detections.confidence) + ] + + annotated_image = image.copy() + annotated_image = sv.BoxAnnotator().annotate(annotated_image, detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) + + sv.plot_image(annotated_image) +``` diff --git a/docs/learn/run/detection.md b/docs/learn/run/detection.md new file mode 100644 index 0000000..2540620 --- /dev/null +++ b/docs/learn/run/detection.md @@ -0,0 +1,185 @@ +--- +description: Run RF-DETR object detection on images, video, and streams. Nano to 2XLarge models with 2.3-17.2 ms latency and up to 60.1 AP on COCO. +--- + +# Run an RF-DETR Object Detection Model + +RF-DETR is a real-time transformer architecture for object detection, built on a DINOv2 vision transformer backbone. The base models are trained on the Microsoft COCO dataset and achieve state-of-the-art accuracy and latency trade-offs. + +## Pre-trained Checkpoints + +RF-DETR offers model sizes from Nano to 2XLarge, allowing trade-offs between accuracy, latency, and parameter count. All latency numbers were measured on an NVIDIA T4 using TensorRT, FP16, and batch size 1. Core models (Nano to Large) are licensed under Apache 2.0. XLarge and 2XLarge (marked with △) are provided by the [`rfdetr_plus`](https://github.com/roboflow/rf-detr-plus) extension (`pip install rfdetr[plus]`) under the Platform Model License 1.0 and require a Roboflow account. + +| Size | RF-DETR package class | Inference package alias | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :--: | :-------------------: | :---------------------- | :------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: | +| N | `RFDETRNano` | `rfdetr-nano` | 67.6 | 48.4 | 2.3 | 30.5 | 384x384 | Apache 2.0 | +| S | `RFDETRSmall` | `rfdetr-small` | 72.1 | 53.0 | 3.5 | 32.1 | 512x512 | Apache 2.0 | +| M | `RFDETRMedium` | `rfdetr-medium` | 73.6 | 54.7 | 4.4 | 33.7 | 576x576 | Apache 2.0 | +| L | `RFDETRLarge` | `rfdetr-large` | 75.1 | 56.5 | 6.8 | 33.9 | 704x704 | Apache 2.0 | +| XL | `RFDETRXLarge` △ | `rfdetr-xlarge` | 77.4 | 58.6 | 11.5 | 126.4 | 700x700 | PML 1.0 | +| 2XL | `RFDETR2XLarge` △ | `rfdetr-2xlarge` | 78.5 | 60.1 | 17.2 | 126.9 | 880x880 | PML 1.0 | + +> △ Requires the `rfdetr_plus` extension: `pip install rfdetr[plus]` + +## Run on an Image + +Perform inference on an image using either the `rfdetr` package or the `inference` package. To use a different model size, select the corresponding class or alias from the table above. + +=== "rfdetr" + + ```python + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + detections = model.predict("https://media.roboflow.com/dog.jpg", threshold=0.5) + + labels = [f"{COCO_CLASSES[class_id]}" for class_id in detections.class_id] + + annotated_image = sv.BoxAnnotator().annotate(detections.metadata["source_image"], detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) + ``` + +=== "inference" + + ```python + import requests + import supervision as sv + from PIL import Image + from inference import get_model + + model = get_model("rfdetr-medium") + + image = Image.open(requests.get("https://media.roboflow.com/dog.jpg", stream=True).raw) + predictions = model.infer(image, confidence=0.5)[0] + detections = sv.Detections.from_inference(predictions) + + annotated_image = sv.BoxAnnotator().annotate(image, detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections) + ``` + +!!! note "Using COCO classes vs. fine-tuned model classes" + + `COCO_CLASSES` works for COCO-pretrained models (80 COCO classes, indexed 0-79). + For fine-tuned models, use `detections.data["class_name"]` instead — it resolves + class names from the checkpoint and works for both COCO and custom datasets. + +For memory-constrained inference-only deployments with the `rfdetr` package, optimize the loaded model in place before +calling `predict()`. Pass `dtype="float16"` to halve weight memory in addition to clearing the base model reference. +This operation is irreversible — to restore the original model, create a new `RFDETR` instance: + +```python +model.optimize_for_inference(compile=False, inplace=True, dtype="float16") +``` + +## Run on video, webcam, or RTSP stream + +These examples use OpenCV for decoding and display. Replace ``, ``, and `` with your inputs. `` is usually `0` for the default camera. + +=== "video" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + video_capture = cv2.VideoCapture("") + if not video_capture.isOpened(): + raise RuntimeError("Failed to open video source: ") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + detections = model.predict(frame_rgb, threshold=0.5) + + labels = [COCO_CLASSES[class_id] for class_id in detections.class_id] + + annotated_frame = sv.BoxAnnotator().annotate(frame_bgr, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RF-DETR Video", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +=== "webcam" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + WEBCAM_INDEX = 0 + video_capture = cv2.VideoCapture(WEBCAM_INDEX) + if not video_capture.isOpened(): + raise RuntimeError(f"Failed to open webcam: {WEBCAM_INDEX}") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + detections = model.predict(frame_rgb, threshold=0.5) + + labels = [COCO_CLASSES[class_id] for class_id in detections.class_id] + + annotated_frame = sv.BoxAnnotator().annotate(frame_bgr, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RF-DETR Webcam", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +=== "stream" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRMedium() + + video_capture = cv2.VideoCapture("") + if not video_capture.isOpened(): + raise RuntimeError("Failed to open RTSP stream: ") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + detections = model.predict(frame_rgb, threshold=0.5) + + labels = [COCO_CLASSES[class_id] for class_id in detections.class_id] + + annotated_frame = sv.BoxAnnotator().annotate(frame_bgr, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RF-DETR RTSP", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` diff --git a/docs/learn/run/keypoints.md b/docs/learn/run/keypoints.md new file mode 100644 index 0000000..05cae7f --- /dev/null +++ b/docs/learn/run/keypoints.md @@ -0,0 +1,203 @@ +--- +description: Run RF-DETR keypoint detection on images, video, and streams. COCO-pretrained preview model predicts 17 person keypoints with 71.8 AP at 9.7 ms on NVIDIA T4. +--- + +# Run an RF-DETR Keypoint Model + +RF-DETR Keypoint is a real-time transformer architecture for keypoint detection, built on a DINOv2 vision transformer backbone. The preview model is pretrained on the Microsoft COCO dataset and predicts 17 body keypoints per detected person. + +![People walking on a bridge with RF-DETR keypoint skeleton overlays and bounding boxes](../../assets/keypoints/bridge-1.jpg) + +!!! note "Preview model" + + `RFDETRKeypointPreview` is an early-access release. Fine-tuning on custom keypoint datasets is the primary intended use case. See [Keypoint Preview Parameters](../train/training-parameters.md#keypoint-preview-parameters) for training configuration. API surface and checkpoint weights may change before the stable release. + +## Pre-trained Checkpoints + +RF-DETR Keypoint outperforms YOLO26-pose X and YOLO11-pose X at comparable latency on MS COCO. Latency measured on NVIDIA T4, TensorRT FP16, batch size 1. + +![RF-DETR Keypoint mAP vs latency chart comparing against YOLO26-pose and YOLO11-pose on MS COCO](../../assets/keypoints/kp-map-latency.png){ width=560 } + +| Model | RF-DETR package class | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | License | +| :----------------: | :---------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: | +| Keypoint (Preview) | `RFDETRKeypointPreview` | 71.8 | 9.7 | 126.4 | 576x576 | Apache 2.0 | + +> The keypoint model is available in the `rfdetr` package only. It is not yet available via the `inference` package. + +> Benchmark evaluated on COCO val2017 person keypoints (AP50:95) with the standard COCO 17-keypoint OKS sigmas; latency on NVIDIA T4, TensorRT FP16, batch size 1. + +## Run on an Image + +Perform inference on an image using the `rfdetr` package. `model.predict()` returns an [`sv.KeyPoints`](https://supervision.roboflow.com/latest/keypoint/core/) object containing skeleton coordinates and per-keypoint confidence scores for each detected person. + +=== "rfdetr" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRKeypointPreview + + model = RFDETRKeypointPreview() + + image_bgr = cv2.imread("/path/to/image.jpg") + image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB) + key_points = model.predict(image_rgb, threshold=0.5) + + annotated_image = sv.VertexAnnotator().annotate(image_rgb, key_points) + ``` + +![People walking on a bridge — RF-DETR keypoint skeleton visualization without bounding boxes](../../assets/keypoints/bridge-2.jpg) + +## Understanding the Output + +`model.predict()` returns an `sv.KeyPoints` object. The fields most commonly used downstream: + +| Field | Shape | Description | +| --------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `key_points.xy` | `(N, K, 2)` | Pixel coordinates of each keypoint per detected instance | +| `key_points.keypoint_confidence` | `(N, K)` | Per-keypoint findability score; use to filter low-confidence points | +| `key_points.detection_confidence` | `(N,)` | Per-instance detection score; this is what `threshold` filters on. For keypoint models it includes the default uncertainty fusion term normalized to `[0, 1)`. | +| `key_points.class_id` | `(N,)` | Model label ID for each detection. COCO-pretrained checkpoints use sparse COCO category IDs (1–90). Fine-tuned active-first keypoint checkpoints use normal 0-based class IDs; in the one-class preview setup, `class_id=0` is the foreground class and `class_id=1` is `"__background__"`. Legacy background-first keypoint checkpoints use slot 0 as `"__background__"` and start foreground classes at slot 1. Use `key_points.data["class_name"]` for name resolution rather than indexing your class list by `class_id`. | +| `key_points.data["class_name"]` | `(N,)` | Class names resolved from `class_id`; prefer this over indexing a class-name list directly. | +| `key_points.data["xyxy"]` | `(N, 4)` | Bounding box for each detected instance in `[x1, y1, x2, y2]` format | +| `key_points.data["source_image"]` | list of arrays | Source frame stored once per detection; all N entries are the same array — use `[0]` to access it | + +`K=17` for the pretrained COCO person-keypoint preview checkpoint. Fine-tuned checkpoints use the keypoint count from their dataset schema, so custom keypoint datasets can return any `K` supported by their COCO keypoint annotations. + +Keypoints with `visible=False` are skipped by supervision annotators. To hide low-confidence joints manually, threshold `key_points.keypoint_confidence` and set matching entries to `False` in `key_points.visible`. + +For fine-tuning on a custom keypoint dataset, see [Keypoint preview custom datasets](../train/index.md#keypoint-preview-custom-datasets). + +## Run on video, webcam, or RTSP stream + +These examples use OpenCV for decoding and display. Replace ``, ``, and `` with your inputs. `` is usually `0` for the default camera. + +=== "video" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRKeypointPreview + + model = RFDETRKeypointPreview() + + video_capture = cv2.VideoCapture("") + if not video_capture.isOpened(): + raise RuntimeError("Failed to open video source: ") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + key_points = model.predict(frame_rgb, threshold=0.5) + + annotated_frame = sv.VertexAnnotator().annotate(frame_bgr, key_points) + + cv2.imshow("RF-DETR Keypoint Video", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +=== "webcam" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRKeypointPreview + + model = RFDETRKeypointPreview() + + WEBCAM_INDEX = 0 # Change this to the desired webcam index (e.g., 1, 2, ...) + video_capture = cv2.VideoCapture(WEBCAM_INDEX) + if not video_capture.isOpened(): + raise RuntimeError(f"Failed to open webcam: {WEBCAM_INDEX}") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + key_points = model.predict(frame_rgb, threshold=0.5) + + annotated_frame = sv.VertexAnnotator().annotate(frame_bgr, key_points) + + cv2.imshow("RF-DETR Keypoint Webcam", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +=== "stream" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRKeypointPreview + + model = RFDETRKeypointPreview() + + video_capture = cv2.VideoCapture("") + if not video_capture.isOpened(): + raise RuntimeError("Failed to open RTSP stream: ") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + key_points = model.predict(frame_rgb, threshold=0.5) + + annotated_frame = sv.VertexAnnotator().annotate(frame_bgr, key_points) + + cv2.imshow("RF-DETR Keypoint RTSP", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +## Visualization + +`supervision` provides several keypoint annotators. Choose based on what you want to draw. + +=== "EdgeAnnotator" + + Draws skeleton edges (lines between connected joints). Edges where either endpoint has `visible=False` are skipped automatically. + + ```python + annotated = sv.EdgeAnnotator().annotate(image, key_points) + ``` + +=== "VertexAnnotator" + + Draws a dot at each keypoint. Keypoints with `visible=False` are skipped automatically. + + ```python + annotated = sv.VertexAnnotator().annotate(image, key_points) + ``` + +=== "VertexEllipseAnnotator" + + Draws covariance ellipses from `key_points.data["covariance"]`, giving a visual footprint of per-keypoint uncertainty. + + ```python + annotated = sv.VertexEllipseAnnotator().annotate(image, key_points) + ``` + +=== "VertexEllipseHaloAnnotator" + + Draws the same covariance uncertainty with a soft halo for improved contrast on busy backgrounds. + + ```python + annotated = sv.VertexEllipseHaloAnnotator().annotate(image, key_points) + ``` diff --git a/docs/learn/run/segmentation.md b/docs/learn/run/segmentation.md new file mode 100644 index 0000000..e5284ea --- /dev/null +++ b/docs/learn/run/segmentation.md @@ -0,0 +1,177 @@ +--- +description: Run RF-DETR instance segmentation on images, video, and streams. Mask predictions with 3.4-21.8 ms latency using DINOv2 backbone. +--- + +# Run an RF-DETR Instance Segmentation Model + +RF-DETR is a real-time transformer architecture for instance segmentation, built on a DINOv2 vision transformer backbone. The base models are trained on the Microsoft COCO dataset and achieve strong accuracy and latency trade-offs. + +## Pre-trained Checkpoints + +RF-DETR-Seg offers model sizes from Nano to 2XLarge, allowing trade-offs between accuracy, latency, and parameter count. All latency numbers were measured on an NVIDIA T4 using TensorRT, FP16, and batch size 1. + +| Size | RF-DETR package class | Inference package alias | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | +| :--: | :-------------------: | :---------------------- | :------------------: | :---------------------: | :----------: | :--------: | :--------: | +| N | `RFDETRSegNano` | `rfdetr-seg-nano` | 63.0 | 40.3 | 3.4 | 33.6 | 312x312 | +| S | `RFDETRSegSmall` | `rfdetr-seg-small` | 66.2 | 43.1 | 4.4 | 33.7 | 384x384 | +| M | `RFDETRSegMedium` | `rfdetr-seg-medium` | 68.4 | 45.3 | 5.9 | 35.7 | 432x432 | +| L | `RFDETRSegLarge` | `rfdetr-seg-large` | 70.5 | 47.1 | 8.8 | 36.2 | 504x504 | +| XL | `RFDETRSegXLarge` | `rfdetr-seg-xlarge` | 72.2 | 48.8 | 13.5 | 38.1 | 624x624 | +| 2XL | `RFDETRSeg2XLarge` | `rfdetr-seg-2xlarge` | 73.1 | 49.9 | 21.8 | 38.6 | 768x768 | + +## Run on an Image + +Perform inference on an image using either the `rfdetr` package or the `inference` package. To use a different model size, select the corresponding class or alias from the table above. + +=== "rfdetr" + + ```python + import supervision as sv + from rfdetr import RFDETRSegMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRSegMedium() + + detections = model.predict("https://media.roboflow.com/dog.jpg", threshold=0.5) + + labels = [f"{COCO_CLASSES[class_id]}" for class_id in detections.class_id] + + annotated_image = sv.MaskAnnotator().annotate(detections.metadata["source_image"], detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels) + ``` + +=== "inference" + + ```python + import requests + import supervision as sv + from PIL import Image + from inference import get_model + + model = get_model("rfdetr-seg-medium") + + image = Image.open(requests.get("https://media.roboflow.com/dog.jpg", stream=True).raw) + predictions = model.infer(image, confidence=0.5)[0] + detections = sv.Detections.from_inference(predictions) + + annotated_image = sv.MaskAnnotator().annotate(image, detections) + annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections) + ``` + +For memory-constrained inference-only deployments with the `rfdetr` package, optimize the loaded model in place before +calling `predict()`. Pass `dtype="float16"` to halve weight memory in addition to clearing the base model reference. +This operation is irreversible — to restore the original model, create a new `RFDETR` instance: + +```python +model.optimize_for_inference(compile=False, inplace=True, dtype="float16") +``` + +## Run on video, webcam, or RTSP stream + +These examples use OpenCV for decoding and display. Replace ``, ``, and `` with your inputs. `` is usually `0` for the default camera. + +=== "video" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRSegMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRSegMedium() + + video_capture = cv2.VideoCapture("") + if not video_capture.isOpened(): + raise RuntimeError("Failed to open video source: ") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + detections = model.predict(frame_rgb, threshold=0.5) + + labels = [COCO_CLASSES[class_id] for class_id in detections.class_id] + + annotated_frame = sv.MaskAnnotator().annotate(frame_bgr, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RF-DETR-Seg Video", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +=== "webcam" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRSegMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRSegMedium() + + WEBCAM_INDEX = 0 # Change this to the desired webcam index (e.g., 1, 2, ...) + video_capture = cv2.VideoCapture(WEBCAM_INDEX) + if not video_capture.isOpened(): + raise RuntimeError(f"Failed to open webcam: {WEBCAM_INDEX}") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + detections = model.predict(frame_rgb, threshold=0.5) + + labels = [COCO_CLASSES[class_id] for class_id in detections.class_id] + + annotated_frame = sv.MaskAnnotator().annotate(frame_bgr, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RF-DETR-Seg Webcam", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` + +=== "stream" + + ```python + import cv2 + import supervision as sv + from rfdetr import RFDETRSegMedium + from rfdetr.assets.coco_classes import COCO_CLASSES + + model = RFDETRSegMedium() + + video_capture = cv2.VideoCapture("") + if not video_capture.isOpened(): + raise RuntimeError("Failed to open RTSP stream: ") + + while True: + success, frame_bgr = video_capture.read() + if not success: + break + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + detections = model.predict(frame_rgb, threshold=0.5) + + labels = [COCO_CLASSES[class_id] for class_id in detections.class_id] + + annotated_frame = sv.MaskAnnotator().annotate(frame_bgr, detections) + annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels) + + cv2.imshow("RF-DETR-Seg RTSP", annotated_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + video_capture.release() + cv2.destroyAllWindows() + ``` diff --git a/docs/learn/train/advanced.md b/docs/learn/train/advanced.md new file mode 100644 index 0000000..e2e5364 --- /dev/null +++ b/docs/learn/train/advanced.md @@ -0,0 +1,381 @@ +--- +description: Advanced RF-DETR training with resume, early stopping, multi-GPU DDP, gradient checkpointing, and memory optimization for large models. +--- + +# Advanced Training + +This page covers advanced training topics including resuming training, early stopping, multi-GPU training, and memory optimization techniques. + +!!! tip "PTL API for deeper customisation" + + All examples on this page use the `RFDETR.train()` high-level API. For custom callbacks, non-default loggers, and fine-grained distributed training control, see the [Custom Training API](customization.md) guide. + +## Resume Training + +You can resume training from a previously saved checkpoint by passing the path to the `checkpoint.pth` file using the `resume` argument. This is useful when training is interrupted or you want to continue fine-tuning an already partially trained model. + +The training loop will automatically load: + +- Model weights +- Optimizer state +- Learning rate scheduler state +- Training epoch number + +=== "Object Detection" + + ```python + from rfdetr import RFDETRMedium + + model = RFDETRMedium() + + model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + resume="output/checkpoint.pth", + ) + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegMedium + + model = RFDETRSegMedium() + + model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + resume="output/checkpoint.pth", + ) + ``` + +!!! tip "Resume vs Pretrain Weights" + + - Use `resume="checkpoint.pth"` to continue training with optimizer state + - Use `pretrain_weights="checkpoint_best_total.pth"` when initializing a model to start fresh training from those weights + +--- + +## Early Stopping + +Early stopping monitors the validation task metric and halts training if improvements remain below a threshold for a +set number of epochs. Detection and segmentation models use box mAP; keypoint preview models use COCO keypoint AP. + +### Basic Usage + +=== "Object Detection" + + ```python + from rfdetr import RFDETRMedium + + model = RFDETRMedium() + + model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + early_stopping=True, + ) + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegMedium + + model = RFDETRSegMedium() + + model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + early_stopping=True, + ) + ``` + +### Configuration Options + +| Parameter | Default | Description | +| -------------------------- | ------- | ---------------------------------------------------- | +| `early_stopping_patience` | 10 | Number of epochs without improvement before stopping | +| `early_stopping_min_delta` | 0.001 | Minimum metric change to count as improvement | +| `early_stopping_use_ema` | False | Use EMA model metrics for comparisons | + +### Advanced Example + +```python +model.train( + dataset_dir="path/to/dataset", + epochs=200, + early_stopping=True, + early_stopping_patience=15, # Wait 15 epochs before stopping + early_stopping_min_delta=0.005, # Require 0.5% validation metric improvement + early_stopping_use_ema=True, # Track EMA model performance +) +``` + +### How It Works + +1. After each epoch, the validation task metric is computed +2. If the metric improves by at least `min_delta`, the patience counter resets +3. If the metric doesn't improve, the patience counter increments +4. When patience counter reaches `patience`, training stops +5. The best checkpoint is already saved as `checkpoint_best_total.pth` + +``` +Epoch 10: mAP = 0.450 (best: 0.450) - counter: 0 +Epoch 11: mAP = 0.455 (best: 0.455) - counter: 0 (improved) +Epoch 12: mAP = 0.454 (best: 0.455) - counter: 1 (no improvement) +Epoch 13: mAP = 0.453 (best: 0.455) - counter: 2 +... +Epoch 22: mAP = 0.452 (best: 0.455) - counter: 10 → STOP +``` + +--- + +## Multi-GPU Training + +RF-DETR's training stack is built on PyTorch Lightning, so multi-GPU and multi-node training use the Lightning `Trainer` strategies directly. You can start multi-GPU runs through the high-level API or by using the Lightning primitives explicitly. + +### Using RFDETR.train() with multiple GPUs + +Create a training script and launch it with `torchrun`: + +```python +# train.py +from rfdetr import RFDETRMedium + +model = RFDETRMedium() + +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, # per-GPU batch size + grad_accum_steps=1, + lr=1e-4, + output_dir="output", + devices="auto", # required — see note below +) +``` + +```bash +torchrun --nproc_per_node=4 train.py +``` + +!!! warning "Pass `devices=` explicitly" + + `build_trainer()` defaults to `devices=1`. Without overriding this, training silently + runs on a single GPU even when `torchrun` launches multiple processes. + + Pass `devices="auto"` to use all GPUs visible to the process, or pass an explicit + integer (e.g. `devices=4`). These values are forwarded to `build_trainer` via + `**trainer_kwargs`: + + ```python + model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=1, + lr=1e-4, + output_dir="output", + devices="auto", # or devices=4 + ) + ``` + +### Batch Size with Multiple GPUs + +When using multiple GPUs, your effective batch size is multiplied by the number of GPUs: + +``` +effective_batch_size = batch_size × grad_accum_steps × num_gpus +``` + +**Example configurations for effective batch size of 16:** + +| GPUs | `batch_size` | `grad_accum_steps` | Effective | +| ---- | ------------ | ------------------ | --------- | +| 1 | 4 | 4 | 16 | +| 2 | 4 | 2 | 16 | +| 4 | 4 | 1 | 16 | +| 8 | 2 | 1 | 16 | + +!!! warning "Adjust for GPU count" + + When switching between single and multi-GPU training, remember to adjust `batch_size` and `grad_accum_steps` to maintain the same effective batch size. + +### Multi-Node Training + +For training across multiple machines, pass the standard `torchrun` flags: + +```bash +torchrun \ + --nproc_per_node=8 \ + --nnodes=2 \ + --node_rank=0 \ + --master_addr="192.168.1.1" \ + --master_port=1234 \ + train.py +``` + +Run this command on each node, changing `--node_rank` accordingly. + +### Advanced multi-GPU options (PTL API) + +For fine-grained control over strategy, sync batch norm, precision, and other distributed settings, use the Lightning API directly. + +→ **[Multi-GPU with the PTL API](customization.md#multi-gpu-training)** + +--- + +## Custom Augmentations + +RF-DETR supports advanced data augmentations using the [Albumentations](https://albumentations.ai/) library, providing access to over 70 different image transformations optimized for object detection. + +→ **[Complete Augmentation Guide](augmentations.md)** - Configuration examples, best practices, troubleshooting, and advanced topics. + +### Quick Start + +Pass an `aug_config` dictionary to `model.train()`. Each key is an Albumentations transform name; the value is a dict of keyword arguments for that transform: + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium() + +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + aug_config={ + "HorizontalFlip": {"p": 0.5}, + "VerticalFlip": {"p": 0.5}, + "Rotate": {"limit": 45, "p": 0.5}, + }, +) +``` + +Use a built-in preset by importing it from `rfdetr.datasets.aug_configs`: + +```python +from rfdetr.datasets.aug_configs import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL + +model.train(dataset_dir="path/to/dataset", aug_config=AUG_AGGRESSIVE) +``` + +To disable all augmentations, pass an empty dict: + +```python +model.train(dataset_dir="path/to/dataset", aug_config={}) +``` + +--- + +## Memory Optimization + +### Gradient Checkpointing + +For large models or high resolutions, enable gradient checkpointing to trade compute for memory. + +!!! warning "Constructor parameter — not a `train()` parameter" + + `gradient_checkpointing` is a `ModelConfig` field and must be passed to the **model constructor**, not to `train()`. Passing it to `train()` will raise a `ValidationError` because `TrainConfig` has `extra="forbid"`. + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium(gradient_checkpointing=True) + +model.train( + dataset_dir="path/to/dataset", + batch_size=2, # May be able to increase with checkpointing +) +``` + +This re-computes activations during the backward pass instead of storing them, reducing memory usage by ~30-40% at the cost of ~20% slower training. + +### Memory-Efficient Configurations + +| Memory Level | Configuration | +| ----------------- | -------------------------------------------------------------------------------------- | +| Very Low (8GB) | `batch_size=1`, `grad_accum_steps=16`, `gradient_checkpointing=True`, `resolution=576` | +| Low (12GB) | `batch_size=2`, `grad_accum_steps=8`, `gradient_checkpointing=True` | +| Medium (16GB) | `batch_size=4`, `grad_accum_steps=4` | +| High (24GB) | `batch_size=8`, `grad_accum_steps=2` | +| Very High (40GB+) | `batch_size=16`, `grad_accum_steps=1`, `resolution=768` | + +--- + +## Training Tips + +### Learning Rate Tuning + +- **Fine-tuning from COCO weights (default):** Use default learning rates (`lr=1e-4`, `lr_encoder=1.5e-4`) +- **Small dataset (\<1000 images):** Consider lower `lr` (e.g., `5e-5`) to prevent overfitting +- **Large dataset (>10000 images):** May benefit from higher `lr` (e.g., `2e-4`) + +### Epoch Count + +| Dataset Size | Recommended Epochs | +| ----------------- | ------------------ | +| < 500 images | 100-200 | +| 500-2000 images | 50-100 | +| 2000-10000 images | 30-50 | +| > 10000 images | 20-30 | + +Use early stopping to automatically determine the optimal stopping point. + +### Data Augmentation + +RF-DETR applies built-in augmentations during training: + +- Random resizing +- Random cropping +- Color jittering +- Horizontal flipping + +These are automatically configured and don't require manual setup. + +--- + +## Troubleshooting + +### Out of Memory (OOM) + +If you encounter CUDA out of memory errors: + +1. Reduce `batch_size` +2. Enable `gradient_checkpointing=True` (pass to the model constructor, not `train()`) +3. Reduce `resolution` +4. Increase `grad_accum_steps` to maintain effective batch size + +### Training Too Slow + +1. Increase `batch_size` (if memory allows) +2. Use multiple GPUs with DDP +3. Ensure you're using GPU (check `device="cuda"`) +4. Consider using a smaller model (e.g., `RFDETRSmall` instead of `RFDETRLarge`) + +### Loss Not Decreasing + +1. Check that your dataset is correctly formatted +2. Verify annotations are correct (bounding boxes in correct format) +3. Try reducing the learning rate +4. Check for class imbalance in your dataset diff --git a/docs/learn/train/augmentations.md b/docs/learn/train/augmentations.md new file mode 100644 index 0000000..f73138f --- /dev/null +++ b/docs/learn/train/augmentations.md @@ -0,0 +1,156 @@ +--- +description: Configure RF-DETR data augmentations with Albumentations. Built-in presets for aerial, industrial, and small datasets plus custom transforms. +--- + +# Augmentations + +RF-DETR supports custom data augmentations via [Albumentations](https://albumentations.ai/), with automatic bounding box and mask handling for geometric transforms. Albumentations 1.4.24+ and 2.x are supported. + +## Quick Start + +Pass `aug_config` to your training call. Import one of the built-in presets: + +```python +from rfdetr import RFDETRSmall +from rfdetr.datasets.aug_configs import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL + +model = RFDETRSmall() +model.train(dataset_dir="path/to/dataset", epochs=100, aug_config=AUG_CONSERVATIVE) +``` + +Or pass a custom dict directly — keys are Albumentations transform names: + +```python +model.train( + dataset_dir="path/to/dataset", + epochs=100, + aug_config={ + "HorizontalFlip": {"p": 0.5}, + "Rotate": {"limit": 15, "p": 0.3}, + "GaussianBlur": {"p": 0.2}, + }, +) +``` + +To disable augmentations: `aug_config={}`. Omitting it uses the default (horizontal flip at 50%). + +## Built-in Presets + +| Preset | Best for | +| ------------------ | --------------------------------- | +| `AUG_CONSERVATIVE` | Small datasets (under 500 images) | +| `AUG_AGGRESSIVE` | Large datasets (2000+ images) | +| `AUG_AERIAL` | Satellite / overhead imagery | +| `AUG_INDUSTRIAL` | Manufacturing / inspection data | + +All presets are plain dicts — inspect or extend them before passing: + +```python +from rfdetr.datasets.aug_configs import AUG_AGGRESSIVE + +my_config = {**AUG_AGGRESSIVE, "VerticalFlip": {"p": 0.1}} +model.train(dataset_dir="...", aug_config=my_config) +``` + +### Recommendations by Dataset Size + +| Dataset Size | Recommended preset | +| ---------------- | --------------------------------------------------------------- | +| Under 500 images | `AUG_CONSERVATIVE` — flip + mild brightness/contrast | +| 500–2000 images | Default or `AUG_CONSERVATIVE` with a few extra transforms added | +| 2000+ images | `AUG_AGGRESSIVE` — rotations, affine, color jitter | + +## Nested Transforms + +RF-DETR supports `OneOf`, `SomeOf`, and `Sequential` container transforms from Albumentations. The most common pattern is `OneOf`, which randomly picks one transform from a group: + +```python +aug_config = { + "HorizontalFlip": {"p": 0.5}, + "OneOf": { + "transforms": [ + {"Rotate": {"limit": 45, "p": 1.0}}, + {"Affine": {"scale": (0.8, 1.2), "p": 1.0}}, + ], + }, + "GaussianBlur": {"p": 0.2}, +} +``` + +Each child's `p` controls its relative selection weight. The container itself always fires. + +If you need the same transform twice, or want explicit ordering, pass a list instead of a dict: + +```python +aug_config = [ + {"HorizontalFlip": {"p": 0.5}}, + {"Rotate": {"limit": 45, "p": 0.3}}, + {"Rotate": {"limit": 5, "p": 0.5}}, # second Rotate — only possible with list format +] +``` + +Bounding boxes are updated automatically when a container holds any geometric transform — no extra configuration needed. + +## Geometric vs. Pixel-Level Transforms + +RF-DETR automatically handles bounding boxes for **geometric transforms** (flips, rotations, crops, affine, perspective). **Pixel-level transforms** (blur, noise, color) preserve coordinates unchanged. You don't need to handle this distinction — it's automatic based on the transform name. + +## Best Practices + +!!! tip "Start Conservative" + + Begin with simple augmentations (horizontal flip, small brightness changes) and gradually add more as needed. + +!!! warning "Geometric Transforms" + + Be careful with aggressive rotations and crops on datasets where object orientation matters (e.g., text detection, oriented objects). + +- **CPU-bound:** Augmentations run on CPU during data loading — more transforms means slower loading +- **Use `num_workers`:** Parallelize augmentation across data loader workers +- **Monitor training mAP vs validation mAP:** With strong augmentations it's normal for training mAP to be lower — validation uses original images while training uses augmented (harder) ones + +## Troubleshooting + +**Training is slow** — reduce the number of transforms or increase `num_workers`. + +**Boxes disappear after augmentation** — aggressive rotations or crops can push boxes outside the image boundary. Reduce rotation angles or avoid large crops. + +**Model not improving** — augmentations may be too aggressive. Start with `AUG_CONSERVATIVE` and add transforms gradually. Try removing geometric transforms first to isolate the cause. + +**Validation mAP is much higher than training mAP** — this is expected with strong augmentations and not a bug. See the monitoring tip above. + +**Upgrading albumentations to 2.x with existing `RandomSizedCrop` configs?** RF-DETR automatically adapts `height`/`width` kwargs to the `size=(height, width)` format required by albumentations 2.x. No config changes needed. + +## Advanced: Custom Transforms + +Any Albumentations transform works by name. If your custom transform is geometric, register it in `rfdetr/datasets/transforms.py` so boxes are updated automatically: + +```python +GEOMETRIC_TRANSFORMS = { + ..., + "YourCustomTransform", +} +``` + +Then use it like any other transform: + +```python +model.train( + dataset_dir="...", + aug_config={ + "HorizontalFlip": {"p": 0.5}, + "YourCustomTransform": {"param": 1, "p": 0.3}, + }, +) +``` + +## Reference + +- [Albumentations docs](https://albumentations.ai/docs/) +- [All available transforms](https://albumentations.ai/docs/api_reference/augmentations/) + +## Next Steps + +- [Monitor training with TensorBoard](loggers.md#tensorboard) +- [Use early stopping](advanced.md#early-stopping) to prevent overfitting +- [Export your trained model](../export.md) for deployment diff --git a/docs/learn/train/customization.md b/docs/learn/train/customization.md new file mode 100644 index 0000000..1e0cb25 --- /dev/null +++ b/docs/learn/train/customization.md @@ -0,0 +1,384 @@ +--- +description: Customize RF-DETR training with PyTorch Lightning primitives. Direct access to RFDETRModelModule, RFDETRDataModule, and build_trainer. +--- + +# Custom Training API + +The high-level `RFDETR.train()` method is the quickest path to fine-tuning, but the underlying training primitives are fully public and are the **recommended path for any customisation**: custom callbacks, alternative loggers, mixed-precision overrides, multi-GPU strategies, or integration with external training frameworks. + +!!! tip "Quickstart vs. customisation" + + If you want to start training with minimal code, use `model.train()` — it sets up and runs the full PTL stack automatically. Come here when you need to take direct control over any part of that stack. + +## How `RFDETR.train()` relates to PTL + +When you call `model.train(...)`, three things happen internally: + +```python +from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer + +module = RFDETRModelModule(model_config, train_config) +datamodule = RFDETRDataModule(model_config, train_config) +trainer = build_trainer(train_config, model_config) +trainer.fit(module, datamodule, ckpt_path=train_config.resume or None) +``` + +Each of these objects is a standard PTL class. You can construct them directly, modify them, swap out callbacks, or replace the trainer entirely. + +--- + +## RFDETRModelModule + +`RFDETRModelModule` is a `pytorch_lightning.LightningModule`. It owns the model weights, the criterion, the postprocessor, and the optimizer/scheduler configuration. + +```python +from rfdetr.config import ( + RFDETRMediumConfig, + TrainConfig, +) # config classes live in rfdetr.config, not the top-level rfdetr namespace +from rfdetr.training import RFDETRModelModule + +model_config = RFDETRMediumConfig(num_classes=10) +train_config = TrainConfig( + dataset_dir="path/to/dataset", + epochs=50, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", +) + +module = RFDETRModelModule(model_config, train_config) +``` + +### Lifecycle hooks + +| Hook | Behaviour | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `on_fit_start` | Seeds RNGs when `train_config.seed` is set. | +| `on_train_batch_start` | Applies multi-scale random resize when `train_config.multi_scale=True`. | +| `transfer_batch_to_device` | Moves `NestedTensor` batches to the target device. | +| `training_step` | Computes loss and logs `train/loss` plus per-term losses. Keypoint models use manual optimization with box-normalized accumulation across microbatches; detection and segmentation use Lightning's automatic optimization path. | +| `validation_step` | Runs forward pass and postprocessing; returns `{results, targets}` for `COCOEvalCallback`. | +| `test_step` | Same as `validation_step`, logs under `test/`. | +| `predict_step` | Runs inference-only forward pass and returns postprocessed detections. | +| `configure_optimizers` | Builds AdamW with layer-wise LR decay and a LambdaLR scheduler (cosine or step). | +| `on_load_checkpoint` | Auto-converts legacy `.pth` checkpoints to PTL format. | + +### Accessing the underlying model + +The raw `nn.Module` is `module.model`. After training completes, `RFDETR.train()` syncs it back onto `self.model.model` so `predict()` and `export()` continue to work. + +--- + +## RFDETRDataModule + +`RFDETRDataModule` is a `pytorch_lightning.LightningDataModule`. It builds train/val/test datasets and wraps them in `DataLoader` objects. + +```python +from rfdetr.training import RFDETRDataModule + +datamodule = RFDETRDataModule(model_config, train_config) +``` + +### Stages + +| Stage | Datasets built | +| ------------ | ------------------------------------------ | +| `"fit"` | `train` + `val` | +| `"validate"` | `val` only | +| `"test"` | `test` (or `val` for COCO-format datasets) | + +The `setup(stage)` method is lazy — each split is built at most once, even if called multiple times. + +### class_names property + +```python +datamodule.setup("fit") +print(datamodule.class_names) # e.g. ["cat", "dog", "person"] +``` + +Returns sorted category names from the COCO annotation file of the first available split, or `None` if the dataset has not been set up yet. + +--- + +## build_trainer + +`build_trainer` assembles a `pytorch_lightning.Trainer` with the full RF-DETR callback and logger stack. All `TrainConfig` fields are wired automatically. + +```python +from rfdetr.training import build_trainer + +trainer = build_trainer(train_config, model_config) +``` + +### What build_trainer configures + +| Concern | Source | +| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Max epochs | `train_config.epochs` | +| Gradient accumulation | Detection/segmentation: `train_config.grad_accum_steps` forwarded to Trainer. Keypoint models: owned by `RFDETRModelModule` manual optimization (Trainer always sees `1`). | +| Gradient clipping | Detection/segmentation: `train_config.clip_max_norm` forwarded to Trainer. Keypoint models: owned by `RFDETRModelModule` manual optimization (Trainer always sees `None`). | +| Mixed precision | `model_config.amp` enables AMP; dtype resolved from `train_config.amp_dtype` (`"auto"` selects `bf16-mixed` on Ampere+, `"bf16"` / `"fp16"` force a specific dtype) | +| Accelerator | `train_config.accelerator` (default `"auto"`) | +| Strategy | Set via `train_config.strategy` (default `"auto"`) or pass `strategy=` as a `**trainer_kwarg` to `build_trainer`. Common values: `"auto"`, `"ddp"`, `"ddp_spawn"`. `TrainConfig` also exposes `devices` and `num_nodes` for multi-GPU and multi-node training. | +| Sync batch norm | `train_config.sync_bn` | +| Progress bar | `train_config.progress_bar` | +| Loggers | CSVLogger always; TensorBoard, WandB, MLflow when their `train_config` flags are `True` | +| Callbacks | `RFDETREMACallback`, `DropPathCallback`, `COCOEvalCallback`, `BestModelCallback`, `RFDETREarlyStopping` (conditional) | + +### Overriding PTL Trainer kwargs + +Pass keyword arguments accepted by `pytorch_lightning.Trainer` via `**trainer_kwargs`. Most keys override the built configuration. + +**Detection and segmentation models** forward `accumulate_grad_batches` and `gradient_clip_val` to the Trainer normally — you can override them via `trainer_kwargs` or configure them on `TrainConfig` (`grad_accum_steps`, `clip_max_norm`). + +**Keypoint models** use manual optimization, so `RFDETRModelModule` owns accumulation and clipping internally. `build_trainer()` forces `accumulate_grad_batches=1` and `gradient_clip_val=None` regardless of what is passed, and emits a `UserWarning` if those keys appear in `trainer_kwargs` so the override is visible: + +```python +trainer = build_trainer( + train_config, + model_config, + fast_dev_run=2, # run 2 batches per epoch for a smoke test + log_every_n_steps=10, +) +``` + +--- + +## Running the training loop + +### Full training run + +```python +from rfdetr.config import ( + RFDETRMediumConfig, + TrainConfig, +) # config classes live in rfdetr.config, not the top-level rfdetr namespace +from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer + +model_config = RFDETRMediumConfig(num_classes=10) +train_config = TrainConfig( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", +) + +module = RFDETRModelModule(model_config, train_config) +datamodule = RFDETRDataModule(model_config, train_config) +trainer = build_trainer(train_config, model_config) + +trainer.fit(module, datamodule) +``` + +### Resume from checkpoint + +Pass the checkpoint path to `trainer.fit` via `ckpt_path`. The path can be a PTL `.ckpt` file or a legacy RF-DETR `.pth` file — `RFDETRModelModule.on_load_checkpoint` converts either format automatically. + +```python +trainer.fit(module, datamodule, ckpt_path="output/last.ckpt") +# or a legacy checkpoint: +trainer.fit(module, datamodule, ckpt_path="output/checkpoint.pth") +``` + +> **Note:** When `checkpoint_interval=1`, no `last.ckpt` is written. Use `checkpoint_{epoch}.ckpt` (e.g. `output/checkpoint_epoch=4.ckpt`) to resume instead. + +If you need to persist a converted checkpoint on disk (for example to inspect it, share it, or use it outside of PTL), convert it explicitly before passing it to `trainer.fit`: + +```python +from rfdetr.training import convert_legacy_checkpoint + +convert_legacy_checkpoint("old_checkpoint.pth", "new_checkpoint.ckpt") +trainer.fit(module, datamodule, ckpt_path="new_checkpoint.ckpt") +``` + +`convert_legacy_checkpoint` reads a pre-PTL `.pth` file produced by the legacy `engine.py` training loop and writes a PTL-compatible `.ckpt` file. Use it when migrating saved checkpoints to the PTL format rather than relying on on-the-fly conversion at load time. + +### Validation only + +```python +trainer.validate(module, datamodule) +``` + +Runs one full validation pass and logs `val/mAP_50_95`, `val/mAP_50`, `val/F1`, and per-class AP metrics to all active loggers. + +### Inference with the data pipeline + +```python +predictions = trainer.predict(module, dataloaders=datamodule.val_dataloader()) +``` + +Calls `module.predict_step` on every batch and returns a list of postprocessed detection results. Pass any `DataLoader` instance — `datamodule.val_dataloader()`, `datamodule.test_dataloader()`, or a custom loader — as the `dataloaders` argument. This is useful for offline evaluation or generating submission files. + +!!! note "predict_dataloader not implemented" + + `RFDETRDataModule` does not define a `predict_dataloader()` method, so `trainer.predict(module, datamodule)` will raise an error. Always pass a dataloader explicitly via the `dataloaders=` argument. + +--- + +## Multi-GPU training + +`build_trainer` configures PyTorch Lightning's `Trainer` directly, so all PTL strategies work out of the box. + +### Data Parallel (DDP) — recommended + +Set `train_config.accelerator = "auto"` and pass `strategy="ddp"` to `build_trainer`, then launch with `torchrun`: + +!!! note "`devices` must be overridden for multi-GPU runs" + + `build_trainer` defaults to `devices=1`. To use all available GPUs, pass `devices="auto"` (or an explicit count) as a `**trainer_kwarg`: + + ```python + trainer = build_trainer(train_config, model_config, strategy="ddp", devices="auto") + ``` + + Without this override, `torchrun` will spawn multiple processes but each process will only see one device, defeating the purpose of the multi-GPU launch. + +```bash +torchrun --nproc_per_node=4 train.py +``` + +where `train.py` contains: + +```python +from rfdetr.config import ( + RFDETRMediumConfig, + TrainConfig, +) # config classes live in rfdetr.config, not the top-level rfdetr namespace +from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer + +model_config = RFDETRMediumConfig(num_classes=10) +train_config = TrainConfig( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, # per-GPU batch size + grad_accum_steps=1, # reduce when using more GPUs + output_dir="output", + sync_bn=True, # sync batch norms across GPUs +) + +module = RFDETRModelModule(model_config, train_config) +datamodule = RFDETRDataModule(model_config, train_config) +trainer = build_trainer(train_config, model_config, strategy="ddp", devices="auto") + +trainer.fit(module, datamodule) +``` + +!!! warning "EMA is not compatible with FSDP or DeepSpeed" + + `build_trainer` automatically disables `RFDETREMACallback` when `strategy` contains `"fsdp"` or `"deepspeed"`, and emits a `UserWarning`. Use `strategy="ddp"` or `strategy="auto"` to keep EMA active. + +### Effective batch size + +``` +effective_batch_size = batch_size × grad_accum_steps × num_gpus +``` + +Maintain an effective batch size of 16 regardless of GPU count: + +| GPUs | `batch_size` | `grad_accum_steps` | Effective | +| ---- | ------------ | ------------------ | --------- | +| 1 | 4 | 4 | 16 | +| 2 | 4 | 2 | 16 | +| 4 | 4 | 1 | 16 | +| 8 | 2 | 1 | 16 | + +--- + +## Custom callbacks + +`build_trainer` builds the default callback stack. To add your own callbacks alongside the built-in ones, pass them via `trainer_kwargs`: + +```python +from pytorch_lightning.callbacks import LearningRateMonitor, ModelSummary +from rfdetr.training import build_trainer + +extra_callbacks = [ + LearningRateMonitor(logging_interval="step"), + ModelSummary(max_depth=3), +] + +trainer = build_trainer( + train_config, + model_config, + callbacks=extra_callbacks, # replaces the default callback list entirely +) +``` + +!!! warning "Replacing vs. extending callbacks" + + Passing `callbacks=` to `build_trainer` via `trainer_kwargs` **replaces** the entire default callback list built inside `build_trainer` (EMA, COCO eval, best-model checkpointing, etc.). To extend rather than replace, build the extra callbacks separately and merge them after calling `build_trainer`: + + ```python + trainer = build_trainer(train_config, model_config) + trainer.callbacks.extend( + [ + LearningRateMonitor(logging_interval="step"), + ] + ) + trainer.fit(module, datamodule) + ``` + +### Built-in callbacks + +| Class | Purpose | Enabled when | +| --------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `RFDETREMACallback` | Maintains an EMA copy of model weights | `train_config.use_ema=True` and strategy is not sharded | +| `DropPathCallback` | Anneals drop-path rate over training | `train_config.drop_path > 0` | +| `COCOEvalCallback` | Computes task validation metrics after each validation epoch | Always | +| `BestModelCallback` | Saves `checkpoint_best_regular.pth`, `checkpoint_best_ema.pth`, `checkpoint_best_total.pth` | Always | +| `RFDETREarlyStopping` | Stops training when the validation task metric stops improving | `train_config.early_stopping=True` | + +--- + +## Custom loggers + +`build_trainer` adds loggers based on `TrainConfig` flags. To attach a logger not supported by `TrainConfig` (for example a custom Neptune or Comet logger), build it yourself and pass it alongside the defaults: + +```python +from pytorch_lightning.loggers import NeptuneLogger # hypothetical +from rfdetr.training import build_trainer + +trainer = build_trainer(train_config, model_config) +trainer.loggers.append(NeptuneLogger(project="my-workspace/rf-detr")) +trainer.fit(module, datamodule) +``` + +All logged keys (`train/loss`, `val/mAP_50_95`, `val/keypoint_map_50_95`, `val/F1`, `val/ema_mAP_50_95`, etc.) are written to every active logger in the list. + +--- + +## Logged metrics reference + +| Key | When logged | Description | +| ------------------------ | ---------------------- | --------------------------------------------------------- | +| `train/loss` | Every step / epoch | Total weighted training loss | +| `train/` | Every step / epoch | Individual loss terms (e.g. `train/loss_bbox`) | +| `val/loss` | Each epoch | Validation loss (if `train_config.compute_val_loss=True`) | +| `val/mAP_50_95` | Each eval epoch | COCO box mAP@[.50:.05:.95] | +| `val/mAP_50` | Each eval epoch | COCO box mAP@.50 | +| `val/mAP_75` | Each eval epoch | COCO box mAP@.75 | +| `val/mAR` | Each eval epoch | COCO mean average recall | +| `val/ema_mAP_50_95` | Each eval epoch | EMA-model mAP@[.50:.05:.95] (if EMA active) | +| `val/F1` | Each eval epoch | Macro F1 at best confidence threshold | +| `val/precision` | Each eval epoch | Precision at best F1 threshold | +| `val/recall` | Each eval epoch | Recall at best F1 threshold | +| `val/AP/` | Each eval epoch | Per-class AP (if `log_per_class_metrics=True`) | +| `val/segm_mAP_50_95` | Each eval epoch | Segmentation mAP (segmentation models only) | +| `val/segm_mAP_50` | Each eval epoch | Segmentation mAP@.50 (segmentation models only) | +| `val/keypoint_map_50_95` | Each eval epoch | COCO keypoint AP@[.50:.05:.95] (keypoint preview only) | +| `val/keypoint_map_50` | Each eval epoch | COCO keypoint AP@.50 (keypoint preview only) | +| `test/*` | After `trainer.test()` | Mirror of `val/*` keys | + +--- + +## See also + +- [RFDETR.train() — high-level API](index.md#quick-start) — the one-liner training path +- [Training parameters](training-parameters.md) — all `TrainConfig` fields +- [Training loggers](loggers.md) — TensorBoard, WandB, MLflow setup +- [Advanced training](advanced.md) — checkpointing, early stopping, memory optimisation +- [PTL primitives API reference](../../reference/training.md) — full docstring reference diff --git a/docs/learn/train/dataset-formats.md b/docs/learn/train/dataset-formats.md new file mode 100644 index 0000000..f4a8187 --- /dev/null +++ b/docs/learn/train/dataset-formats.md @@ -0,0 +1,493 @@ +--- +description: RF-DETR dataset format guide for COCO JSON and YOLO. Auto-detection, directory structure, annotation schemas, and format conversion. +--- + +# Dataset Formats + +RF-DETR supports training on datasets in two popular formats: **COCO** and **YOLO**. The format is automatically detected based on your dataset's directory structure—simply pass your dataset directory to the `train()` method. + +## Automatic Format Detection + +When you call `model.train(dataset_dir=)`, RF-DETR checks the following: + +1. **COCO format**: Looks for `train/_annotations.coco.json` +2. **YOLO format**: Looks for `data.yaml` (or `data.yml`) and `train/images/` directory + +If neither format is detected, an error is raised with instructions on what's expected. + +!!! tip "Roboflow Export" + + [Roboflow](https://roboflow.com/annotate) can export datasets in both COCO and YOLO formats. When downloading from Roboflow, select the appropriate format based on your preference. + +--- + +## COCO Format + +COCO (Common Objects in Context) format uses JSON files to store annotations in a structured format with images, categories, and annotations. + +### Directory Structure + +``` +dataset/ +├── train/ +│ ├── _annotations.coco.json +│ ├── image1.jpg +│ ├── image2.jpg +│ └── ... (other image files) +├── valid/ +│ ├── _annotations.coco.json +│ ├── image1.jpg +│ ├── image2.jpg +│ └── ... (other image files) +└── test/ + ├── _annotations.coco.json + ├── image1.jpg + ├── image2.jpg + └── ... (other image files) +``` + +### Annotation File Structure + +Each `_annotations.coco.json` file contains: + +```json +{ + "info": { + "description": "Dataset description", + "version": "1.0" + }, + "licenses": [], + "images": [ + { + "id": 1, + "file_name": "image1.jpg", + "width": 640, + "height": 480 + } + ], + "categories": [ + { + "id": 1, + "name": "cat", + "supercategory": "animal" + }, + { + "id": 2, + "name": "dog", + "supercategory": "animal" + } + ], + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "bbox": [ + 100, + 150, + 200, + 180 + ], + "area": 36000, + "iscrowd": 0 + } + ] +} +``` + +#### Key Fields + +| Field | Description | +| ------------- | --------------------------------------------------------------------- | +| `images` | List of image metadata including `id`, `file_name`, `width`, `height` | +| `categories` | List of object categories with `id` and `name` | +| `annotations` | List of object annotations linking images to categories | +| `bbox` | Bounding box in `[x, y, width, height]` format (top-left corner) | +| `area` | Area of the bounding box | +| `iscrowd` | 0 for individual objects, 1 for crowd regions | + +### Segmentation Annotations + +For training segmentation models, your COCO annotations must include a `segmentation` key with polygon coordinates: + +```json +{ + "id": 1, + "image_id": 1, + "category_id": 1, + "bbox": [ + 100, + 150, + 200, + 180 + ], + "area": 36000, + "iscrowd": 0, + "segmentation": [ + [ + 100, + 150, + 150, + 150, + 200, + 200, + 150, + 250, + 100, + 200 + ] + ] +} +``` + +The `segmentation` field contains a list of polygons, where each polygon is a flat list of coordinates: `[x1, y1, x2, y2, x3, y3, ...]`. + +--- + +### Keypoint Annotations + +For training the keypoint preview model, use COCO JSON keypoint annotations. Roboflow-style COCO exports are supported +when the split files are named `train/_annotations.coco.json` and `valid/_annotations.coco.json`. + +Each keypoint annotation must include a bounding box plus COCO keypoint fields: + +```json +{ + "id": 1, + "image_id": 1, + "category_id": 0, + "bbox": [ + 100, + 150, + 200, + 180 + ], + "area": 36000, + "iscrowd": 0, + "num_keypoints": 17, + "keypoints": [ + 110, + 160, + 2, + 125, + 158, + 2 + ] +} +``` + +The category should declare the keypoint schema: + +```json +{ + "id": 0, + "name": "person", + "supercategory": "person", + "keypoints": [ + "nose", + "left_eye", + "right_eye" + ], + "skeleton": [] +} +``` + +The `keypoints` array above is shortened for readability. In a valid COCO person-keypoint annotation it contains +`17 * 3` values: `x`, `y`, and visibility for each keypoint. + +The keypoint preview model is pretrained on COCO person-style keypoints. Its default COCO schema is `[17]`, so +keypoint-bearing categories are mapped onto the active keypoint label slot during COCO loading. Legacy checkpoints may +still report a background-first `[0, 17]` schema, which RF-DETR accepts for compatibility. Custom keypoint training can +also use YOLO pose labels, described below. + +--- + +## YOLO Format + +YOLO format uses separate text files for each image's annotations and a `data.yaml` configuration file that defines class names. + +### Directory Structure + +``` +dataset/ +├── data.yaml +├── train/ +│ ├── images/ +│ │ ├── image1.jpg +│ │ ├── image2.jpg +│ │ └── ... +│ └── labels/ +│ ├── image1.txt +│ ├── image2.txt +│ └── ... +├── valid/ +│ ├── images/ +│ │ ├── image1.jpg +│ │ ├── image2.jpg +│ │ └── ... +│ └── labels/ +│ ├── image1.txt +│ ├── image2.txt +│ └── ... +└── test/ + ├── images/ + │ ├── image1.jpg + │ └── ... + └── labels/ + ├── image1.txt + └── ... +``` + +### data.yaml Configuration + +The `data.yaml` file at the root of your dataset directory defines the class names: + +```yaml +names: + - cat + - dog + - bird + +nc: 3 + +train: train/images +val: valid/images +test: test/images +``` + +| Field | Description | +| ---------------------- | -------------------------------------------------- | +| `names` | List of class names (0-indexed) | +| `nc` | Number of classes | +| `train`, `val`, `test` | Paths to image directories (relative to data.yaml) | + +!!! note "Alternative format" + + Some YOLO datasets use a dictionary format for names: + + ```yaml + names: + 0: cat + 1: dog + 2: bird + ``` + + Both formats are supported. + +### Label File Format + +Each image has a corresponding `.txt` file in the `labels/` directory with the same base name. Each line in the label file represents one object: + +``` + +``` + +**Example** (`image1.txt`): + +``` +0 0.5 0.4 0.3 0.2 +1 0.2 0.6 0.15 0.25 +``` + +#### Coordinate Format + +| Field | Range | Description | +| ---------- | ------------ | ----------------------------------------------- | +| `class_id` | 0, 1, 2, ... | Zero-indexed class ID from `names` in data.yaml | +| `x_center` | 0.0 - 1.0 | Normalized x-coordinate of bounding box center | +| `y_center` | 0.0 - 1.0 | Normalized y-coordinate of bounding box center | +| `width` | 0.0 - 1.0 | Normalized width of bounding box | +| `height` | 0.0 - 1.0 | Normalized height of bounding box | + +All coordinates are normalized relative to image dimensions. For example, if an image is 640×480 pixels and the bounding box center is at (320, 240): + +- `x_center` = 320 / 640 = 0.5 +- `y_center` = 240 / 480 = 0.5 + +### Segmentation Labels (YOLO-Seg) + +For segmentation, YOLO format extends the label format with polygon coordinates: + +``` + ... +``` + +**Example** (`image1.txt` with segmentation): + +``` +0 0.1 0.2 0.3 0.2 0.4 0.5 0.2 0.6 0.1 0.4 +``` + +The coordinates after the class ID represent the polygon vertices in normalized format. + +--- + +### Pose Labels (YOLO Pose) + +For keypoint preview training, RF-DETR supports Ultralytics YOLO pose labels in the same directory layout shown above. +The `data.yaml` file must declare `kpt_shape`: + +```yaml +names: + 0: person + +kpt_shape: [17, 3] # [number_of_keypoints, dimensions]; dimensions must be 2 or 3 +flip_idx: [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] +kpt_names: + 0: + - nose + - left_eye + - right_eye +``` + +`kpt_names` is optional. When omitted, RF-DETR creates placeholder names such as `keypoint_0`. `flip_idx` is an +Ultralytics-style length-`K` permutation used to infer RF-DETR's flat `keypoint_flip_pairs` for horizontal-flip +augmentation. + +Each pose label row contains a bounding box followed by keypoints: + +```text + ... +``` + +For `kpt_shape: [K, 2]`, omit the visibility value: + +```text + ... +``` + +All box and keypoint coordinates are normalized to `[0, 1]`. RF-DETR converts keypoints to COCO-style `(x, y, visibility)` tensors internally. For `[K, 3]`, the visibility values are preserved. For `[K, 2]`, visibility is +synthesized: nonzero points are marked visible (`2`) and `(0, 0)` points are marked absent (`0`). + +Use the YOLO schema helper when you want to configure a model explicitly: + +```python +from pathlib import Path + +from rfdetr import RFDETRKeypointPreview +from rfdetr.datasets._keypoint_schema import infer_yolo_keypoint_schema + +DATASET_DIR = Path("/path/to/yolo-pose-dataset") +schema = infer_yolo_keypoint_schema(DATASET_DIR / "data.yaml") + +model = RFDETRKeypointPreview( + num_classes=len(schema.class_names), + num_keypoints_per_class=schema.num_keypoints_per_class, +) + +model.train( + dataset_file="yolo", + dataset_dir=str(DATASET_DIR), + class_names=schema.class_names, + keypoint_oks_sigmas=schema.keypoint_oks_sigmas, +) +``` + +!!! note "flip_idx and keypoint_flip_pairs" + + `flip_idx` is a permutation, while `keypoint_flip_pairs` is a flat pair list. During `model.train()`, RF-DETR infers + the pair list automatically from `flip_idx` when no explicit `keypoint_flip_pairs` is provided. + +--- + +## Converting Between Formats + +### YOLO to COCO + +You can use the [supervision](https://github.com/roboflow/supervision) library to convert datasets: + +```python +import supervision as sv + +# Load YOLO dataset +dataset = sv.DetectionDataset.from_yolo( + images_directory_path="path/to/images", + annotations_directory_path="path/to/labels", + data_yaml_path="path/to/data.yaml", +) + +# Save as COCO +dataset.as_coco(images_directory_path="output/images", annotations_path="output/annotations.json") +``` + +### COCO to YOLO + +```python +import supervision as sv + +# Load COCO dataset +dataset = sv.DetectionDataset.from_coco( + images_directory_path="path/to/images", annotations_path="path/to/annotations.json" +) + +# Save as YOLO +dataset.as_yolo( + images_directory_path="output/images", annotations_directory_path="output/labels", data_yaml_path="output/data.yaml" +) +``` + +### Using Roboflow + +[Roboflow](https://roboflow.com) provides a web interface to: + +1. Upload datasets in any format +2. Annotate new images or edit existing annotations +3. Export in COCO, YOLO, or other formats + +This is often the easiest way to convert between formats while also having the option to augment your data. + +--- + +## Which Format Should I Use? + +Both formats work equally well with RF-DETR. Choose based on your workflow: + +| Consideration | COCO | YOLO | +| --------------------------------- | -------------------------- | ----------------------- | +| **Annotation storage** | Single JSON file per split | One text file per image | +| **Human readability** | JSON structure, verbose | Simple text, compact | +| **Other framework compatibility** | DETR family, MMDetection | Ultralytics YOLO | +| **Segmentation support** | Full polygon support | Full polygon support | +| **Editing annotations** | Requires JSON parsing | Simple text editing | + +!!! tip "Recommendation" + + If you're exporting from Roboflow or already have a dataset in one format, simply use that format. RF-DETR handles both identically. + +--- + +## Troubleshooting + +### Format Detection Fails + +If you see an error like: + +``` +Could not detect dataset format in /path/to/dataset +``` + +Check that: + +**For COCO format:** + +- `train/_annotations.coco.json` exists +- The JSON file is valid + +**For YOLO format:** + +- `data.yaml` or `data.yml` exists at the root +- `train/images/` directory exists with images + +### Empty Annotations + +If images have no objects, handle them as follows: + +**COCO format:** Include the image in the `images` array but don't add any annotations for it. + +**YOLO format:** Create an empty `.txt` file (0 bytes) for the image, or omit the label file entirely. + +### Class ID Mismatch + +**COCO format:** Category IDs in annotations must match IDs defined in the `categories` array. + +**YOLO format:** Class IDs in label files must be valid indices (0 to `nc-1`) based on the `names` list in `data.yaml`. diff --git a/docs/learn/train/index.md b/docs/learn/train/index.md new file mode 100644 index 0000000..aba32c1 --- /dev/null +++ b/docs/learn/train/index.md @@ -0,0 +1,247 @@ +--- +description: Train RF-DETR detection and segmentation models on custom datasets. Supports COCO and YOLO formats with one-line Python API and PyTorch Lightning. +--- + +# Train an RF-DETR Model + +!!! tip "Key Takeaways" + + - Train detection, segmentation, or keypoint preview models with a single `model.train(dataset_dir=...)` call + - Detection and segmentation support COCO JSON and YOLO dataset formats with automatic detection + - Keypoint preview training supports COCO keypoint JSON and Ultralytics YOLO pose datasets + - Fine-tune from COCO-pretrained checkpoints (Nano to 2XLarge) for fastest convergence + - Built on PyTorch Lightning — use the high-level API or access PTL primitives directly for full control + - EMA weights, early stopping, and best-model checkpointing are included by default + +You can train RF-DETR object detection and segmentation models on a custom dataset using the `rfdetr` Python package, or in the cloud using Roboflow. + +This guide describes how to train both an object detection and segmentation RF-DETR model. + +## Training paths + +RF-DETR provides two training paths: + +| Path | When to use | +| ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`RFDETR.train()`** (this page) | Quickstart, fine-tuning with standard options, Colab notebooks. One call sets up and runs everything. | +| **[Custom Training API](customization.md)** | Custom callbacks, alternative loggers, multi-GPU strategies, integration with external frameworks, or any other customisation of the training loop. | + +Both paths run the same underlying PyTorch Lightning stack. `RFDETR.train()` constructs `RFDETRModelModule`, `RFDETRDataModule`, and a `Trainer` internally; the Lightning API page shows how to do the same thing explicitly so you can modify each component. + +## Quick Start + +!!! info "Training requires the `train` extra" + + Training dependencies are not included in the base install. Install them with: + + ```bash + pip install "rfdetr[train]" + ``` + + For experiment tracking, also add `pip install "rfdetr[train,loggers]"`. + +RF-DETR supports training on datasets in both **COCO** and **YOLO** formats. The format is automatically detected based on the structure of your dataset directory. + +=== "Object Detection" + + ```python + from rfdetr import RFDETRMedium + + model = RFDETRMedium() + + model.train( + dataset_dir="", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="", + ) + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegMedium + + model = RFDETRSegMedium() + + model.train( + dataset_dir="", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="", + ) + ``` + +=== "Keypoint Preview" + + ```python + from rfdetr import RFDETRKeypointPreview + + model = RFDETRKeypointPreview() + + model.train( + dataset_dir="", + epochs=50, + batch_size=2, + grad_accum_steps=8, + lr=1e-5, + output_dir="", + ) + ``` + +Different GPUs have different VRAM capacities, so adjust batch_size and grad_accum_steps to maintain a total batch size of 16. For example, on a powerful GPU like the A100, use `batch_size=16` and `grad_accum_steps=1`; on smaller GPUs like the T4, use `batch_size=4` and `grad_accum_steps=4`. This gradient accumulation strategy helps train effectively even with limited memory. + +Each model class downloads its COCO-pretrained checkpoint automatically when instantiated. To get started quickly with training an object detection model, please refer to our fine-tuning Google Colab [notebook](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/how-to-finetune-rf-detr-on-detection-dataset.ipynb). + +## Keypoint preview custom datasets + +The pretrained keypoint preview checkpoint predicts 17 COCO person keypoints. Fine-tuned keypoint preview models can use the keypoint schema from your own COCO or YOLO pose dataset, so the output keypoint count is not limited to 17. + +Use COCO keypoint JSON or Ultralytics YOLO pose labels for custom keypoint training. Roboflow COCO exports are supported when split annotations are named `train/_annotations.coco.json`, `valid/_annotations.coco.json`, and optionally `test/_annotations.coco.json`. YOLO pose datasets use the existing RF-DETR YOLO directory layout with `data.yaml`, `train/images`, `train/labels`, `valid/images`, and `valid/labels`. + +The keypoint fine-tuning demo infers the class names and keypoint schema from the training annotation file, then passes those values into `RFDETRKeypointPreview` and `model.train()`: + +```python +from pathlib import Path + +from rfdetr import RFDETRKeypointPreview +from rfdetr.datasets._keypoint_schema import infer_coco_keypoint_schema + +DATASET_DIR = Path("/path/to/coco-keypoint-dataset") +schema = infer_coco_keypoint_schema(DATASET_DIR / "train" / "_annotations.coco.json") + +model = RFDETRKeypointPreview( + num_classes=len(schema.class_names), + num_keypoints_per_class=schema.num_keypoints_per_class, +) + +model.train( + dataset_file="roboflow", + dataset_dir=str(DATASET_DIR), + class_names=schema.class_names, + keypoint_oks_sigmas=schema.keypoint_oks_sigmas, + epochs=50, + batch_size=8, + grad_accum_steps=2, + lr=2e-5, + lr_encoder=2e-5, + output_dir="output/keypoint_custom", + use_ema=False, + run_test=False, +) +``` + +Set `keypoint_flip_pairs` if horizontal flips should swap left/right keypoints for your schema. + +For YOLO pose datasets, use `infer_yolo_keypoint_schema(DATASET_DIR / "data.yaml")` instead. RF-DETR also infers YOLO pose schema automatically during `model.train()` when `data.yaml` declares `kpt_shape`. + +## Dataset Format + +RF-DETR **automatically detects** whether your dataset is in COCO or YOLO format. Simply pass your dataset directory to the `train()` method and the appropriate data loader will be used. + +| Format | Detection Method | Learn More | +| -------- | ---------------------------------------- | --------------------------------------------------- | +| **COCO** | Looks for `train/_annotations.coco.json` | [COCO Format Guide](dataset-formats.md#coco-format) | +| **YOLO** | Looks for `data.yaml` + `train/images/` | [YOLO Format Guide](dataset-formats.md#yolo-format) | + +For keypoint preview training, use COCO keypoint JSON or YOLO pose labels. YOLO pose datasets must declare +`kpt_shape` in `data.yaml`; detection-only YOLO datasets still fail clearly in keypoint mode instead of being treated as +pose labels. + +[Roboflow](https://roboflow.com/annotate) allows you to create object detection datasets from scratch and export them in either COCO JSON or YOLO format for training. You can also explore [Roboflow Universe](https://universe.roboflow.com/) to find pre-labeled datasets for a range of use cases. + +→ **[Learn more about dataset formats](dataset-formats.md)** + +## Training Configuration + +RF-DETR provides many configuration options to customize your training run. See the complete reference for all available parameters. + +→ **[View all training parameters](training-parameters.md)** + +## Advanced Topics + +- [Resume training](advanced.md#resume-training) from a checkpoint +- [Early stopping](advanced.md#early-stopping) to prevent overfitting +- [Multi-GPU training](advanced.md#multi-gpu-training) with PyTorch Lightning DDP +- [Custom augmentations with Albumentations](augmentations.md) - Dedicated guide +- [Memory optimization](advanced.md#memory-optimization) with gradient checkpointing + +→ **[Learn more about advanced training](advanced.md)** + +## Custom Training API + +RF-DETR's training stack is built on PyTorch Lightning. The `RFDETR.train()` call above constructs and runs PTL primitives internally. Use them directly when you need custom callbacks, non-default loggers, multi-GPU strategies, or full control over the training loop. + +→ **[Custom Training API guide](customization.md)** + +## Training Loggers + +Track your experiments with popular logging platforms: + +- [TensorBoard](loggers.md#tensorboard) for local visualization +- [Weights and Biases](loggers.md#weights-and-biases) for cloud-based tracking +- [ClearML](loggers.md#clearml) workaround for SDK auto-binding +- [MLflow](loggers.md#mlflow) for experiment lifecycle management + +→ **[Learn more about training loggers](loggers.md)** + +## Result Checkpoints + +During training, multiple model checkpoints are saved to the output directory: + +- `checkpoint.pth` – the most recent checkpoint, saved at the end of the latest epoch. + +- `checkpoint_.pth` – periodic checkpoints saved every N epochs (default is every 10). + +- `checkpoint_best_ema.pth` – best checkpoint based on validation score, using the EMA (Exponential Moving Average) weights. EMA weights are a smoothed version of the model's parameters across training steps, often yielding better generalization. + +- `checkpoint_best_regular.pth` – best checkpoint based on validation score, using the raw (non-EMA) model weights. + +- `checkpoint_best_total.pth` – final checkpoint selected for inference and benchmarking. It contains only the model weights (no optimizer state or scheduler) and is chosen as the better of the EMA and non-EMA models based on validation performance. + +For detection and segmentation models, the validation score is box mAP (`val/mAP_50_95`). For keypoint preview models, +best-checkpoint selection uses COCO keypoint AP (`val/keypoint_map_50_95`) and checkpoints persist the model keypoint +schema so `RFDETR.from_checkpoint()` can reconstruct the same label/keypoint slots. + +??? note "Checkpoint file sizes" + + Checkpoint sizes vary based on what they contain: + + - **Training checkpoints** (e.g. `checkpoint.pth`, `checkpoint_.pth`) include model weights, optimizer state, scheduler state, and training metadata. Use these to resume training. + + - **Evaluation checkpoints** (e.g. `checkpoint_best_ema.pth`, `checkpoint_best_regular.pth`) store only the model weights — either EMA or raw — and are used to track the best-performing models. These may come from different epochs depending on which version achieved the highest validation score. + + - **Stripped checkpoint** (e.g. `checkpoint_best_total.pth`) contains only the final model weights and is optimized for inference and deployment. + +## Load and Run Fine-Tuned Model + +=== "Object Detection" + + ```python + from rfdetr import RFDETRMedium + + model = RFDETRMedium(pretrain_weights="") + + detections = model.predict("") + ``` + +=== "Image Segmentation" + + ```python + from rfdetr import RFDETRSegMedium + + model = RFDETRSegMedium(pretrain_weights="") + + detections = model.predict("") + ``` + +## Next Steps + +After training your model, you can: + +- [Export your model to ONNX](../export.md) for deployment with various inference frameworks +- [Deploy to Roboflow](../deploy.md) for cloud-based inference and workflow integration diff --git a/docs/learn/train/loggers.md b/docs/learn/train/loggers.md new file mode 100644 index 0000000..ab69e2f --- /dev/null +++ b/docs/learn/train/loggers.md @@ -0,0 +1,316 @@ +--- +description: Track RF-DETR training with TensorBoard, Weights and Biases, and MLflow. Configure multiple experiment loggers simultaneously. +--- + +# Training Loggers + +RF-DETR supports integration with popular experiment tracking and visualization platforms. You can enable one or more supported loggers to monitor your training runs, compare experiments, and track metrics over time. + +## CSV (always active) + +A `CSVLogger` is always active regardless of any flags. It requires no extra packages and writes all metrics to `{output_dir}/metrics.csv` on every validation step. + +--- + +## TensorBoard + +[TensorBoard](https://www.tensorflow.org/tensorboard) is a powerful toolkit for visualizing and tracking training metrics. + +TensorBoard logging is enabled by default. Pass `tensorboard=False` to disable it. + +!!! note "Missing package behaviour" + + If the `tensorboard` package is not installed, training continues without error — a + `UserWarning` is emitted and TensorBoard logging is silently suppressed. Install + `rfdetr[loggers]` to avoid this. + +### Setup + +Install the required packages: + +```bash +pip install "rfdetr[loggers]" +``` + +### Usage + +TensorBoard is active unless you explicitly disable it: + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium() + +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + # tensorboard=True is the default; pass tensorboard=False to disable +) +``` + +### Viewing Logs + +**Local environment:** + +```bash +tensorboard --logdir output +``` + +Then open `http://localhost:6006/` in your browser. + +**Google Colab:** + +```ipython +%load_ext tensorboard +%tensorboard --logdir output +``` + +### Logged Metrics + +All logged metric keys are listed in the [Logged Metrics Reference](customization.md#logged-metrics-reference). + +--- + +## Weights and Biases + +[Weights and Biases (W&B)](https://www.wandb.ai) is a cloud-based platform for experiment tracking and visualization. + +### Setup + +Install the required packages: + +```bash +pip install "rfdetr[loggers]" +``` + +Log in to W&B: + +```bash +wandb login +``` + +You can retrieve your API key at [wandb.ai/authorize](https://wandb.ai/authorize). + +### Usage + +Enable W&B logging in your training: + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium() + +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + wandb=True, + project="my-detection-project", + run="experiment-001", +) +``` + +### Configuration + +| Parameter | Description | +| --------- | --------------------------------------- | +| `project` | Groups related experiments together | +| `run` | Identifies individual training sessions | + +If you don't specify a run name, W&B assigns a random one automatically. + +### Features + +Access your runs at [wandb.ai](https://wandb.ai). W&B provides: + +- Real-time metric visualization +- Experiment comparison +- Hyperparameter tracking +- System metrics (GPU usage, memory) +- Training config logging + +### Logged Metrics + +All logged metric keys are listed in the [Logged Metrics Reference](customization.md#logged-metrics-reference). + +--- + +## ClearML + +[ClearML](https://clear.ml) is an open-source platform for managing, tracking, and automating machine learning experiments. + +**ClearML is not yet integrated as a native PTL logger.** Passing `clearml=True` to `model.train()` raises `NotImplementedError`; metrics are not logged to ClearML through RF-DETR's built-in logger wiring. + +### Workaround: ClearML SDK auto-binding + +ClearML's SDK captures PyTorch Lightning metrics automatically when a `Task` is initialised before training begins: + +```python +from clearml import Task +from rfdetr import RFDETRMedium + +# Initialise before model.train() — ClearML auto-binds to PTL logging +task = Task.init(project_name="my-detection-project", task_name="experiment-001") + +model = RFDETRMedium() +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + # Do NOT pass clearml=True — RF-DETR raises NotImplementedError for that flag +) +``` + +Alternatively, attach a ClearML callback directly using the [Custom Training API](#attaching-loggers-via-the-custom-training-api). + +--- + +## MLflow + +[MLflow](https://mlflow.org/) is an open-source platform for the machine learning lifecycle that helps track experiments, package code into reproducible runs, and share and deploy models. + +### Setup + +Install the required packages: + +```bash +pip install "rfdetr[loggers]" +``` + +### Usage + +Enable MLflow logging in your training: + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium() + +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", + mlflow=True, + project="my-detection-project", + run="experiment-001", +) +``` + +### Configuration + +| Parameter | Description | +| --------- | --------------------------------------------------- | +| `project` | Sets the experiment name in MLflow | +| `run` | Sets the run name (auto-generated if not specified) | + +### Custom Tracking Server + +To use a custom MLflow tracking server, set environment variables: + +```python +import os + +# Set MLflow tracking URI +os.environ["MLFLOW_TRACKING_URI"] = "https://your-mlflow-server.com" + +# For authentication with tracking servers that require it +os.environ["MLFLOW_TRACKING_TOKEN"] = "your-auth-token" + +# Then initialize and train your model +model = RFDETRMedium() +model.train(..., mlflow=True) +``` + +For teams using a hosted MLflow service (like Databricks), you'll typically need to set: + +- `MLFLOW_TRACKING_URI`: The URL of your MLflow tracking server +- `MLFLOW_TRACKING_TOKEN`: Authentication token for your MLflow server + +### Viewing Logs + +Start the MLflow UI: + +```bash +mlflow ui --backend-store-uri +``` + +Then open `http://localhost:5000` in your browser to access the MLflow dashboard. + +### Logged Metrics + +All logged metric keys are listed in the [Logged Metrics Reference](customization.md#logged-metrics-reference). + +--- + +## Using Multiple Loggers + +You can enable multiple logging systems simultaneously: + +```python +model.train( + dataset_dir="path/to/dataset", + epochs=100, + tensorboard=True, + wandb=True, + mlflow=True, + project="my-project", + run="experiment-001", +) +``` + +This allows you to leverage the strengths of different platforms: + +- **TensorBoard**: Local visualization and debugging +- **W&B**: Cloud-based collaboration and experiment comparison +- **MLflow**: Model registry and deployment tracking + +Note: `clearml=True` is accepted by the config schema but raises `NotImplementedError` when the trainer is built. Use the [ClearML SDK workaround](#clearml) instead. + +--- + +## Attaching loggers via the Custom Training API + +`build_trainer` automatically creates loggers from `TrainConfig` flags. To attach a logger not listed above (for example Neptune, Comet, or a fully custom logger), build it separately and append it to `trainer.loggers` before calling `trainer.fit`: + +```python +from rfdetr.config import RFDETRMediumConfig, TrainConfig +from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer + +model_config = RFDETRMediumConfig(num_classes=10) +train_config = TrainConfig( + dataset_dir="path/to/dataset", + epochs=100, + output_dir="output", + tensorboard=True, # built-in loggers still work +) + +module = RFDETRModelModule(model_config, train_config) +datamodule = RFDETRDataModule(model_config, train_config) +trainer = build_trainer(train_config, model_config) + +# Attach any additional PTL-compatible logger +from pytorch_lightning.loggers import CSVLogger # example — use any PTL logger + +trainer.loggers.append(CSVLogger(save_dir="output", name="extra")) + +trainer.fit(module, datamodule) +``` + +CSVLogger is always active (it requires no extra packages). All logged metric keys — `train/loss`, `val/mAP_50_95`, +`val/keypoint_map_50_95`, `val/F1`, `val/ema_mAP_50_95`, `val/AP/`, etc. — are written to every logger in the +list. + +→ **[Full list of logged metrics](customization.md#logged-metrics-reference)** diff --git a/docs/learn/train/training-parameters.md b/docs/learn/train/training-parameters.md new file mode 100644 index 0000000..3e5712f --- /dev/null +++ b/docs/learn/train/training-parameters.md @@ -0,0 +1,288 @@ +--- +description: Complete RF-DETR training parameter reference. Learning rate, batch size, EMA, early stopping, resolution, and hardware configuration. +--- + +# Training Parameters + +This page provides a complete reference of all parameters available when training RF-DETR models. + +## Basic Example + +```python +from rfdetr import RFDETRMedium + +model = RFDETRMedium() + +model.train( + dataset_dir="path/to/dataset", + epochs=100, + batch_size=4, + grad_accum_steps=4, + lr=1e-4, + output_dir="output", +) +``` + +## Core Parameters + +These are the essential parameters for training: + +| Parameter | Type | Default | Description | +| ------------------ | --------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dataset_dir` | `str` | Required | Path to your dataset directory. RF-DETR auto-detects if it's in COCO or YOLO format. See [Dataset Formats](dataset-formats.md). | +| `output_dir` | `str` | `"output"` | Directory where training artifacts (checkpoints, logs) are saved. | +| `epochs` | `int` | `100` | Number of full passes over the training dataset. | +| `batch_size` | `int or "auto"` | `4` | Number of samples processed per iteration. Higher values require more GPU memory. Set to `"auto"` to probe the GPU for the largest safe batch size automatically. | +| `grad_accum_steps` | `int` | `4` | Accumulates gradients over multiple mini-batches. Use with `batch_size` to achieve effective batch size. | +| `resume` | `str` | `None` | Path to a saved checkpoint to continue training. Restores model weights, optimizer state, and scheduler. | + +### Understanding Batch Size + +The **effective batch size** is calculated as: + +``` +effective_batch_size = batch_size × grad_accum_steps × num_gpus +``` + +Recommended configurations for different GPUs (targeting effective batch size of 16): + +| GPU | VRAM | `batch_size` | `grad_accum_steps` | +| -------- | ------- | ------------ | ------------------ | +| A100 | 40-80GB | 16 | 1 | +| RTX 4090 | 24GB | 8 | 2 | +| RTX 3090 | 24GB | 8 | 2 | +| T4 | 16GB | 4 | 4 | +| RTX 3070 | 8GB | 2 | 8 | + +## Learning Rate Parameters + +| Parameter | Type | Default | Description | +| ------------ | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lr` | `float` | `1e-4` | Learning rate for most parts of the model. | +| `lr_encoder` | `float` | `1.5e-4` | Learning rate specifically for the backbone encoder. Can be set lower than `lr` if you want to fine-tune the encoder more conservatively than the rest of the model. | + +!!! tip "Learning rate tips" + + - Start with the default values for fine-tuning + - If the model doesn't converge, try reducing `lr` by half + - For training from scratch (not recommended), you may need higher learning rates + +## Resolution Parameters + +| Parameter | Type | Default | Description | +| ------------ | ----- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `resolution` | `int` | Model-dependent | Input image resolution. Higher values can improve accuracy but require more memory. Each model has its own valid block size: current standard detection checkpoints use multiples of 32, current segmentation checkpoints use multiples of 24 (most variants) or 12 (`RFDETRSegNano`), and the definitive rule is that the resolution must be divisible by `patch_size * num_windows` for the selected model. | + +Common resolution values for currently documented checkpoints: + +- Detection: `384`, `512`, `576`, `704` +- Segmentation: `312`, `384`, `432`, `504`, `624`, `768` + +For example, `RFDETRSegXLarge` uses `624x624`, which is valid because `624` is divisible by `24`. + +## Regularization Parameters + +| Parameter | Type | Default | Description | +| -------------- | ------- | ------- | ------------------------------------------------------------------------------------- | +| `weight_decay` | `float` | `1e-4` | L2 regularization coefficient. Helps prevent overfitting by penalizing large weights. | + +## Hardware Parameters + +| Parameter | Type | Default | Description | +| ------------------------ | ------ | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `device` | `str` | `None` | Device to run training on. `None` means auto-detected by PyTorch Lightning. Options: `"cuda"`, `"cpu"`, `"mps"` (Apple Silicon). | +| `gradient_checkpointing` | `bool` | `False` | **Constructor-only parameter** — pass to the model constructor (`RFDETRMedium(gradient_checkpointing=True)`), not to `train()`. Re-computes activations during backprop to reduce memory usage by ~30-40% at the cost of ~20% slower training. | + +## EMA (Exponential Moving Average) + +| Parameter | Type | Default | Description | +| --------- | ------ | ------- | -------------------------------------------------------------------------------------------------------------------- | +| `use_ema` | `bool` | `True` | Enables Exponential Moving Average of weights. Produces a smoothed checkpoint that often improves final performance. | + +!!! info "What is EMA?" + + EMA maintains a moving average of the model weights throughout training. This smoothed version often generalizes better than the raw weights and is commonly used for the final model. + +## Checkpoint Parameters + +| Parameter | Type | Default | Description | +| --------------------- | ----- | ------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `checkpoint_interval` | `int` | `10` | Frequency (in epochs) at which model checkpoints are saved. More frequent saves provide better coverage but consume more storage. | +| `skip_best_epochs` | `int` | `0` | Ignore the first N epochs when tracking best checkpoints and early-stopping patience. Useful when fine-tuning from a prior checkpoint. | + +### Checkpoint Files + +During training, multiple checkpoints are saved: + +| File | Description | +| ----------------------------- | ----------------------------------------- | +| `checkpoint.pth` | Most recent checkpoint (for resuming) | +| `checkpoint_.pth` | Periodic checkpoint at epoch N | +| `checkpoint_best_ema.pth` | Best validation performance (EMA weights) | +| `checkpoint_best_regular.pth` | Best validation performance (raw weights) | +| `checkpoint_best_total.pth` | Final best model for inference | + +Best validation performance uses the task metric for the model family: box mAP for detection/segmentation and COCO +keypoint AP for keypoint preview. + +## Early Stopping Parameters + +| Parameter | Type | Default | Description | +| -------------------------- | ------- | ------- | ---------------------------------------------------------------------------------------- | +| `early_stopping` | `bool` | `False` | Enable early stopping based on the validation task metric. | +| `early_stopping_patience` | `int` | `10` | Number of epochs without improvement before stopping. | +| `early_stopping_min_delta` | `float` | `0.001` | Minimum metric change to qualify as an improvement. | +| `early_stopping_use_ema` | `bool` | `False` | Whether to track improvements using EMA model metrics. | +| `skip_best_epochs` | `int` | `0` | Ignore the first N epochs (0..N-1) for best-model selection and early-stopping patience. | + +### Early Stopping Example + +```python +model.train( + dataset_dir="path/to/dataset", + epochs=200, + batch_size=4, + early_stopping=True, + early_stopping_patience=15, + early_stopping_min_delta=0.005, + skip_best_epochs=3, +) +``` + +This configuration will: + +- Train for up to 200 epochs +- Ignore epochs 0-2 for best-checkpoint tracking and patience counting +- Stop early if the validation metric doesn't improve by at least 0.005 for 15 consecutive epochs + +!!! note "Transfer learning with `pretrain_weights`" + + When fine-tuning from `pretrain_weights`, the pretrained model's epoch-0 validation metric can be artificially high + relative to the training trajectory on the new dataset. This causes `checkpoint_best_total.pth` to always contain + the untrained pretrained weights and may trigger early stopping prematurely. Use `skip_best_epochs` to defer + best-checkpoint selection and patience counting until the model has had time to adapt. + +## Logging Parameters + +| Parameter | Type | Default | Description | +| ------------- | ------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tensorboard` | `bool` | `True` | Enable TensorBoard logging. Requires `pip install "rfdetr[loggers]"`. If the `tensorboard` package is not installed, training continues with a `UserWarning` and TensorBoard output is silently suppressed. | +| `wandb` | `bool` | `False` | Enable Weights & Biases logging. Requires `pip install "rfdetr[loggers]"`. | +| `project` | `str` | `None` | Project name for W&B logging. | +| `run` | `str` | `None` | Run name for W&B logging. If not specified, W&B assigns a random name. | + +### Logging Example + +```python +model.train( + dataset_dir="path/to/dataset", + epochs=100, + tensorboard=True, + wandb=True, + project="my-detection-project", + run="experiment-001", +) +``` + +## Evaluation Parameters + +| Parameter | Type | Default | Description | +| ----------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------ | +| `eval_max_dets` | `int` | `500` | Maximum number of detections per image considered during COCO evaluation. Lower values speed up evaluation. | +| `eval_interval` | `int` | `1` | Run COCO evaluation every N epochs. Set to a higher value to reduce evaluation overhead during long training runs. | +| `log_per_class_metrics` | `bool` | `True` | Log per-class AP metrics to the console and loggers. Disable to reduce log verbosity when there are many classes. | +| `progress_bar` | str \| bool \| None | `None` | Progress bar style: `"tqdm"`, `"rich"`, or `None`. Legacy booleans are still accepted. | + +## Keypoint Preview Parameters + +These parameters apply when training `RFDETRKeypointPreview` on COCO keypoint annotations or Ultralytics YOLO pose labels. + +| Parameter | Type | Default | Description | +| ----------------------------- | --------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `num_keypoints_per_class` | `list[int]` | `[17]` | **Constructor parameter** — pass to `RFDETRKeypointPreview(num_keypoints_per_class=...)`. Keypoint schema by model label slot. A zero entry marks a detection-only class slot; legacy checkpoints may use a background-first `[0, 17]` schema. | +| `keypoint_flip_pairs` | `list[int]` | `[]` | Flat left/right keypoint index pairs used to swap joints after horizontal-flip augmentation. YOLO `flip_idx` metadata is a permutation; RF-DETR converts it to this pair-list form during automatic schema inference when possible — it extracts only symmetric mutual pairs where `flip_idx[i] == j` and `flip_idx[j] == i`. Asymmetric entries and self-mapped keypoints (`flip_idx[i] == i`) are silently omitted; supply `keypoint_flip_pairs` explicitly when your `flip_idx` includes such entries. | +| `keypoint_l1_loss_coef` | `float` | `1.0` | Weight for keypoint coordinate L1 loss in keypoint preview training. | +| `keypoint_findable_loss_coef` | `float` | `1.0` | Weight for keypoint findable/objectness loss. | +| `keypoint_visible_loss_coef` | `float` | `1.0` | Weight for keypoint visibility loss. | +| `keypoint_nll_loss_coef` | `float` | `1.0` | Weight for keypoint negative-log-likelihood loss. | +| `keypoint_oks_sigmas` | `list[float] \| None` | `None` | Per-keypoint OKS sigma values used for COCO AP evaluation. When `None`, 17-keypoint person datasets use the evaluator's standard COCO sigmas and custom keypoint counts use RF-DETR's uniform custom fallback. Pass explicit values, such as schema-inferred sigmas, when you need a specific custom OKS policy. | + +!!! note "OKS sigma values: flat vs per-keypoint" + + `infer_coco_keypoint_schema` and `infer_yolo_keypoint_schema` return a flat sigma of 0.1 for all inferred keypoints, and the keypoint demos pass those values explicitly for custom datasets. If `keypoint_oks_sigmas=None`, COCO person-keypoint evaluation uses the standard 17-keypoint COCO sigmas, while non-17 custom keypoint counts use RF-DETR's uniform custom fallback. Flat custom sigmas are not directly comparable to official COCO benchmark numbers. + +## Advanced Parameters + +The parameters below are available for fine-grained control over training behaviour. Most users can leave these at their defaults. + +### Scheduler and Regularization + +| Parameter | Type | Default | Description | +| --------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------------- | +| `lr_scheduler` | `str` | `"step"` | Learning rate scheduler type. Options: `"step"` (step decay at `lr_drop`) or `"cosine"` (cosine annealing). | +| `lr_min_factor` | `float` | `0.0` | Floor for the cosine scheduler, expressed as a fraction of the initial LR. Ignored when using `"step"`. | +| `warmup_epochs` | `float` | `0.0` | Number of epochs for linear learning rate warmup at the start of training. | +| `drop_path` | `float` | `0.0` | Stochastic depth drop-path rate applied to the backbone. Higher values add more regularization. | + +### Runtime and Accelerator + +| Parameter | Type | Default | Description | +| ------------------- | ------ | -------- | ------------------------------------------------------------------------------------------------ | +| `accelerator` | `str` | `"auto"` | PyTorch Lightning accelerator selection. `"auto"` picks GPU if available, then MPS, then CPU. | +| `seed` | `int` | `None` | Global random seed for reproducibility. `None` means no fixed seed is set. | +| `fp16_eval` | `bool` | `False` | Run evaluation passes in FP16 precision. Reduces memory usage but may lower numerical precision. | +| `compute_val_loss` | `bool` | `True` | Compute and log the detection loss on the validation set each epoch. | +| `compute_test_loss` | `bool` | `True` | Compute and log the detection loss during the final test run. | + +### DataLoader Tuning + +| Parameter | Type | Default | Description | +| -------------------- | ------ | ------- | --------------------------------------------------------------------------------------------------------- | +| `pin_memory` | `bool` | `None` | Pin host memory in the DataLoader for faster GPU transfers. `None` defers to PyTorch Lightning's default. | +| `persistent_workers` | `bool` | `None` | Keep DataLoader worker processes alive between epochs. `None` defers to PyTorch Lightning's default. | +| `prefetch_factor` | `int` | `None` | Number of batches to prefetch per DataLoader worker. `None` uses PyTorch's built-in default. | + +## Complete Parameter Reference + +Below is a summary table of all training parameters: + +| Parameter | Type | Default | Description | +| -------------------------- | ------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `dataset_dir` | str | Required | Path to COCO or YOLO formatted dataset with train/valid/test splits. | +| `output_dir` | str | "output" | Directory for checkpoints, logs, and other training artifacts. | +| `epochs` | int | 100 | Number of full passes over the dataset. | +| `batch_size` | int or "auto" | 4 | Samples per iteration. Set to `"auto"` to let RF-DETR probe the GPU for the largest safe batch size. Balance with `grad_accum_steps`. | +| `grad_accum_steps` | int | 4 | Gradient accumulation steps for effective larger batch sizes. | +| `lr` | float | 1e-4 | Learning rate for the model (excluding encoder). | +| `lr_encoder` | float | 1.5e-4 | Learning rate for the backbone encoder. | +| `resolution` | int | Model-specific | Input image size (must be divisible by the selected model's `patch_size * num_windows`). | +| `weight_decay` | float | 1e-4 | L2 regularization coefficient. | +| `device` | str | "cuda" | Training device: cuda, cpu, or mps. | +| `use_ema` | bool | True | Enable Exponential Moving Average of weights. | +| `gradient_checkpointing` | bool | False | Trade compute for memory during backprop. | +| `checkpoint_interval` | int | 10 | Save checkpoint every N epochs. | +| `resume` | str | None | Path to checkpoint for resuming training. | +| `tensorboard` | bool | True | Enable TensorBoard logging. | +| `wandb` | bool | False | Enable Weights & Biases logging. | +| `project` | str | None | W&B project name. | +| `run` | str | None | W&B run name. | +| `early_stopping` | bool | False | Enable early stopping. | +| `early_stopping_patience` | int | 10 | Epochs without improvement before stopping. | +| `early_stopping_min_delta` | float | 0.001 | Minimum validation metric change to qualify as improvement. | +| `early_stopping_use_ema` | bool | False | Use EMA model for early stopping metrics. | +| `eval_max_dets` | int | 500 | Maximum detections per image considered during COCO evaluation. | +| `eval_interval` | int | 1 | Run COCO evaluation every N epochs. | +| `log_per_class_metrics` | bool | True | Log per-class AP metrics to the console and loggers. | +| `progress_bar` | str \| bool \| None | None | Progress bar style: `"tqdm"`, `"rich"`, or `None`. Legacy booleans are still accepted. | +| `accelerator` | str | "auto" | PyTorch Lightning accelerator. "auto" selects GPU/MPS/CPU automatically. | +| `seed` | int | None | Random seed for reproducibility. None means no fixed seed. | +| `lr_scheduler` | str | "step" | Learning rate scheduler type: "step" or "cosine". | +| `lr_min_factor` | float | 0.0 | Minimum LR as a fraction of the initial LR (cosine scheduler floor). | +| `warmup_epochs` | float | 0.0 | Number of linear warmup epochs at the start of training. | +| `drop_path` | float | 0.0 | Stochastic depth drop-path rate for the backbone. | +| `compute_val_loss` | bool | True | Compute and log loss during validation. | +| `compute_test_loss` | bool | True | Compute and log loss during the test run. | +| `fp16_eval` | bool | False | Run evaluation in FP16 precision to reduce memory usage. | +| `pin_memory` | bool | None | Pin DataLoader memory. None defers to PyTorch Lightning's default. | +| `persistent_workers` | bool | None | Keep DataLoader workers alive between epochs. None uses PTL default. | +| `prefetch_factor` | int | None | Number of batches prefetched per worker. None uses PyTorch default. | diff --git a/docs/reference/2xlarge.md b/docs/reference/2xlarge.md new file mode 100644 index 0000000..6f4ced1 --- /dev/null +++ b/docs/reference/2xlarge.md @@ -0,0 +1,10 @@ +--- +description: RF-DETR 2XLarge API reference for the highest-accuracy detection model. Requires rfdetr[plus] and the Platform Model License. +--- + +!!! warning "License Notice" + This model is licensed under the Platform Model License (PML-1.0) and requires `pip install rfdetr[plus]`. See the [rfdetr_plus repository](https://github.com/roboflow/rf-detr-plus) for license details. + +:::rfdetr.platform.models.RFDETR2XLarge + options: + inherited_members: true diff --git a/docs/reference/keypoint_train_config.md b/docs/reference/keypoint_train_config.md new file mode 100644 index 0000000..5b27d5d --- /dev/null +++ b/docs/reference/keypoint_train_config.md @@ -0,0 +1,7 @@ +--- +description: KeypointTrainConfig API reference for configuring RF-DETR keypoint training parameters with keypoint loss coefficients and optimization settings. +--- + +:::rfdetr.config.KeypointTrainConfig + options: + inherited_members: true diff --git a/docs/reference/kp_preview.md b/docs/reference/kp_preview.md new file mode 100644 index 0000000..d3c87f6 --- /dev/null +++ b/docs/reference/kp_preview.md @@ -0,0 +1,7 @@ +--- +description: API reference for the RF-DETR keypoint preview model — keypoint prediction and fine-tuning with inherited training and inference methods. +--- + +:::rfdetr.variants.RFDETRKeypointPreview + options: + inherited_members: true diff --git a/docs/reference/large.md b/docs/reference/large.md new file mode 100644 index 0000000..de99981 --- /dev/null +++ b/docs/reference/large.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Large API reference for high-accuracy real-time object detection with inherited training, prediction, export, and deployment methods. +--- + +:::rfdetr.variants.RFDETRLarge + options: + inherited_members: true diff --git a/docs/reference/medium.md b/docs/reference/medium.md new file mode 100644 index 0000000..fb2c6dd --- /dev/null +++ b/docs/reference/medium.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Medium API reference for balanced real-time object detection with inherited training, prediction, export, and deployment methods. +--- + +:::rfdetr.variants.RFDETRMedium + options: + inherited_members: true diff --git a/docs/reference/nano.md b/docs/reference/nano.md new file mode 100644 index 0000000..b8241c5 --- /dev/null +++ b/docs/reference/nano.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Nano API reference for the fastest real-time object detection model with inherited training, prediction, export, and deployment methods. +--- + +:::rfdetr.variants.RFDETRNano + options: + inherited_members: true diff --git a/docs/reference/rfdetr.md b/docs/reference/rfdetr.md new file mode 100644 index 0000000..2a93837 --- /dev/null +++ b/docs/reference/rfdetr.md @@ -0,0 +1,5 @@ +--- +description: RFDETR base class API reference covering common train, predict, export, optimize, and deploy methods for RF-DETR models. +--- + +:::rfdetr.detr.RFDETR diff --git a/docs/reference/seg_2xlarge.md b/docs/reference/seg_2xlarge.md new file mode 100644 index 0000000..26a0834 --- /dev/null +++ b/docs/reference/seg_2xlarge.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Seg 2XLarge API reference for the highest-accuracy instance segmentation model with inherited training and inference methods. +--- + +:::rfdetr.variants.RFDETRSeg2XLarge + options: + inherited_members: true diff --git a/docs/reference/seg_large.md b/docs/reference/seg_large.md new file mode 100644 index 0000000..6de0af7 --- /dev/null +++ b/docs/reference/seg_large.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Seg Large API reference for high-accuracy real-time instance segmentation with inherited training and inference methods. +--- + +:::rfdetr.variants.RFDETRSegLarge + options: + inherited_members: true diff --git a/docs/reference/seg_medium.md b/docs/reference/seg_medium.md new file mode 100644 index 0000000..d191bfb --- /dev/null +++ b/docs/reference/seg_medium.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Seg Medium API reference for balanced real-time instance segmentation with inherited training and inference methods. +--- + +:::rfdetr.variants.RFDETRSegMedium + options: + inherited_members: true diff --git a/docs/reference/seg_nano.md b/docs/reference/seg_nano.md new file mode 100644 index 0000000..19decb3 --- /dev/null +++ b/docs/reference/seg_nano.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Seg Nano API reference for the fastest instance segmentation model with inherited training, prediction, and export methods. +--- + +:::rfdetr.variants.RFDETRSegNano + options: + inherited_members: true diff --git a/docs/reference/seg_small.md b/docs/reference/seg_small.md new file mode 100644 index 0000000..97e7558 --- /dev/null +++ b/docs/reference/seg_small.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Seg Small API reference for efficient real-time instance segmentation with inherited training and inference methods. +--- + +:::rfdetr.variants.RFDETRSegSmall + options: + inherited_members: true diff --git a/docs/reference/seg_xlarge.md b/docs/reference/seg_xlarge.md new file mode 100644 index 0000000..ba292aa --- /dev/null +++ b/docs/reference/seg_xlarge.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Seg XLarge API reference for high-capacity instance segmentation with inherited training, prediction, and export methods. +--- + +:::rfdetr.variants.RFDETRSegXLarge + options: + inherited_members: true diff --git a/docs/reference/segmentation_train_config.md b/docs/reference/segmentation_train_config.md new file mode 100644 index 0000000..b903960 --- /dev/null +++ b/docs/reference/segmentation_train_config.md @@ -0,0 +1,7 @@ +--- +description: SegmentationTrainConfig API reference for configuring RF-DETR segmentation training parameters, dataset options, and optimization settings. +--- + +:::rfdetr.config.SegmentationTrainConfig + options: + inherited_members: true diff --git a/docs/reference/small.md b/docs/reference/small.md new file mode 100644 index 0000000..86b87cf --- /dev/null +++ b/docs/reference/small.md @@ -0,0 +1,7 @@ +--- +description: RF-DETR Small API reference for efficient real-time object detection with inherited training, prediction, export, and deployment methods. +--- + +:::rfdetr.variants.RFDETRSmall + options: + inherited_members: true diff --git a/docs/reference/train_config.md b/docs/reference/train_config.md new file mode 100644 index 0000000..e70b1a7 --- /dev/null +++ b/docs/reference/train_config.md @@ -0,0 +1,7 @@ +--- +description: TrainConfig API reference for RF-DETR detection training parameters, dataset configuration, optimization settings, and checkpoint behavior. +--- + +:::rfdetr.config.TrainConfig + options: + inherited_members: true diff --git a/docs/reference/training.md b/docs/reference/training.md new file mode 100644 index 0000000..ba7c332 --- /dev/null +++ b/docs/reference/training.md @@ -0,0 +1,126 @@ +--- +description: RF-DETR Lightning training API reference for RFDETRModelModule, RFDETRDataModule, build_trainer, callbacks, and training primitives. +--- + +# Training API Reference + +This page documents the training primitives that power RF-DETR. For a narrative guide with runnable examples, see [Custom Training API](../learn/train/customization.md). + +## RFDETRModelModule + +::: rfdetr.training.module_model.RFDETRModelModule + options: + show_source: false + members: + - __init__ + - on_fit_start + - on_train_batch_start + - transfer_batch_to_device + - training_step + - validation_step + - test_step + - predict_step + - configure_optimizers + - clip_gradients + - on_load_checkpoint + - reinitialize_detection_head + +--- + +## RFDETRDataModule + +::: rfdetr.training.module_data.RFDETRDataModule + options: + show_source: false + members: + - __init__ + - setup + - train_dataloader + - val_dataloader + - test_dataloader + - class_names + +--- + +## build_trainer + +::: rfdetr.training.trainer.build_trainer + options: + show_source: false + +--- + +## Callbacks + +### RFDETREMACallback + +::: rfdetr.training.callbacks.ema.RFDETREMACallback + options: + show_source: false + members: + - __init__ + +### BestModelCallback + +::: rfdetr.training.callbacks.best_model.BestModelCallback + options: + show_source: false + members: + - __init__ + +### RFDETREarlyStopping + +::: rfdetr.training.callbacks.best_model.RFDETREarlyStopping + options: + show_source: false + members: + - __init__ + +### DropPathCallback + +::: rfdetr.training.callbacks.drop_schedule.DropPathCallback + options: + show_source: false + members: + - __init__ + +### COCOEvalCallback + +::: rfdetr.training.callbacks.coco_eval.COCOEvalCallback + options: + show_source: false + members: + - __init__ + +--- + +## RFDETRCli + +!!! info "CLI requires the `train` and `cli` extras" + + ```bash + pip install "rfdetr[train,cli]" + ``` + + The `rfdetr` console script and its `--config` / `--print_config` flags are provided by `jsonargparse`, which is included in the `cli` extra. + +`RFDETRCli` is the command-line entry point for RF-DETR. It wraps +`RFDETRModelModule` and `RFDETRDataModule` under a single `rfdetr` command and +auto-generates four subcommands from the PyTorch Lightning CLI machinery: + +```bash +rfdetr fit --config configs/rfdetr_base.yaml +rfdetr validate --ckpt_path output/best.ckpt +rfdetr test --ckpt_path output/best.ckpt +rfdetr predict --ckpt_path output/best.ckpt +``` + +Both `model_config` and `train_config` are specified once; `RFDETRCli` +automatically links them to the datamodule so you do not need to repeat the +same arguments under `--data.*`. + +::: rfdetr.training.cli.RFDETRCli + options: + show_source: false + members: + - __init__ diff --git a/docs/reference/xlarge.md b/docs/reference/xlarge.md new file mode 100644 index 0000000..42868f0 --- /dev/null +++ b/docs/reference/xlarge.md @@ -0,0 +1,10 @@ +--- +description: RF-DETR XLarge API reference for high-capacity detection. Requires rfdetr[plus] and the Platform Model License. +--- + +!!! warning "License Notice" + This model is licensed under the Platform Model License (PML-1.0) and requires `pip install rfdetr[plus]`. See the [rfdetr_plus repository](https://github.com/roboflow/rf-detr-plus) for license details. + +:::rfdetr.platform.models.RFDETRXLarge + options: + inherited_members: true diff --git a/docs/resources/index.md b/docs/resources/index.md new file mode 100644 index 0000000..49feaad --- /dev/null +++ b/docs/resources/index.md @@ -0,0 +1,103 @@ +--- +description: RF-DETR resources — blog posts, videos, and guides for training, deploying, and running detection and segmentation models across cloud, edge, iOS, Jetson, and Lightning workflows. +hide: + - toc + - navigation +--- + +# Resources + +Use the resources below to learn how to train, build with, and deploy RF-DETR models both in the cloud and on your own hardware. + +
+ +- **SOTA Instance Segmentation with RF-DETR Seg (Preview) [October 2025]** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/10/rfdetr-seg.png) + + Read more about the RF-DETR Segmentation architecture and how to train an RF-DETR Seg (Preview) model. + + [:octicons-arrow-right-24: Learn more](https://blog.roboflow.com/rf-detr-segmentation-preview/) + +- **Announcing RF-DETR Nano, Small, and Medium [July 2025]** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/07/img-blog-rf-detr-1.png) + + Read more about the state-of-the-art RF-DETR Nano, Small, and Medium models we released in July 2025. + + [:octicons-arrow-right-24: Learn more](https://blog.roboflow.com/rf-detr-nano-small-medium/) + +- **RF-DETR: How to Train SOTA for Object Detection on a Custom Dataset [video]** + + --- + + ![](https://i.ytimg.com/vi/-OvpdLAElFA/maxresdefault.jpg) + + Learn how to train an RF-DETR model on a custom dataset. + + [:octicons-arrow-right-24: Watch the video](https://www.youtube.com/watch?v=-OvpdLAElFA) + +- **How to Train RF-DETR on a Custom Dataset [article]** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/03/img-blog-how-to-send-slack-notification-workflows-v4-1.png) + + Learn how to train an RF-DETR model on a custom dataset. + + [:octicons-arrow-right-24: Read the guide](https://blog.roboflow.com/train-rf-detr-on-a-custom-dataset/) + +- **Deploy RF-DETR on iOS [tutorial & example application]** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/07/img-blog-deep-learning-solves-frustrations--5--min.png) + + Read this guide to learn how to run RF-DETR models on an iPhone using the Roboflow iOS SDK. + + [:octicons-arrow-right-24: Get started](https://blog.roboflow.com/ios-rf-detr-nano/) + +- **How to Deploy RF-DETR to an NVIDIA Jetson [article]** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/06/inst-3-.png) + + Learn how to deploy an RF-DETR model to an NVIDIA Jetson using Roboflow Inference. + + [:octicons-arrow-right-24: Read the tutorial](https://blog.roboflow.com/how-to-deploy-rf-detr-to-an-nvidia-jetson/) + +- **Deploy RF-DETR with LitServe [Lightning AI Studio]** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/03/img-blog-nycerebro--2--min.png) + Learn how to deploy RF-DETR as a scalable inference server using LitServe, the AI model serving framework from Lightning AI. + + [:octicons-arrow-right-24: Open the Studio](https://lightning.ai/bhimrajyadav/studios/deploy-rf-detr-a-sota-real-time-object-detection-model-using-litserve) + +- **Train and Deploy RF-DETR Models with Roboflow** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/03/img-blog-nycerebro-2.png) + + Learn how to train RF-DETR models in the cloud with Roboflow and deploy your models on your own hardware with Roboflow Inference and Workflows. + + [:octicons-arrow-right-24: Get started](https://blog.roboflow.com/train-deploy-rf-detr/) + +- **RF-DETR: A SOTA Real-Time Object Detection Model** + + --- + + ![](https://blog.roboflow.com/content/images/size/w1000/format/webp/2025/03/img-blog-nycerebro--2--min.png) + + Read our announcement for RF-DETR, the first real-time model to achieve 60+ mean Average Precision when benchmarked on the COCO dataset. + + [:octicons-arrow-right-24: Read the announcement](https://blog.roboflow.com/rf-detr/) + +
diff --git a/docs/root-static/cace9ef287cb8d641db90d4295fcb1e1.txt b/docs/root-static/cace9ef287cb8d641db90d4295fcb1e1.txt new file mode 100644 index 0000000..b8a3721 --- /dev/null +++ b/docs/root-static/cace9ef287cb8d641db90d4295fcb1e1.txt @@ -0,0 +1 @@ +cace9ef287cb8d641db90d4295fcb1e1 diff --git a/docs/root-static/llms-full.txt b/docs/root-static/llms-full.txt new file mode 100644 index 0000000..3e7d6c3 --- /dev/null +++ b/docs/root-static/llms-full.txt @@ -0,0 +1,361 @@ +# RF-DETR — Full Documentation + +> RF-DETR is a real-time transformer architecture for object detection and instance segmentation by Roboflow. +> DINOv2 vision transformer backbone. ICLR 2026. SOTA on COCO (60.1 AP50:95, RF-DETR-2XL). +> Apache 2.0 license for core models (Nano through Large). Python 3.10+. +> Source: https://github.com/roboflow/rf-detr | Paper: https://arxiv.org/abs/2511.09554 + +--- + +## Overview + +RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy–latency trade-offs on Microsoft COCO and RF100-VL. + +RF-DETR is the first real-time model to exceed 60 mean Average Precision (mAP) when benchmarked on the COCO dataset. RF-DETR-2XL achieves 60.1 AP50:95 at 17.2 ms latency on NVIDIA T4 (TensorRT FP16, batch 1). + +**Licensing:** +- Core models (Nano, Small, Medium, Large) and all code: Apache 2.0 +- XLarge and 2XLarge detection models: PML 1.0 (requires rfdetr[plus]) +- Segmentation models follow the same tier structure + +--- + +## Installation + +**Requirements:** Python 3.10 or higher, pip or uv. + +```bash +pip install rfdetr +``` + +For uv projects: +```bash +uv add rfdetr +``` + +For XLarge/2XLarge models (PML 1.0): +```bash +pip install "rfdetr[plus]" +``` + +--- + +## Run Detection + +```python +from rfdetr import RFDETRLarge + +model = RFDETRLarge() +detections = model.predict("image.jpg", threshold=0.5) +print(detections) +``` + +**Available detection model classes:** +- `RFDETRNano` — 2.3 ms, 67.6 AP50, 30.5M params, 384×384 +- `RFDETRSmall` — 3.5 ms, 72.1 AP50, 32.1M params, 512×512 +- `RFDETRMedium` — 4.4 ms, 73.6 AP50, 33.7M params, 576×576 +- `RFDETRLarge` — 6.8 ms, 75.1 AP50 (56.5 AP50:95), 33.9M params, 704×704 +- `RFDETRXLarge` — 11.5 ms, 77.4 AP50, 126.4M params, 700×700 (requires rfdetr[plus]) +- `RFDETR2XLarge` — 17.2 ms, 78.5 AP50 (60.1 AP50:95), 126.9M params, 880×880 (requires rfdetr[plus]) + +All latency figures: NVIDIA T4, TensorRT FP16, batch size 1. + +**Load fine-tuned checkpoint:** +```python +model = RFDETRLarge(pretrain_weights="path/to/checkpoint_best_total.pth") +``` + +--- + +## Run Segmentation + +```python +from rfdetr import RFDETRSegLarge + +model = RFDETRSegLarge() +detections = model.predict("image.jpg", threshold=0.5) +``` + +**Available segmentation model classes:** +- `RFDETRSegNano` — 3.4 ms, 63.0 AP50, 33.6M params, 312×312 +- `RFDETRSegSmall` — 4.4 ms, 66.2 AP50, 33.7M params, 384×384 +- `RFDETRSegMedium` — 5.9 ms, 68.4 AP50, 35.7M params, 432×432 +- `RFDETRSegLarge` — 8.8 ms, 70.5 AP50 (47.1 AP50:95), 36.2M params, 504×504 +- `RFDETRSegXLarge` — 13.5 ms, 72.2 AP50, 38.1M params, 624×624 +- `RFDETRSeg2XLarge` — 21.8 ms, 73.1 AP50 (49.9 AP50:95), 38.6M params, 768×768 + +Segmentation models output instance masks in addition to bounding boxes. Masks are returned as a `torch.Tensor` or dict with `spatial_features`, `query_features`, and `bias` keys. + +--- + +## Train a Custom Model + +### Quick Start (Detection) + +```python +from rfdetr import RFDETRLarge + +model = RFDETRLarge() +model.train( + dataset_dir="./dataset", + epochs=50, + batch_size=4, + grad_accum_steps=4, # effective batch = batch_size * grad_accum_steps = 16 + lr=1e-4, + output_dir="./output" +) +``` + +### Quick Start (Segmentation) + +```python +from rfdetr import RFDETRSegLarge + +model = RFDETRSegLarge() +model.train(dataset_dir="./dataset", epochs=50, batch_size=4) +``` + +### Dataset Formats + +**COCO JSON format** (auto-detected): +``` +dataset/ + train/ + _annotations.coco.json + image1.jpg + ... + valid/ + _annotations.coco.json + image1.jpg + ... +``` + +**YOLO format** (set `dataset_file="yolo"`): +``` +dataset/ + data.yaml + train/ + images/ + labels/ + valid/ + images/ + labels/ +``` + +### Key Training Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `epochs` | 50 | Training epochs | +| `batch_size` | 4 | Images per GPU step | +| `grad_accum_steps` | 4 | Gradient accumulation steps (effective batch = batch_size × grad_accum_steps) | +| `lr` | 1e-4 | Peak learning rate | +| `output_dir` | `"output"` | Checkpoint directory | +| `checkpoint_interval` | 1 | Save checkpoint every N epochs | + +**Recommended effective batch size: 16** (e.g., batch_size=4, grad_accum_steps=4 on 8GB VRAM GPU). + +### Checkpoint Types + +After training, the `output_dir` contains: +- `checkpoint_best_total.pth` — best checkpoint by total loss (use for production) +- `checkpoint_best_ap.pth` — best checkpoint by COCO AP +- `checkpoint_epoch_N.pth` — periodic snapshots + +**Load fine-tuned model:** +```python +model = RFDETRLarge(pretrain_weights="output/checkpoint_best_total.pth") +``` + +### Advanced Training + +**Resume from checkpoint:** +```python +model.train(dataset_dir="./dataset", resume="output/checkpoint_epoch_20.pth") +``` + +**Multi-GPU DDP training:** +```bash +torchrun --nproc_per_node=4 train.py +``` + +Here, `train.py` refers to your own PyTorch distributed training entrypoint. In this standalone reference, it should be a script that initializes RF-DETR and calls `model.train(...)` with your dataset and training arguments, so `torchrun` can launch one process per GPU. +**Gradient checkpointing** (reduces VRAM at cost of speed): +```python +model.train(dataset_dir="./dataset", gradient_checkpointing=True) +``` + +### Training Loggers + +```python +# TensorBoard +model.train(dataset_dir="./dataset", tensorboard=True) + +# Weights and Biases +model.train(dataset_dir="./dataset", wandb=True) + +# ClearML is not yet integrated as a native RF-DETR logger. +# Do not pass clearml=True; that flag raises NotImplementedError +# when the trainer is built. + +# MLflow +model.train(dataset_dir="./dataset", mlflow=True) +``` + +--- + +## Export and Deploy + +### Export to ONNX + +```python +model.export(format="onnx") +``` + +Produces a single `.onnx` file compatible with ONNX Runtime and OpenCV DNN. Export works on CPU. + +### Export to TFLite + +```python +model.export(format="tflite") +``` + +### TensorRT Deployment + +Export to ONNX first, then convert with TensorRT tooling: +```bash +trtexec --onnx=model.onnx --saveEngine=model.trt --fp16 +``` + +### Deploy to Roboflow + +```python +model.deploy_to_roboflow( + workspace="your-workspace", + project_id="your-project-id", + version=1, + api_key="YOUR_API_KEY" +) +``` + +--- + +## Benchmarks + +**Benchmark methodology:** Accuracy measured with standard COCO metrics (pycocotools) on COCO val2017 split. Latency measured on NVIDIA T4 GPU, TensorRT 10.4, CUDA 12.4, FP16 precision, batch size 1, with a 200ms buffer between passes to reduce thermal variance. All accuracy and latency measurements use the same model artifact and numerical precision. + +### Detection Results + +| Architecture | COCO AP50 | COCO AP50:95 | RF100VL AP50 | RF100VL AP50:95 | Latency (ms) | Params (M) | Resolution | +|---|---|---|---|---|---|---|---| +| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384×384 | +| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512×512 | +| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576×576 | +| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704×704 | +| RF-DETR-XL | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700×700 | +| RF-DETR-2XL | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880×880 | + +### Segmentation Results + +| Architecture | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution | +|---|---|---|---|---|---| +| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312×312 | +| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384×384 | +| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432×432 | +| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504×504 | +| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624×624 | +| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768×768 | + +--- + +## Frequently Asked Questions + +**What is RF-DETR?** +RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy–latency trade-offs on COCO (60.1 AP50:95 for RF-DETR-2XL) and RF100-VL. + +**How does RF-DETR compare to YOLO11?** +RF-DETR-L achieves 56.5 AP50:95 on COCO at 6.8 ms latency on an NVIDIA T4, outperforming YOLO11x (54.7 AP) at lower latency. The DINOv2 backbone gives RF-DETR stronger performance on domain-shift benchmarks such as RF100-VL. + +**What GPU is required to train RF-DETR?** +A CUDA-capable GPU with at least 8 GB VRAM (e.g., NVIDIA RTX 3060, T4, A10) is recommended for fine-tuning. Smaller models (RF-DETR-N and RF-DETR-S) can fit in 6 GB VRAM with reduced batch size. CPU inference is supported for evaluation. + +**Which dataset formats does RF-DETR support?** +RF-DETR supports COCO JSON and YOLO-format datasets. Roboflow datasets export directly to both formats. Detection and segmentation datasets use the same format — the model variant determines the task. + +**Can RF-DETR run in real time?** +Yes. RF-DETR-N runs at 2.3 ms per frame on a T4 GPU (TensorRT FP16, batch 1), and RF-DETR-L at 6.8 ms — both well within real-time thresholds. ONNX and TFLite exports are available for edge deployment. + +**What is the difference between RF-DETR detection and segmentation models?** +Detection models output bounding boxes. Segmentation models additionally output instance masks. Both share the same backbone and training API; segmentation adds a mask head and requires COCO-format segmentation annotations. + +**Is RF-DETR open source?** +Yes. Core models (Nano through Large) and all training/inference code are released under the Apache 2.0 license. XLarge and 2XLarge models require the rfdetr[plus] package (PML 1.0 license). + +**How do I fine-tune RF-DETR on a custom dataset?** +Instantiate a model and call model.train() with your dataset directory in COCO JSON or YOLO format. Example: model = RFDETRLarge(); model.train(dataset_dir='./dataset', epochs=50, batch_size=4). The model downloads pretrained weights automatically and resumes from the best checkpoint. + +**How do I export RF-DETR to ONNX or TensorRT?** +Call model.export(format='onnx') after training or loading a checkpoint. ONNX export works on CPU. For TensorRT, export to ONNX first, then convert with trtexec --onnx=model.onnx --saveEngine=model.trt --fp16. + +**Which RF-DETR model size should I use?** +RF-DETR-Nano (2.3 ms, 67.6 AP50) is best for edge/real-time. RF-DETR-Large (6.8 ms, 56.5 AP50:95) is best accuracy–latency trade-off for server deployment. RF-DETR-2XLarge (17.2 ms, 60.1 AP50:95) maximizes accuracy when latency budget allows. + +Checkpoint note: current RFDETRLarge defaults to rf-detr-large-2026.pth. The older rf-detr-large.pth checkpoint is a legacy Large release kept for backward compatibility and has been superseded by the current release. + +--- + +## API Reference + +### RFDETR base class + +All model classes inherit from `RFDETR` and share these methods: + +- `predict(image, threshold=0.5)` — run inference on a single image (path, URL, numpy array, or PIL Image) +- `train(dataset_dir, epochs, batch_size, ...)` — fine-tune on custom dataset +- `export(format)` — export to ONNX, TFLite, or TensorRT +- `deploy_to_roboflow(workspace, project_id, version)` — deploy model to Roboflow hosted inference + +### TrainConfig + +Key fields for detection training configuration: + +| Field | Type | Description | +|---|---|---| +| `epochs` | int | Training epochs | +| `batch_size` | int | Images per GPU step | +| `grad_accum_steps` | int | Gradient accumulation steps | +| `lr` | float | Peak learning rate | +| `output_dir` | str | Checkpoint output directory | +| `resume` | str | Path to checkpoint to resume from | +| `gradient_checkpointing` | bool | Reduce VRAM at cost of speed | +| `tensorboard` | bool | Enable TensorBoard logging | +| `wandb` | bool | Enable Weights & Biases logging | + +--- + +## Migration Guide + +If upgrading from rfdetr < 1.4.0, update these imports: + +```python +# Old model class (deprecated) +from rfdetr import RFDETRBase +# New +from rfdetr import RFDETRLarge + +# Old +from rfdetr.util.misc import get_rank +# New (unchanged — still works) +from rfdetr.util.misc import get_rank +``` + +--- + +## External Links + +- [GitHub](https://github.com/roboflow/rf-detr) — Source code, issues, pull requests (Apache 2.0) +- [arXiv Paper](https://arxiv.org/abs/2511.09554) — RF-DETR: Real-Time Detection Transformers, ICLR 2026 +- [PyPI](https://pypi.org/project/rfdetr/) — pip install rfdetr +- [Hugging Face Demo](https://huggingface.co/spaces/Roboflow/RF-DETR) — Interactive demo +- [Discord](https://discord.gg/GbfgXGJ8Bk) — Community support +- [Documentation](https://rfdetr.roboflow.com/) — Full docs diff --git a/docs/root-static/llms.txt b/docs/root-static/llms.txt new file mode 100644 index 0000000..9cc2fde --- /dev/null +++ b/docs/root-static/llms.txt @@ -0,0 +1,67 @@ +# RF-DETR +> RF-DETR is a real-time object detection and instance segmentation transformer by Roboflow. +> DINOv2 backbone. ICLR 2026. SOTA on COCO (60.1 AP50:95, RF-DETR-2XL). +> Apache 2.0 for base models (Nano-Large). Install: pip install rfdetr. + +## Canonical Facts +- Product: RF-DETR (Roboflow Detection Transformer) +- Python package: `rfdetr` +- Maintainer: Roboflow +- Canonical docs: https://rfdetr.roboflow.com/ +- Source repository: https://github.com/roboflow/rf-detr +- Paper: https://arxiv.org/abs/2511.09554 +- Runtime: Python 3.10+, torch >=2.2.0, torchvision >=0.17.0, transformers >=5.1.0 and <6.0.0 +- Tasks: object detection and instance segmentation +- Backbone: DINOv2 vision transformer +- Dataset formats: COCO JSON and YOLO +- Export targets: ONNX, TensorRT, and TFLite +- License: Apache 2.0 for code and core Nano through Large models; XLarge and 2XLarge detection models require `rfdetr[plus]` and PML 1.0 + +## Getting Started +- [Install](https://rfdetr.roboflow.com/latest/getting-started/install/): pip install rfdetr +- [Run Detection](https://rfdetr.roboflow.com/latest/learn/run/detection/): Run RF-DETR detection on images, video, webcam, and streams +- [Run Segmentation](https://rfdetr.roboflow.com/latest/learn/run/segmentation/): Run RF-DETR Seg instance segmentation on images and video +- [Pretrained Models](https://rfdetr.roboflow.com/latest/learn/pretrained/): Nano, Small, Medium, Large, XLarge, and 2XLarge checkpoints + +## Train +- [Train Overview](https://rfdetr.roboflow.com/latest/learn/train/): Fine-tune RF-DETR detection and segmentation models +- [Dataset Formats](https://rfdetr.roboflow.com/latest/learn/train/dataset-formats/): COCO JSON and YOLO dataset formats +- [Training Parameters](https://rfdetr.roboflow.com/latest/learn/train/training-parameters/): RF-DETR hyperparameters and TrainConfig options +- [Advanced Training](https://rfdetr.roboflow.com/latest/learn/train/advanced/): Resume, early stopping, multi-GPU DDP, and memory optimization +- [Custom Training API](https://rfdetr.roboflow.com/latest/learn/train/customization/): PyTorch Lightning training primitives +- [Augmentations](https://rfdetr.roboflow.com/latest/learn/train/augmentations/): Albumentations presets and custom transforms +- [Training Loggers](https://rfdetr.roboflow.com/latest/learn/train/loggers/): TensorBoard, Weights and Biases, MLflow, and a ClearML workaround + +## Export and Deploy +- [Export](https://rfdetr.roboflow.com/latest/learn/export/): ONNX, TFLite, TensorRT +- [Deploy](https://rfdetr.roboflow.com/latest/learn/deploy/): deploy_to_roboflow() API + +## Benchmarks +- [COCO and RF100-VL Results](https://rfdetr.roboflow.com/latest/learn/benchmarks/): T4 TensorRT FP16, batch 1 +- Detection model range: RF-DETR-Nano 2.3 ms and 48.4 COCO AP50:95 through RF-DETR-2XLarge 17.2 ms and 60.1 COCO AP50:95 +- Segmentation model range: RF-DETR-Seg-Nano 3.4 ms and 40.3 COCO AP50:95 through RF-DETR-Seg-2XLarge 21.8 ms and 49.9 COCO AP50:95 + +## Model Selection Answers +- Use RF-DETR-Nano for lowest latency edge detection: 2.3 ms on T4 TensorRT FP16, 48.4 AP50:95 on COCO, 30.5 M params. +- Use RF-DETR-Small (3.5 ms, 53.0 AP50:95) or RF-DETR-Medium (4.4 ms, 54.7 AP50:95) when latency is tight but accuracy must improve over Nano. +- Use RF-DETR-Large for the best open Apache 2.0 accuracy-latency trade-off: 6.8 ms, 56.5 AP50:95 on COCO. +- Use RF-DETR-XLarge (11.5 ms, 58.6 AP50:95) or RF-DETR-2XLarge (17.2 ms, 60.1 AP50:95) when maximum accuracy matters; requires rfdetr[plus] and PML 1.0 license. +- Use RF-DETR-Seg-Large (8.8 ms, 47.1 AP50:95) or RF-DETR-Seg-2XLarge (21.8 ms, 49.9 AP50:95) when instance masks are required; detection-only models return bounding boxes only. + +## API Reference +- [RFDETR base class](https://rfdetr.roboflow.com/latest/reference/rfdetr/): train(), predict(), export(), deploy_to_roboflow() +- [Detection models](https://rfdetr.roboflow.com/latest/reference/nano/): RFDETRNano through RFDETR2XLarge +- [Segmentation models](https://rfdetr.roboflow.com/latest/reference/seg_nano/): RFDETRSegNano through RFDETRSeg2XLarge +- [TrainConfig](https://rfdetr.roboflow.com/latest/reference/train_config/): Detection training configuration schema +- [SegmentationTrainConfig](https://rfdetr.roboflow.com/latest/reference/segmentation_train_config/): Segmentation training configuration schema +- [Lightning Training](https://rfdetr.roboflow.com/latest/reference/training/): PyTorch Lightning training module, datamodule, callbacks, and trainer builder + +## Migration +- [Migration Guide](https://rfdetr.roboflow.com/latest/learn/migration/): Update imports from deprecated module paths + +## Optional +- [GitHub](https://github.com/roboflow/rf-detr): 6.4k stars, Apache 2.0 +- [arXiv Paper](https://arxiv.org/abs/2511.09554): ICLR 2026 +- [PyPI](https://pypi.org/project/rfdetr/): rfdetr package +- [Hugging Face Demo](https://huggingface.co/spaces/Roboflow/RF-DETR): Interactive RF-DETR demo +- [Discord](https://discord.gg/GbfgXGJ8Bk): RF-DETR community diff --git a/docs/root-static/robots.txt b/docs/root-static/robots.txt new file mode 100644 index 0000000..2b5f09c --- /dev/null +++ b/docs/root-static/robots.txt @@ -0,0 +1,34 @@ +User-agent: * +Allow: / +# Block numeric-versioned paths (/0.x/, /1.x/, etc.) — canonical URL is /latest/ +# robots.txt has no character classes, so one Disallow per leading digit is required. +Disallow: /0.*/ +Disallow: /1.*/ +Disallow: /2.*/ +Disallow: /3.*/ +Disallow: /4.*/ +Disallow: /5.*/ +Disallow: /6.*/ +Disallow: /7.*/ +Disallow: /8.*/ +Disallow: /9.*/ + +# AI crawlers — explicitly allow all paths including numeric-versioned ones. +# Named groups are independent of User-agent: * — they do NOT inherit its Disallow rules. +# Listing Allow: / here is sufficient; no Disallow needed. +User-agent: GPTBot +User-agent: ClaudeBot +User-agent: PerplexityBot +User-agent: CCBot +User-agent: Google-Extended +User-agent: Bytespider +User-agent: GoogleOther +User-agent: OAI-SearchBot +User-agent: ChatGPT-User +User-agent: Amazonbot +User-agent: FacebookBot +User-agent: Cohere-ai +User-agent: Applebot-Extended +Allow: / + +Sitemap: https://rfdetr.roboflow.com/sitemap.xml diff --git a/docs/stylesheets/cookbooks_card.css b/docs/stylesheets/cookbooks_card.css new file mode 100644 index 0000000..db809d8 --- /dev/null +++ b/docs/stylesheets/cookbooks_card.css @@ -0,0 +1,123 @@ +.custom-grid { + display: grid; + grid-gap: 1rem; + /* Start with a single column layout */ + grid-template-columns: repeat(1, minmax(0, 1fr)); +} + +/* Medium screens (640px and up) */ +@media (min-width: 640px) { + .custom-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +/* Large screens (1024px and up) */ +@media (min-width: 1024px) { + .custom-grid { + grid-template-columns: repeat(4, minmax(0, 1fr)); + } +} + +.custom-grid a { + color: inherit; + text-decoration: none; + display: block; +} + +.custom-grid a:hover { + color: inherit; /* Ensure color does not change on hover */ +} + +.cookbooks-subtitle { + color: var(--md-default-fg-color--light); + font-size: 1rem; + margin-bottom: 2rem; +} + +.repo-card { + background: radial-gradient(at right top, #A351FB66, #A5F9EA66); + border-radius: 0.5rem; + padding: 1rem; + transition: transform 0.2s ease-in-out; /* Smooth transition for the transform */ + cursor: pointer; +} + +.repo-card:hover { + transform: translateY(-0.125rem); /* Move up by -0.125rem on hover */ +} + +.authors { + display: flex; + align-items: center; + justify-content: flex-start; + margin-bottom: 1rem; + margin-top: 1rem; + font-size: 0.75rem; + line-height: 1rem; +} + +.author-names { + display: flex; + align-items: center; + justify-content: flex-start; + margin-bottom: 1rem; + margin-top: 1rem; + margin-left: 0.5rem; +} + +.author-container { + display: inline-flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + padding: 0; + margin: 0; + transition: transform 0.2s; +} + +.author-container.hover { + transform: translateY(-0.125rem); +} + +.author-container:hover { + transform: translateY(-0.125rem); +} + +.author-container a { + padding: 0; + margin: 0; + display: block; +} + +.author-avatar { + width: 32px; + height: 32px; + border-radius: 50%; + display: block; +} + +.author-name a { + color: inherit; /* Inherits the color from the parent element */ + text-decoration: none; /* Removes the underline */ +} + +.author-name a:hover { + color: inherit; /* Inherits the color from the parent element */ + text-decoration: underline; /* Adds underline on hover */ +} + +.label { + color: #fff; + padding: 2px 6px; + border-radius: 4px; + margin-right: 4px; +} + +.non-selectable-text { + -webkit-user-select: none; /* Safari */ + -moz-user-select: none; /* Firefox */ + -ms-user-select: none; /* Internet Explorer/Edge */ + user-select: none; /* Non-prefixed version, currently supported by Chrome, Opera, and Edge */ +} diff --git a/docs/stylesheets/rf.css b/docs/stylesheets/rf.css new file mode 100644 index 0000000..214208e --- /dev/null +++ b/docs/stylesheets/rf.css @@ -0,0 +1,270 @@ +:root, body { + /* Default to light theme */ + --md-primary-fg-color: #8315F9; + --md-code-hl-color: #8315F9 !important; + --md-accent-fg-color: #8315F9 !important; + --md-code-hl-color--light: #e8d2ff89 !important; + --md-footer-fg-color--light: rgb(111, 108, 121) !important; +} + +body.light { + /* Light theme */ + --md-text-color: #000000; + --md-h2-color: #000000; +} +.md-grid { + max-width: 85%; + margin: auto; +} +.sublist { + display: none; + list-style: none; + padding-left: 0; + background: white; + position: absolute; + border-radius: 8px; + margin-top: 0.25rem; +} +.sublist { + transition: opacity 0.5s ease-in-out; + display: none; + position: absolute; /* Ensure it overlaps and doesn't break flow */ + background: white; /* So it's visible */ + z-index: 1000; +} +.sublist li { + padding: 0.5rem; +} +#products-list *:hover .products-sublist { + display: block; +} +#resources-list *, #products-list * { + cursor: pointer; +} +.products-sublist, .resources-sublist { + padding: 0.25rem; +} +.products-sublist li:hover, .resources-sublist li:hover, .md-nav__link[href]:hover { + background: rgb(242, 241, 247) !important; + border-radius: 6px; + color: initial !important; +} +.md-search { + flex-grow: 2; +} +.portfolio-section .md-grid { + max-width: 100%; +} +.md-header__inner { + align-items: center; + display: grid; + grid-template-columns: 0.1fr 1.4fr 2fr 2fr; + padding-right: 1rem; +} +.md-search__inner { + max-width: 600px; + width: 100%; + min-width: 100%; +} +.md-search__input { + background: white; + border: 1px solid rgb(229, 231, 235); + border-radius: 8px; + color: rgb(111, 108, 121); +} +.md-search__form *, .md-search__icon, .md-search__input { + color: rgb(111, 108, 121); +} +.md-search__input::placeholder { + color: rgb(156, 163, 175); +} +.md-search__form { + background: none !important; +} +.md-footer, .md-footer-meta { + background-color: transparent; + color: rgb(111, 108, 121); +} +.md-typeset .tabbed-set > input:first-child:checked ~ .tabbed-labels > :first-child, .md-typeset .tabbed-set > input:nth-child(10):checked ~ .tabbed-labels > :nth-child(10), .md-typeset .tabbed-set > input:nth-child(11):checked ~ .tabbed-labels > :nth-child(11), .md-typeset .tabbed-set > input:nth-child(12):checked ~ .tabbed-labels > :nth-child(12), .md-typeset .tabbed-set > input:nth-child(13):checked ~ .tabbed-labels > :nth-child(13), .md-typeset .tabbed-set > input:nth-child(14):checked ~ .tabbed-labels > :nth-child(14), .md-typeset .tabbed-set > input:nth-child(15):checked ~ .tabbed-labels > :nth-child(15), .md-typeset .tabbed-set > input:nth-child(16):checked ~ .tabbed-labels > :nth-child(16), .md-typeset .tabbed-set > input:nth-child(17):checked ~ .tabbed-labels > :nth-child(17), .md-typeset .tabbed-set > input:nth-child(18):checked ~ .tabbed-labels > :nth-child(18), .md-typeset .tabbed-set > input:nth-child(19):checked ~ .tabbed-labels > :nth-child(19), .md-typeset .tabbed-set > input:nth-child(2):checked ~ .tabbed-labels > :nth-child(2), .md-typeset .tabbed-set > input:nth-child(20):checked ~ .tabbed-labels > :nth-child(20), .md-typeset .tabbed-set > input:nth-child(3):checked ~ .tabbed-labels > :nth-child(3), .md-typeset .tabbed-set > input:nth-child(4):checked ~ .tabbed-labels > :nth-child(4), .md-typeset .tabbed-set > input:nth-child(5):checked ~ .tabbed-labels > :nth-child(5), .md-typeset .tabbed-set > input:nth-child(6):checked ~ .tabbed-labels > :nth-child(6), .md-typeset .tabbed-set > input:nth-child(7):checked ~ .tabbed-labels > :nth-child(7), .md-typeset .tabbed-set > input:nth-child(8):checked ~ .tabbed-labels > :nth-child(8), .md-typeset .tabbed-set > input:nth-child(9):checked ~ .tabbed-labels > :nth-child(9) { + color: #8315F9; + border-bottom: 1px solid #8315F9; +} +.md-footer *, html .md-footer-meta.md-typeset a { + color: rgb(111, 108, 121); +} +.repo-card { + height: 100%; +} +.header-btn { + text-align: center; +} +.header-btn, .sublist { + box-shadow: rgb(255, 255, 255) 0px 0px 0px 0px, rgb(217, 215, 226) 0px 0px 0px 1px, rgb(217, 215, 226) 0px 1px 2px 0px; +} +.header-btn:hover { + box-shadow: rgb(255, 255, 255) 0px 0px 0px 0px, rgb(217, 215, 226) 0px 0px 0px 1px, rgb(217, 215, 226) 0px 1.0001px 2.00013px -0.0000327245px, rgba(0, 0, 0, 0) 0px 0.000065449px 0.000130898px -0.000065449px; +} +.md-typeset .headerlink:hover, .md-typeset .headerlink:target { + color: #8315F9; +} +.md-typeset h1, .md-header__title { + color: black; + font-weight: 800; +} +.md-typeset h1 { + font-weight: normal; + margin-bottom: 1rem; +} +body { + background: linear-gradient(to left bottom, rgb(243, 238, 255), rgb(255, 255, 255) 60%) no-repeat; +} + +/* .md-nav__link:has([tabindex=""]) { + text-transform: uppercase; +} */ + +.header-list { + display: flex; + align-items: center; + gap: 1rem; + list-style: none; + font-size: 0.75rem; + justify-content: flex-end; +} +.header-chevron { + border: solid currentcolor; + border-width: 0 1px 1px 0; + display: inline-block; + height: 0.4rem; + margin-left: 0.35rem; + transform: rotate(45deg) translateY(-0.1rem); + width: 0.4rem; +} +#dropdown-resources:checked ~ .resources-sublist, +#dropdown-products:checked ~ .products-sublist { + display: block; +} +body:not(:has(#dropdown-resources:checked)) .resources-sublist, +body:not(:has(#dropdown-products:checked)) .products-sublist { + display: none; +} +.md-typeset code .na { + color: #4a176f !important; +} +.md-nav__list label, .md-nav--secondary label { + /* text-transform: uppercase; */ + color: rgb(29, 29, 31) !important; + font-size: 0.7rem; + margin-bottom: 0; +} +.md-nav--secondary label { + margin-left: 0.5rem; +} + +.md-nav__link { + padding: 0.25rem; + padding-left: 0.5rem; + padding-right: 0.5rem; +} + +.md-nav__link--active { + background: rgb(243, 238, 255); + border-radius: 6px; + padding-top: 0.25rem; +} + +.md-tabs__item--active { + color: var(--md-primary-fg-color); + border-bottom: 2px solid var(--md-primary-fg-color); +} + +.md-nav--secondary .md-nav__title { + background: transparent; + box-shadow: none; +} + +.md-header, .md-tabs { + color: rgb(111, 108, 121); + background-color: transparent; +} +.md-header--shadow { + background: linear-gradient(to left bottom, rgb(243, 238, 255), rgb(255, 255, 255) 60%); + box-shadow: none; + border-bottom: 1px solid rgb(229, 231, 235); +} + +#item-logo { + display: none; +} +.md-main__inner, .md-header__inner, .md-grid { + max-width: 100%; +} +@media (max-width: 1200px) { + .md-header__inner { + display: flex; + } + .header-list { + display: none; + } + #item-logo { + display: block; + } +} +.md-content { + max-width: 40rem; + margin: auto; +} +/* // if no md-sidebar--primary, make .md-content full width */ +.md-main__inner:has(.md-sidebar--primary[hidden]) .md-content { + max-width: 100%; +} +.md-sidebar--primary { + flex: 0 20%; +} +.md-tabs { + border-bottom: 1px solid rgb(229, 231, 235); +} +.md-main__inner { + padding-top: 1rem; + margin-top: 0; +} + +body.dark { + /* Dark theme */ + --md-text-color: #FFFFFF; + --md-h2-color: #add8e6; +} + +body.light .md-content *, body.dark .md-content * { + color: var(--md-text-color) !important; +} + +body[data-md-url$="/cookbooks/"] .md-sidebar--primary, +body[data-md-url$="/cookbooks/"] .md-sidebar--secondary { + display: none; +} + +body[data-md-url$="/cookbooks/"] .md-content { + margin-left: 0; + width: 100%; +} + +.md-main, nav .md-grid, .md-header__inner { + max-width: 1600px; + width: 100%; + margin: auto; +} +.md-search__scrollwrap { + width: 100% !important; +} +.md-nav--secondary .md-nav__title { + position: initial !important; +} + +.md-header__title .md-ellipsis { + overflow: initial !important; + text-overflow: initial !important; +} +.md-search { + flex-grow: 0; +} diff --git a/docs/theme/main.html b/docs/theme/main.html new file mode 100644 index 0000000..323d738 --- /dev/null +++ b/docs/theme/main.html @@ -0,0 +1,283 @@ +{% extends "base.html" %} + +{% block htmltitle %} + {% if page and page.is_homepage %} + RF-DETR — Real-Time Object Detection & Segmentation by Roboflow + {% else %} + {{ super() }} + {% endif %} +{% endblock %} + +{% block extrahead %} + {{ super() }} + + + {% if page %} + {% set og_image_url = page.meta.image | default(config.extra.og_image | default('')) %} + {# Open Graph — all pages #} + + + + + {% if og_image_url %} + {% if og_image_url == config.extra.og_image %} + {% endif %}{% endif %} + + {# Twitter Card — all pages #} + + + + + {% if og_image_url %}{% endif %} + + {# JSON-LD: Organization + SoftwareApplication + WebSite — homepage only #} + {% if page.is_homepage %} + + + {% else %} + {# article:modified_time — content pages only; emit only when a full ISO-8601 date-time is available #} + {# striptags: plugin may return a