chore: import upstream snapshot with attribution
CPU tests Workflow / Testing (ubuntu-latest, 3.12) (push) Failing after 1s
CPU tests Workflow / Testing (ubuntu-latest, 3.13) (push) Failing after 0s
Mypy Type Check / Type Check (push) Failing after 0s
Docs/Test WorkFlow / Test docs build (push) Failing after 1s
PR Conflict Labeler / labeling (push) Failing after 1s
Dependency resolution / Resolve [tflite] extra — Python 3.12 (push) Failing after 0s
Smoke Tests / try-all-models (ubuntu-latest, 3.10) (push) Failing after 0s
Smoke Tests / try-all-models (ubuntu-latest, 3.13) (push) Failing after 1s
CPU tests Workflow / build-pkg (push) Failing after 1s
CPU tests Workflow / Testing (ubuntu-latest, 3.10) (push) Failing after 0s
CPU tests Workflow / Testing (ubuntu-latest, 3.11) (push) Failing after 0s
Smoke Tests / try-all-models (macos-latest, 3.10) (push) Has been cancelled
Smoke Tests / try-all-models (macos-latest, 3.13) (push) Has been cancelled
Smoke Tests / try-all-models (windows-latest, 3.10) (push) Has been cancelled
Smoke Tests / try-all-models (windows-latest, 3.13) (push) Has been cancelled
CPU tests Workflow / Testing (macos-latest, 3.10) (push) Has been cancelled
CPU tests Workflow / Testing (macos-latest, 3.13) (push) Has been cancelled
CPU tests Workflow / Testing (windows-latest, 3.10) (push) Has been cancelled
CPU tests Workflow / Testing (windows-latest, 3.13) (push) Has been cancelled
CPU tests Workflow / testing-guardian (push) Has been cancelled
GPU tests Workflow / Testing (push) Has been cancelled
CPU tests Workflow / Testing (ubuntu-latest, 3.12) (push) Failing after 1s
CPU tests Workflow / Testing (ubuntu-latest, 3.13) (push) Failing after 0s
Mypy Type Check / Type Check (push) Failing after 0s
Docs/Test WorkFlow / Test docs build (push) Failing after 1s
PR Conflict Labeler / labeling (push) Failing after 1s
Dependency resolution / Resolve [tflite] extra — Python 3.12 (push) Failing after 0s
Smoke Tests / try-all-models (ubuntu-latest, 3.10) (push) Failing after 0s
Smoke Tests / try-all-models (ubuntu-latest, 3.13) (push) Failing after 1s
CPU tests Workflow / build-pkg (push) Failing after 1s
CPU tests Workflow / Testing (ubuntu-latest, 3.10) (push) Failing after 0s
CPU tests Workflow / Testing (ubuntu-latest, 3.11) (push) Failing after 0s
Smoke Tests / try-all-models (macos-latest, 3.10) (push) Has been cancelled
Smoke Tests / try-all-models (macos-latest, 3.13) (push) Has been cancelled
Smoke Tests / try-all-models (windows-latest, 3.10) (push) Has been cancelled
Smoke Tests / try-all-models (windows-latest, 3.13) (push) Has been cancelled
CPU tests Workflow / Testing (macos-latest, 3.10) (push) Has been cancelled
CPU tests Workflow / Testing (macos-latest, 3.13) (push) Has been cancelled
CPU tests Workflow / Testing (windows-latest, 3.10) (push) Has been cancelled
CPU tests Workflow / Testing (windows-latest, 3.13) (push) Has been cancelled
CPU tests Workflow / testing-guardian (push) Has been cancelled
GPU tests Workflow / Testing (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
rfdetr.roboflow.com
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
@@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 25.3.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 600 600" style="enable-background:new 0 0 600 600;" xml:space="preserve">
|
||||
<style type="text/css">
|
||||
.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#6706CE;}
|
||||
.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#FFFFFF;}
|
||||
</style>
|
||||
<g>
|
||||
<circle class="st0" cx="300" cy="300" r="300"/>
|
||||
</g>
|
||||
<g>
|
||||
<path class="st1" d="M279.34,267.67c27.7-21.58,15.55-51.11-30.24-45.4c-53.03,6.6-120.62,58.45-137.46,120.7
|
||||
c-1.38,5.09-2.45,10.27-3.19,15.5c-0.73,5.15-1.12,10.35-1.17,15.55c-0.05,5.15,0.25,10.3,0.89,15.4
|
||||
c0.64,5.11,1.64,10.17,2.98,15.14c1.36,5.03,3.06,9.97,5.09,14.76c2.09,4.92,4.51,9.68,7.24,14.26c2.83,4.76,5.98,9.31,9.4,13.64
|
||||
c11.67,14.78,26.65,27.16,42.63,37.02c3.46,2.14,7.18,4.21,11.25,4.41c5.94,0.28,10.44-4.44,10.68-10.23
|
||||
c0.09-2.18-0.46-4.24-1.46-6c-13.42-23.67-24.52-58.81-22.24-80.99C179.43,336.29,235.28,301.98,279.34,267.67z"/>
|
||||
<path class="st1" d="M500.28,363.84c-2.9-6.91-10.16-10.81-17.51-10.15c-5.02,0.45-9.17,3.13-12.55,6.68
|
||||
c-5.49,4.97-9.31,11.89-13.43,17.91c-4.86,7.08-9.94,14.02-15.41,20.63c-10.8,13.04-23.24,24.79-37.76,33.61
|
||||
c-24.63,14.95-47.17,22.12-74.27,8.31c-19.83-15.86-13.13-34.6,3.29-49.87c28.08-26.1,61.93-44.42,88.52-72.44
|
||||
c25-26.34,47.05-59.11,52.24-95.78c1.46-10.27,1.48-20.74-0.17-31c-7.8-48.31-51.48-78.97-95.59-92.01
|
||||
C321.47,83.13,259.6,88,206.48,112.56c-21.86,10.11-42.23,23.65-59.94,39.98c-7.85,7.24-15.24,15.01-22.1,23.2
|
||||
c-11.1,13.27-23.73,28.77-25.89,46.57c-1.33,10.99,8.58,18.59,18.99,17.18c6.98-0.94,13.16-5.15,18.3-9.74
|
||||
c5.97-5.33,11.45-11.25,17.21-16.82c17.38-16.82,35.94-32.47,58.01-42.79c28.11-13.15,59.61-19.55,90.6-16.32
|
||||
c31.43,3.27,69.67,17.44,81.15,49.98c11.97,33.97-13.73,67.36-37.18,89.06c-38.29,35.44-110.99,71.63-102.98,134.19
|
||||
c4.29,33.51,28.4,62.19,59.24,74.96c53.66,22.23,111.74-4.84,148.89-46.18c12.46-13.24,30.38-35.82,46.68-69.46
|
||||
C500.61,379.84,503.23,370.87,500.28,363.84z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1,19 @@
|
||||
---
|
||||
title: Changelog
|
||||
description: RF-DETR release history and changelog.
|
||||
hide:
|
||||
- navigation
|
||||
---
|
||||
|
||||
# Changelog
|
||||
|
||||
RF-DETR release notes are maintained on [GitHub Releases](https://github.com/roboflow/rf-detr/releases).
|
||||
Use the release feed to review versioned package changes, migration notes, and model updates.
|
||||
|
||||
- [Install the latest PyPI package](https://pypi.org/project/rfdetr/)
|
||||
- [Migration guide](getting-started/migration.md) — upgrade steps between major versions
|
||||
- [Cookbooks](cookbooks.md) — runnable notebooks for training, fine-tuning, export, and deployment
|
||||
|
||||
---
|
||||
|
||||
--8<-- "CHANGELOG.md"
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
template: notebooks.html
|
||||
description: Practical RF-DETR cookbooks — runnable notebooks covering training, fine-tuning, export, and deployment workflows.
|
||||
hide:
|
||||
- navigation
|
||||
- toc
|
||||
---
|
||||
@@ -0,0 +1,58 @@
|
||||
# Notebooks
|
||||
|
||||
Each `.ipynb` file here is rendered as a page under `/cookbooks/` in the docs site.
|
||||
|
||||
Cards on the cookbooks landing page are driven by [`cards.yaml`](cards.yaml). The MkDocs hook
|
||||
`docs/hooks/cookbooks_cards.py` loads that file and exposes it to `docs/theme/notebooks.html`,
|
||||
which renders each entry as a card via a Jinja loop.
|
||||
|
||||
## Converting a jupytext `.py` to `.ipynb`
|
||||
|
||||
Cookbook source files live as jupytext percent-format `.py` scripts (e.g. `fine-tune_keypoints.py`) inside `docs/cookbooks/`. Each script requires at minimum a **docs render copy**; some also have a `notebooks/` copy for users who want to run it directly. Regenerate every existing copy after each edit:
|
||||
|
||||
```bash
|
||||
# Docs render copy (served by mkdocs-jupyter at /cookbooks/) — always required
|
||||
jupytext --to notebook fine-tune_keypoints.py --output docs/cookbooks/fine-tune_keypoints.ipynb
|
||||
|
||||
# Runnable copy in notebooks/ — only for notebooks explicitly placed there
|
||||
jupytext --to notebook fine-tune_keypoints.py --output notebooks/fine-tune_keypoints.ipynb
|
||||
```
|
||||
|
||||
New notebooks default to the docs-only copy. Add a `notebooks/` copy only when there is an explicit need (e.g. a runnable starter notebook shipped with the repo). Omit `--execute` — notebooks are rendered statically by `mkdocs-jupyter` with `execute: false`, so pre-run outputs in the `.ipynb` are displayed as-is.
|
||||
|
||||
If jupytext is not installed: `pip install jupytext` (or `uv add jupytext --dev`).
|
||||
|
||||
## Adding a notebook
|
||||
|
||||
1. Add the `.ipynb` file here, named after its content (e.g. `custom-augmentations.ipynb`, `onnx-export.ipynb`).
|
||||
2. Add a new entry to `docs/cookbooks/cards.yaml` under the `cards:` list:
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
|
||||
```yaml
|
||||
- href: content-slug/
|
||||
name: Short Title
|
||||
labels: [LABEL1, LABEL2]
|
||||
version: vX.Y.0
|
||||
author: GitHubUsername
|
||||
description: One sentence describing what the notebook demonstrates.
|
||||
```
|
||||
|
||||
Available labels (reuse these to keep tags standardised): `TRAINING`, `AUGMENTATION`, `EXPORT`, `TFLITE`, `PYTORCH LIGHTNING`, `INFERENCE`, `SEGMENTATION`, `DEPLOY`.
|
||||
Current tag colours are assigned dynamically by the docs UI, so they may change if cards or labels are added or reordered.
|
||||
|
||||
## Removing a notebook
|
||||
|
||||
1. Delete the `.ipynb` file.
|
||||
2. Remove the matching entry (the `- href: content-slug/` block) from `docs/cookbooks/cards.yaml`.
|
||||
|
||||
## Current notebooks
|
||||
|
||||
| File | Card title | Version |
|
||||
| ----------------------------------- | ----------------------------------------------- | ------- |
|
||||
| `custom-augmentations.ipynb` | Custom Augmentations and Live Training Progress | v1.5.0 |
|
||||
| `fine-tune_detection.ipynb` | Fine-Tune RF-DETR Object Detection | v1.8.0 |
|
||||
| `fine-tune_keypoints.ipynb` | Fine-Tune RF-DETR Keypoint Detection | v1.8.0 |
|
||||
| `fine-tune_segmentation.ipynb` | Fine-Tune RF-DETR Instance Segmentation | v1.8.2 |
|
||||
| `inference-latency-benchmark.ipynb` | Inference Latency Benchmark | v1.8.2 |
|
||||
| `pytorch-lightning.ipynb` | Training with PyTorch Lightning | v1.6.0 |
|
||||
@@ -0,0 +1,37 @@
|
||||
cards:
|
||||
- href: fine-tune_detection/
|
||||
name: "Fine-Tune RF-DETR Object Detection"
|
||||
labels: [TRAINING, PYTORCH LIGHTNING, INFERENCE]
|
||||
version: v1.8.2
|
||||
author: Borda
|
||||
description: "Fine-tune RF-DETR object detection on any COCO detection dataset from Roboflow Universe — covers PPE safety, traffic, sports, infrastructure, and marine-wildlife use cases."
|
||||
- href: inference-latency-benchmark/
|
||||
name: "Inference Latency Benchmark"
|
||||
labels: [INFERENCE, ONNX, GPU]
|
||||
version: v1.8.1
|
||||
author: Borda
|
||||
description: "Benchmark detection, segmentation, and keypoint models across FP32, FP16+JIT, and ONNX Runtime — measures latency and FPS on GPU using CUDA events."
|
||||
- href: fine-tune_segmentation/
|
||||
name: "Fine-Tune RF-DETR Instance Segmentation"
|
||||
labels: [TRAINING, PYTORCH LIGHTNING, SEGMENTATION, INFERENCE]
|
||||
version: v1.8.1
|
||||
author: Borda
|
||||
description: "Fine-tune RF-DETR instance segmentation on any COCO segmentation dataset from Roboflow Universe — covers training, mask loss metrics, and inference with mask overlays."
|
||||
- href: fine-tune_keypoints/
|
||||
name: "Fine-Tune RF-DETR Keypoint Detection"
|
||||
labels: [TRAINING, PYTORCH LIGHTNING, INFERENCE]
|
||||
version: v1.8.0
|
||||
author: Borda
|
||||
description: "Fine-tune RF-DETR keypoint detection on any COCO keypoint dataset from Roboflow Universe — covers training, metrics, and inference with uncertainty ellipses."
|
||||
- href: custom-augmentations/
|
||||
name: "Custom Augmentations and Live Training Progress"
|
||||
labels: [TRAINING, AUGMENTATION]
|
||||
version: v1.5.0
|
||||
author: Borda
|
||||
description: "Add custom Albumentations augmentations and monitor training metrics in real time."
|
||||
- href: pytorch-lightning/
|
||||
name: "Training with PyTorch Lightning"
|
||||
labels: [TRAINING, PYTORCH LIGHTNING]
|
||||
version: v1.6.0
|
||||
author: Borda
|
||||
description: "Train RF-DETR with PyTorch Lightning — callbacks, checkpointing, and the LightningModule API."
|
||||
@@ -0,0 +1,417 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0f72b166",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Custom Augmentations and Live Training Progress\n",
|
||||
"\n",
|
||||
"[](https://colab.research.google.com/github/roboflow/rf-detr/blob/develop/docs/cookbooks/custom-augmentations.ipynb)\n",
|
||||
"\n",
|
||||
"**RF-DETR** is a real-time object detection model that combines the accuracy of\n",
|
||||
"transformer-based detectors with inference speeds suitable for production.\n",
|
||||
"1.5.0 brings two headline additions:\n",
|
||||
"\n",
|
||||
"1. **Custom training augmentations via Albumentations** — a flexible system that\n",
|
||||
" lets you control exactly how images are transformed during training, with bounding\n",
|
||||
" boxes and segmentation masks kept in sync automatically. Four ready-made presets\n",
|
||||
" cover the most common scenarios out of the box.\n",
|
||||
"2. **Live progress bars and structured epoch logs** — per-epoch rich / tqdm progress\n",
|
||||
" so you can monitor batch-level metrics without parsing raw log output.\n",
|
||||
"\n",
|
||||
"You will learn how to:\n",
|
||||
"- Explore and compare the four built-in augmentation presets\n",
|
||||
"- Visually inspect augmented samples *before* committing to a full training run\n",
|
||||
"- Define a fully custom augmentation pipeline\n",
|
||||
"- Train RF-DETR with your chosen augmentation config\n",
|
||||
"- Run inference with the trained model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "49ad10f9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Install RF-DETR 1.5.0\n",
|
||||
"\n",
|
||||
"`rfdetr` includes the augmentation system and progress-bar support introduced in\n",
|
||||
"this release. `supervision` handles visualization."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5c1df817",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install -q rfdetr==1.5.0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "075cf0b4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Check GPU availability\n",
|
||||
"\n",
|
||||
"RF-DETR trains on GPU when one is available and falls back to CPU otherwise.\n",
|
||||
"The cell below detects your device and prints VRAM size — a useful sanity check\n",
|
||||
"before choosing batch size."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d005aa12",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"\n",
|
||||
"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
|
||||
"print(f\"Using device: {device}\")\n",
|
||||
"if device == \"cuda\":\n",
|
||||
" print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n",
|
||||
" print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n",
|
||||
"\n",
|
||||
"# Scale data-loader workers with available CPUs so the GPU is kept fed.\n",
|
||||
"num_workers = max(os.cpu_count() or 0, 2)\n",
|
||||
"print(f\"Data loader workers: {num_workers}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c926124e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Download COCO 2017\n",
|
||||
"\n",
|
||||
"We use the official train/val splits — no manual splitting needed.\n",
|
||||
"\n",
|
||||
"| Split | Images | Size |\n",
|
||||
"|---|---|---|\n",
|
||||
"| `train2017` | ~118 000 | ~18 GB |\n",
|
||||
"| `val2017` | 5 000 | ~1 GB |\n",
|
||||
"| annotations | — | ~241 MB |"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a3cc746d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!wget -q --show-progress http://images.cocodataset.org/zips/train2017.zip -O train2017.zip\n",
|
||||
"!wget -q --show-progress http://images.cocodataset.org/zips/val2017.zip -O val2017.zip\n",
|
||||
"!wget -q --show-progress http://images.cocodataset.org/annotations/annotations_trainval2017.zip -O annotations.zip"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "985d1e18",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!unzip -q train2017.zip\n",
|
||||
"!unzip -q val2017.zip\n",
|
||||
"!unzip -q annotations.zip"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "27b7ae27",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Set up the dataset directory structure\n",
|
||||
"\n",
|
||||
"`model.train()` expects:\n",
|
||||
"```\n",
|
||||
"dataset/\n",
|
||||
" train/ _annotations.coco.json + images\n",
|
||||
" valid/ _annotations.coco.json + images\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "da25896a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!mkdir -p coco_demo\n",
|
||||
"!mv train2017 coco_demo/train\n",
|
||||
"!mv val2017 coco_demo/valid\n",
|
||||
"!cp annotations/instances_train2017.json coco_demo/train/_annotations.coco.json\n",
|
||||
"!cp annotations/instances_val2017.json coco_demo/valid/_annotations.coco.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "69343c86",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Explore the built-in augmentation presets\n",
|
||||
"\n",
|
||||
"RF-DETR ships four ready-made presets tuned for common use cases. Each preset\n",
|
||||
"is a plain Python dict mapping Albumentations transform names to their constructor\n",
|
||||
"kwargs (including `p`, the probability of applying the transform). You can\n",
|
||||
"inspect, merge, or extend them like any other dict.\n",
|
||||
"\n",
|
||||
"| Preset | When to use |\n",
|
||||
"|---|---|\n",
|
||||
"| `AUG_CONSERVATIVE` | Small datasets (< 500 images) — gentle transforms to avoid overfitting |\n",
|
||||
"| `AUG_AGGRESSIVE` | Large datasets (2 000+ images) — stronger augmentations for better generalisation |\n",
|
||||
"| `AUG_AERIAL` | Satellite and overhead imagery — rotation-invariant transforms |\n",
|
||||
"| `AUG_INDUSTRIAL` | Manufacturing and inspection data — handles structured textures |"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "409e006a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import rfdetr.datasets.aug_configs as aug_config\n",
|
||||
"from rfdetr.datasets.aug_configs import AUG_AGGRESSIVE\n",
|
||||
"\n",
|
||||
"for name in (\"AUG_CONSERVATIVE\", \"AUG_AGGRESSIVE\", \"AUG_AERIAL\", \"AUG_INDUSTRIAL\"):\n",
|
||||
" preset = getattr(aug_config, name)\n",
|
||||
" print(f\"\\n{name}:\")\n",
|
||||
" for transform, params in preset.items():\n",
|
||||
" print(f\" {transform}: {params}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d10837d5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Inspect augmented samples before training\n",
|
||||
"\n",
|
||||
"Setting `save_dataset_grids=True` writes 3×3 JPEG grids of augmented training\n",
|
||||
"and validation images to `output_dir` *before any weight updates occur*. Use\n",
|
||||
"this to visually sanity-check your pipeline in seconds — catching problems like\n",
|
||||
"flipped labels or extreme colour shifts before committing to a full training run.\n",
|
||||
"\n",
|
||||
"Saved files:\n",
|
||||
"```\n",
|
||||
"output/\n",
|
||||
" train_batch0_grid.jpg train_batch1_grid.jpg train_batch2_grid.jpg\n",
|
||||
" val_batch0_grid.jpg val_batch1_grid.jpg val_batch2_grid.jpg\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d7797651",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from rfdetr import RFDETRNano # smallest & fastest model — ideal for demos\n",
|
||||
"\n",
|
||||
"DATASET_DIR = \"coco_demo\"\n",
|
||||
"OUTPUT_DIR = \"output\"\n",
|
||||
"os.makedirs(OUTPUT_DIR, exist_ok=True)\n",
|
||||
"\n",
|
||||
"model = RFDETRNano()\n",
|
||||
"model.train(\n",
|
||||
" dataset_dir=str(DATASET_DIR),\n",
|
||||
" epochs=1, # one epoch is enough to generate the grids\n",
|
||||
" batch_size=12,\n",
|
||||
" aug_config=AUG_AGGRESSIVE,\n",
|
||||
" save_dataset_grids=True,\n",
|
||||
" output_dir=OUTPUT_DIR,\n",
|
||||
" device=device,\n",
|
||||
" num_workers=num_workers,\n",
|
||||
" run_test=False,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7b7cb782",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Display the saved grids inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "28df5b59",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import matplotlib.image as mpimg\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"grids = sorted(Path(OUTPUT_DIR).glob(\"*_grid.jpg\"))\n",
|
||||
"if grids:\n",
|
||||
" fig, axes = plt.subplots(len(grids), 1, figsize=(6, 6 * len(grids)))\n",
|
||||
" if len(grids) == 1:\n",
|
||||
" axes = [axes]\n",
|
||||
" for ax, grid_path in zip(axes, grids):\n",
|
||||
" ax.imshow(mpimg.imread(grid_path))\n",
|
||||
" ax.set_title(grid_path.name)\n",
|
||||
" ax.axis(\"off\")\n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d7f10179",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. Choose your augmentation config\n",
|
||||
"\n",
|
||||
"Pick one of the four options below. Option A (a built-in preset) is the fastest\n",
|
||||
"way to get started. Options B–D give increasing levels of control. The selected\n",
|
||||
"`AUG_CONFIG` is passed straight to `model.train()` in the next section."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f62b76de",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# --- Option A: built-in preset ---\n",
|
||||
"AUG_CONFIG = AUG_AGGRESSIVE\n",
|
||||
"\n",
|
||||
"# --- Option B: extend a preset ---\n",
|
||||
"# AUG_CONFIG = {**AUG_AGGRESSIVE, \"VerticalFlip\": {\"p\": 0.3}}\n",
|
||||
"\n",
|
||||
"# --- Option C: fully custom ---\n",
|
||||
"# AUG_CONFIG = {\n",
|
||||
"# \"HorizontalFlip\": {\"p\": 0.5},\n",
|
||||
"# \"Rotate\": {\"limit\": 15, \"p\": 0.3},\n",
|
||||
"# \"RandomBrightnessContrast\": {\"brightness_limit\": 0.2, \"contrast_limit\": 0.2, \"p\": 0.4},\n",
|
||||
"# \"GaussianBlur\": {\"blur_limit\": 3, \"p\": 0.2},\n",
|
||||
"# }\n",
|
||||
"\n",
|
||||
"# --- Option D: no augmentations ---\n",
|
||||
"# AUG_CONFIG = {}\n",
|
||||
"\n",
|
||||
"print(\"Selected augmentation config:\")\n",
|
||||
"for transform, params in AUG_CONFIG.items():\n",
|
||||
" print(f\" {transform}: {params}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5a5ff7b9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Train with augmentations\n",
|
||||
"\n",
|
||||
"Run a full training pass with the augmentation config chosen above. The\n",
|
||||
"`progress_bar=\"rich\"` setting activates per-epoch live progress bars — a batch\n",
|
||||
"counter, loss, and learning rate are updated in real time. Structured metric\n",
|
||||
"tables are printed at the end of each epoch.\n",
|
||||
"\n",
|
||||
"`aug_config` accepts any dict of Albumentations transforms, including the\n",
|
||||
"`AUG_CONFIG` defined above. Passing `{}` disables all augmentations."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "79d29fd9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = RFDETRNano()\n",
|
||||
"model.train(\n",
|
||||
" dataset_dir=str(DATASET_DIR),\n",
|
||||
" epochs=2,\n",
|
||||
" batch_size=24,\n",
|
||||
" aug_config=AUG_CONFIG,\n",
|
||||
" output_dir=OUTPUT_DIR,\n",
|
||||
" device=device,\n",
|
||||
" num_workers=num_workers,\n",
|
||||
" progress_bar=\"rich\",\n",
|
||||
" run_test=False,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "beccaf40",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 8. Run inference\n",
|
||||
"\n",
|
||||
"Load a sample image and run `model.predict()`. The method returns a\n",
|
||||
"`supervision.Detections` object, which makes it straightforward to draw\n",
|
||||
"bounding boxes and labels with the `supervision` annotators."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d79c7612",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"import supervision as sv\n",
|
||||
"from PIL import Image\n",
|
||||
"\n",
|
||||
"from rfdetr.util.coco_classes import COCO_CLASSES\n",
|
||||
"\n",
|
||||
"image_url = \"https://media.roboflow.com/dog.jpg\"\n",
|
||||
"image = Image.open(requests.get(image_url, stream=True).raw)\n",
|
||||
"\n",
|
||||
"detections = model.predict(image, threshold=0.5)\n",
|
||||
"\n",
|
||||
"labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]\n",
|
||||
"annotated = sv.BoxAnnotator().annotate(image.copy(), detections)\n",
|
||||
"annotated = sv.LabelAnnotator().annotate(annotated, detections, labels)\n",
|
||||
"sv.plot_image(annotated)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d69f8215",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 9. Next steps\n",
|
||||
"\n",
|
||||
"You have seen the complete 1.5.0 augmentation workflow — from exploring presets\n",
|
||||
"through live inspection to a full training run. From here:\n",
|
||||
"\n",
|
||||
"- [Augmentation docs](https://rfdetr.roboflow.com/1.5.0/learn/train/augmentations/) — full transform reference and custom pipeline guide\n",
|
||||
"- [Advanced training options](https://rfdetr.roboflow.com/1.5.0/learn/train/advanced/) — EMA, gradient accumulation, learning rate schedules\n",
|
||||
"- [Logger integrations (ClearML, MLflow, W&B)](https://rfdetr.roboflow.com/1.5.0/learn/train/loggers/) — experiment tracking\n",
|
||||
"- [Export your model](https://rfdetr.roboflow.com/1.5.0/learn/export/) — ONNX, TensorRT, CoreML\n",
|
||||
"- [RF-DETR 1.6.0 notebook](https://colab.research.google.com/github/roboflow/rf-detr/blob/develop/notebooks/release-demo_1-6.ipynb) — PyTorch Lightning building blocks"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"jupytext": {
|
||||
"cell_metadata_filter": "-all",
|
||||
"formats": "ipynb,py",
|
||||
"main_language": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,944 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "920e5414",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# RF-DETR object detection fine-tuning on Roboflow Universe datasets\n",
|
||||
"\n",
|
||||
"Select one dataset with `DATASET_KEY`, then run the same fine-tuning, plotting, checkpoint loading, and inference cells.\n",
|
||||
"\n",
|
||||
"## What is object detection?\n",
|
||||
"\n",
|
||||
"Object detection answers two questions simultaneously: *what* objects are in an image and *where* they are.\n",
|
||||
"Each detected object gets a **bounding box** (four coordinates) and a **class label** (e.g. \"car\", \"person\").\n",
|
||||
"This is the most common computer vision task — it powers traffic monitoring, safety compliance, retail analytics,\n",
|
||||
"medical imaging, environmental surveys, and virtually every real-time vision application.\n",
|
||||
"\n",
|
||||
"## Why fine-tune instead of train from scratch?\n",
|
||||
"\n",
|
||||
"RF-DETR ships pre-trained on the COCO dataset (80 common object classes). Fine-tuning reuses the backbone's\n",
|
||||
"rich visual representations — edges, textures, object parts — learned from millions of images.\n",
|
||||
"Starting from these weights instead of random initialisation typically:\n",
|
||||
"- **Reduces training time** from days to minutes or hours\n",
|
||||
"- **Requires far fewer labelled images** (hundreds instead of tens of thousands)\n",
|
||||
"- **Achieves higher accuracy** than training from scratch on small datasets\n",
|
||||
"\n",
|
||||
"The process is straightforward: swap out the final classification head for your number of classes,\n",
|
||||
"then continue training with a small learning rate so the pre-trained features are preserved.\n",
|
||||
"\n",
|
||||
"## Dataset overview\n",
|
||||
"\n",
|
||||
"Five datasets covering different domains are pre-configured. Set `DATASET_KEY` to one of:\n",
|
||||
"\n",
|
||||
"| Key | Dataset | Domain | Classes |\n",
|
||||
"|-----|---------|--------|---------|\n",
|
||||
"| `\"hard_hat_ppes\"` | Hard Hat Worker Safety | Construction / PPE | hat, person, vest |\n",
|
||||
"| `\"road_damage\"` | Pavement Distress Detection | Infrastructure Inspection | pot_hole, cracking, ravelling, ... |\n",
|
||||
"| `\"traffic\"` | Traffic Detection | Autonomous Driving | car, bus, truck, motorbike, bike, traffic lights |\n",
|
||||
"| `\"football\"` | Football Player Detection | Sports Analytics | player, ball, goalkeeper, referee |\n",
|
||||
"| `\"brackish_underwater\"` | Brackish Underwater | Marine Wildlife | fish, crab, jellyfish, shrimp, starfish, small_fish |\n",
|
||||
"\n",
|
||||
"Every subsequent cell (fine-tuning, plotting, checkpoint loading, inference) adapts automatically.\n",
|
||||
"By the end you will have a fine-tuned model, training curves, and annotated inference images with bounding boxes."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b6eb7cd5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"Install `rfdetr` with the `train` and `visual` extras. The `train` extra pulls in the training loop dependencies\n",
|
||||
"(PyTorch Lightning, COCO evaluation tools, and data augmentation libraries), while `visual` adds the visualisation\n",
|
||||
"helpers used later in the notebook. The `roboflow` package handles dataset download; `pandas` and `seaborn` are\n",
|
||||
"needed for the metrics table and optional plot styling. If you hit an `ImportError` after running this cell,\n",
|
||||
"the most likely cause is a stale in-memory import — restart the Python runtime once and re-run from the top.\n",
|
||||
"\n",
|
||||
"**GPU recommended.** Fine-tuning 50 epochs on a ~1,500-image dataset takes roughly 10–25 minutes on a modern GPU\n",
|
||||
"(RTX 3090, A10, T4). CPU-only is possible for a quick test but will be much slower."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e79facfd",
|
||||
"metadata": {
|
||||
"title": "[bash]"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": "!pip install -q \"rfdetr[train,visual]>=1.8.0\" roboflow pandas seaborn"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "62aff634",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\"\"\"Shared RF-DETR object detection fine-tuning demo for several Roboflow COCO detection exports.\"\"\"\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"from pathlib import Path\n",
|
||||
"from typing import Any\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import supervision as sv\n",
|
||||
"import torch\n",
|
||||
"from IPython.display import display\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"from matplotlib.figure import Figure\n",
|
||||
"from PIL import Image\n",
|
||||
"from roboflow import Roboflow\n",
|
||||
"\n",
|
||||
"from rfdetr import RFDETRSmall\n",
|
||||
"from rfdetr.config import TrainConfig\n",
|
||||
"from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer\n",
|
||||
"from rfdetr.training.callbacks.best_model import BestModelCallback\n",
|
||||
"from rfdetr.utilities.reproducibility import seed_all\n",
|
||||
"from rfdetr.visualize.training import plot_loss_metrics, plot_map_metrics"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c9c78daa",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ROOT = Path(__file__).resolve().parent if \"__file__\" in globals() else Path.cwd()\n",
|
||||
"DATASETS_DIR = PROJECT_ROOT / \"datasets\"\n",
|
||||
"\n",
|
||||
"# Resolution must be divisible by patch_size × num_windows (32 for RFDETRSmall: patch_size=16, num_windows=2).\n",
|
||||
"# Higher resolution improves detection of small objects at the cost of GPU memory and training time.\n",
|
||||
"# Practical range: 384–640. The default 512 suits most fine-tuning runs.\n",
|
||||
"DATASETS: dict[str, dict[str, Any]] = {\n",
|
||||
" \"hard_hat_ppes\": {\n",
|
||||
" \"name\": \"Hard Hat Worker Safety\",\n",
|
||||
" \"workspace\": \"safety-first\",\n",
|
||||
" \"project\": \"hard-hat-worker-safety-equipments\",\n",
|
||||
" \"version\": 9,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/safety-first/hard-hat-worker-safety-equipments\",\n",
|
||||
" \"output_name\": \"det_hard_hat_ppes\",\n",
|
||||
" # 512 — outdoor construction scenes; workers appear at medium scale so the\n",
|
||||
" # default resolution is sufficient to distinguish helmet vs no-helmet.\n",
|
||||
" \"resolution\": 512,\n",
|
||||
" },\n",
|
||||
" \"road_damage\": {\n",
|
||||
" \"name\": \"Pavement Distress Detection\",\n",
|
||||
" \"workspace\": \"pavement-distresses\",\n",
|
||||
" \"project\": \"distress-detection\",\n",
|
||||
" \"version\": 2,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/pavement-distresses/distress-detection\",\n",
|
||||
" \"output_name\": \"det_road_damage\",\n",
|
||||
" # 544 — road-surface images captured from dashcams and drones; fine cracks\n",
|
||||
" # and potholes are small relative to the frame, so the extra resolution\n",
|
||||
" # helps the model distinguish closely spaced defect categories.\n",
|
||||
" \"resolution\": 544,\n",
|
||||
" },\n",
|
||||
" \"traffic\": {\n",
|
||||
" \"name\": \"Traffic Detection\",\n",
|
||||
" \"workspace\": \"redlightrunningdection\",\n",
|
||||
" \"project\": \"traffic-detection-sutq6\",\n",
|
||||
" \"version\": 36,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/redlightrunningdection/traffic-detection-sutq6\",\n",
|
||||
" \"output_name\": \"det_traffic\",\n",
|
||||
" # 512 — street-level traffic scenes; vehicles span a wide size range\n",
|
||||
" # (large buses to distant motorbikes); 512 gives a good speed/accuracy trade-off.\n",
|
||||
" \"resolution\": 512,\n",
|
||||
" },\n",
|
||||
" \"football\": {\n",
|
||||
" \"name\": \"Football Player Detection\",\n",
|
||||
" \"workspace\": \"football-gozni\",\n",
|
||||
" \"project\": \"football-player-detection-bfswn\",\n",
|
||||
" \"version\": 1,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/football-gozni/football-player-detection-bfswn\",\n",
|
||||
" \"output_name\": \"det_football\",\n",
|
||||
" # 544 — broadcast and stadium-angle shots; the ball is a tiny object relative\n",
|
||||
" # to the field; the extra resolution meaningfully reduces false negatives.\n",
|
||||
" \"resolution\": 544,\n",
|
||||
" },\n",
|
||||
" \"brackish_underwater\": {\n",
|
||||
" \"name\": \"Brackish Underwater\",\n",
|
||||
" \"workspace\": \"brad-dwyer\",\n",
|
||||
" \"project\": \"brackish-underwater\",\n",
|
||||
" \"version\": 2,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/brad-dwyer/brackish-underwater\",\n",
|
||||
" \"output_name\": \"det_brackish_underwater\",\n",
|
||||
" # 512 — sonar and underwater camera frames; marine creatures appear at\n",
|
||||
" # medium-to-large scale; the default resolution captures enough detail.\n",
|
||||
" \"resolution\": 512,\n",
|
||||
" },\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"DATASET_KEY = \"hard_hat_ppes\"\n",
|
||||
"DATASET_INFO = DATASETS[DATASET_KEY]\n",
|
||||
"\n",
|
||||
"OUTPUT_DIR = PROJECT_ROOT / \"output\" / str(DATASET_INFO[\"output_name\"])\n",
|
||||
"METRICS_CSV = OUTPUT_DIR / \"metrics.csv\"\n",
|
||||
"VALIDATION_METRICS_JSON = OUTPUT_DIR / \"validation_metrics.json\"\n",
|
||||
"FINAL_CHECKPOINT_PATH = OUTPUT_DIR / \"checkpoint_final_demo.pth\"\n",
|
||||
"\n",
|
||||
"RESOLUTION = int(DATASET_INFO[\"resolution\"])\n",
|
||||
"\n",
|
||||
"SEED = 7\n",
|
||||
"EPOCHS = 50\n",
|
||||
"BATCH_SIZE = 8\n",
|
||||
"GRAD_ACCUM_STEPS = 2\n",
|
||||
"NUM_WORKERS = 8\n",
|
||||
"LR = 1e-4\n",
|
||||
"LR_ENCODER = 1e-4\n",
|
||||
"SAMPLE_PREVIEW_COUNT = 6\n",
|
||||
"SAMPLE_PREVIEW_COLUMNS = 3\n",
|
||||
"SAMPLE_PREVIEW_FIGURE_SIZE: tuple[float, float] = (15.0, 10.0)\n",
|
||||
"INFERENCE_COUNT = 6\n",
|
||||
"INFERENCE_COLUMNS = 3\n",
|
||||
"INFERENCE_THRESHOLD = 0.3\n",
|
||||
"PLOT_LOSS_LOG_SCALE = False\n",
|
||||
"\n",
|
||||
"print(f\"dataset_key={DATASET_KEY}\")\n",
|
||||
"print(f\"dataset={DATASET_INFO['name']}\")\n",
|
||||
"print(f\"source_url={DATASET_INFO['source_url']}\")\n",
|
||||
"print(f\"resolution={RESOLUTION}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cdce84bd",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## Notebook display\n",
|
||||
"\n",
|
||||
"This helper registers the `%matplotlib inline` magic so figures render directly beneath each cell when you run the\n",
|
||||
"notebook in Jupyter or Google Colab. If you are running this file as a plain Python script the function detects the\n",
|
||||
"absence of an IPython kernel and exits silently, so it is always safe to call."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "993a411e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _enable_notebook_inline_matplotlib() -> None:\n",
|
||||
" \"\"\"Enable inline matplotlib figures when running in IPython.\"\"\"\n",
|
||||
" get_ipython_func = globals().get(\"get_ipython\")\n",
|
||||
" if not callable(get_ipython_func):\n",
|
||||
" return\n",
|
||||
" ipython = get_ipython_func()\n",
|
||||
" if ipython is not None:\n",
|
||||
" ipython.run_line_magic(\"matplotlib\", \"inline\")\n",
|
||||
" ipython.run_line_magic(\"config\", \"InlineBackend.close_figures = True\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"_enable_notebook_inline_matplotlib()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9c7f944e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 1 - Download dataset\n",
|
||||
"\n",
|
||||
"Roboflow exports object detection datasets in **COCO format**: a JSON file containing image metadata and per-image\n",
|
||||
"annotations, each with a `bbox` field (x, y, width, height in pixels) and a `category_id` referencing the class.\n",
|
||||
"The download cell fetches the exact dataset version listed in `DATASETS` and places it under\n",
|
||||
"`datasets/<DATASET_KEY>/`. You need a Roboflow API key — get one for free at `app.roboflow.com/settings/api`\n",
|
||||
"and set it as the `ROBOFLOW_API_KEY` environment variable (or as a Colab secret with the same name).\n",
|
||||
"\n",
|
||||
"The download is **idempotent**: if the target directory already exists Roboflow skips the network transfer,\n",
|
||||
"so re-running this cell after a successful download is fast.\n",
|
||||
"\n",
|
||||
"> **Tip:** All five datasets in this notebook are open-access on Roboflow Universe. You can browse them in your\n",
|
||||
"> browser to explore image samples and annotation statistics before downloading."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "20fea837",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"try:\n",
|
||||
" from google.colab import userdata\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" ROBOFLOW_API_KEY = userdata.get(\"ROBOFLOW_API_KEY\") or \"\"\n",
|
||||
" except Exception:\n",
|
||||
" ROBOFLOW_API_KEY = \"\"\n",
|
||||
"except ImportError:\n",
|
||||
" ROBOFLOW_API_KEY = \"\"\n",
|
||||
"\n",
|
||||
"if not ROBOFLOW_API_KEY:\n",
|
||||
" ROBOFLOW_API_KEY = os.environ.get(\"ROBOFLOW_API_KEY\", \"\")\n",
|
||||
"if not ROBOFLOW_API_KEY:\n",
|
||||
" raise RuntimeError(\n",
|
||||
" \"ROBOFLOW_API_KEY not found. \"\n",
|
||||
" \"In Colab: add it via Secrets (key icon). \"\n",
|
||||
" \"Locally: set the environment variable before running.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"rf = Roboflow(api_key=ROBOFLOW_API_KEY)\n",
|
||||
"dataset = (\n",
|
||||
" rf.workspace(str(DATASET_INFO[\"workspace\"]))\n",
|
||||
" .project(str(DATASET_INFO[\"project\"]))\n",
|
||||
" .version(int(DATASET_INFO[\"version\"]))\n",
|
||||
" .download(\"coco\", location=str(DATASETS_DIR / DATASET_KEY))\n",
|
||||
")\n",
|
||||
"DATASET_DIR = Path(dataset.location)\n",
|
||||
"TRAIN_ANNOTATIONS = DATASET_DIR / \"train\" / \"_annotations.coco.json\"\n",
|
||||
"print(f\"dataset_dir={DATASET_DIR}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6516f6ee",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 2 - Infer class names\n",
|
||||
"\n",
|
||||
"The COCO annotation file defines a `categories` list mapping integer IDs to human-readable names. RF-DETR uses\n",
|
||||
"**0-based class indices** internally, so we sort the categories by their original `id` field and read names in order.\n",
|
||||
"The resulting `CLASS_NAMES` list maps each index back to a label, which is used during inference visualisation.\n",
|
||||
"\n",
|
||||
"If your own dataset has a single category, this list will have one element — that is expected and valid."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5ca32aa1",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with TRAIN_ANNOTATIONS.open() as _f:\n",
|
||||
" _coco = json.load(_f)\n",
|
||||
"\n",
|
||||
"CLASS_NAMES: list[str] = [cat[\"name\"] for cat in sorted(_coco[\"categories\"], key=lambda c: c[\"id\"])]\n",
|
||||
"NUM_CLASSES = len(CLASS_NAMES)\n",
|
||||
"\n",
|
||||
"print(f\"class_names={CLASS_NAMES}\")\n",
|
||||
"print(f\"num_classes={NUM_CLASSES}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3e39e02e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_save_final_checkpoint` writes a self-contained `.pth` file that bundles model weights with the full training and\n",
|
||||
"model config. This is separate from the PTL checkpoint because it can be loaded with a single `from_checkpoint`\n",
|
||||
"call on any machine, without reconstructing the original config."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "89450ec9",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _save_final_checkpoint(\n",
|
||||
" module: RFDETRModelModule,\n",
|
||||
" trainer: Any,\n",
|
||||
" train_config: TrainConfig,\n",
|
||||
" model_config: Any,\n",
|
||||
" output_path: Path,\n",
|
||||
") -> Path:\n",
|
||||
" \"\"\"Save a final RF-DETR .pth checkpoint that can be loaded with ``from_checkpoint``.\"\"\"\n",
|
||||
" raw_model: Any = getattr(module.model, \"_orig_mod\", module.model)\n",
|
||||
" output_path.parent.mkdir(parents=True, exist_ok=True)\n",
|
||||
" torch.save(\n",
|
||||
" BestModelCallback._build_checkpoint_payload(\n",
|
||||
" raw_model.state_dict(),\n",
|
||||
" train_config.model_dump(),\n",
|
||||
" trainer,\n",
|
||||
" model_name=\"RFDETRSmall\",\n",
|
||||
" model_config_dict=model_config.model_dump(),\n",
|
||||
" ),\n",
|
||||
" output_path,\n",
|
||||
" )\n",
|
||||
" return output_path"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e0a0ad62",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_detection_grid_figure` arranges multiple annotated images into a fixed-column matplotlib grid.\n",
|
||||
"Bounding boxes are drawn with `sv.BoxAnnotator` and labels with `sv.LabelAnnotator`, showing the class name and\n",
|
||||
"confidence score for each detection. Using a grid keeps the visual summary compact and makes it easy to compare\n",
|
||||
"predictions across images at a glance."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "efae67bb",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _detection_grid_figure(\n",
|
||||
" items: list[tuple[str, np.ndarray, sv.Detections]],\n",
|
||||
" columns: int,\n",
|
||||
" class_names: list[str],\n",
|
||||
") -> Figure:\n",
|
||||
" \"\"\"Render bounding box detections in a fixed-column subplot grid.\"\"\"\n",
|
||||
" if columns <= 0:\n",
|
||||
" raise ValueError(f\"columns must be positive, got {columns}.\")\n",
|
||||
"\n",
|
||||
" box_annotator = sv.BoxAnnotator(thickness=2)\n",
|
||||
" label_annotator = sv.LabelAnnotator(text_scale=0.5, text_thickness=1, text_padding=4)\n",
|
||||
"\n",
|
||||
" rows = max(1, (len(items) + columns - 1) // columns)\n",
|
||||
" figure, axes = plt.subplots(rows, columns, figsize=(5 * columns, 5 * rows))\n",
|
||||
" axes_array = np.asarray(axes, dtype=object).reshape(-1)\n",
|
||||
" for axis in axes_array:\n",
|
||||
" axis.axis(\"off\")\n",
|
||||
"\n",
|
||||
" for axis, (title, image, detections) in zip(axes_array, items, strict=False):\n",
|
||||
" scene = image.copy()\n",
|
||||
" scene = box_annotator.annotate(scene=scene, detections=detections)\n",
|
||||
" if len(detections) > 0 and detections.class_id is not None:\n",
|
||||
" conf_list = (\n",
|
||||
" detections.confidence.tolist() if detections.confidence is not None else [None] * len(detections)\n",
|
||||
" )\n",
|
||||
" labels = [\n",
|
||||
" f\"{class_names[cid] if cid < len(class_names) else cid}\" + (f\" {conf:.2f}\" if conf is not None else \"\")\n",
|
||||
" for cid, conf in zip(detections.class_id.tolist(), conf_list)\n",
|
||||
" ]\n",
|
||||
" scene = label_annotator.annotate(scene=scene, detections=detections, labels=labels)\n",
|
||||
" axis.imshow(scene)\n",
|
||||
" axis.set_title(title, fontsize=10)\n",
|
||||
" axis.axis(\"off\")\n",
|
||||
"\n",
|
||||
" figure.tight_layout()\n",
|
||||
" return figure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "38359554",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 3 - Configure training\n",
|
||||
"\n",
|
||||
"`TrainConfig` centralises every hyperparameter the training loop needs. The most important fields to understand\n",
|
||||
"before customising them for your own dataset:\n",
|
||||
"\n",
|
||||
"**Learning rates** — `lr` and `lr_encoder` are both set to `1e-4` for fine-tuning. The `lr` controls the\n",
|
||||
"detection head and decoder; `lr_encoder` controls the vision backbone (DINOv2). For fine-tuning, both are usually\n",
|
||||
"equal. If you notice the backbone overfitting early — validation loss rising while training loss keeps falling —\n",
|
||||
"halve `lr_encoder` to protect the pre-trained features while the head continues to adapt.\n",
|
||||
"\n",
|
||||
"**Batch size and gradient accumulation** — `batch_size=8` with `grad_accum_steps=2` gives an effective batch size\n",
|
||||
"of 16 without requiring extra GPU memory. Effective batch size affects how smoothly gradients are estimated each\n",
|
||||
"update; too small (< 8) leads to noisy updates; too large (> 64) may over-smooth and slow convergence.\n",
|
||||
"\n",
|
||||
"**EMA** — `use_ema=False` keeps this demo fast. For a production run you can enable EMA\n",
|
||||
"(`use_ema=True`) to maintain an exponential moving average of the weights, which typically adds 0.5–1.0 mAP points\n",
|
||||
"on small datasets by reducing the impact of noisy late-epoch updates.\n",
|
||||
"\n",
|
||||
"**Multi-scale training** — `multi_scale=False` and `expanded_scales=False` disable the multi-resolution\n",
|
||||
"augmentation used during pre-training. Turning them off shortens epoch time by up to 40 % and is fine for\n",
|
||||
"most fine-tuning runs on small custom datasets. Re-enable if your validation mAP plateaus early.\n",
|
||||
"\n",
|
||||
"**Notes dict** — stored verbatim in the checkpoint file alongside the weights, giving you a lightweight experiment\n",
|
||||
"log that travels with the model file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a8c0bbcf",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"seed_all(SEED)\n",
|
||||
"variant = RFDETRSmall( # type: ignore[no-untyped-call]\n",
|
||||
" num_classes=NUM_CLASSES,\n",
|
||||
" resolution=RESOLUTION,\n",
|
||||
")\n",
|
||||
"variant.model_config.model_name = type(variant).__name__\n",
|
||||
"\n",
|
||||
"train_config = TrainConfig(\n",
|
||||
" dataset_file=\"roboflow\",\n",
|
||||
" dataset_dir=str(DATASET_DIR),\n",
|
||||
" output_dir=str(OUTPUT_DIR),\n",
|
||||
" epochs=EPOCHS,\n",
|
||||
" batch_size=BATCH_SIZE,\n",
|
||||
" grad_accum_steps=GRAD_ACCUM_STEPS,\n",
|
||||
" num_workers=NUM_WORKERS,\n",
|
||||
" lr=LR,\n",
|
||||
" lr_encoder=LR_ENCODER,\n",
|
||||
" use_ema=False,\n",
|
||||
" run_test=False,\n",
|
||||
" compute_train_metrics=True,\n",
|
||||
" compute_val_loss=True,\n",
|
||||
" multi_scale=False,\n",
|
||||
" expanded_scales=False,\n",
|
||||
" do_random_resize_via_padding=False,\n",
|
||||
" tensorboard=False,\n",
|
||||
" wandb=False,\n",
|
||||
" mlflow=False,\n",
|
||||
" clearml=False,\n",
|
||||
" class_names=CLASS_NAMES,\n",
|
||||
" notes={\n",
|
||||
" \"demo\": f\"detection PTL fine-tune on Roboflow Universe {DATASET_INFO['project']}\",\n",
|
||||
" \"source_url\": DATASET_INFO[\"source_url\"],\n",
|
||||
" \"roboflow_workspace\": DATASET_INFO[\"workspace\"],\n",
|
||||
" \"roboflow_project\": DATASET_INFO[\"project\"],\n",
|
||||
" \"roboflow_version\": DATASET_INFO[\"version\"],\n",
|
||||
" \"num_classes\": NUM_CLASSES,\n",
|
||||
" \"class_names\": CLASS_NAMES,\n",
|
||||
" },\n",
|
||||
" progress_bar=\"tqdm\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"datamodule = RFDETRDataModule(variant.model_config, train_config)\n",
|
||||
"model = RFDETRModelModule(variant.model_config, train_config)\n",
|
||||
"trainer = build_trainer(train_config, variant.model_config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c1eaa77a",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 4 - Preview dataset inputs\n",
|
||||
"\n",
|
||||
"**Always look at your data before starting a long training run.** This cell renders a grid of annotated training\n",
|
||||
"images drawn from the augmented pipeline — the same view the model will see during training.\n",
|
||||
"\n",
|
||||
"What to check here:\n",
|
||||
"- **Box alignment** — do bounding boxes tightly surround the objects, or are they shifted or too large?\n",
|
||||
"- **Class correctness** — are labels correct for each box? Mixed-up classes are one of the most common\n",
|
||||
" annotation errors and are hard to recover from after training.\n",
|
||||
"- **Augmentation sanity** — horizontal flips, colour jitter, and random crops should look plausible;\n",
|
||||
" cropped boxes at image edges are normal.\n",
|
||||
"- **Class imbalance** — if one class dominates the preview, the model will likely be biased toward it;\n",
|
||||
" rebalancing or oversampling rare classes can help.\n",
|
||||
"\n",
|
||||
"Catching label noise here costs a few seconds; catching it after 50 epochs of training costs much more."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a42a8135",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"sample_figure = datamodule._show_samples(\n",
|
||||
" SAMPLE_PREVIEW_COUNT,\n",
|
||||
" split=\"train\",\n",
|
||||
" columns=SAMPLE_PREVIEW_COLUMNS,\n",
|
||||
" figure_size=SAMPLE_PREVIEW_FIGURE_SIZE,\n",
|
||||
")\n",
|
||||
"display(sample_figure)\n",
|
||||
"plt.close(sample_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "58343884",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 5 - Fine-tune\n",
|
||||
"\n",
|
||||
"`trainer.fit` hands control to PyTorch Lightning, which drives the full training loop: forward pass, bipartite\n",
|
||||
"matching loss (classification + bounding-box L1 + GIoU), gradient accumulation, weight updates,\n",
|
||||
"learning-rate scheduling, and periodic validation with COCO bounding-box mAP.\n",
|
||||
"\n",
|
||||
"After each epoch the trainer appends a row to `metrics.csv` and, if validation mAP improves,\n",
|
||||
"saves a new best checkpoint via `BestModelCallback`.\n",
|
||||
"\n",
|
||||
"**Typical training times on a single GPU** (RTX 3090 or similar):\n",
|
||||
"- PPE / hard hat dataset (~2,600 images): ~20–30 min for 50 epochs\n",
|
||||
"- Traffic dataset (~1,400 images): ~15–20 min for 50 epochs\n",
|
||||
"- Underwater dataset (~12,000 images): ~60–90 min for 50 epochs\n",
|
||||
"\n",
|
||||
"To **resume an interrupted run**, set `train_config.resume` to the path of the last PTL checkpoint\n",
|
||||
"(e.g. `OUTPUT_DIR / \"last.ckpt\"`) before calling this cell again."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8c78f17f",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer.fit(model, datamodule=datamodule, ckpt_path=train_config.resume or None)\n",
|
||||
"print(f\"output_dir={OUTPUT_DIR}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "92918976",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 6 - Save checkpoint/model\n",
|
||||
"\n",
|
||||
"PTL saves its own checkpoints during training (optimizer state, scheduler state, epoch counter), but those files\n",
|
||||
"are not directly portable — they require the same class hierarchy to load. The `.pth` file written here is a\n",
|
||||
"**self-contained RF-DETR checkpoint**: it bundles the model weights together with the full training and model\n",
|
||||
"configs, including the class names. You can share the file with a colleague and they can run inference with a\n",
|
||||
"single `RFDETRSmall.from_checkpoint(path)` call, with no need to reconstruct the original config.\n",
|
||||
"\n",
|
||||
"For full reproducibility, keep this checkpoint alongside the dataset version number printed in section 1."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "47156e82",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"final_checkpoint = _save_final_checkpoint(model, trainer, train_config, variant.model_config, FINAL_CHECKPOINT_PATH)\n",
|
||||
"print(f\"saved_checkpoint={final_checkpoint}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9dbf2232",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 7 - Validate metrics\n",
|
||||
"\n",
|
||||
"This cell runs a clean post-training validation: no augmentation, the best checkpoint loaded from\n",
|
||||
"`checkpoint_best_total.pth` written by `BestModelCallback`, and results serialised to JSON for downstream\n",
|
||||
"comparison or reporting.\n",
|
||||
"\n",
|
||||
"**Key metrics to inspect:**\n",
|
||||
"\n",
|
||||
"- **`bbox/map`** — the primary metric: standard COCO bounding-box mAP averaged across IoU thresholds 0.50–0.95\n",
|
||||
" in steps of 0.05. A value of 0.50 means the model correctly localises and classifies 50 % of objects at strict\n",
|
||||
" IoU thresholds. This is the most commonly reported number in object detection benchmarks.\n",
|
||||
"- **`bbox/map_50`** — mAP at IoU ≥ 0.50 (a box is \"correct\" if it overlaps the ground-truth box by at least 50 %).\n",
|
||||
" Rises fastest and is the clearest early signal of whether the model is learning at all.\n",
|
||||
"- **`bbox/map_75`** — mAP at the stricter IoU ≥ 0.75 threshold. Reflects how precisely the model localises\n",
|
||||
" objects, not just whether it finds them.\n",
|
||||
"\n",
|
||||
"A model with high `bbox/map_50` but low `bbox/map_75` finds objects reliably but draws imprecise boxes.\n",
|
||||
"This usually improves with more training epochs or by increasing `RESOLUTION`.\n",
|
||||
"\n",
|
||||
"> **Note:** `checkpoint_best_total.pth` is written after the first completed validation epoch. If training was\n",
|
||||
"> interrupted before any epoch finished, this file will not exist and the cell below will raise `FileNotFoundError`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b35e7f96",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# checkpoint_best_total.pth stores plain model weights under key \"model\", not a full\n",
|
||||
"# PTL checkpoint. Passing it as ckpt_path to trainer.validate() triggers a KeyError\n",
|
||||
"# on \"validate_loop\" because PTL tries to restore loop state that isn't present.\n",
|
||||
"# Load the weights directly and run validate without ckpt_path instead.\n",
|
||||
"_ckpt = torch.load(OUTPUT_DIR / \"checkpoint_best_total.pth\", map_location=\"cpu\", weights_only=False)\n",
|
||||
"model.model.load_state_dict(_ckpt[\"model\"], strict=True)\n",
|
||||
"del _ckpt\n",
|
||||
"validation_results = trainer.validate(model, datamodule=datamodule)\n",
|
||||
"validation_metrics = {key: float(value) for key, value in validation_results[0].items()} if validation_results else {}\n",
|
||||
"VALIDATION_METRICS_JSON.write_text(json.dumps(validation_metrics, indent=2, sort_keys=True), encoding=\"utf-8\")\n",
|
||||
"print(f\"validation_metrics={validation_metrics}\")\n",
|
||||
"print(f\"validation_metrics_json={VALIDATION_METRICS_JSON}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cbb8112e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 8 - Plot CSVLogger metrics\n",
|
||||
"\n",
|
||||
"Reading loss and mAP curves is the fastest way to diagnose training problems.\n",
|
||||
"\n",
|
||||
"**Loss curves (first plot):**\n",
|
||||
"- A healthy run shows both training loss and validation loss decreasing together.\n",
|
||||
"- A **growing gap** between train and val loss is a classic overfitting signal — the model is memorising the\n",
|
||||
" training set. Add augmentation (`multi_scale=True`, or custom Albumentations), reduce learning rate, or collect\n",
|
||||
" more data.\n",
|
||||
"- A **flat or rising loss from epoch 1** usually means the learning rate is too high. Try reducing `LR` to `5e-5`.\n",
|
||||
"\n",
|
||||
"**mAP curves (second plot):**\n",
|
||||
"- `bbox_map_50` (solid line) rises fastest and is the clearest signal of learning.\n",
|
||||
"- On small datasets (a few hundred images) you typically see rapid improvement in epochs 1–20 followed by\n",
|
||||
" a plateau around epoch 30–50.\n",
|
||||
"- If mAP is **still rising at epoch 50**, extend `EPOCHS` to 75 or 100.\n",
|
||||
"- If mAP **never rises above 0.05**, check class names, annotation quality, and whether\n",
|
||||
" `RESOLUTION` is appropriate for the objects' scale in your dataset.\n",
|
||||
"\n",
|
||||
"Set `PLOT_LOSS_LOG_SCALE = True` if the loss drops by an order of magnitude in the first few epochs and the later,\n",
|
||||
"more meaningful portion of the curve gets compressed into a flat line."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "278892c2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(f\"metrics_csv={METRICS_CSV}\")\n",
|
||||
"loss_figure = plot_loss_metrics(str(METRICS_CSV), loss_log_scale=PLOT_LOSS_LOG_SCALE)\n",
|
||||
"display(loss_figure)\n",
|
||||
"plt.close(loss_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d0f5f468",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"map_figure = plot_map_metrics(str(METRICS_CSV))\n",
|
||||
"display(map_figure)\n",
|
||||
"plt.close(map_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "550cf8f7",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 9 - Load checkpoint/model\n",
|
||||
"\n",
|
||||
"`from_checkpoint` is the standard entry point for loading a saved RF-DETR model. It reads both the weights and the\n",
|
||||
"stored config from the `.pth` file, so the reconstructed model has the correct number of classes and resolution\n",
|
||||
"without you having to pass them explicitly.\n",
|
||||
"\n",
|
||||
"This cell also confirms that the save–load round trip works before you proceed to inference, so any file corruption\n",
|
||||
"or version mismatch surfaces here rather than silently producing wrong predictions later.\n",
|
||||
"\n",
|
||||
"To deploy this model in a production pipeline, you only need to ship the `.pth` file and call `from_checkpoint`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "dd0a0623",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loaded_model = RFDETRSmall.from_checkpoint(FINAL_CHECKPOINT_PATH)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "929edc54",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 10 - Select inference images\n",
|
||||
"\n",
|
||||
"This cell implements a fallback chain — test split first, then validation, then train — so the notebook always\n",
|
||||
"finds images to run inference on even when the dataset has no dedicated test split.\n",
|
||||
"\n",
|
||||
"**Using test images gives an unbiased view of model performance** because those images were never seen during\n",
|
||||
"training or used to select the best checkpoint. If your deployment images look different from the training set\n",
|
||||
"(different lighting, camera angle, resolution), replace `inference_image_paths` with a list of `Path` objects\n",
|
||||
"pointing to your own files to assess real-world performance before shipping the model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cc90e306",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"_IMAGE_EXTS = {\".jpg\", \".jpeg\", \".png\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _find_images(directory: Path) -> list[Path]:\n",
|
||||
" if not directory.is_dir():\n",
|
||||
" return []\n",
|
||||
" return sorted(p for p in directory.glob(\"**/*\") if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"inference_image_paths = _find_images(DATASET_DIR / \"test\")\n",
|
||||
"if not inference_image_paths:\n",
|
||||
" inference_image_paths = _find_images(DATASET_DIR / \"valid\")\n",
|
||||
"if not inference_image_paths:\n",
|
||||
" inference_image_paths = _find_images(DATASET_DIR / \"train\")\n",
|
||||
"if not inference_image_paths:\n",
|
||||
" raise FileNotFoundError(f\"No images ({', '.join(sorted(_IMAGE_EXTS))}) found under {DATASET_DIR}\")\n",
|
||||
"\n",
|
||||
"inference_image_paths = inference_image_paths[:INFERENCE_COUNT]\n",
|
||||
"print(f\"inference_images={[str(p) for p in inference_image_paths]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f49ac0a4",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 11 - Visualize inference\n",
|
||||
"\n",
|
||||
"`INFERENCE_THRESHOLD` is a **detection confidence threshold**: only detections whose bounding-box score exceeds\n",
|
||||
"this value are shown. The default `0.3` is a reasonable starting point; adjust it based on what you see:\n",
|
||||
"\n",
|
||||
"- **Too many false positives** (boxes on background or wrong objects) → raise the threshold towards `0.5`–`0.7`.\n",
|
||||
"- **Missing true objects** (objects in the image not detected) → lower the threshold towards `0.1`–`0.2`.\n",
|
||||
"- **Zero detections on most images** → lower the threshold; the model may be well-calibrated to lower confidence\n",
|
||||
" scores on a custom dataset than the COCO-pretrained default.\n",
|
||||
"\n",
|
||||
"The confidence score reflects how certain the model is about each box. On a well-fine-tuned model you will\n",
|
||||
"typically see scores above 0.7 for clear, unoccluded objects of the target classes."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4f7739a5",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"inference_grid_items: list[tuple[str, np.ndarray, sv.Detections]] = []\n",
|
||||
"for image_path in inference_image_paths:\n",
|
||||
" with Image.open(image_path) as image:\n",
|
||||
" image_rgb = image.convert(\"RGB\")\n",
|
||||
" detections = loaded_model.predict(image_rgb, threshold=INFERENCE_THRESHOLD)\n",
|
||||
" if not isinstance(detections, sv.Detections):\n",
|
||||
" raise RuntimeError(f\"Expected RFDETRSmall.predict() to return sv.Detections, got {type(detections)!r}.\")\n",
|
||||
" inference_grid_items.append((image_path.name, np.array(image_rgb), detections))\n",
|
||||
"\n",
|
||||
"if inference_grid_items:\n",
|
||||
" figure = _detection_grid_figure(inference_grid_items, columns=INFERENCE_COLUMNS, class_names=CLASS_NAMES)\n",
|
||||
" display(figure)\n",
|
||||
" plt.close(figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "40582975",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 12 - Detection summary\n",
|
||||
"\n",
|
||||
"Print per-image detection counts and confidence scores as a quick sanity check. This table is useful for:\n",
|
||||
"\n",
|
||||
"- **Spotting images with zero detections** — if many images produce no detections at your chosen threshold,\n",
|
||||
" the model may be under-confident; try lowering `INFERENCE_THRESHOLD` first.\n",
|
||||
"- **Checking average confidence** — a well-calibrated fine-tuned model typically reports average confidence\n",
|
||||
" between 0.5 and 0.9 on in-distribution images. Scores consistently below 0.3 suggest the model is uncertain\n",
|
||||
" and may benefit from more training epochs or additional labelled data.\n",
|
||||
"- **Class distribution** — if certain classes never appear in inference results but were present in training data,\n",
|
||||
" they may be under-represented in the training set. Consider adding more examples for those classes."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aa58d24e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summary_rows = []\n",
|
||||
"for image_path, (_, _, detections) in zip(inference_image_paths, inference_grid_items):\n",
|
||||
" num_det = len(detections)\n",
|
||||
" avg_conf = (\n",
|
||||
" float(np.mean(detections.confidence)) if detections.confidence is not None and num_det > 0 else float(\"nan\")\n",
|
||||
" )\n",
|
||||
" class_counts: dict[str, int] = {}\n",
|
||||
" if num_det > 0 and detections.class_id is not None:\n",
|
||||
" for cid in detections.class_id.tolist():\n",
|
||||
" label = CLASS_NAMES[cid] if cid < len(CLASS_NAMES) else str(cid)\n",
|
||||
" class_counts[label] = class_counts.get(label, 0) + 1\n",
|
||||
" summary_rows.append(\n",
|
||||
" {\n",
|
||||
" \"image\": image_path.name,\n",
|
||||
" \"detections\": num_det,\n",
|
||||
" \"avg_confidence\": round(avg_conf, 3),\n",
|
||||
" \"classes\": \", \".join(f\"{cls}×{cnt}\" for cls, cnt in sorted(class_counts.items())),\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"summary_df = pd.DataFrame(summary_rows)\n",
|
||||
"display(summary_df)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"jupytext": {
|
||||
"cell_metadata_filter": "title,-all",
|
||||
"main_language": "python",
|
||||
"notebook_metadata_filter": "-all"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,878 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7e465973",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# RF-DETR keypoint training demo on multiple Roboflow Universe datasets\n",
|
||||
"\n",
|
||||
"Select one dataset with `DATASET_KEY`, then run the same fine-tuning, plotting, checkpoint loading, and inference cells.\n",
|
||||
"\n",
|
||||
"RF-DETR extends bounding-box detection to predict structured keypoint skeletons — useful for human pose\n",
|
||||
"estimation, sports field calibration, product inspection, and any task where *where* matters as much as *what*.\n",
|
||||
"\n",
|
||||
"**Datasets** — set `DATASET_KEY` to one of:\n",
|
||||
"\n",
|
||||
"| Key | Dataset |\n",
|
||||
"|-----|---------|\n",
|
||||
"| `\"dart\"` | Darts Detection |\n",
|
||||
"| `\"human_pose\"` | Human Body Pose |\n",
|
||||
"| `\"basketball_court\"` | Basketball Court Detection |\n",
|
||||
"| `\"football_field\"` | Football Field Detection |\n",
|
||||
"| `\"tennis_court\"` | Tennis Court Keypoint |\n",
|
||||
"\n",
|
||||
"Every subsequent cell (fine-tuning, plotting, checkpoint loading, inference) adapts automatically.\n",
|
||||
"By the end you will have a fine-tuned model, training curves, and annotated inference images with\n",
|
||||
"optional uncertainty ellipses."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6d361242",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"Install `rfdetr` with the `train` and `visual` extras. The `train` extra pulls in the training loop dependencies (PyTorch Lightning, COCO evaluation tools, and data augmentation libraries), while `visual` adds the visualisation helpers used later in the notebook. The `roboflow` package handles dataset download; `pandas` and `seaborn` are needed for the metrics table and optional plot styling. If you hit an `ImportError` after running this cell, the most likely cause is a stale in-memory import — restart the Python runtime once and re-run from the top."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ca1b5461",
|
||||
"metadata": {
|
||||
"title": "[bash]"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install -q \"rfdetr[train,visual]==1.8.2\" roboflow pandas seaborn"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d89771c9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\"\"\"Shared RF-DETR keypoint fine-tuning demo for several Roboflow COCO keypoint exports.\"\"\"\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"from pathlib import Path\n",
|
||||
"from typing import Any\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import supervision as sv\n",
|
||||
"import torch\n",
|
||||
"from IPython.display import display\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"from matplotlib.figure import Figure\n",
|
||||
"from PIL import Image\n",
|
||||
"from roboflow import Roboflow\n",
|
||||
"\n",
|
||||
"from rfdetr import RFDETRKeypointPreview\n",
|
||||
"from rfdetr.config import KeypointTrainConfig\n",
|
||||
"from rfdetr.datasets._keypoint_schema import infer_coco_keypoint_schema, infer_yolo_keypoint_schema\n",
|
||||
"from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer\n",
|
||||
"from rfdetr.training.callbacks.best_model import BestModelCallback\n",
|
||||
"from rfdetr.utilities.reproducibility import seed_all\n",
|
||||
"from rfdetr.visualize.keypoints import _key_points_for_display, _keypoint_prediction_records\n",
|
||||
"from rfdetr.visualize.training import plot_loss_metrics, plot_map_metrics"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0b7de4e2",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ROOT = Path(__file__).resolve().parent if \"__file__\" in globals() else Path.cwd()\n",
|
||||
"DATASETS_DIR = PROJECT_ROOT / \"datasets\"\n",
|
||||
"\n",
|
||||
"# Resolution must be divisible by the model patch_size (12 for RFDETRKeypointPreview).\n",
|
||||
"# Larger resolutions improve fine-grained keypoint localisation at the cost of GPU memory\n",
|
||||
"# and training time. Recommended choices per dataset are annotated below.\n",
|
||||
"DATASETS: dict[str, dict[str, Any]] = {\n",
|
||||
" \"dart\": {\n",
|
||||
" \"name\": \"Darts Detection\",\n",
|
||||
" \"workspace\": \"dartdetection-vfgjd\",\n",
|
||||
" \"project\": \"darts_detection-yp7lt\",\n",
|
||||
" \"version\": 5,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/dartdetection-vfgjd/darts_detection-yp7lt\",\n",
|
||||
" \"output_name\": \"keypoint_darts_detection_demo\",\n",
|
||||
" \"keypoint_flip_pairs\": [],\n",
|
||||
" # 576 — close-up shots where the dart tip occupies a large fraction of the frame;\n",
|
||||
" # default resolution provides sufficient detail for tip localisation.\n",
|
||||
" \"resolution\": 576,\n",
|
||||
" },\n",
|
||||
" \"human_pose\": {\n",
|
||||
" \"name\": \"Dataset Ridimensionato\",\n",
|
||||
" \"workspace\": \"poseestimation-wzidb\",\n",
|
||||
" \"project\": \"dataset-ridimensionato\",\n",
|
||||
" \"version\": 65,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/poseestimation-wzidb/dataset-ridimensionato/dataset/65\",\n",
|
||||
" \"output_name\": \"keypoint_dataset_ridimensionato_demo\",\n",
|
||||
" \"format\": \"yolo\",\n",
|
||||
" \"keypoint_flip_pairs\": [],\n",
|
||||
" # 768 — body joints (wrists, ankles) require fine localisation across a range of\n",
|
||||
" # person scales; a larger canvas reduces positional error for distant people.\n",
|
||||
" \"resolution\": 768,\n",
|
||||
" },\n",
|
||||
" \"basketball_court\": {\n",
|
||||
" \"name\": \"Basketball Court Detection 2\",\n",
|
||||
" \"workspace\": \"roboflow-jvuqo\",\n",
|
||||
" \"project\": \"basketball-court-detection-2\",\n",
|
||||
" \"version\": 19,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/roboflow-jvuqo/basketball-court-detection-2\",\n",
|
||||
" \"output_name\": \"keypoint_basketball_court_detection_demo\",\n",
|
||||
" \"keypoint_flip_pairs\": [],\n",
|
||||
" # 576 — court lines are coarse features that fill the frame; the default resolution\n",
|
||||
" # is sufficient to locate corners and centre-circle keypoints accurately.\n",
|
||||
" \"resolution\": 576,\n",
|
||||
" },\n",
|
||||
" \"football_field\": {\n",
|
||||
" \"name\": \"Football Field Detection\",\n",
|
||||
" \"workspace\": \"roboflow-jvuqo\",\n",
|
||||
" \"project\": \"football-field-detection-f07vi\",\n",
|
||||
" \"version\": 18,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/roboflow-jvuqo/football-field-detection-f07vi\",\n",
|
||||
" \"output_name\": \"keypoint_football_field_detection_demo\",\n",
|
||||
" \"keypoint_flip_pairs\": [],\n",
|
||||
" # 768 — a full football pitch captured in a single frame compresses many line\n",
|
||||
" # intersections into a small pixel area; the higher resolution separates\n",
|
||||
" # closely spaced markings and reduces keypoint confusion near the centre circle.\n",
|
||||
" \"resolution\": 768,\n",
|
||||
" },\n",
|
||||
" \"tennis_court\": {\n",
|
||||
" \"name\": \"Tennis Court Keypoint\",\n",
|
||||
" \"workspace\": \"shj-pk1wt\",\n",
|
||||
" \"project\": \"tennis_court_keypoint\",\n",
|
||||
" \"version\": 6,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/shj-pk1wt/tennis_court_keypoint\",\n",
|
||||
" \"output_name\": \"keypoint_tennis_court_demo\",\n",
|
||||
" \"format\": \"yolo\",\n",
|
||||
" \"keypoint_flip_pairs\": [],\n",
|
||||
" # 768 — a full court viewed from broadcast angles packs many line intersections\n",
|
||||
" # into a small pixel area; the higher resolution reduces confusion between\n",
|
||||
" # closely spaced baseline and service-box corners.\n",
|
||||
" \"resolution\": 768,\n",
|
||||
" },\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"DATASET_KEY = \"dart\"\n",
|
||||
"DATASET_INFO = DATASETS[DATASET_KEY]\n",
|
||||
"DATASET_FORMAT = DATASET_INFO.get(\"format\", \"coco\") # \"coco\" or \"yolo\"\n",
|
||||
"\n",
|
||||
"OUTPUT_DIR = PROJECT_ROOT / \"output\" / str(DATASET_INFO[\"output_name\"])\n",
|
||||
"METRICS_CSV = OUTPUT_DIR / \"metrics.csv\"\n",
|
||||
"VALIDATION_METRICS_JSON = OUTPUT_DIR / \"validation_metrics.json\"\n",
|
||||
"FINAL_CHECKPOINT_PATH = OUTPUT_DIR / \"checkpoint_final_demo.pth\"\n",
|
||||
"\n",
|
||||
"RESOLUTION = int(DATASET_INFO[\"resolution\"])\n",
|
||||
"\n",
|
||||
"SEED = 7\n",
|
||||
"EPOCHS = 50\n",
|
||||
"BATCH_SIZE = 8\n",
|
||||
"GRAD_ACCUM_STEPS = 2\n",
|
||||
"NUM_WORKERS = 8\n",
|
||||
"LR = 1e-4\n",
|
||||
"LR_ENCODER = 1e-4\n",
|
||||
"SAMPLE_PREVIEW_COUNT = 6\n",
|
||||
"SAMPLE_PREVIEW_COLUMNS = 3\n",
|
||||
"SAMPLE_PREVIEW_FIGURE_SIZE: tuple[float, float] | None = None\n",
|
||||
"INFERENCE_COUNT = 6\n",
|
||||
"INFERENCE_COLUMNS = 3\n",
|
||||
"INFERENCE_THRESHOLD = 0.25\n",
|
||||
"KEYPOINT_THRESHOLD = 0.1\n",
|
||||
"PLOT_LOSS_LOG_SCALE = False\n",
|
||||
"DRAW_UNCERTAINTY_ELLIPSES = True\n",
|
||||
"ELLIPSE_SIGMA = 1.0\n",
|
||||
"MAX_ELLIPSE_RADIUS = 36.0\n",
|
||||
"\n",
|
||||
"print(f\"dataset_key={DATASET_KEY}\")\n",
|
||||
"print(f\"dataset={DATASET_INFO['name']}\")\n",
|
||||
"print(f\"dataset_format={DATASET_FORMAT}\")\n",
|
||||
"print(f\"source_url={DATASET_INFO['source_url']}\")\n",
|
||||
"print(f\"resolution={RESOLUTION}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3d1ca469",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## Notebook display\n",
|
||||
"\n",
|
||||
"This helper registers the `%matplotlib inline` magic so figures render directly beneath each cell when you run the notebook in Jupyter or Google Colab. If you are running this file as a plain Python script the function detects the absence of an IPython kernel and exits silently, so it is always safe to call."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b7920201",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _enable_notebook_inline_matplotlib() -> None:\n",
|
||||
" \"\"\"Enable inline matplotlib figures when running in IPython.\"\"\"\n",
|
||||
" get_ipython_func = globals().get(\"get_ipython\")\n",
|
||||
" if not callable(get_ipython_func):\n",
|
||||
" return\n",
|
||||
" ipython = get_ipython_func()\n",
|
||||
" if ipython is not None:\n",
|
||||
" ipython.run_line_magic(\"matplotlib\", \"inline\")\n",
|
||||
" ipython.run_line_magic(\"config\", \"InlineBackend.close_figures = True\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"_enable_notebook_inline_matplotlib()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "08de3f94",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 1 - Download dataset\n",
|
||||
"\n",
|
||||
"Roboflow exports datasets in COCO keypoint format: a JSON file where each annotation contains a flat `keypoints` array of `[x, y, visibility, x, y, visibility, ...]` triplets, one per defined skeleton joint. The download cell fetches the exact dataset version listed in `DATASETS` and places it under `datasets/<DATASET_KEY>/`. You need a Roboflow API key — get one for free at app.roboflow.com/settings/api and set it as the `ROBOFLOW_API_KEY` environment variable (or as a Colab secret with the same name). The download is idempotent: if the target directory already exists Roboflow skips the network transfer, so re-running this cell after a successful download is fast."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b2109a1e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"try:\n",
|
||||
" from google.colab import userdata\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" ROBOFLOW_API_KEY = userdata.get(\"ROBOFLOW_API_KEY\") or \"\"\n",
|
||||
" except Exception:\n",
|
||||
" ROBOFLOW_API_KEY = \"\"\n",
|
||||
"except ImportError:\n",
|
||||
" ROBOFLOW_API_KEY = \"\"\n",
|
||||
"\n",
|
||||
"if not ROBOFLOW_API_KEY:\n",
|
||||
" ROBOFLOW_API_KEY = os.environ.get(\"ROBOFLOW_API_KEY\", \"\")\n",
|
||||
"if not ROBOFLOW_API_KEY:\n",
|
||||
" raise RuntimeError(\n",
|
||||
" \"ROBOFLOW_API_KEY not found. \"\n",
|
||||
" \"In Colab: add it via Secrets (key icon). \"\n",
|
||||
" \"Locally: set the environment variable before running.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"rf = Roboflow(api_key=ROBOFLOW_API_KEY)\n",
|
||||
"_rf_format = \"yolov8\" if DATASET_FORMAT == \"yolo\" else \"coco\"\n",
|
||||
"dataset = (\n",
|
||||
" rf.workspace(str(DATASET_INFO[\"workspace\"]))\n",
|
||||
" .project(str(DATASET_INFO[\"project\"]))\n",
|
||||
" .version(int(DATASET_INFO[\"version\"]))\n",
|
||||
" .download(_rf_format, location=str(DATASETS_DIR / DATASET_KEY))\n",
|
||||
")\n",
|
||||
"DATASET_DIR = Path(dataset.location)\n",
|
||||
"print(f\"dataset_dir={DATASET_DIR}\")\n",
|
||||
"print(f\"rf_format={_rf_format}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2f59cc9e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 2 - Infer keypoint schema\n",
|
||||
"\n",
|
||||
"`infer_coco_keypoint_schema` reads the training annotation JSON and extracts three things the model needs: the class names, the number of keypoints per class, and the OKS sigmas. OKS stands for Object Keypoint Similarity — it is the keypoint analogue of IoU and ranges from 0 to 1. Each sigma is a per-keypoint scale factor that controls how strictly localisation is penalised during evaluation: a smaller sigma (e.g. 0.025 for a wrist) means the model must predict that joint more precisely to score well, while a larger sigma (e.g. 0.107 for a hip) is more forgiving. `VALIDATE_KEYPOINT_METRICS` will be `False` when your dataset has categories with different keypoint counts, because the standard COCO OKS evaluator assumes a uniform skeleton across all instances; bounding-box mAP is always computed regardless."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3111b71a",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if DATASET_FORMAT == \"yolo\":\n",
|
||||
" schema = infer_yolo_keypoint_schema(DATASET_DIR / \"data.yaml\")\n",
|
||||
" _flip_idx = schema.flip_idx\n",
|
||||
" _pairs: list[int] = []\n",
|
||||
" _seen: set[int] = set()\n",
|
||||
" for _i, _j in enumerate(_flip_idx):\n",
|
||||
" if _i in _seen or _j in _seen or _i == _j:\n",
|
||||
" _seen.add(_i)\n",
|
||||
" continue\n",
|
||||
" if _j < len(_flip_idx) and _flip_idx[_j] == _i:\n",
|
||||
" _pairs.extend([_i, _j])\n",
|
||||
" _seen.update({_i, _j})\n",
|
||||
" KEYPOINT_FLIP_PAIRS = _pairs\n",
|
||||
"else:\n",
|
||||
" TRAIN_ANNOTATIONS = DATASET_DIR / \"train\" / \"_annotations.coco.json\"\n",
|
||||
" schema = infer_coco_keypoint_schema(TRAIN_ANNOTATIONS) # type: ignore[assignment]\n",
|
||||
" KEYPOINT_FLIP_PAIRS = list(DATASET_INFO.get(\"keypoint_flip_pairs\", []))\n",
|
||||
"\n",
|
||||
"CLASS_NAMES = schema.class_names\n",
|
||||
"NUM_KEYPOINTS_PER_CLASS = schema.num_keypoints_per_class\n",
|
||||
"NUM_CLASSES = len(CLASS_NAMES)\n",
|
||||
"KEYPOINT_OKS_SIGMAS = schema.keypoint_oks_sigmas\n",
|
||||
"ACTIVE_KEYPOINT_COUNTS = [count for count in NUM_KEYPOINTS_PER_CLASS if count > 0]\n",
|
||||
"VALIDATE_KEYPOINT_METRICS = len(set(ACTIVE_KEYPOINT_COUNTS)) <= 1\n",
|
||||
"\n",
|
||||
"print(f\"class_names={CLASS_NAMES}\")\n",
|
||||
"print(f\"num_keypoints_per_class={NUM_KEYPOINTS_PER_CLASS}\")\n",
|
||||
"print(\"bbox_validation=True\")\n",
|
||||
"print(f\"keypoint_oks_validation={VALIDATE_KEYPOINT_METRICS}\")\n",
|
||||
"if VALIDATE_KEYPOINT_METRICS:\n",
|
||||
" print(f\"keypoint_oks_sigmas={len(KEYPOINT_OKS_SIGMAS)} values\")\n",
|
||||
"else:\n",
|
||||
" print(\"keypoint_oks_validation=skipped because COCO OKS requires one keypoint count across categories\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4c9b5a6b",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_save_final_checkpoint` writes a self-contained `.pth` file that bundles model weights with the full training and model config. This is separate from the PTL checkpoint because it can be loaded with a single `from_checkpoint` call on any machine, without reconstructing the original config."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e0bd03c1",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _save_final_checkpoint(\n",
|
||||
" module: RFDETRModelModule,\n",
|
||||
" trainer: Any,\n",
|
||||
" train_config: KeypointTrainConfig,\n",
|
||||
" model_config: Any,\n",
|
||||
" output_path: Path,\n",
|
||||
") -> Path:\n",
|
||||
" \"\"\"Save a final RF-DETR .pth checkpoint that can be loaded with ``from_checkpoint``.\"\"\"\n",
|
||||
" raw_model: Any = getattr(module.model, \"_orig_mod\", module.model)\n",
|
||||
" output_path.parent.mkdir(parents=True, exist_ok=True)\n",
|
||||
" torch.save(\n",
|
||||
" BestModelCallback._build_checkpoint_payload(\n",
|
||||
" raw_model.state_dict(),\n",
|
||||
" train_config.model_dump(),\n",
|
||||
" trainer,\n",
|
||||
" model_name=\"RFDETRKeypointPreview\",\n",
|
||||
" model_config_dict=model_config.model_dump(),\n",
|
||||
" ),\n",
|
||||
" output_path,\n",
|
||||
" )\n",
|
||||
" return output_path"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f516409d",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_annotated_keypoint_scene` renders keypoint dots and, when covariance data is available, uncertainty ellipses on top of an image. Drawing ellipses here rather than in the grid helper keeps the annotation logic in one place and makes it easy to toggle `DRAW_UNCERTAINTY_ELLIPSES` without touching the layout code."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "80f4317b",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _annotated_keypoint_scene(image: np.ndarray, key_points: sv.KeyPoints) -> np.ndarray:\n",
|
||||
" \"\"\"Return an image with visible vertices and optional uncertainty ellipses.\"\"\"\n",
|
||||
" scene = image.copy()\n",
|
||||
" if DRAW_UNCERTAINTY_ELLIPSES and \"covariance\" in key_points.data:\n",
|
||||
" scene = sv.VertexEllipseAnnotator(\n",
|
||||
" sigma=ELLIPSE_SIGMA,\n",
|
||||
" color=sv.Color.ROBOFLOW,\n",
|
||||
" max_axis=MAX_ELLIPSE_RADIUS,\n",
|
||||
" ).annotate(scene=scene, key_points=key_points)\n",
|
||||
"\n",
|
||||
" return sv.VertexAnnotator(radius=3).annotate(scene=scene, key_points=key_points)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "80f50cd8",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_keypoint_grid_figure` arranges multiple annotated images into a fixed-column matplotlib grid. Using a grid rather than individual figures keeps the visual summary compact and makes it easy to compare predictions across images at a glance."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "51734ce1",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _keypoint_grid_figure(items: list[tuple[str, np.ndarray, sv.KeyPoints]], columns: int) -> Figure:\n",
|
||||
" \"\"\"Render keypoint annotations in a fixed-column subplot grid.\"\"\"\n",
|
||||
" if columns <= 0:\n",
|
||||
" raise ValueError(f\"columns must be positive, got {columns}.\")\n",
|
||||
"\n",
|
||||
" rows = max(1, (len(items) + columns - 1) // columns)\n",
|
||||
" figure, axes = plt.subplots(rows, columns, figsize=(5 * columns, 5 * rows))\n",
|
||||
" axes_array = np.asarray(axes, dtype=object).reshape(-1)\n",
|
||||
" for axis in axes_array:\n",
|
||||
" axis.axis(\"off\")\n",
|
||||
"\n",
|
||||
" for axis, (title, image, key_points) in zip(axes_array, items, strict=False):\n",
|
||||
" axis.imshow(_annotated_keypoint_scene(image, key_points))\n",
|
||||
" axis.set_title(title, fontsize=10)\n",
|
||||
" axis.axis(\"off\")\n",
|
||||
"\n",
|
||||
" figure.tight_layout()\n",
|
||||
" return figure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2e548c8c",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_display_keypoint_records` prints per-image keypoint predictions as a pandas table. Fields that are constant across all keypoints in an image (such as the filename or detection score) are printed once as a header line rather than repeated in every row, keeping the output readable for images with many joints."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "757c151a",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def _display_keypoint_records(records: list[dict[str, Any]]) -> None:\n",
|
||||
" \"\"\"Display keypoint rows while printing per-image constant fields once.\"\"\"\n",
|
||||
" if not records:\n",
|
||||
" print(\"keypoint_rows=[]\")\n",
|
||||
" return\n",
|
||||
"\n",
|
||||
" keypoint_columns = {\"detection_index\", \"keypoint_index\", \"x\", \"y\", \"keypoint_confidence\"}\n",
|
||||
" records_frame = pd.DataFrame(records)\n",
|
||||
" for image_name, image_frame in records_frame.groupby(\"image\", dropna=False, sort=False):\n",
|
||||
" constant_columns = [\n",
|
||||
" column\n",
|
||||
" for column in image_frame.columns\n",
|
||||
" if column not in keypoint_columns and image_frame[column].nunique(dropna=False) == 1\n",
|
||||
" ]\n",
|
||||
" if constant_columns:\n",
|
||||
" constants = image_frame.iloc[0][constant_columns]\n",
|
||||
" print(\", \".join(f\"{column}={constants[column]}\" for column in constant_columns))\n",
|
||||
" elif image_name is not None:\n",
|
||||
" print(f\"image={image_name}\")\n",
|
||||
"\n",
|
||||
" display(image_frame.drop(columns=constant_columns).reset_index(drop=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "105aefc9",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 3 - Configure training\n",
|
||||
"\n",
|
||||
"`KeypointTrainConfig` centralises every hyperparameter the training loop needs. A few fields are worth understanding before you customise them for your own dataset. `lr` and `lr_encoder` are usually set to the same value for fine-tuning; if you notice the backbone overfitting early you can halve `lr_encoder` to protect the pre-trained features while the detection head continues to adapt. `grad_accum_steps=2` means gradients are accumulated over two mini-batches before a weight update, which effectively doubles the batch size without requiring extra GPU memory — useful on smaller GPUs. `use_ema=False` keeps the demo fast; for a production run you can set this to `True` to maintain an exponential moving average of weights, which often improves final accuracy by a few tenths of a mAP point. `multi_scale=False` and `expanded_scales=False` disable the multi-resolution augmentation that RF-DETR uses during pre-training; turning them off shortens epoch time significantly and is fine for most fine-tuning runs. The `notes` dict is stored in the checkpoint alongside the weights, giving you a lightweight experiment log that travels with the model file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fe060e29",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"seed_all(SEED)\n",
|
||||
"variant = RFDETRKeypointPreview( # type: ignore[no-untyped-call]\n",
|
||||
" num_classes=NUM_CLASSES,\n",
|
||||
" num_keypoints_per_class=NUM_KEYPOINTS_PER_CLASS,\n",
|
||||
" resolution=RESOLUTION,\n",
|
||||
")\n",
|
||||
"variant.model_config.model_name = type(variant).__name__\n",
|
||||
"\n",
|
||||
"train_config = KeypointTrainConfig(\n",
|
||||
" dataset_file=\"yolo\" if DATASET_FORMAT == \"yolo\" else \"roboflow\",\n",
|
||||
" dataset_dir=str(DATASET_DIR),\n",
|
||||
" output_dir=str(OUTPUT_DIR),\n",
|
||||
" epochs=EPOCHS,\n",
|
||||
" batch_size=BATCH_SIZE,\n",
|
||||
" grad_accum_steps=GRAD_ACCUM_STEPS,\n",
|
||||
" num_workers=NUM_WORKERS,\n",
|
||||
" lr=LR,\n",
|
||||
" lr_encoder=LR_ENCODER,\n",
|
||||
" use_ema=False,\n",
|
||||
" run_test=False,\n",
|
||||
" compute_train_metrics=True,\n",
|
||||
" compute_val_loss=True,\n",
|
||||
" multi_scale=False,\n",
|
||||
" expanded_scales=False,\n",
|
||||
" do_random_resize_via_padding=False,\n",
|
||||
" tensorboard=False,\n",
|
||||
" wandb=False,\n",
|
||||
" mlflow=False,\n",
|
||||
" clearml=False,\n",
|
||||
" class_names=CLASS_NAMES,\n",
|
||||
" keypoint_flip_pairs=KEYPOINT_FLIP_PAIRS,\n",
|
||||
" keypoint_oks_sigmas=KEYPOINT_OKS_SIGMAS if VALIDATE_KEYPOINT_METRICS else None,\n",
|
||||
" notes={\n",
|
||||
" \"demo\": f\"keypoint-preview PTL fine-tune on Roboflow Universe {DATASET_INFO['project']}\",\n",
|
||||
" \"source_url\": DATASET_INFO[\"source_url\"],\n",
|
||||
" \"roboflow_workspace\": DATASET_INFO[\"workspace\"],\n",
|
||||
" \"roboflow_project\": DATASET_INFO[\"project\"],\n",
|
||||
" \"roboflow_version\": DATASET_INFO[\"version\"],\n",
|
||||
" \"num_keypoints_per_class\": NUM_KEYPOINTS_PER_CLASS,\n",
|
||||
" \"keypoint_flip_pairs\": KEYPOINT_FLIP_PAIRS,\n",
|
||||
" \"bbox_validation_enabled\": True,\n",
|
||||
" \"keypoint_oks_validation_enabled\": VALIDATE_KEYPOINT_METRICS,\n",
|
||||
" \"keypoint_oks_sigmas\": KEYPOINT_OKS_SIGMAS if VALIDATE_KEYPOINT_METRICS else None,\n",
|
||||
" },\n",
|
||||
" progress_bar=\"tqdm\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"datamodule = RFDETRDataModule(variant.model_config, train_config)\n",
|
||||
"model = RFDETRModelModule(variant.model_config, train_config)\n",
|
||||
"trainer = build_trainer(train_config, variant.model_config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4cdfa696",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 4 - Preview dataset inputs\n",
|
||||
"\n",
|
||||
"Always look at your data before you start a long training run. This cell renders a grid of annotated training images so you can confirm that keypoint skeletons are aligned with objects, that flips and colour jitter look reasonable, and that there are no systematic labelling errors such as swapped left/right joints or keypoints sitting outside their bounding box. Catching label noise here costs a few seconds; catching it after 50 epochs of training costs much more. If the preview shows empty images or missing annotations, the most common cause is a mismatch between the image filenames in the COCO JSON and the files on disk — check that the download completed fully."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "df126db6",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"sample_figure = datamodule._show_samples(\n",
|
||||
" SAMPLE_PREVIEW_COUNT,\n",
|
||||
" split=\"train\",\n",
|
||||
" columns=SAMPLE_PREVIEW_COLUMNS,\n",
|
||||
" figure_size=SAMPLE_PREVIEW_FIGURE_SIZE,\n",
|
||||
")\n",
|
||||
"display(sample_figure)\n",
|
||||
"plt.close(sample_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "76be07a3",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 5 - Fine-tune\n",
|
||||
"\n",
|
||||
"`trainer.fit` hands control to PyTorch Lightning, which drives the full training loop: forward pass, loss computation, gradient accumulation, weight updates, learning-rate scheduling, and periodic validation. After each epoch the trainer appends a row to `metrics.csv` and, if validation mAP improves, saves a new best checkpoint via `BestModelCallback`. On a single consumer GPU (RTX 3090 or similar) 50 epochs on the dart dataset takes roughly 15–30 minutes depending on `BATCH_SIZE` and `NUM_WORKERS`. If you need to resume an interrupted run, set `train_config.resume` to the path of the last PTL checkpoint before calling this cell again."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "71d28853",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer.fit(model, datamodule=datamodule, ckpt_path=train_config.resume or None)\n",
|
||||
"print(f\"output_dir={OUTPUT_DIR}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e1e72b80",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 6 - Save checkpoint/model\n",
|
||||
"\n",
|
||||
"PTL saves its own checkpoints during training (optimizer state, scheduler state, epoch counter), but those files are not directly portable — they require the same class hierarchy to load. The `.pth` file written here is a self-contained RF-DETR checkpoint: it bundles the model weights together with the full training and model configs, including the keypoint schema and class names. This means you can share the file with a colleague and they can run inference with a single `RFDETRKeypointPreview.from_checkpoint(path)` call, with no need to reconstruct the original config. For full reproducibility, keep this checkpoint alongside the dataset version number printed in section 1."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5a052495",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"final_checkpoint = _save_final_checkpoint(model, trainer, train_config, variant.model_config, FINAL_CHECKPOINT_PATH)\n",
|
||||
"print(f\"saved_checkpoint_model={final_checkpoint}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1482fa79",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 7 - Validate metrics\n",
|
||||
"\n",
|
||||
"The validation pass that runs at the end of every training epoch uses the best weights seen so far and applies augmentation. This cell runs a clean post-training validation: no augmentation, the best checkpoint loaded from `checkpoint_best_total.pth` written by `BestModelCallback`, and results serialised to JSON for downstream comparison. Two metrics are most important to inspect here. `bbox/map` is the standard COCO bounding-box mAP at IoU 0.50:0.95 — it tells you how reliably the detector finds and bounds each object. `keypoint/oks` is the OKS-based mAP — it measures how precisely the model places each joint within the detected bounding boxes. A model can have a high `bbox/map` but a low `keypoint/oks` if it finds objects well but struggles to localise their joints; addressing that usually means more labelled data or tighter OKS sigmas.\n",
|
||||
"\n",
|
||||
"**Note:** `checkpoint_best_total.pth` is written after the first completed validation epoch. If training was interrupted before any epoch finished, this file will not exist and the cell below will raise `FileNotFoundError`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7aa26516",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# checkpoint_best_total.pth stores plain model weights under key \"model\", not a full\n",
|
||||
"# PTL checkpoint. Passing it as ckpt_path to trainer.validate() triggers a KeyError\n",
|
||||
"# on \"validate_loop\" because PTL tries to restore loop state that isn't present.\n",
|
||||
"# Load the weights directly and run validate without ckpt_path instead.\n",
|
||||
"_ckpt = torch.load(OUTPUT_DIR / \"checkpoint_best_total.pth\", map_location=\"cpu\", weights_only=False)\n",
|
||||
"model.model.load_state_dict(_ckpt[\"model\"], strict=True)\n",
|
||||
"del _ckpt\n",
|
||||
"validation_results = trainer.validate(model, datamodule=datamodule)\n",
|
||||
"validation_metrics = {key: float(value) for key, value in validation_results[0].items()} if validation_results else {}\n",
|
||||
"if not VALIDATE_KEYPOINT_METRICS:\n",
|
||||
" validation_metrics[\"keypoint_oks_skipped_mixed_keypoint_counts\"] = 1.0\n",
|
||||
"VALIDATION_METRICS_JSON.write_text(json.dumps(validation_metrics, indent=2, sort_keys=True), encoding=\"utf-8\")\n",
|
||||
"print(f\"validation_metrics={validation_metrics}\")\n",
|
||||
"print(f\"validation_metrics_json={VALIDATION_METRICS_JSON}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0aa1a943",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 8 - Plot CSVLogger metrics\n",
|
||||
"\n",
|
||||
"Reading loss curves is one of the fastest ways to diagnose training problems. Healthy runs show both training loss and validation loss decreasing together; a large and growing gap between the two is a classic overfitting signal — the model is memorising the training set rather than generalising. For the mAP curves, on small datasets (a few hundred images) you typically see rapid improvement in the first 10–20 epochs followed by a plateau around epoch 30–50; if mAP is still climbing at epoch 50, consider extending `EPOCHS`. Set `PLOT_LOSS_LOG_SCALE = True` if the loss drops by an order of magnitude in the first few epochs and the later, more meaningful portion of the curve gets compressed into a flat line at the bottom of the plot."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "37589b33",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(f\"metrics_csv={METRICS_CSV}\")\n",
|
||||
"loss_figure = plot_loss_metrics(str(METRICS_CSV), loss_log_scale=PLOT_LOSS_LOG_SCALE)\n",
|
||||
"display(loss_figure)\n",
|
||||
"plt.close(loss_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a5608045",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"map_figure = plot_map_metrics(str(METRICS_CSV))\n",
|
||||
"display(map_figure)\n",
|
||||
"plt.close(map_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "690a200a",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 9 - Load checkpoint/model\n",
|
||||
"\n",
|
||||
"`from_checkpoint` is the standard entry point for loading a saved RF-DETR model. It reads both the weights and the stored config from the `.pth` file, so the reconstructed model has the correct number of classes and keypoints without you having to pass them explicitly. You can share this checkpoint file with teammates and they can run inference immediately — the keypoint schema, class names, and OKS sigmas are all embedded in the file alongside the weights. This cell also confirms that the save-load round trip works before you proceed to inference, so any corruption or version mismatch surfaces here rather than silently producing wrong predictions later."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c8d75b0f",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loaded_model = RFDETRKeypointPreview.from_checkpoint(FINAL_CHECKPOINT_PATH)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7bee8d28",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 10 - Select inference images\n",
|
||||
"\n",
|
||||
"This cell implements a fallback chain — test split first, then validation, then train — so the notebook always finds images to run inference on even when the dataset has no dedicated test split. Using images from the test set gives you an unbiased view of model performance because those images were never seen during training or used to pick the best checkpoint. If you want to run inference on your own images, replace `inference_image_paths` with a list of `Path` objects pointing to your files before running the next cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a5182f47",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"_IMAGE_EXTS = {\".jpg\", \".jpeg\", \".png\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _find_images(directory: Path) -> list[Path]:\n",
|
||||
" if not directory.is_dir():\n",
|
||||
" return []\n",
|
||||
" # Iterate through all subfolders to find images\n",
|
||||
" return sorted(p for p in directory.glob(\"**/*\") if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"validation_image_paths = _find_images(DATASET_DIR / \"test\")\n",
|
||||
"if not validation_image_paths:\n",
|
||||
" validation_image_paths = _find_images(DATASET_DIR / \"valid\")\n",
|
||||
"if not validation_image_paths:\n",
|
||||
" validation_image_paths = _find_images(DATASET_DIR / \"train\")\n",
|
||||
"if not validation_image_paths:\n",
|
||||
" raise FileNotFoundError(f\"No images ({', '.join(sorted(_IMAGE_EXTS))}) found under {DATASET_DIR}\")\n",
|
||||
"\n",
|
||||
"inference_image_paths = validation_image_paths[:INFERENCE_COUNT]\n",
|
||||
"print(f\"inference_images={[str(path) for path in inference_image_paths]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1c7654a9",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 11 - Visualize inference\n",
|
||||
"\n",
|
||||
"Two thresholds control what you see here. `INFERENCE_THRESHOLD` is a detection confidence threshold: only detections whose bounding-box score exceeds this value are shown. Raise it to suppress false positives; lower it to surface low-confidence detections. `KEYPOINT_THRESHOLD` operates within each accepted detection: individual keypoints whose confidence falls below this value are treated as not visible and omitted from the overlay. The uncertainty ellipses (drawn when `DRAW_UNCERTAINTY_ELLIPSES = True`) visualise the predicted covariance for each keypoint — a larger ellipse means the model is less certain about that joint's exact location."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e520b84c",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"inference_grid_items = []\n",
|
||||
"keypoint_rows = []\n",
|
||||
"for image_path in inference_image_paths:\n",
|
||||
" with Image.open(image_path) as image:\n",
|
||||
" image = image.convert(\"RGB\")\n",
|
||||
" key_points_raw = loaded_model.predict(image, threshold=INFERENCE_THRESHOLD)\n",
|
||||
" if not isinstance(key_points_raw, sv.KeyPoints):\n",
|
||||
" raise RuntimeError(\n",
|
||||
" f\"Expected RFDETRKeypointPreview.predict() to return sv.KeyPoints, got {type(key_points_raw)!r}.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" key_points = _key_points_for_display(key_points_raw, keypoint_threshold=KEYPOINT_THRESHOLD)\n",
|
||||
" rows = _keypoint_prediction_records(key_points, image=image_path, keypoint_threshold=KEYPOINT_THRESHOLD)\n",
|
||||
" inference_grid_items.append((image_path.name, np.array(image), key_points))\n",
|
||||
" keypoint_rows.extend(rows)\n",
|
||||
"\n",
|
||||
"if inference_grid_items:\n",
|
||||
" figure = _keypoint_grid_figure(inference_grid_items, columns=INFERENCE_COLUMNS)\n",
|
||||
" display(figure)\n",
|
||||
" plt.close(figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "93df94eb",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 12 - Keypoint prediction table\n",
|
||||
"\n",
|
||||
"The table below gives you the raw coordinates and per-keypoint confidence scores for every detection in the grid above. Fields that are constant across all keypoints in an image (filename, detection score) are printed once as a header line rather than repeated in every row. This format is easy to copy into a spreadsheet or feed into downstream analysis."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "de9b2707",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"_display_keypoint_records(keypoint_rows)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"jupytext": {
|
||||
"cell_metadata_filter": "title,-all",
|
||||
"main_language": "python",
|
||||
"notebook_metadata_filter": "-all"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,846 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e6fd4da4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# RF-DETR instance segmentation fine-tuning on Roboflow Universe datasets\n",
|
||||
"\n",
|
||||
"Select one dataset with `DATASET_KEY`, then run the same fine-tuning, plotting, checkpoint loading, and inference cells.\n",
|
||||
"\n",
|
||||
"RF-DETR extends bounding-box detection with a sparse segmentation head that predicts per-object binary masks — useful\n",
|
||||
"for facade analysis, infrastructure damage inspection, vehicle part assessment, dental imaging, litter mapping, and\n",
|
||||
"any task where *what shape* matters as much as *what class*.\n",
|
||||
"\n",
|
||||
"**Datasets** — set `DATASET_KEY` to one of:\n",
|
||||
"\n",
|
||||
"| Key | Dataset | Classes |\n",
|
||||
"|-----|---------|---------|\n",
|
||||
"| `\"building_facade\"` | Building Facade Segmentation | facade, window, street, vegetation, car, fence, ... |\n",
|
||||
"| `\"spalling_rebar\"` | Spalling and Exposed Rebar | exposed_rebar, spalling |\n",
|
||||
"| `\"car_damage_parts\"` | Car Part Damage Segmentation | bumper, wheel, door, headlight, dent, scratch, ... |\n",
|
||||
"| `\"teeth_numbering\"` | Teeth Numbering Segmentation | tooth IDs 11-48 |\n",
|
||||
"| `\"taco_trash\"` | TACO Trash | bottle, can, carton, cup, lid, straw, wrapper, ... |\n",
|
||||
"\n",
|
||||
"Every subsequent cell (fine-tuning, plotting, checkpoint loading, inference) adapts automatically.\n",
|
||||
"By the end you will have a fine-tuned model, training curves, and annotated inference images with segmentation masks."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6eaacf22",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"Install `rfdetr` with the `train` and `visual` extras. The `train` extra pulls in the training loop dependencies\n",
|
||||
"(PyTorch Lightning, COCO evaluation tools, and data augmentation libraries), while `visual` adds the visualisation\n",
|
||||
"helpers used later in the notebook. The `roboflow` package handles dataset download; `pandas` and `seaborn` are needed\n",
|
||||
"for the metrics table and optional plot styling. If you hit an `ImportError` after running this cell, the most likely\n",
|
||||
"cause is a stale in-memory import — restart the Python runtime once and re-run from the top."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "64736ca1",
|
||||
"metadata": {},
|
||||
"source": "!pip install -q \"rfdetr[train,visual]>=1.8.2\" roboflow pandas seaborn",
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6b4797e9",
|
||||
"metadata": {
|
||||
"title": "[bash]"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\"\"\"Shared RF-DETR instance segmentation fine-tuning demo for several Roboflow COCO segmentation exports.\"\"\"\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"from pathlib import Path\n",
|
||||
"from typing import Any\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import supervision as sv\n",
|
||||
"import torch\n",
|
||||
"from IPython.display import display\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"from matplotlib.figure import Figure\n",
|
||||
"from PIL import Image\n",
|
||||
"from roboflow import Roboflow\n",
|
||||
"\n",
|
||||
"from rfdetr import RFDETRSegSmall\n",
|
||||
"from rfdetr.config import SegmentationTrainConfig\n",
|
||||
"from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer\n",
|
||||
"from rfdetr.training.callbacks.best_model import BestModelCallback\n",
|
||||
"from rfdetr.utilities.reproducibility import seed_all\n",
|
||||
"from rfdetr.visualize.training import plot_loss_metrics, plot_map_metrics"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "99f1f539",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ROOT = Path(__file__).resolve().parent if \"__file__\" in globals() else Path.cwd()\n",
|
||||
"DATASETS_DIR = PROJECT_ROOT / \"datasets\"\n",
|
||||
"\n",
|
||||
"# Resolution must be divisible by patch_size × num_windows (24 for RFDETRSegSmall).\n",
|
||||
"# Larger resolutions capture fine-grained mask boundaries at the cost of GPU memory and time.\n",
|
||||
"DATASETS: dict[str, dict[str, Any]] = {\n",
|
||||
" \"building_facade\": {\n",
|
||||
" \"name\": \"Building Facade Segmentation\",\n",
|
||||
" \"workspace\": \"building-facade\",\n",
|
||||
" \"project\": \"building-facade-segmentation-instance\",\n",
|
||||
" \"version\": 4,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/building-facade/building-facade-segmentation-instance\",\n",
|
||||
" \"output_name\": \"seg_building_facade\",\n",
|
||||
" # 432 — structured street scenes with medium-to-large facade regions plus\n",
|
||||
" # smaller windows, cars, vegetation, and infrastructure boundaries.\n",
|
||||
" \"resolution\": 432,\n",
|
||||
" },\n",
|
||||
" \"spalling_rebar\": {\n",
|
||||
" \"name\": \"Spalling and Exposed Rebar\",\n",
|
||||
" \"workspace\": \"uni-of-birmingham\",\n",
|
||||
" \"project\": \"spalling-and-exposed-rebar\",\n",
|
||||
" \"version\": 7,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/uni-of-birmingham/spalling-and-exposed-rebar\",\n",
|
||||
" \"output_name\": \"seg_spalling_rebar\",\n",
|
||||
" # 432 — civil-infrastructure damage masks with irregular thin regions and\n",
|
||||
" # exposed-rebar details that stress boundary quality.\n",
|
||||
" \"resolution\": 432,\n",
|
||||
" },\n",
|
||||
" \"car_damage_parts\": {\n",
|
||||
" \"name\": \"Car Part Damage Segmentation\",\n",
|
||||
" \"workspace\": \"car-damaged-detection-e66m0\",\n",
|
||||
" \"project\": \"car-part-detection-with-damage-part\",\n",
|
||||
" \"version\": 1,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/car-damaged-detection-e66m0/car-part-detection-with-damage-part\",\n",
|
||||
" \"output_name\": \"seg_car_damage_parts\",\n",
|
||||
" # 432 — fine-grained automotive part masks with many adjacent classes and\n",
|
||||
" # damage regions, useful for checking class-specific mask boundaries.\n",
|
||||
" \"resolution\": 432,\n",
|
||||
" },\n",
|
||||
" \"teeth_numbering\": {\n",
|
||||
" \"name\": \"Teeth Numbering Segmentation\",\n",
|
||||
" \"workspace\": \"godento2\",\n",
|
||||
" \"project\": \"teeth-seg-3537-iaky1\",\n",
|
||||
" \"version\": 1,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/godento2/teeth-seg-3537-iaky1\",\n",
|
||||
" \"output_name\": \"seg_teeth_numbering\",\n",
|
||||
" # 432 — dental images with many repeated small instances whose labels are\n",
|
||||
" # tied to position, stressing class confusion and small-mask quality.\n",
|
||||
" \"resolution\": 432,\n",
|
||||
" },\n",
|
||||
" \"taco_trash\": {\n",
|
||||
" \"name\": \"TACO Trash\",\n",
|
||||
" \"workspace\": \"mohamed-traore-2ekkp\",\n",
|
||||
" \"project\": \"taco-trash-annotations-in-context\",\n",
|
||||
" \"version\": 13,\n",
|
||||
" \"source_url\": \"https://universe.roboflow.com/mohamed-traore-2ekkp/taco-trash-annotations-in-context\",\n",
|
||||
" \"output_name\": \"seg_taco_trash\",\n",
|
||||
" # 432 — long-tail litter segmentation with many small, cluttered object\n",
|
||||
" # categories. Use as a harder stress test after the simpler datasets.\n",
|
||||
" \"resolution\": 432,\n",
|
||||
" },\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"DATASET_KEY = \"building_facade\"\n",
|
||||
"DATASET_INFO = DATASETS[DATASET_KEY]\n",
|
||||
"\n",
|
||||
"OUTPUT_DIR = PROJECT_ROOT / \"output\" / str(DATASET_INFO[\"output_name\"])\n",
|
||||
"METRICS_CSV = OUTPUT_DIR / \"metrics.csv\"\n",
|
||||
"VALIDATION_METRICS_JSON = OUTPUT_DIR / \"validation_metrics.json\"\n",
|
||||
"FINAL_CHECKPOINT_PATH = OUTPUT_DIR / \"checkpoint_final_demo.pth\"\n",
|
||||
"\n",
|
||||
"RESOLUTION = int(DATASET_INFO[\"resolution\"])\n",
|
||||
"\n",
|
||||
"SEED = 7\n",
|
||||
"EPOCHS = 50\n",
|
||||
"BATCH_SIZE = 8\n",
|
||||
"GRAD_ACCUM_STEPS = 2\n",
|
||||
"NUM_WORKERS = 8\n",
|
||||
"LR = 1e-4\n",
|
||||
"LR_ENCODER = 1e-4\n",
|
||||
"SAMPLE_PREVIEW_COUNT = 6\n",
|
||||
"SAMPLE_PREVIEW_COLUMNS = 3\n",
|
||||
"SAMPLE_PREVIEW_FIGURE_SIZE: tuple[float, float] = (15.0, 10.0)\n",
|
||||
"INFERENCE_COUNT = 6\n",
|
||||
"INFERENCE_COLUMNS = 3\n",
|
||||
"INFERENCE_THRESHOLD = 0.3\n",
|
||||
"PLOT_LOSS_LOG_SCALE = False\n",
|
||||
"\n",
|
||||
"print(f\"dataset_key={DATASET_KEY}\")\n",
|
||||
"print(f\"dataset={DATASET_INFO['name']}\")\n",
|
||||
"print(f\"source_url={DATASET_INFO['source_url']}\")\n",
|
||||
"print(f\"resolution={RESOLUTION}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b60971f9",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## Notebook display\n",
|
||||
"\n",
|
||||
"This helper registers the `%matplotlib inline` magic so figures render directly beneath each cell when you run the\n",
|
||||
"notebook in Jupyter or Google Colab. If you are running this file as a plain Python script the function detects the\n",
|
||||
"absence of an IPython kernel and exits silently, so it is always safe to call."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "1c40c67d",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"def _enable_notebook_inline_matplotlib() -> None:\n",
|
||||
" \"\"\"Enable inline matplotlib figures when running in IPython.\"\"\"\n",
|
||||
" get_ipython_func = globals().get(\"get_ipython\")\n",
|
||||
" if not callable(get_ipython_func):\n",
|
||||
" return\n",
|
||||
" ipython = get_ipython_func()\n",
|
||||
" if ipython is not None:\n",
|
||||
" ipython.run_line_magic(\"matplotlib\", \"inline\")\n",
|
||||
" ipython.run_line_magic(\"config\", \"InlineBackend.close_figures = True\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"_enable_notebook_inline_matplotlib()"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7f99c55d",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 1 - Download dataset\n",
|
||||
"\n",
|
||||
"Roboflow exports instance segmentation datasets in COCO format: a JSON file where each annotation contains a\n",
|
||||
"`segmentation` field with polygon coordinates that trace the precise object boundary. The download cell fetches the\n",
|
||||
"exact dataset version listed in `DATASETS` and places it under `datasets/<DATASET_KEY>/`. You need a Roboflow API key\n",
|
||||
"— get one for free at app.roboflow.com/settings/api and set it as the `ROBOFLOW_API_KEY` environment variable (or as\n",
|
||||
"a Colab secret with the same name). The download is idempotent: if the target directory already exists Roboflow skips\n",
|
||||
"the network transfer, so re-running this cell after a successful download is fast."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "ab8fdeb6",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"try:\n",
|
||||
" from google.colab import userdata\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" ROBOFLOW_API_KEY = userdata.get(\"ROBOFLOW_API_KEY\") or \"\"\n",
|
||||
" except Exception:\n",
|
||||
" ROBOFLOW_API_KEY = \"\"\n",
|
||||
"except ImportError:\n",
|
||||
" ROBOFLOW_API_KEY = \"\"\n",
|
||||
"\n",
|
||||
"if not ROBOFLOW_API_KEY:\n",
|
||||
" ROBOFLOW_API_KEY = os.environ.get(\"ROBOFLOW_API_KEY\", \"\")\n",
|
||||
"if not ROBOFLOW_API_KEY:\n",
|
||||
" raise RuntimeError(\n",
|
||||
" \"ROBOFLOW_API_KEY not found. \"\n",
|
||||
" \"In Colab: add it via Secrets (key icon). \"\n",
|
||||
" \"Locally: set the environment variable before running.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"rf = Roboflow(api_key=ROBOFLOW_API_KEY)\n",
|
||||
"dataset = (\n",
|
||||
" rf.workspace(str(DATASET_INFO[\"workspace\"]))\n",
|
||||
" .project(str(DATASET_INFO[\"project\"]))\n",
|
||||
" .version(int(DATASET_INFO[\"version\"]))\n",
|
||||
" .download(\"coco\", location=str(DATASETS_DIR / DATASET_KEY))\n",
|
||||
")\n",
|
||||
"DATASET_DIR = Path(dataset.location)\n",
|
||||
"TRAIN_ANNOTATIONS = DATASET_DIR / \"train\" / \"_annotations.coco.json\"\n",
|
||||
"print(f\"dataset_dir={DATASET_DIR}\")"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e38db150",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 2 - Infer class names\n",
|
||||
"\n",
|
||||
"Read the class names directly from the COCO annotation file. RF-DETR uses 0-based class indices internally; the\n",
|
||||
"`class_names` list maps each index to a human-readable label so the inference visualisation shows category names\n",
|
||||
"rather than numbers. If your dataset has a single category the list will have one element — that is expected."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "2c6b8dc4",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"with TRAIN_ANNOTATIONS.open() as _f:\n",
|
||||
" _coco = json.load(_f)\n",
|
||||
"\n",
|
||||
"CLASS_NAMES: list[str] = [cat[\"name\"] for cat in sorted(_coco[\"categories\"], key=lambda c: c[\"id\"])]\n",
|
||||
"NUM_CLASSES = len(CLASS_NAMES)\n",
|
||||
"\n",
|
||||
"print(f\"class_names={CLASS_NAMES}\")\n",
|
||||
"print(f\"num_classes={NUM_CLASSES}\")"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "945822fa",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_save_final_checkpoint` writes a self-contained `.pth` file that bundles model weights with the full training and\n",
|
||||
"model config. This is separate from the PTL checkpoint because it can be loaded with a single `from_checkpoint` call\n",
|
||||
"on any machine, without reconstructing the original config."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "ed36df66",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"def _save_final_checkpoint(\n",
|
||||
" module: RFDETRModelModule,\n",
|
||||
" trainer: Any,\n",
|
||||
" train_config: SegmentationTrainConfig,\n",
|
||||
" model_config: Any,\n",
|
||||
" output_path: Path,\n",
|
||||
") -> Path:\n",
|
||||
" \"\"\"Save a final RF-DETR .pth checkpoint that can be loaded with ``from_checkpoint``.\"\"\"\n",
|
||||
" raw_model: Any = getattr(module.model, \"_orig_mod\", module.model)\n",
|
||||
" output_path.parent.mkdir(parents=True, exist_ok=True)\n",
|
||||
" torch.save(\n",
|
||||
" BestModelCallback._build_checkpoint_payload(\n",
|
||||
" raw_model.state_dict(),\n",
|
||||
" train_config.model_dump(),\n",
|
||||
" trainer,\n",
|
||||
" model_name=\"RFDETRSegSmall\",\n",
|
||||
" model_config_dict=model_config.model_dump(),\n",
|
||||
" ),\n",
|
||||
" output_path,\n",
|
||||
" )\n",
|
||||
" return output_path"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7000d9a4",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"`_segmentation_grid_figure` arranges multiple annotated images into a fixed-column matplotlib grid.\n",
|
||||
"Masks are drawn with `sv.MaskAnnotator` (translucent colour fill) and bounding boxes with\n",
|
||||
"`sv.BoxAnnotator`. Labels show the class name and confidence score."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "477cd2f6",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"def _segmentation_grid_figure(\n",
|
||||
" items: list[tuple[str, np.ndarray, sv.Detections]],\n",
|
||||
" columns: int,\n",
|
||||
" class_names: list[str],\n",
|
||||
") -> Figure:\n",
|
||||
" \"\"\"Render segmentation masks and bounding boxes in a fixed-column subplot grid.\"\"\"\n",
|
||||
" if columns <= 0:\n",
|
||||
" raise ValueError(f\"columns must be positive, got {columns}.\")\n",
|
||||
"\n",
|
||||
" mask_annotator = sv.MaskAnnotator()\n",
|
||||
" box_annotator = sv.BoxAnnotator(thickness=2)\n",
|
||||
" label_annotator = sv.LabelAnnotator(text_scale=0.5, text_thickness=1, text_padding=4)\n",
|
||||
"\n",
|
||||
" rows = max(1, (len(items) + columns - 1) // columns)\n",
|
||||
" figure, axes = plt.subplots(rows, columns, figsize=(5 * columns, 5 * rows))\n",
|
||||
" axes_array = np.asarray(axes, dtype=object).reshape(-1)\n",
|
||||
" for axis in axes_array:\n",
|
||||
" axis.axis(\"off\")\n",
|
||||
"\n",
|
||||
" for axis, (title, image, detections) in zip(axes_array, items, strict=False):\n",
|
||||
" scene = image.copy()\n",
|
||||
" if detections.mask is not None and len(detections) > 0:\n",
|
||||
" scene = mask_annotator.annotate(scene=scene, detections=detections)\n",
|
||||
" scene = box_annotator.annotate(scene=scene, detections=detections)\n",
|
||||
" if len(detections) > 0 and detections.class_id is not None:\n",
|
||||
" conf_list = (\n",
|
||||
" detections.confidence.tolist() if detections.confidence is not None else [None] * len(detections)\n",
|
||||
" )\n",
|
||||
" labels = [\n",
|
||||
" f\"{class_names[cid] if cid < len(class_names) else cid}\" + (f\" {conf:.2f}\" if conf is not None else \"\")\n",
|
||||
" for cid, conf in zip(detections.class_id.tolist(), conf_list)\n",
|
||||
" ]\n",
|
||||
" scene = label_annotator.annotate(scene=scene, detections=detections, labels=labels)\n",
|
||||
" axis.imshow(scene)\n",
|
||||
" axis.set_title(title, fontsize=10)\n",
|
||||
" axis.axis(\"off\")\n",
|
||||
"\n",
|
||||
" figure.tight_layout()\n",
|
||||
" return figure"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8e53a5a5",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 3 - Configure training\n",
|
||||
"\n",
|
||||
"`SegmentationTrainConfig` centralises every hyperparameter the training loop needs, including mask-specific loss\n",
|
||||
"coefficients (`mask_ce_loss_coef` and `mask_dice_loss_coef`). The defaults (5.0 each) work well for most datasets;\n",
|
||||
"if the segmentation head overfits early you can reduce them towards 2.0 while keeping `cls_loss_coef=5.0`.\n",
|
||||
"\n",
|
||||
"`lr` and `lr_encoder` control the head and backbone learning rates respectively. For fine-tuning both are usually\n",
|
||||
"set to the same value; if you notice the backbone overfitting early, halve `lr_encoder` to protect the pre-trained\n",
|
||||
"features. `grad_accum_steps=2` doubles the effective batch size without extra GPU memory.\n",
|
||||
"\n",
|
||||
"`multi_scale=False` and `expanded_scales=False` disable multi-resolution augmentation to shorten epoch time during\n",
|
||||
"fine-tuning — they rarely help on small custom datasets. The `notes` dict is stored alongside the weights in the\n",
|
||||
"checkpoint, giving you a lightweight experiment log that travels with the model file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "3e71bc58",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"seed_all(SEED)\n",
|
||||
"variant = RFDETRSegSmall( # type: ignore[no-untyped-call]\n",
|
||||
" num_classes=NUM_CLASSES,\n",
|
||||
" resolution=RESOLUTION,\n",
|
||||
")\n",
|
||||
"variant.model_config.model_name = type(variant).__name__\n",
|
||||
"\n",
|
||||
"train_config = SegmentationTrainConfig(\n",
|
||||
" dataset_file=\"roboflow\",\n",
|
||||
" dataset_dir=str(DATASET_DIR),\n",
|
||||
" output_dir=str(OUTPUT_DIR),\n",
|
||||
" epochs=EPOCHS,\n",
|
||||
" batch_size=BATCH_SIZE,\n",
|
||||
" grad_accum_steps=GRAD_ACCUM_STEPS,\n",
|
||||
" num_workers=NUM_WORKERS,\n",
|
||||
" lr=LR,\n",
|
||||
" lr_encoder=LR_ENCODER,\n",
|
||||
" use_ema=False,\n",
|
||||
" run_test=False,\n",
|
||||
" compute_train_metrics=True,\n",
|
||||
" compute_val_loss=True,\n",
|
||||
" multi_scale=False,\n",
|
||||
" expanded_scales=False,\n",
|
||||
" do_random_resize_via_padding=False,\n",
|
||||
" tensorboard=False,\n",
|
||||
" wandb=False,\n",
|
||||
" mlflow=False,\n",
|
||||
" clearml=False,\n",
|
||||
" class_names=CLASS_NAMES,\n",
|
||||
" notes={\n",
|
||||
" \"demo\": f\"segmentation PTL fine-tune on Roboflow Universe {DATASET_INFO['project']}\",\n",
|
||||
" \"source_url\": DATASET_INFO[\"source_url\"],\n",
|
||||
" \"roboflow_workspace\": DATASET_INFO[\"workspace\"],\n",
|
||||
" \"roboflow_project\": DATASET_INFO[\"project\"],\n",
|
||||
" \"roboflow_version\": DATASET_INFO[\"version\"],\n",
|
||||
" \"num_classes\": NUM_CLASSES,\n",
|
||||
" \"class_names\": CLASS_NAMES,\n",
|
||||
" },\n",
|
||||
" progress_bar=\"tqdm\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"datamodule = RFDETRDataModule(variant.model_config, train_config)\n",
|
||||
"model = RFDETRModelModule(variant.model_config, train_config)\n",
|
||||
"trainer = build_trainer(train_config, variant.model_config)"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5f0ccbd3",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 4 - Preview dataset inputs\n",
|
||||
"\n",
|
||||
"Always look at your data before you start a long training run. This cell renders a grid of annotated training images\n",
|
||||
"so you can confirm that segmentation masks align with objects, colour jitter and flips look reasonable, and there are\n",
|
||||
"no systematic labelling errors such as masks placed on the wrong object or masks that bleed outside the object\n",
|
||||
"boundary. Catching label noise here costs a few seconds; catching it after 50 epochs costs much more."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "71e1a7db",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"sample_figure = datamodule._show_samples(\n",
|
||||
" SAMPLE_PREVIEW_COUNT,\n",
|
||||
" split=\"train\",\n",
|
||||
" columns=SAMPLE_PREVIEW_COLUMNS,\n",
|
||||
" figure_size=SAMPLE_PREVIEW_FIGURE_SIZE,\n",
|
||||
")\n",
|
||||
"display(sample_figure)\n",
|
||||
"plt.close(sample_figure)"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "993b1cd1",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 5 - Fine-tune\n",
|
||||
"\n",
|
||||
"`trainer.fit` hands control to PyTorch Lightning, which drives the full training loop: forward pass, loss\n",
|
||||
"computation (cross-entropy + Dice on sampled mask points), gradient accumulation, weight updates,\n",
|
||||
"learning-rate scheduling, and periodic validation with COCO mask mAP. After each epoch the trainer appends a row to\n",
|
||||
"`metrics.csv` and, if validation mAP improves, saves a new best checkpoint via `BestModelCallback`. On a single\n",
|
||||
"consumer GPU (RTX 3090 or similar) 50 epochs on the crack dataset takes roughly 20–35 minutes depending on\n",
|
||||
"`BATCH_SIZE` and `NUM_WORKERS`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "0564b2c6",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"trainer.fit(model, datamodule=datamodule, ckpt_path=train_config.resume or None)\n",
|
||||
"print(f\"output_dir={OUTPUT_DIR}\")"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c746e44d",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 6 - Save checkpoint/model\n",
|
||||
"\n",
|
||||
"PTL saves its own checkpoints during training (optimizer state, scheduler state, epoch counter), but those files are\n",
|
||||
"not directly portable — they require the same class hierarchy to load. The `.pth` file written here is a\n",
|
||||
"self-contained RF-DETR checkpoint: it bundles the model weights together with the full training and model configs,\n",
|
||||
"including the class names. You can share the file with a colleague and they can run inference with a single\n",
|
||||
"`RFDETRSegSmall.from_checkpoint(path)` call. For full reproducibility, keep this checkpoint alongside the dataset\n",
|
||||
"version number printed in section 1."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "684059b3",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"final_checkpoint = _save_final_checkpoint(model, trainer, train_config, variant.model_config, FINAL_CHECKPOINT_PATH)\n",
|
||||
"print(f\"saved_checkpoint={final_checkpoint}\")"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dadba8e3",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 7 - Validate metrics\n",
|
||||
"\n",
|
||||
"This cell runs a clean post-training validation with no augmentation, using the best checkpoint loaded from\n",
|
||||
"`checkpoint_best_total.pth` written by `BestModelCallback`, and serialises results to JSON. Two metrics are most\n",
|
||||
"important to inspect. `bbox/map` is the standard COCO bounding-box mAP at IoU 0.50:0.95. `segm/map` (when logged)\n",
|
||||
"is the mask mAP — it measures how precisely the predicted binary masks overlap with the annotated polygons. A model\n",
|
||||
"can have high `bbox/map` but low `segm/map` if the detector locates objects well but the mask boundaries are coarse;\n",
|
||||
"this usually improves with more training epochs or by increasing `resolution`.\n",
|
||||
"\n",
|
||||
"**Note:** `checkpoint_best_total.pth` is written after the first completed validation epoch. If training was\n",
|
||||
"interrupted before any epoch finished, this file will not exist and the cell below will raise `FileNotFoundError`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "ea69ee92",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"# checkpoint_best_total.pth stores plain model weights under key \"model\", not a full\n",
|
||||
"# PTL checkpoint. Passing it as ckpt_path to trainer.validate() triggers a KeyError\n",
|
||||
"# on \"validate_loop\" because PTL tries to restore loop state that isn't present.\n",
|
||||
"# Load the weights directly and run validate without ckpt_path instead.\n",
|
||||
"_ckpt = torch.load(OUTPUT_DIR / \"checkpoint_best_total.pth\", map_location=\"cpu\", weights_only=False)\n",
|
||||
"model.model.load_state_dict(_ckpt[\"model\"], strict=True)\n",
|
||||
"del _ckpt\n",
|
||||
"validation_results = trainer.validate(model, datamodule=datamodule)\n",
|
||||
"validation_metrics = {key: float(value) for key, value in validation_results[0].items()} if validation_results else {}\n",
|
||||
"VALIDATION_METRICS_JSON.write_text(json.dumps(validation_metrics, indent=2, sort_keys=True), encoding=\"utf-8\")\n",
|
||||
"print(f\"validation_metrics={validation_metrics}\")\n",
|
||||
"print(f\"validation_metrics_json={VALIDATION_METRICS_JSON}\")"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e9b7a1cb",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 8 - Plot CSVLogger metrics\n",
|
||||
"\n",
|
||||
"Reading loss curves is one of the fastest ways to diagnose training problems. Healthy segmentation runs show the\n",
|
||||
"combined detection + mask loss decreasing together for train and val. A loss that rises or plateaus from epoch 1 is\n",
|
||||
"usually a sign that the learning rate is too high or the dataset is too small for the chosen `BATCH_SIZE`. For the\n",
|
||||
"mAP curves, `segm mAP 50` (mask IoU ≥ 0.50) rises fastest and is the clearest signal of whether the model is\n",
|
||||
"learning to segment objects; `segm mAP 50:95` (averaged across stricter thresholds) follows more slowly.\n",
|
||||
"Set `PLOT_LOSS_LOG_SCALE = True` if the loss drops by an order of magnitude in the first few epochs and the later,\n",
|
||||
"more meaningful portion of the curve gets compressed into a flat line."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "abab244f",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"print(f\"metrics_csv={METRICS_CSV}\")\n",
|
||||
"loss_figure = plot_loss_metrics(str(METRICS_CSV), loss_log_scale=PLOT_LOSS_LOG_SCALE)\n",
|
||||
"display(loss_figure)\n",
|
||||
"plt.close(loss_figure)"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c93477c9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"map_figure = plot_map_metrics(str(METRICS_CSV))\n",
|
||||
"display(map_figure)\n",
|
||||
"plt.close(map_figure)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e46c2424",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 9 - Load checkpoint/model\n",
|
||||
"\n",
|
||||
"`from_checkpoint` is the standard entry point for loading a saved RF-DETR model. It reads both the weights and the\n",
|
||||
"stored config from the `.pth` file, so the reconstructed model has the correct number of classes and resolution\n",
|
||||
"without you having to pass them explicitly. You can share this file with teammates and they can run inference\n",
|
||||
"immediately — the class names and architecture are all embedded alongside the weights."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "58b00247",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": "loaded_model = RFDETRSegSmall.from_checkpoint(FINAL_CHECKPOINT_PATH)",
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d2252a7a",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 10 - Select inference images\n",
|
||||
"\n",
|
||||
"This cell implements a fallback chain — test split first, then validation, then train — so the notebook always finds\n",
|
||||
"images to run inference on even when the dataset has no dedicated test split. Using images from the test set gives\n",
|
||||
"you an unbiased view of model performance because those images were never seen during training or used to pick the\n",
|
||||
"best checkpoint. Replace `inference_image_paths` with a list of `Path` objects pointing to your own files to run\n",
|
||||
"inference on custom images."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "f03e8dcc",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"_IMAGE_EXTS = {\".jpg\", \".jpeg\", \".png\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _find_images(directory: Path) -> list[Path]:\n",
|
||||
" if not directory.is_dir():\n",
|
||||
" return []\n",
|
||||
" return sorted(p for p in directory.glob(\"**/*\") if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"validation_image_paths = _find_images(DATASET_DIR / \"test\")\n",
|
||||
"if not validation_image_paths:\n",
|
||||
" validation_image_paths = _find_images(DATASET_DIR / \"valid\")\n",
|
||||
"if not validation_image_paths:\n",
|
||||
" validation_image_paths = _find_images(DATASET_DIR / \"train\")\n",
|
||||
"if not validation_image_paths:\n",
|
||||
" raise FileNotFoundError(f\"No images ({', '.join(sorted(_IMAGE_EXTS))}) found under {DATASET_DIR}\")\n",
|
||||
"\n",
|
||||
"inference_image_paths = validation_image_paths[:INFERENCE_COUNT]\n",
|
||||
"print(f\"inference_images={[str(p) for p in inference_image_paths]}\")"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "74116bbe",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 11 - Visualize inference\n",
|
||||
"\n",
|
||||
"`INFERENCE_THRESHOLD` is a detection confidence threshold: only detections whose bounding-box score exceeds this\n",
|
||||
"value are shown. Raise it to suppress false positives; lower it to surface low-confidence detections. Each accepted\n",
|
||||
"detection includes a binary mask resized to the original image dimensions — `sv.MaskAnnotator` renders it as a\n",
|
||||
"translucent colour overlay so you can see both the mask shape and the image beneath it.\n",
|
||||
"\n",
|
||||
"If the mask boundaries look jagged, consider increasing `RESOLUTION` (in multiples of 24) and retraining; higher\n",
|
||||
"resolution gives the segmentation head more spatial detail to work with."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "f3a68bfe",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"inference_grid_items: list[tuple[str, np.ndarray, sv.Detections]] = []\n",
|
||||
"for image_path in inference_image_paths:\n",
|
||||
" with Image.open(image_path) as image:\n",
|
||||
" image_rgb = image.convert(\"RGB\")\n",
|
||||
" detections = loaded_model.predict(image_rgb, threshold=INFERENCE_THRESHOLD)\n",
|
||||
" if not isinstance(detections, sv.Detections):\n",
|
||||
" raise RuntimeError(f\"Expected RFDETRSegSmall.predict() to return sv.Detections, got {type(detections)!r}.\")\n",
|
||||
" inference_grid_items.append((image_path.name, np.array(image_rgb), detections))\n",
|
||||
"\n",
|
||||
"if inference_grid_items:\n",
|
||||
" figure = _segmentation_grid_figure(inference_grid_items, columns=INFERENCE_COLUMNS, class_names=CLASS_NAMES)\n",
|
||||
" display(figure)\n",
|
||||
" plt.close(figure)"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "63027bc2",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 12 - Detection summary\n",
|
||||
"\n",
|
||||
"Print per-image detection counts and confidence scores as a quick sanity check. If you see zero detections on most\n",
|
||||
"images, try lowering `INFERENCE_THRESHOLD` (towards 0.1) — the model may be calibrated to lower confidence scores\n",
|
||||
"on a custom dataset than the COCO-pretrained default. If you see too many false positives, raise the threshold or\n",
|
||||
"add more negative examples to your training set."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "d723ad36",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"summary_rows = []\n",
|
||||
"for image_path, (_, _, detections) in zip(inference_image_paths, inference_grid_items):\n",
|
||||
" num_det = len(detections)\n",
|
||||
" has_masks = detections.mask is not None and detections.mask.shape[0] > 0\n",
|
||||
" avg_conf = (\n",
|
||||
" float(np.mean(detections.confidence)) if detections.confidence is not None and num_det > 0 else float(\"nan\")\n",
|
||||
" )\n",
|
||||
" summary_rows.append(\n",
|
||||
" {\n",
|
||||
" \"image\": image_path.name,\n",
|
||||
" \"detections\": num_det,\n",
|
||||
" \"has_masks\": has_masks,\n",
|
||||
" \"avg_confidence\": round(avg_conf, 3),\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"summary_df = pd.DataFrame(summary_rows)\n",
|
||||
"display(summary_df)"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"jupytext": {
|
||||
"cell_metadata_filter": "title,-all",
|
||||
"main_language": "python",
|
||||
"notebook_metadata_filter": "-all"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a5720dea",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ------------------------------------------------------------------------\n",
|
||||
"# RF-DETR\n",
|
||||
"# Copyright (c) 2025 Roboflow. All Rights Reserved.\n",
|
||||
"# Licensed under the Apache License, Version 2.0 [see LICENSE for details]\n",
|
||||
"# ------------------------------------------------------------------------"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "66c62889",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# RF-DETR Inference Latency Benchmark\n",
|
||||
"\n",
|
||||
"Measures inference latency for three RF-DETR families across three configs:\n",
|
||||
"\n",
|
||||
"| Config | Description |\n",
|
||||
"|--------|-------------|\n",
|
||||
"| **FP32** | `predict()` — unoptimized baseline |\n",
|
||||
"| **FP16+JIT** | `optimize_for_inference(dtype=torch.float16)` |\n",
|
||||
"| **ONNX** | exported `.onnx` via `onnxruntime-gpu` |"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d1568b82",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Install\n",
|
||||
"\n",
|
||||
"We need `onnxruntime-gpu` built against CUDA 12 — the default PyPI wheel targets CUDA 11.8 and silently\n",
|
||||
"falls back to CPU on modern GPUs. The Microsoft CUDA-12 package index ships the correct build.\n",
|
||||
"\n",
|
||||
"> **Colab**: after running this cell, go to **Runtime → Restart session**, then run from the next cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6a116013",
|
||||
"metadata": {
|
||||
"title": "[bash]"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip uninstall -y onnxruntime onnxruntime-gpu\n",
|
||||
"!pip install -q \"rfdetr[onnx]\" pillow pandas\n",
|
||||
"!pip install -q onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1ca0838b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Config\n",
|
||||
"\n",
|
||||
"`WARMUP_RUNS` discards the first N inferences — GPU kernels are JIT-compiled on first use, so early timings\n",
|
||||
"are outliers. `MEASURE_RUNS` then collects the steady-state distribution. 20 + 100 is a reasonable balance\n",
|
||||
"between statistical stability and total wall-clock time per model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0e47b983",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from collections.abc import Callable\n",
|
||||
"from pathlib import Path\n",
|
||||
"from typing import Any, NamedTuple\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"import torch\n",
|
||||
"from PIL import Image\n",
|
||||
"\n",
|
||||
"WARMUP_RUNS = 20\n",
|
||||
"MEASURE_RUNS = 100\n",
|
||||
"EXPORT_DIR = Path(\"benchmark_output\")\n",
|
||||
"EXPORT_DIR.mkdir(exist_ok=True)\n",
|
||||
"\n",
|
||||
"if not torch.cuda.is_available():\n",
|
||||
" raise RuntimeError(\"This benchmark requires a CUDA GPU.\")\n",
|
||||
"print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n",
|
||||
"\n",
|
||||
"import onnxruntime as ort\n",
|
||||
"\n",
|
||||
"_ort_providers = ort.get_available_providers()\n",
|
||||
"print(f\"ORT {ort.__version__}, providers: {_ort_providers}\")\n",
|
||||
"if \"CUDAExecutionProvider\" not in _ort_providers:\n",
|
||||
" raise RuntimeError(\n",
|
||||
" f\"onnxruntime-gpu with CUDA support required. Available providers: {_ort_providers}. \"\n",
|
||||
" \"Fix: reinstall from the CUDA-12 index (see install cell) then restart runtime.\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2ccdc87e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Sample images\n",
|
||||
"\n",
|
||||
"Latency depends on resolution, not pixel content, so synthetic noise images are equivalent to real photos\n",
|
||||
"for benchmarking purposes. Using a fixed seed makes results reproducible across runs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "06a16bb0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"rng = np.random.default_rng(42)\n",
|
||||
"images: list[Image.Image] = [Image.fromarray(rng.integers(0, 256, (640, 640, 3), dtype=np.uint8)) for _ in range(10)]\n",
|
||||
"print(f\"Generated {len(images)} synthetic 640×640 RGB images\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "db0d3d7c",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 4. Latency helpers\n",
|
||||
"\n",
|
||||
"GPU kernels execute asynchronously — `time.perf_counter()` returns before the GPU finishes, giving\n",
|
||||
"misleadingly low numbers. CUDA events are inserted directly into the GPU command stream and timestamped\n",
|
||||
"on the device, so `elapsed_time()` measures actual kernel execution. `torch.cuda.synchronize()` after\n",
|
||||
"each run flushes the stream and ensures the event fires before we read it."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7f928319",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class BenchmarkResult(NamedTuple):\n",
|
||||
" \"\"\"Single benchmark measurement.\"\"\"\n",
|
||||
"\n",
|
||||
" label: str\n",
|
||||
" mean_ms: float\n",
|
||||
" std_ms: float\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def fps(self) -> float:\n",
|
||||
" \"\"\"Frames per second.\"\"\"\n",
|
||||
" return 1000.0 / self.mean_ms\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def measure_latency_gpu(\n",
|
||||
" fn: Callable[[], object],\n",
|
||||
" warmup: int = WARMUP_RUNS,\n",
|
||||
" runs: int = MEASURE_RUNS,\n",
|
||||
") -> tuple[float, float]:\n",
|
||||
" \"\"\"Return (mean_ms, std_ms) using CUDA events.\"\"\"\n",
|
||||
" for _ in range(warmup):\n",
|
||||
" fn()\n",
|
||||
" torch.cuda.synchronize()\n",
|
||||
" start = torch.cuda.Event(enable_timing=True)\n",
|
||||
" end = torch.cuda.Event(enable_timing=True)\n",
|
||||
" timings: list[float] = []\n",
|
||||
" for _ in range(runs):\n",
|
||||
" start.record()\n",
|
||||
" fn()\n",
|
||||
" end.record()\n",
|
||||
" torch.cuda.synchronize()\n",
|
||||
" timings.append(start.elapsed_time(end))\n",
|
||||
" arr = np.array(timings)\n",
|
||||
" return float(arr.mean()), float(arr.std())\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"_measure = measure_latency_gpu"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8f7ee4f5",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 5. Per-config benchmark functions\n",
|
||||
"\n",
|
||||
"Three inference paths are compared:\n",
|
||||
"\n",
|
||||
"- **FP32** — `predict()` as shipped. Full 32-bit arithmetic on GPU. Baseline.\n",
|
||||
"- **FP16+JIT** — `optimize_for_inference(dtype=torch.float16)` fuses layers with `torch.jit.script` and\n",
|
||||
" halves the arithmetic precision. Typically 1.5–2× faster than FP32 with negligible accuracy loss on\n",
|
||||
" modern tensor cores.\n",
|
||||
"- **ONNX** — the model is exported to the Open Neural Network Exchange format and run through ONNX Runtime,\n",
|
||||
" bypassing PyTorch entirely. ORT applies its own graph optimisations and can use the TensorRT execution\n",
|
||||
" provider for additional speedup. Benchmarked separately on CPU and GPU to show the provider impact."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "533f037b",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from rfdetr.export._onnx.inference import _onnx_runtime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _predict_fp32(model: Any, image: Image.Image) -> BenchmarkResult:\n",
|
||||
" \"\"\"Baseline FP32 predict() latency.\"\"\"\n",
|
||||
" mean, std = _measure(lambda: model.predict(image))\n",
|
||||
" return BenchmarkResult(\"predict() FP32\", mean, std)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _predict_fp16(model: Any, image: Image.Image) -> BenchmarkResult:\n",
|
||||
" \"\"\"FP16+JIT latency — applies and removes optimize_for_inference.\"\"\"\n",
|
||||
" model.optimize_for_inference(dtype=torch.float16)\n",
|
||||
" mean, std = _measure(lambda: model.predict(image))\n",
|
||||
" model.remove_optimized_model()\n",
|
||||
" return BenchmarkResult(\"predict() FP16+JIT\", mean, std)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _export_onnx(model: Any, export_dir: Path) -> Path:\n",
|
||||
" \"\"\"Export model to ONNX and return the path.\"\"\"\n",
|
||||
" return Path(model.export(output_dir=str(export_dir)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2f6bc92a",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"source": [
|
||||
"## 6. Model benchmark runner\n",
|
||||
"\n",
|
||||
"Each model is loaded fresh, exported once, and then each inference config is timed independently.\n",
|
||||
"The ONNX path reuses the same exported file for both CPU and CUDA providers so export cost is not\n",
|
||||
"counted in latency. CUDA ONNX is skipped gracefully when no GPU is available; missing `onnxruntime-gpu`\n",
|
||||
"raises immediately so misconfigured environments are caught early."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8afd179a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def run_model_benchmark(\n",
|
||||
" model_cls: type,\n",
|
||||
" model_name: str,\n",
|
||||
" images: list[Image.Image],\n",
|
||||
") -> list[BenchmarkResult]:\n",
|
||||
" \"\"\"Run FP32 / FP16 / ONNX benchmarks for one model and print results.\"\"\"\n",
|
||||
" print(f\"\\n{'=' * 62}\")\n",
|
||||
" print(f\" {model_name}\")\n",
|
||||
" print(\"=\" * 62)\n",
|
||||
"\n",
|
||||
" model: Any = model_cls()\n",
|
||||
" export_dir = EXPORT_DIR / model_name.split()[0]\n",
|
||||
" export_dir.mkdir(exist_ok=True)\n",
|
||||
" image = images[0]\n",
|
||||
"\n",
|
||||
" fp32 = _predict_fp32(model, image)\n",
|
||||
" fp16 = _predict_fp16(model, image)\n",
|
||||
" results: list[BenchmarkResult] = [fp32, fp16]\n",
|
||||
"\n",
|
||||
" onnx_path = _export_onnx(model, export_dir)\n",
|
||||
" for providers in ([\"CPUExecutionProvider\"], [\"CUDAExecutionProvider\", \"CPUExecutionProvider\"]):\n",
|
||||
" if providers[0] == \"CUDAExecutionProvider\" and not torch.cuda.is_available():\n",
|
||||
" print(\" ⚠ ONNX (CUDA) skipped — no CUDA GPU\")\n",
|
||||
" continue\n",
|
||||
" mean_ms, std_ms, label = _onnx_runtime(onnx_path, image, providers, WARMUP_RUNS, MEASURE_RUNS)\n",
|
||||
" results.append(BenchmarkResult(f\"ONNX ({label})\", mean_ms, std_ms))\n",
|
||||
"\n",
|
||||
" for r in results:\n",
|
||||
" print(f\" {r.label:<30} {r.mean_ms:6.2f} ms ± {r.std_ms:5.2f} ({r.fps:6.1f} FPS)\")\n",
|
||||
"\n",
|
||||
" onnx_results = [r for r in results if r.label.startswith(\"ONNX\")]\n",
|
||||
" speedups = [f\"FP16 {fp32.mean_ms / fp16.mean_ms:.1f}×\"]\n",
|
||||
" if onnx_results:\n",
|
||||
" speedups.append(f\"ONNX {fp32.mean_ms / onnx_results[0].mean_ms:.1f}×\")\n",
|
||||
" print(f\" Speedup vs FP32: {' | '.join(speedups)}\")\n",
|
||||
"\n",
|
||||
" return results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e7ccd41c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Benchmark loop — detection · segmentation · keypoint\n",
|
||||
"\n",
|
||||
"Three model families are benchmarked to show how task complexity affects latency. Detection (`RFDETRMedium`)\n",
|
||||
"outputs boxes only; segmentation (`RFDETRSegSmall`) additionally predicts per-object masks, which adds\n",
|
||||
"decoder cost; keypoint (`RFDETRKeypointPreview`) predicts skeleton joints and is typically the lightest\n",
|
||||
"of the three at smaller resolutions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2ab7ffbc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from rfdetr import RFDETRKeypointPreview, RFDETRMedium, RFDETRSegSmall\n",
|
||||
"\n",
|
||||
"MODELS: list[tuple[type, str]] = [\n",
|
||||
" (RFDETRMedium, \"RFDETRMedium — detection\"),\n",
|
||||
" (RFDETRSegSmall, \"RFDETRSegSmall — segmentation\"),\n",
|
||||
" (RFDETRKeypointPreview, \"RFDETRKeypointPreview — keypoint\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"all_results: dict[str, list[BenchmarkResult]] = {}\n",
|
||||
"for _model_cls, _model_name in MODELS:\n",
|
||||
" all_results[_model_name] = run_model_benchmark(_model_cls, _model_name, images)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ebbba970",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 8. Summary\n",
|
||||
"\n",
|
||||
"The table shows FPS (frames per second) for each model × config combination. Higher is better.\n",
|
||||
"Compare columns to see which model fits your latency budget; compare rows to choose the right\n",
|
||||
"inference backend for your deployment target (Python server, edge CPU, or ONNX Runtime service)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5d8eddd9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"summary = {\n",
|
||||
" model_name.split()[0]: {r.label: round(r.fps, 1) for r in results} for model_name, results in all_results.items()\n",
|
||||
"}\n",
|
||||
"df = pd.DataFrame(summary)\n",
|
||||
"df.index.name = \"Config \\\\ Model\"\n",
|
||||
"print(df.to_string())\n",
|
||||
"print(f\"\\nFPS — {MEASURE_RUNS} timed + {WARMUP_RUNS} warmup runs, batch 1, GPU: {torch.cuda.get_device_name(0)}.\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"jupytext": {
|
||||
"cell_metadata_filter": "title,-all",
|
||||
"main_language": "python",
|
||||
"notebook_metadata_filter": "-all"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1777c5f9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Training with PyTorch Lightning\n",
|
||||
"\n",
|
||||
"[](https://colab.research.google.com/github/roboflow/rf-detr/blob/develop/docs/cookbooks/pytorch-lightning.ipynb)\n",
|
||||
"\n",
|
||||
"**RF-DETR** is a real-time object detection model that combines the accuracy of\n",
|
||||
"transformer-based detectors with inference speeds suitable for production.\n",
|
||||
"The training stack is built on **PyTorch Lightning** — giving you\n",
|
||||
"composable building blocks you can adopt incrementally, without changing your\n",
|
||||
"existing code.\n",
|
||||
"\n",
|
||||
"| Building block | Role |\n",
|
||||
"|---|---|\n",
|
||||
"| `RFDETRModelModule` | `LightningModule` — model, loss, optimizer, scheduler |\n",
|
||||
"| `RFDETRDataModule` | `LightningDataModule` — datasets and dataloaders |\n",
|
||||
"| `build_trainer()` | Factory that assembles a `Trainer` with all RF-DETR callbacks |\n",
|
||||
"\n",
|
||||
"**Key design principle:** start simple, then pick up building blocks without losing\n",
|
||||
"your trained weights.\n",
|
||||
"\n",
|
||||
"- **Phase 1** — `model.train()` one-liner (`EPOCHS_PHASE_1` epochs)\n",
|
||||
"- **Phase 2** — swap in the PTL components and continue for `EPOCHS_PHASE_2` more\n",
|
||||
" epochs from the same checkpoint, same output folder — no conversion required\n",
|
||||
"- **End** — full training curve, single-image inference, and batch inference with\n",
|
||||
" `trainer.predict()`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a750078f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Install RF-DETR 1.6.0\n",
|
||||
"\n",
|
||||
"`rfdetr[train,loggers]` pulls in PyTorch Lightning, torchmetrics, and the full\n",
|
||||
"callback stack. `roboflow` downloads the demo dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a4688278",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install -q rfdetr[train,loggers]==1.6.0 roboflow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "77fab4e5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Config\n",
|
||||
"\n",
|
||||
"All notebook-level knobs in one place. Adjust `EPOCHS_PHASE_*` and `BATCH_SIZE`\n",
|
||||
"to match your hardware — every downstream cell reads from these variables.\n",
|
||||
"\n",
|
||||
"`num_workers` is set to `os.cpu_count()` inside a Jupyter/Colab kernel where\n",
|
||||
"process forking is safe, and to `0` when running as a plain Python script.\n",
|
||||
"On macOS and Windows, spawn-based multiprocessing would otherwise re-import\n",
|
||||
"this module as `__main__` and retrigger training."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "174445e1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"DATASET_DIR = os.environ.get(\"DATASET_DIR\", \"\")\n",
|
||||
"OUTPUT_DIR = \"output\"\n",
|
||||
"EPOCHS_PHASE_1 = 20\n",
|
||||
"EPOCHS_PHASE_2 = 10\n",
|
||||
"BATCH_SIZE = 12\n",
|
||||
"THRESHOLD = 0.3\n",
|
||||
"\n",
|
||||
"os.makedirs(OUTPUT_DIR, exist_ok=True)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" from IPython import get_ipython\n",
|
||||
"\n",
|
||||
" _in_notebook = get_ipython() is not None\n",
|
||||
"except Exception:\n",
|
||||
" _in_notebook = False\n",
|
||||
"\n",
|
||||
"# Outside a notebook kernel, macOS/Windows spawn-based multiprocessing will\n",
|
||||
"# re-import this script as __main__, triggering training again.\n",
|
||||
"# Use 0 workers in that case; inside a kernel the usual forking rules apply safely.\n",
|
||||
"num_workers = (os.cpu_count() or 0) if _in_notebook else 0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3a43c723",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Dataset\n",
|
||||
"\n",
|
||||
"[Aquarium Combined](https://universe.roboflow.com/brad-dwyer/aquarium-combined)\n",
|
||||
"— 638 images across 7 classes (`fish`, `jellyfish`, `penguin`, `puffin`,\n",
|
||||
"`shark`, `starfish`, `stingray`). It is small enough to complete a demo run\n",
|
||||
"in a few minutes yet diverse enough to produce meaningful detection results.\n",
|
||||
"\n",
|
||||
"Set `ROBOFLOW_API_KEY` as a Colab secret (Secrets panel, key icon) or as an\n",
|
||||
"environment variable before running this cell. The dataset is downloaded in\n",
|
||||
"COCO format, which RF-DETR reads natively."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "34478ada",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"from roboflow import Roboflow\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" from google.colab import userdata # type: ignore[import]\n",
|
||||
"\n",
|
||||
" API_KEY = userdata.get(\"ROBOFLOW_API_KEY\")\n",
|
||||
"except Exception:\n",
|
||||
" API_KEY = os.environ[\"ROBOFLOW_API_KEY\"]\n",
|
||||
"\n",
|
||||
"rf = Roboflow(api_key=API_KEY)\n",
|
||||
"dataset = rf.workspace(\"brad-dwyer\").project(\"aquarium-combined\").version(1).download(\"coco\", location=\"datasets\")\n",
|
||||
"DATASET_DIR = dataset.location\n",
|
||||
"\n",
|
||||
"with open(Path(DATASET_DIR) / \"train\" / \"_annotations.coco.json\") as f:\n",
|
||||
" _ann = json.load(f)\n",
|
||||
"\n",
|
||||
"CLASS_NAMES = [c[\"name\"] for c in sorted(_ann[\"categories\"], key=lambda c: c[\"id\"])]\n",
|
||||
"NUM_CLASSES = len(CLASS_NAMES)\n",
|
||||
"print(f\"Dataset : {DATASET_DIR}\")\n",
|
||||
"print(f\"Classes : {NUM_CLASSES} — {CLASS_NAMES}\")\n",
|
||||
"\n",
|
||||
"with open(Path(DATASET_DIR) / \"valid\" / \"_annotations.coco.json\") as f:\n",
|
||||
" _val_ann = json.load(f)\n",
|
||||
"\n",
|
||||
"val_images_dir = Path(DATASET_DIR) / \"valid\"\n",
|
||||
"val_image_files = [img[\"file_name\"] for img in _val_ann[\"images\"]]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "44bbcce0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Phase 1 — `model.train()` one-liner\n",
|
||||
"\n",
|
||||
"The same high-level API that has been in RF-DETR since v1.0 — nothing here\n",
|
||||
"changes from previous releases. If you have existing training scripts, they\n",
|
||||
"keep working without modification.\n",
|
||||
"\n",
|
||||
"`pretrain_weights=\"rf-detr-medium.pth\"` downloads COCO-pretrained backbone\n",
|
||||
"weights automatically on first run and caches them locally. `use_ema=True`\n",
|
||||
"maintains an exponential moving average of the weights to stabilise validation\n",
|
||||
"metrics. `run_test=False` skips the final test-set evaluation to keep Phase 1\n",
|
||||
"fast; Phase 2 turns it back on.\n",
|
||||
"\n",
|
||||
"After this cell completes, `OUTPUT_DIR/checkpoint_best_total.pth` holds the\n",
|
||||
"best weights seen so far — the starting point for Phase 2."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bf7b136e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from rfdetr import RFDETRMedium\n",
|
||||
"\n",
|
||||
"model = RFDETRMedium(num_classes=NUM_CLASSES, pretrain_weights=\"rf-detr-medium.pth\")\n",
|
||||
"model.train(\n",
|
||||
" dataset_dir=DATASET_DIR,\n",
|
||||
" epochs=EPOCHS_PHASE_1,\n",
|
||||
" batch_size=BATCH_SIZE,\n",
|
||||
" grad_accum_steps=4,\n",
|
||||
" lr=1e-4,\n",
|
||||
" num_workers=num_workers,\n",
|
||||
" output_dir=OUTPUT_DIR,\n",
|
||||
" use_ema=True,\n",
|
||||
" run_test=False,\n",
|
||||
" progress_bar=\"rich\",\n",
|
||||
" tensorboard=True,\n",
|
||||
" seed=42,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "178dd044",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Phase 2 — PTL building blocks\n",
|
||||
"\n",
|
||||
"Pick up the three PTL components and call `trainer.fit()` pointing at the Phase 1\n",
|
||||
"checkpoint. No weight conversion is needed — `RFDETRModelModule.on_load_checkpoint`\n",
|
||||
"detects the `.pth` format and remaps keys automatically.\n",
|
||||
"\n",
|
||||
"A lower learning rate (`5e-5`) is used because the model is already partially\n",
|
||||
"converged. `epochs=EPOCHS_PHASE_1 + EPOCHS_PHASE_2` sets the *absolute* epoch\n",
|
||||
"ceiling; because the loaded checkpoint records the last completed epoch, PTL\n",
|
||||
"runs exactly `EPOCHS_PHASE_2` additional epochs before stopping. The same\n",
|
||||
"`OUTPUT_DIR` is reused so checkpoints and metrics all land in one place."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d4526ab0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"from rfdetr import RFDETRDataModule, RFDETRModelModule, build_trainer\n",
|
||||
"from rfdetr.config import RFDETRMediumConfig, TrainConfig\n",
|
||||
"\n",
|
||||
"# Read Phase 1 metrics before Phase 2 overwrites the CSV.\n",
|
||||
"df1 = pd.read_csv(f\"{OUTPUT_DIR}/metrics.csv\")\n",
|
||||
"\n",
|
||||
"model_config = RFDETRMediumConfig(\n",
|
||||
" num_classes=NUM_CLASSES,\n",
|
||||
" pretrain_weights=\"rf-detr-medium.pth\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# epochs = EPOCHS_1 + EPOCHS_2 so PTL (which resumes the epoch counter from the\n",
|
||||
"# checkpoint) runs exactly EPOCHS_2 additional epochs before reaching max_epochs.\n",
|
||||
"train_config = TrainConfig(\n",
|
||||
" dataset_dir=DATASET_DIR,\n",
|
||||
" epochs=EPOCHS_PHASE_1 + EPOCHS_PHASE_2,\n",
|
||||
" batch_size=BATCH_SIZE,\n",
|
||||
" grad_accum_steps=4,\n",
|
||||
" lr=5e-5,\n",
|
||||
" num_workers=num_workers,\n",
|
||||
" output_dir=OUTPUT_DIR,\n",
|
||||
" use_ema=True,\n",
|
||||
" run_test=True,\n",
|
||||
" progress_bar=\"tqdm\",\n",
|
||||
" tensorboard=True,\n",
|
||||
" seed=42,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"module = RFDETRModelModule(model_config=model_config, train_config=train_config)\n",
|
||||
"datamodule = RFDETRDataModule(model_config=model_config, train_config=train_config)\n",
|
||||
"trainer = build_trainer(train_config, model_config)\n",
|
||||
"\n",
|
||||
"# Resume directly from the Phase 1 .pth — no conversion needed.\n",
|
||||
"trainer.fit(module, datamodule, ckpt_path=f\"{OUTPUT_DIR}/checkpoint_best_total.pth\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "344147ee",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. Training curve\n",
|
||||
"\n",
|
||||
"Phase 1 and Phase 2 each emit their own `metrics.csv` (Phase 2 overwrites Phase 1's\n",
|
||||
"file when it starts). We captured the Phase 1 copy before fitting so we can\n",
|
||||
"concatenate both DataFrames and plot a single continuous curve across all epochs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "751e8277",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"import supervision as sv\n",
|
||||
"from IPython.display import display\n",
|
||||
"from PIL import Image\n",
|
||||
"\n",
|
||||
"from rfdetr.visualize.training import plot_metrics\n",
|
||||
"\n",
|
||||
"df2 = pd.read_csv(f\"{OUTPUT_DIR}/metrics.csv\")\n",
|
||||
"\n",
|
||||
"combined_csv = f\"{OUTPUT_DIR}/metrics_combined.csv\"\n",
|
||||
"pd.concat([df1, df2], ignore_index=True).to_csv(combined_csv, index=False)\n",
|
||||
"print(f\"Combined CSV: {combined_csv} ({len(df1) + len(df2)} rows)\")\n",
|
||||
"\n",
|
||||
"fig = plot_metrics(combined_csv)\n",
|
||||
"display(fig)\n",
|
||||
"print(f\"Saved: {OUTPUT_DIR}/metrics_plot.png\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d3f29ab1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Single-image inference — `model.predict()`\n",
|
||||
"\n",
|
||||
"`RFDETRMedium` can be instantiated directly from a checkpoint path for inference\n",
|
||||
"— no training config needed. `model.predict()` accepts a PIL `Image`, runs\n",
|
||||
"preprocessing, the forward pass, and postprocessing internally, and returns a\n",
|
||||
"`supervision.Detections` object that is ready to annotate and display."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "90e06abb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"model = RFDETRMedium(pretrain_weights=f\"{OUTPUT_DIR}/checkpoint_best_total.pth\", num_classes=NUM_CLASSES)\n",
|
||||
"\n",
|
||||
"image = Image.open(val_images_dir / val_image_files[0])\n",
|
||||
"detections = model.predict(image, threshold=THRESHOLD)\n",
|
||||
"\n",
|
||||
"annotated = sv.BoxAnnotator().annotate(image.copy(), detections)\n",
|
||||
"annotated = sv.LabelAnnotator().annotate(annotated, detections, labels=[CLASS_NAMES[c] for c in detections.class_id])\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(10, 7))\n",
|
||||
"plt.imshow(np.array(annotated))\n",
|
||||
"plt.axis(\"off\")\n",
|
||||
"plt.show()\n",
|
||||
"\n",
|
||||
"print(f\"Detected {len(detections)} object(s)\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "32baa2d8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 8. Batch inference — `trainer.predict()`\n",
|
||||
"\n",
|
||||
"Instead of calling `model.predict()` one image at a time, `trainer.predict()`\n",
|
||||
"streams the entire validation set through the model in batches and collects\n",
|
||||
"all results — useful for dataset-level evaluation or offline export pipelines.\n",
|
||||
"\n",
|
||||
"**What happens under the hood:**\n",
|
||||
"\n",
|
||||
"1. PTL calls `datamodule.setup(\"predict\")` — this builds `_dataset_val` if it\n",
|
||||
" does not exist yet. Because `trainer.fit()` already ran above, the dataset\n",
|
||||
" is already in memory and `setup` is a no-op.\n",
|
||||
"2. PTL calls `datamodule.predict_dataloader()` — this returns the *validation*\n",
|
||||
" dataset wrapped in a `SequentialSampler` (no shuffle, no augmentation),\n",
|
||||
" identical to `val_dataloader`.\n",
|
||||
"3. For each batch, `RFDETRModelModule.predict_step()` runs a forward pass under\n",
|
||||
" `torch.no_grad()` and returns a list of `{\"scores\", \"labels\", \"boxes\"}`\n",
|
||||
" dicts — one dict per image in the batch.\n",
|
||||
"4. `trainer.predict()` collects all batch results into a\n",
|
||||
" `List[List[dict]]` (outer = batches, inner = images).\n",
|
||||
"\n",
|
||||
"Flatten, apply a confidence threshold, and wrap in `sv.Detections`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7d7543ff",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import itertools\n",
|
||||
"\n",
|
||||
"# Returns List[List[dict]] — outer: batches, inner: one dict per image\n",
|
||||
"# Each dict has keys: \"scores\" (N,), \"labels\" (N,), \"boxes\" (N, 4) — all tensors\n",
|
||||
"all_preds = trainer.predict(module, datamodule)\n",
|
||||
"\n",
|
||||
"# Flatten the batch dimension → one result dict per validation image\n",
|
||||
"flat_preds = [img_result for batch in all_preds for img_result in batch]\n",
|
||||
"print(f\"Ran predict on {len(flat_preds)} validation images\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ed942d37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Build sv.Detections from raw tensors and visualise the first four images\n",
|
||||
"annotated_images = []\n",
|
||||
"for img_file, result in itertools.islice(zip(val_image_files, flat_preds), 4):\n",
|
||||
" keep = result[\"scores\"] > THRESHOLD\n",
|
||||
" detections = sv.Detections(\n",
|
||||
" xyxy=result[\"boxes\"][keep].cpu().float().numpy(),\n",
|
||||
" confidence=result[\"scores\"][keep].cpu().float().numpy(),\n",
|
||||
" class_id=result[\"labels\"][keep].cpu().long().numpy(),\n",
|
||||
" )\n",
|
||||
" image = Image.open(val_images_dir / img_file)\n",
|
||||
" annotated = sv.BoxAnnotator().annotate(image.copy(), detections)\n",
|
||||
" annotated = sv.LabelAnnotator().annotate(\n",
|
||||
" annotated, detections, labels=[CLASS_NAMES[c] for c in detections.class_id]\n",
|
||||
" )\n",
|
||||
" annotated_images.append(np.array(annotated))\n",
|
||||
" print(f\" {img_file}: {len(detections)} detection(s)\")\n",
|
||||
"\n",
|
||||
"fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n",
|
||||
"for ax, img in zip(axes.flat, annotated_images):\n",
|
||||
" ax.imshow(img)\n",
|
||||
" ax.axis(\"off\")\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3387e060",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 9. Next steps\n",
|
||||
"\n",
|
||||
"You have now seen the complete 1.6.0 stack — from a one-liner `model.train()`\n",
|
||||
"through composable PTL components to batch inference. From here:\n",
|
||||
"\n",
|
||||
"- [PyTorch Lightning training docs](https://rfdetr.roboflow.com/1.6.0/reference/training/) — custom callbacks, multi-GPU, mixed precision\n",
|
||||
"- [Advanced training options](https://rfdetr.roboflow.com/1.6.0/learn/train/advanced/) — augmentations, EMA, learning rate schedules\n",
|
||||
"- [Logger integrations (ClearML, MLflow, W&B)](https://rfdetr.roboflow.com/1.6.0/learn/train/loggers/) — experiment tracking\n",
|
||||
"- [Export your model](https://rfdetr.roboflow.com/1.6.0/learn/export/) — ONNX, TensorRT, CoreML"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"jupytext": {
|
||||
"cell_metadata_filter": "-all",
|
||||
"formats": "ipynb,py",
|
||||
"main_language": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
---
|
||||
description: Install RF-DETR via pip, uv, or from source. Set up a development environment for contributing to Roboflow's real-time detection transformer.
|
||||
---
|
||||
|
||||
# Installation
|
||||
|
||||
Welcome to RF-DETR! This guide will help you install and set up RF-DETR for your projects. Whether you're a developer looking to contribute or an end-user ready to start using RF-DETR, we've got you covered.
|
||||
|
||||
## Installation Methods
|
||||
|
||||
RF-DETR supports several installation methods. Choose the option which best fits your workflow.
|
||||
|
||||
!!! example "Installation"
|
||||
|
||||
=== "pip (recommended)"
|
||||
|
||||
The easiest way to install RF-DETR is using `pip`. This method is recommended for most users.
|
||||
|
||||
```bash
|
||||
pip install rfdetr
|
||||
```
|
||||
|
||||
=== "uv"
|
||||
|
||||
If you are using `uv`, you can install RF-DETR using the following command:
|
||||
|
||||
```bash
|
||||
uv pip install rfdetr
|
||||
```
|
||||
|
||||
For `uv` projects, you can also use:
|
||||
|
||||
```bash
|
||||
uv add rfdetr
|
||||
```
|
||||
|
||||
=== "Source Archive"
|
||||
|
||||
To install the latest development version of RF-DETR from source without cloning the full repository, run the command below.
|
||||
|
||||
```bash
|
||||
pip install https://github.com/roboflow/rf-detr/archive/refs/heads/develop.zip
|
||||
```
|
||||
|
||||
## Dev Environment
|
||||
|
||||
If you plan to contribute to RF-DETR or modify the codebase locally, set up a local development environment using the steps below.
|
||||
|
||||
!!! example "Development Setup"
|
||||
|
||||
=== "virtualenv"
|
||||
|
||||
```bash
|
||||
# Clone the repository and navigate to the root directory
|
||||
git clone --depth 1 -b develop https://github.com/roboflow/rf-detr.git
|
||||
cd rf-detr
|
||||
|
||||
# Set up a Python virtual environment with a specific Python version (e.g., 3.10)
|
||||
python3.10 -m venv venv
|
||||
|
||||
# Activate the virtual environment
|
||||
source venv/bin/activate
|
||||
|
||||
# Upgrade pip
|
||||
pip install --upgrade pip
|
||||
|
||||
# Install the package in development mode
|
||||
pip install -e "."
|
||||
```
|
||||
|
||||
=== "uv"
|
||||
|
||||
```bash
|
||||
# Clone the repository and navigate to the root directory
|
||||
git clone --depth 1 -b develop https://github.com/roboflow/rf-detr.git
|
||||
cd rf-detr
|
||||
|
||||
# Pin Python version (optional but recommended)
|
||||
uv python pin 3.11
|
||||
|
||||
# Sync environment (creates .venv, installs pinned Python, and installs dependencies)
|
||||
uv sync
|
||||
|
||||
# Install the package in development mode with all extras
|
||||
uv pip install -e . --all-extras
|
||||
```
|
||||
|
||||
## Optional Extras
|
||||
|
||||
RF-DETR provides several optional extras for additional functionality:
|
||||
|
||||
| Extra | Install command | Purpose |
|
||||
| --------- | ----------------------------------- | --------------------------------------------------------------- |
|
||||
| `train` | `pip install "rfdetr[train]"` | Training dependencies (PyTorch Lightning, albumentations, etc.) |
|
||||
| `loggers` | `pip install "rfdetr[loggers]"` | Experiment tracking (TensorBoard, W&B, MLflow, ClearML) |
|
||||
| `onnx` | `pip install "rfdetr[onnx]"` | ONNX export |
|
||||
| `tflite` | `pip install "rfdetr[onnx,tflite]"` | TFLite export (Python 3.12 only) |
|
||||
| `trt` | `pip install "rfdetr[trt]"` | TensorRT inference (pycuda, onnxruntime-gpu, tensorrt) |
|
||||
| `kornia` | `pip install "rfdetr[kornia]"` | GPU-accelerated augmentations via Kornia |
|
||||
| `lora` | `pip install "rfdetr[lora]"` | LoRA fine-tuning with PEFT |
|
||||
| `visual` | `pip install "rfdetr[visual]"` | Visualization utilities (matplotlib, pandas, seaborn) |
|
||||
| `cli` | `pip install "rfdetr[cli]"` | CLI with typed argument parsing (jsonargparse) |
|
||||
| `plus` | `pip install "rfdetr[plus]"` | XLarge and 2XLarge detection models (PML 1.0 license) |
|
||||
|
||||
## Additional Notes
|
||||
|
||||
- Ensure you have Python 3.10 or higher installed.
|
||||
- For development, it is recommended to use a virtual environment to avoid conflicts with other packages.
|
||||
- If you encounter any issues during installation, refer to the [troubleshooting](#troubleshooting) section or open an issue on the [GitHub repository](https://github.com/roboflow/rf-detr).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If you encounter any issues during installation, here are some common solutions:
|
||||
|
||||
- **Permission Issues**: Use `pip install --user rfdetr` to install the package for your user only.
|
||||
- **Dependency Conflicts**: Use a virtual environment to isolate the installation.
|
||||
- **Python Version**: Ensure you are using Python 3.10 or higher.
|
||||
@@ -0,0 +1,404 @@
|
||||
---
|
||||
description: Per-version migration guide for RF-DETR. Covers breaking changes and deprecated APIs for each release series.
|
||||
---
|
||||
|
||||
# Migration Guide
|
||||
|
||||
Read each section between your current version and your target — every section covers
|
||||
only the delta between two adjacent releases.
|
||||
|
||||
```
|
||||
1.4.x → 1.5 → 1.6 → 1.7 → 1.8
|
||||
```
|
||||
|
||||
You can apply all changes in one go; working through sections one release at a time
|
||||
and verifying between each step is optional but makes failures easier to isolate.
|
||||
Deprecated APIs emit a `DeprecationWarning` until the version marked for removal.
|
||||
See the [Changelog](../changelog.md) for the full list of changes in each release.
|
||||
|
||||
---
|
||||
|
||||
## Upgrade 1.8 → 1.9
|
||||
|
||||
### Planned for Removal in v1.9
|
||||
|
||||
The following APIs were deprecated in earlier releases and will be removed in v1.9. They still work in the current release (v1.8.x) but emit `DeprecationWarning`. Update your code before upgrading.
|
||||
|
||||
!!! warning "Planned for removal: `rfdetr.util.*` and `rfdetr.deploy.*` import paths"
|
||||
|
||||
Deprecated since v1.6. Use the canonical replacements listed in the [Upgrade 1.5 → 1.6](#upgrade-15--16) section.
|
||||
|
||||
```python
|
||||
# These imports still work in v1.8 but emit DeprecationWarning; update before v1.9
|
||||
from rfdetr.util.coco_classes import COCO_CLASSES # → rfdetr.assets.coco_classes
|
||||
from rfdetr.util.misc import get_rank # → rfdetr.utilities
|
||||
from rfdetr.deploy import export_onnx # → rfdetr.export.main
|
||||
```
|
||||
|
||||
!!! warning "Planned for removal: `build_namespace(model_config, train_config)`"
|
||||
|
||||
Deprecated since v1.7. Use `build_model_from_config` and `build_criterion_from_config` instead.
|
||||
|
||||
!!! warning "Planned for removal: `load_pretrain_weights(nn_model, model_config, train_config)` with `train_config`"
|
||||
|
||||
Deprecated since v1.7. Drop the `train_config` positional argument.
|
||||
|
||||
!!! warning "Planned for removal: `start_epoch` kwarg in `train()`"
|
||||
|
||||
Deprecated since v1.7. PyTorch Lightning resumes automatically via `resume=`.
|
||||
|
||||
!!! warning "Planned for removal: `do_benchmark` kwarg in `train()`"
|
||||
|
||||
Deprecated since v1.7. Use the `rfdetr.export.benchmark` module instead.
|
||||
|
||||
!!! warning "Planned for removal: `callbacks` dict kwarg in `train()`"
|
||||
|
||||
Deprecated since v1.7. Pass PTL `Callback` objects directly via the Lightning API instead.
|
||||
|
||||
!!! warning "Planned for removal: misplaced config fields"
|
||||
|
||||
The following `TrainConfig` and `ModelConfig` fields moved to their correct config class in v1.7 and the deprecated compatibility shims will be removed in v1.9. Update any direct references:
|
||||
|
||||
| Field | Removed from | Use in |
|
||||
| ------------------- | ------------- | ------------- |
|
||||
| `group_detr` | `TrainConfig` | `ModelConfig` |
|
||||
| `ia_bce_loss` | `TrainConfig` | `ModelConfig` |
|
||||
| `segmentation_head` | `TrainConfig` | `ModelConfig` |
|
||||
| `num_select` | `TrainConfig` | `ModelConfig` |
|
||||
| `cls_loss_coef` | `ModelConfig` | `TrainConfig` |
|
||||
|
||||
---
|
||||
|
||||
## Upgrade 1.7 → 1.8
|
||||
|
||||
### Breaking changes
|
||||
|
||||
!!! note "Breaking in v1.8.2: default keypoint schema changed to active-first `[17]`"
|
||||
|
||||
New checkpoints created from v1.8.2 onwards use `class_id=0` for person. Legacy `[0, 17]` checkpoints
|
||||
are still supported — RF-DETR auto-detects the schema from the checkpoint at load time.
|
||||
|
||||
If your post-processing code offsets class IDs by 1 (common for background-first models), update it:
|
||||
|
||||
```python
|
||||
# Before (background-first [0, 17]: person was at class_id=1)
|
||||
class_name = "person" if detection.class_id == 1 else "other"
|
||||
|
||||
# After (active-first [17]: person is at class_id=0)
|
||||
class_name = "person" if detection.class_id == 0 else "other"
|
||||
```
|
||||
|
||||
Use `detection.data["class_name"]` for schema-agnostic name resolution.
|
||||
|
||||
!!! warning "Breaking: `rfdetr.datasets.aug_config` renamed to `rfdetr.datasets.aug_configs`"
|
||||
|
||||
The augmentation config module was renamed (singular → plural). If you import from it directly:
|
||||
|
||||
```python
|
||||
# Before
|
||||
from rfdetr.datasets.aug_config import AUG_AGGRESSIVE
|
||||
|
||||
# After
|
||||
from rfdetr.datasets.aug_configs import AUG_AGGRESSIVE
|
||||
```
|
||||
|
||||
All preset constants (`AUG_AGGRESSIVE`, `AUG_CONSERVATIVE`, etc.) are unchanged.
|
||||
|
||||
!!! warning "Breaking: `supervision>=0.29.0` now required"
|
||||
|
||||
Required for `sv.KeyPoints` support. `pip install rfdetr==1.8.0` pulls this automatically.
|
||||
If another dependency pins `supervision<0.29.0`, resolve the conflict manually.
|
||||
|
||||
!!! warning "Breaking: `pyDeprecate` constraint narrowed to `>=0.9,<0.10`"
|
||||
|
||||
Was `>=0.6,<0.8`. If another package pins an older version, resolve with:
|
||||
|
||||
```bash
|
||||
pip install "rfdetr==1.8.0" "pyDeprecate>=0.9,<0.10"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Upgrade 1.6 → 1.7
|
||||
|
||||
### Breaking changes
|
||||
|
||||
!!! warning "Breaking: `peft` removed from the default install"
|
||||
|
||||
LoRA fine-tuning now requires the `lora` extra. If you use LoRA adapters during
|
||||
training, update your install command.
|
||||
|
||||
```bash
|
||||
# Before
|
||||
pip install rfdetr
|
||||
|
||||
# After
|
||||
pip install 'rfdetr[lora]'
|
||||
```
|
||||
|
||||
!!! warning "Breaking: `predict()` stores source image in `detections.metadata`"
|
||||
|
||||
**`predict()` stores the source image in `detections.metadata`, not `detections.data`.**
|
||||
|
||||
```python
|
||||
# Before (1.6.4 and earlier)
|
||||
source = detections.data["source_image"]
|
||||
|
||||
# After
|
||||
source = detections.metadata["source_image"]
|
||||
```
|
||||
|
||||
!!! warning "Breaking: `pyDeprecate` constraint changed to `>=0.6,<0.8`"
|
||||
|
||||
Was `>=0.3,<0.6`. If another package pins an older version, resolve with:
|
||||
|
||||
```bash
|
||||
pip install "rfdetr==1.7.0" "pyDeprecate>=0.6,<0.8"
|
||||
```
|
||||
|
||||
### Deprecated in v1.7 → Remove in v1.9
|
||||
|
||||
!!! note "Deprecated: `build_namespace()` split into two functions"
|
||||
|
||||
**`build_namespace(model_config, train_config)`** — use `build_model_from_config` or
|
||||
`build_criterion_from_config` instead.
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
from rfdetr.models import build_namespace
|
||||
|
||||
ns = build_namespace(model_config, train_config)
|
||||
|
||||
# After
|
||||
from rfdetr.models import build_model_from_config, build_criterion_from_config
|
||||
|
||||
model = build_model_from_config(model_config)
|
||||
criterion = build_criterion_from_config(model_config, train_config)
|
||||
```
|
||||
|
||||
!!! note "Deprecated: `load_pretrain_weights()` no longer takes `train_config`"
|
||||
|
||||
**`load_pretrain_weights(nn_model, model_config, train_config)`** — drop the
|
||||
`train_config` positional argument.
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
from rfdetr.models import load_pretrain_weights
|
||||
|
||||
load_pretrain_weights(nn_model, model_config, train_config)
|
||||
|
||||
# After
|
||||
from rfdetr.models import load_pretrain_weights
|
||||
|
||||
load_pretrain_weights(nn_model, model_config)
|
||||
```
|
||||
|
||||
!!! note "Deprecated: config fields moved between `ModelConfig` and `TrainConfig`"
|
||||
|
||||
**Config fields placed in the wrong config object.** Move them as shown:
|
||||
|
||||
| Field | Was in | Move to |
|
||||
| ------------------- | ------------- | ------------- |
|
||||
| `group_detr` | `TrainConfig` | `ModelConfig` |
|
||||
| `ia_bce_loss` | `TrainConfig` | `ModelConfig` |
|
||||
| `segmentation_head` | `TrainConfig` | `ModelConfig` |
|
||||
| `num_select` | `TrainConfig` | `ModelConfig` |
|
||||
| `cls_loss_coef` | `ModelConfig` | `TrainConfig` |
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
train_config = TrainConfig(group_detr=13, cls_loss_coef=2.0)
|
||||
|
||||
# After
|
||||
model_config = ModelConfig(group_detr=13)
|
||||
train_config = TrainConfig(cls_loss_coef=2.0)
|
||||
```
|
||||
|
||||
### Deprecated in v1.7 → Remove in v2.0
|
||||
|
||||
!!! note "Deprecated: `RFDETRBase` replaced by size-specific classes"
|
||||
|
||||
**`RFDETRBase`** defaulted to the small variant and is replaced by size-specific
|
||||
classes. Choose the variant that matches your previous model size. If you used
|
||||
`RFDETRBase()` without arguments, switch to `RFDETRSmall()`.
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
from rfdetr import RFDETRBase
|
||||
|
||||
model = RFDETRBase()
|
||||
|
||||
# After — pick one
|
||||
from rfdetr import RFDETRNano, RFDETRSmall, RFDETRMedium, RFDETRLarge
|
||||
|
||||
model = RFDETRSmall()
|
||||
```
|
||||
|
||||
!!! note "Deprecated: `RFDETRSegPreview` replaced by size-specific segmentation classes"
|
||||
|
||||
**`RFDETRSegPreview`** defaulted to the small variant and is replaced by size-specific
|
||||
segmentation classes. If you used `RFDETRSegPreview()` without arguments, switch to
|
||||
`RFDETRSegSmall()`.
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
from rfdetr import RFDETRSegPreview
|
||||
|
||||
model = RFDETRSegPreview()
|
||||
|
||||
# After — pick one
|
||||
from rfdetr import RFDETRSegNano, RFDETRSegSmall, RFDETRSegMedium, RFDETRSegLarge
|
||||
|
||||
model = RFDETRSegSmall()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Upgrade 1.5 → 1.6
|
||||
|
||||
### Breaking changes
|
||||
|
||||
!!! warning "Breaking: `transformers` minimum version raised to `>=5.1.0`"
|
||||
|
||||
**`transformers` minimum version raised to `>=5.1.0,<6.0.0`.**
|
||||
|
||||
Projects pinned to `transformers<5.0.0` must upgrade. If upgrading is not possible,
|
||||
pin `rfdetr<1.6.0`.
|
||||
|
||||
```bash
|
||||
pip install 'transformers>=5.1.0,<6.0.0'
|
||||
```
|
||||
|
||||
!!! warning "Breaking: PyPI extras renamed"
|
||||
|
||||
**PyPI extras renamed.**
|
||||
|
||||
Update your `pip install` commands and `requirements*.txt` files.
|
||||
|
||||
| Old extra | New extra |
|
||||
| -------------------- | ----------------- |
|
||||
| `rfdetr[metrics]` | `rfdetr[loggers]` |
|
||||
| `rfdetr[onnxexport]` | `rfdetr[onnx]` |
|
||||
|
||||
```bash
|
||||
# Before
|
||||
pip install 'rfdetr[metrics]'
|
||||
pip install 'rfdetr[onnxexport]'
|
||||
|
||||
# After
|
||||
pip install 'rfdetr[loggers]'
|
||||
pip install 'rfdetr[onnx]'
|
||||
```
|
||||
|
||||
!!! warning "Breaking: `draw_synthetic_shape()` now returns a tuple"
|
||||
|
||||
**`draw_synthetic_shape()` now returns `(image, polygon)` instead of `image`.**
|
||||
|
||||
Update every call site that unpacks only the image.
|
||||
|
||||
```python
|
||||
# Before
|
||||
img = draw_synthetic_shape(...)
|
||||
|
||||
# After
|
||||
img, polygon = draw_synthetic_shape(...)
|
||||
```
|
||||
|
||||
### Deprecated in v1.6 → Removed in v1.8
|
||||
|
||||
!!! note "Deprecated: `simplify` and `force` arguments in `RFDETR.export()`"
|
||||
|
||||
**`RFDETR.export(..., simplify=..., force=...)`** — both arguments are no-ops.
|
||||
Remove them from your calls.
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
model.export("model.onnx", simplify=True, force=True)
|
||||
|
||||
# After
|
||||
model.export("model.onnx")
|
||||
```
|
||||
|
||||
### Deprecated in v1.6 → Remove in v1.9
|
||||
|
||||
!!! note "Deprecated: `rfdetr.util.*` and `rfdetr.deploy.*` import paths"
|
||||
|
||||
Backward-compatibility shims are still active but emit `DeprecationWarning` on import.
|
||||
Replace with the canonical paths listed in the table below.
|
||||
|
||||
| Deprecated module | Canonical replacement |
|
||||
| --------------------------------- | ---------------------------------- |
|
||||
| `rfdetr.util.coco_classes` | `rfdetr.assets.coco_classes` |
|
||||
| `rfdetr.util.misc` | `rfdetr.utilities` |
|
||||
| `rfdetr.util.logger` | `rfdetr.utilities.logger` |
|
||||
| `rfdetr.util.box_ops` | `rfdetr.utilities.box_ops` |
|
||||
| `rfdetr.util.files` | `rfdetr.utilities.files` |
|
||||
| `rfdetr.util.package` | `rfdetr.utilities.package` |
|
||||
| `rfdetr.util.get_param_dicts` | `rfdetr.training.param_groups` |
|
||||
| `rfdetr.util.drop_scheduler` | `rfdetr.training.drop_schedule` |
|
||||
| `rfdetr.util.visualize` | `rfdetr.visualize.data` |
|
||||
| `rfdetr.deploy` | `rfdetr.export` |
|
||||
| `rfdetr.models.segmentation_head` | `rfdetr.models.heads.segmentation` |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
from rfdetr.util.coco_classes import COCO_CLASSES
|
||||
from rfdetr.util.misc import get_rank, get_world_size, is_main_process, save_on_master
|
||||
from rfdetr.util.logger import get_logger
|
||||
from rfdetr.util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou
|
||||
from rfdetr.util.get_param_dicts import get_param_dict
|
||||
from rfdetr.util.drop_scheduler import drop_scheduler
|
||||
from rfdetr.util.visualize import save_gt_predictions_visualization
|
||||
from rfdetr.deploy import export_onnx
|
||||
from rfdetr.models.segmentation_head import SegmentationHead
|
||||
|
||||
# After
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
from rfdetr.utilities.distributed import get_rank, get_world_size, is_main_process, save_on_master
|
||||
from rfdetr.utilities.logger import get_logger
|
||||
from rfdetr.utilities.box_ops import box_cxcywh_to_xyxy, generalized_box_iou
|
||||
from rfdetr.training.param_groups import get_param_dict
|
||||
from rfdetr.training.drop_schedule import drop_scheduler
|
||||
from rfdetr.visualize.data import save_gt_predictions_visualization
|
||||
from rfdetr.export.main import export_onnx
|
||||
from rfdetr.models.heads.segmentation import SegmentationHead
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Upgrade 1.4 → 1.5
|
||||
|
||||
### Breaking changes
|
||||
|
||||
!!! warning "Breaking: `ModelConfig` rejects unknown keyword arguments"
|
||||
|
||||
**`ModelConfig` now raises `ValidationError` on unknown keyword arguments.**
|
||||
|
||||
Previously, unrecognised fields were silently ignored. Remove or rename any
|
||||
unrecognised keys you pass to `ModelConfig(...)`.
|
||||
|
||||
```python
|
||||
# Before — silently accepted
|
||||
config = ModelConfig(unknown_field=True)
|
||||
|
||||
# Now raises ValidationError — remove the unknown key
|
||||
config = ModelConfig()
|
||||
```
|
||||
|
||||
### Deprecated in v1.5 → Removed in v1.7
|
||||
|
||||
!!! note "Deprecated: `OPEN_SOURCE_MODELS` replaced by `ModelWeights` enum"
|
||||
|
||||
**`OPEN_SOURCE_MODELS` constant** — use the `ModelWeights` enum instead. A
|
||||
`DeprecationWarning` is emitted on access. See the
|
||||
[API reference](../reference/rfdetr.md) for available enum values.
|
||||
|
||||
```python
|
||||
# Before (deprecated)
|
||||
from rfdetr import OPEN_SOURCE_MODELS
|
||||
|
||||
# After
|
||||
from rfdetr.assets.model_weights import ModelWeights
|
||||
```
|
||||
@@ -0,0 +1,72 @@
|
||||
# ------------------------------------------------------------------------
|
||||
# RF-DETR
|
||||
# Copyright (c) 2025 Roboflow. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
|
||||
# ------------------------------------------------------------------------
|
||||
"""MkDocs hook for exposing cookbook card data to documentation templates."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml # type: ignore[import-untyped] # yaml stubs not in docs group
|
||||
|
||||
|
||||
def _load_cards(cards_path: Path) -> list[dict[str, Any]]:
|
||||
"""Load cookbook card definitions from a YAML file.
|
||||
|
||||
Reads the YAML file and returns the ``cards`` list, which is consumed by the
|
||||
cookbook landing-page template. Centralising card data in YAML keeps content
|
||||
decoupled from presentation and avoids repeated edits to the HTML template.
|
||||
|
||||
Args:
|
||||
cards_path: Path to the cookbook cards YAML file.
|
||||
|
||||
Returns:
|
||||
Ordered list of card dictionaries from the YAML ``cards`` key.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the YAML file does not exist.
|
||||
RuntimeError: If the YAML payload does not expose a list under ``cards``.
|
||||
yaml.YAMLError: If the YAML file cannot be parsed.
|
||||
|
||||
Example:
|
||||
>>> cards = _load_cards(Path("docs/cookbooks/cards.yaml"))
|
||||
>>> isinstance(cards, list)
|
||||
True
|
||||
"""
|
||||
with cards_path.open("r", encoding="utf-8") as cards_file:
|
||||
payload = yaml.safe_load(cards_file)
|
||||
cards = (payload or {}).get("cards")
|
||||
if not isinstance(cards, list):
|
||||
msg = f"Missing list 'cards' in {cards_path}"
|
||||
raise RuntimeError(msg)
|
||||
return cards
|
||||
|
||||
|
||||
def on_config(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Expose cookbook card data to MkDocs templates.
|
||||
|
||||
Adds ``config.extra.cookbooks_cards`` so the cookbooks landing-page template
|
||||
can render cards from a single YAML source of truth instead of hardcoded
|
||||
HTML blocks.
|
||||
|
||||
Args:
|
||||
config: MkDocs configuration object.
|
||||
|
||||
Returns:
|
||||
Updated MkDocs configuration object.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the cookbook cards YAML file is missing.
|
||||
RuntimeError: If the YAML file does not expose a ``cards`` list.
|
||||
|
||||
Example:
|
||||
>>> cfg = {"extra": {}}
|
||||
>>> updated = on_config(cfg)
|
||||
>>> isinstance(updated["extra"]["cookbooks_cards"], list)
|
||||
True
|
||||
"""
|
||||
cards_path = Path(__file__).resolve().parents[2] / "docs" / "cookbooks" / "cards.yaml"
|
||||
extra = config.setdefault("extra", {})
|
||||
extra["cookbooks_cards"] = _load_cards(cards_path)
|
||||
return config
|
||||
@@ -0,0 +1,97 @@
|
||||
# ------------------------------------------------------------------------
|
||||
# RF-DETR
|
||||
# Copyright (c) 2025 Roboflow. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
|
||||
# ------------------------------------------------------------------------
|
||||
"""MkDocs hooks for exposing package metadata to documentation templates."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
import tomli
|
||||
|
||||
|
||||
def _load_pyproject(pyproject_path: Path) -> dict[str, Any]:
|
||||
"""Load project metadata from a pyproject file.
|
||||
|
||||
Reads the TOML file in binary mode, matching the parser API. The returned
|
||||
dictionary is used by MkDocs hooks to avoid duplicating package metadata.
|
||||
|
||||
Args:
|
||||
pyproject_path: Path to the repository's pyproject file.
|
||||
|
||||
Returns:
|
||||
Parsed pyproject data.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the pyproject file does not exist.
|
||||
tomllib.TOMLDecodeError: If the pyproject file is invalid TOML on Python 3.11+.
|
||||
tomli.TOMLDecodeError: If the pyproject file is invalid TOML on Python 3.10.
|
||||
|
||||
Example:
|
||||
>>> data = _load_pyproject(Path("pyproject.toml"))
|
||||
>>> isinstance(data["project"]["version"], str)
|
||||
True
|
||||
"""
|
||||
with pyproject_path.open("rb") as pyproject_file:
|
||||
if sys.version_info >= (3, 11):
|
||||
return tomllib.load(pyproject_file)
|
||||
return tomli.load(pyproject_file)
|
||||
|
||||
|
||||
def _read_project_version(pyproject_path: Path) -> str:
|
||||
"""Read the package version from pyproject project metadata.
|
||||
|
||||
Validates that the version exists and is a string before exposing it to the
|
||||
documentation templates.
|
||||
|
||||
Args:
|
||||
pyproject_path: Path to the repository's pyproject file.
|
||||
|
||||
Returns:
|
||||
Package version from the ``[project]`` table.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``[project].version`` is missing or not a string.
|
||||
|
||||
Example:
|
||||
>>> _read_project_version(Path("pyproject.toml"))
|
||||
'1.6.0'
|
||||
"""
|
||||
pyproject = _load_pyproject(pyproject_path)
|
||||
version = pyproject.get("project", {}).get("version")
|
||||
if not isinstance(version, str):
|
||||
msg = f"Missing string [project].version in {pyproject_path}"
|
||||
raise RuntimeError(msg)
|
||||
return version
|
||||
|
||||
|
||||
def on_config(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Expose the package version to MkDocs templates.
|
||||
|
||||
Adds ``config.extra.software_version`` so schema.org JSON-LD can stay in
|
||||
sync with the package metadata in ``pyproject.toml``.
|
||||
|
||||
Args:
|
||||
config: MkDocs configuration object.
|
||||
|
||||
Returns:
|
||||
Updated MkDocs configuration object.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If package metadata cannot provide a valid version.
|
||||
|
||||
Example:
|
||||
>>> cfg = {"extra": {}}
|
||||
>>> updated = on_config(cfg)
|
||||
>>> updated["extra"]["software_version"]
|
||||
'1.6.0'
|
||||
"""
|
||||
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
|
||||
extra = config.setdefault("extra", {})
|
||||
extra["software_version"] = _read_project_version(pyproject_path)
|
||||
return config
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
---
|
||||
description: RF-DETR is a real-time transformer for object detection, instance segmentation, and keypoint detection (preview) by Roboflow. DINOv2 backbone, SOTA on COCO (60.1 AP50:95). Apache 2.0.
|
||||
hide:
|
||||
- navigation
|
||||
---
|
||||
|
||||
# RF-DETR: Real-Time SOTA Object Detection, Instance Segmentation, and Keypoint Detection
|
||||
|
||||
RF-DETR is a real-time transformer architecture for object detection, instance segmentation, and keypoint detection (preview) developed by Roboflow. Built on a DINOv2 vision transformer backbone, RF-DETR achieves state-of-the-art accuracy–latency trade-offs: RF-DETR-L reaches 56.5 AP50:95 on COCO at 6.8 ms (NVIDIA T4, TensorRT FP16), and RF-DETR-2XL achieves 60.1 AP50:95 — the first real-time model to exceed 60 AP on COCO. Accepted at [ICLR 2026](https://arxiv.org/abs/2511.09554).
|
||||
|
||||
RF-DETR uses a DINOv2 vision transformer backbone and supports object detection, instance segmentation, and keypoint detection (preview) in a single, consistent API. Core models (Nano through Large) and all code are released under the Apache 2.0 license; XL and 2XLarge detection models require `rfdetr[plus]` and are provided under PML 1.0.
|
||||
|
||||
Developed by Isaac Robinson, Peter Robicheaux, Matvei Popov, Deva Ramanan (CMU), and Neehar Peri (CMU) at [Roboflow](https://roboflow.com). If you use RF-DETR in your research, please cite:
|
||||
|
||||
```bibtex
|
||||
@inproceedings{robinson2026rfdetr,
|
||||
title = {RF-DETR: Real-Time Detection Transformer},
|
||||
author = {Robinson, Isaac and Robicheaux, Peter and Popov, Matvei and Ramanan, Deva and Peri, Neehar},
|
||||
booktitle = {International Conference on Learning Representations (ICLR)},
|
||||
year = {2026},
|
||||
url = {https://arxiv.org/abs/2511.09554}
|
||||
}
|
||||
```
|
||||
|
||||
## Install
|
||||
|
||||
You can install and use `rfdetr` in a [**Python>=3.10**](https://www.python.org/) environment. For detailed installation instructions, including installing from source, and setting up a local development environment, check out our [install](getting-started/install.md) page.
|
||||
|
||||
!!! example "Installation"
|
||||
|
||||
<a href="https://badge.fury.io/py/rfdetr"><img alt="version" src="https://badge.fury.io/py/rfdetr.svg" width="125" height="20" /></a>
|
||||
<a href="https://badge.fury.io/py/rfdetr"><img alt="python-version" src="https://img.shields.io/pypi/pyversions/rfdetr" width="198" height="20" /></a>
|
||||
<a href="https://github.com/roboflow/rf-detr/blob/main/LICENSE"><img alt="license" src="https://img.shields.io/pypi/l/rfdetr" width="164" height="20" /></a>
|
||||
<a href="https://pypistats.org/packages/rfdetr"><img alt="downloads" src="https://img.shields.io/pypi/dm/rfdetr" width="258" height="20" /></a>
|
||||
|
||||
=== "pip"
|
||||
|
||||
```bash
|
||||
pip install rfdetr
|
||||
```
|
||||
|
||||
=== "uv"
|
||||
|
||||
```bash
|
||||
uv pip install rfdetr
|
||||
```
|
||||
|
||||
For uv projects:
|
||||
|
||||
```bash
|
||||
uv add rfdetr
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
<div class="grid cards" markdown>
|
||||
|
||||
- **Run Detection Models**
|
||||
|
||||
---
|
||||
|
||||
Load and run pre-trained RF-DETR detection models.
|
||||
|
||||
[:octicons-arrow-right-24: Tutorial](learn/run/detection.md)
|
||||
|
||||
- **Run Segmentation Models**
|
||||
|
||||
---
|
||||
|
||||
Load and run pre-trained RF-DETR-Seg segmentation models.
|
||||
|
||||
[:octicons-arrow-right-24: Tutorial](learn/run/segmentation.md)
|
||||
|
||||
- **Train Models**
|
||||
|
||||
---
|
||||
|
||||
Learn how to fine-tune RF-DETR models for detection and segmentation.
|
||||
|
||||
[:octicons-arrow-right-24: Tutorial](learn/train/index.md)
|
||||
|
||||
</div>
|
||||
|
||||
## Tutorials
|
||||
|
||||
<div class="grid cards" markdown>
|
||||
|
||||
- **Train RF-DETR on a Custom Dataset. Video**
|
||||
|
||||
---
|
||||
|
||||
{ width="1280" height="720" loading="lazy" }
|
||||
|
||||
End to end walkthrough of training RF-DETR on a custom dataset.
|
||||
|
||||
[:octicons-arrow-right-24: Watch the video](https://www.youtube.com/watch?v=-OvpdLAElFA)
|
||||
|
||||
- **Deploy RF-DETR to NVIDIA Jetson. Article**
|
||||
|
||||
---
|
||||
|
||||
{ width="1000" height="563" loading="lazy" }
|
||||
|
||||
Instructions for deploying RF-DETR on NVIDIA Jetson with Roboflow Inference.
|
||||
|
||||
[:octicons-arrow-right-24: Read the tutorial](https://blog.roboflow.com/how-to-deploy-rf-detr-to-an-nvidia-jetson/)
|
||||
|
||||
- **Train and Deploy RF-DETR with Roboflow**
|
||||
|
||||
---
|
||||
|
||||
{ width="1000" height="563" loading="lazy" }
|
||||
|
||||
Cloud training and hardware deployment workflow using Roboflow.
|
||||
|
||||
[:octicons-arrow-right-24: Read the tutorial](https://blog.roboflow.com/train-and-deploy-rf-detr-models-with-roboflow/)
|
||||
|
||||
</div>
|
||||
|
||||
## Benchmarks
|
||||
|
||||
RF-DETR achieves the best accuracy–latency trade-off among real-time object detection and instance segmentation models. It also provides keypoint detection (preview) on COCO person keypoints. For detailed benchmark tables and methodology, check out our [benchmarks](learn/benchmarks.md) page.
|
||||
|
||||
### Detection
|
||||
|
||||
<img alt="Pareto front — detection accuracy vs latency: RF-DETR-2XL achieves 78.5 COCO AP50 (60.1 AP50:95) at 17.2 ms; RF-DETR-L achieves 75.1 AP50 at 6.8 ms, outperforming YOLO11x at comparable latency" src="https://storage.googleapis.com/com-roboflow-marketing/rf-detr/rf_detr_1-4_latency_accuracy_object_detection.png" width="840" height="630" style="max-width: 840px; height: auto;" />
|
||||
|
||||
| Architecture | COCO AP<sub>50</sub> | COCO AP<sub>50:95</sub> | RF100VL AP<sub>50</sub> | RF100VL AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution |
|
||||
| ------------ | -------------------- | ----------------------- | ----------------------- | -------------------------- | ------------ | ---------- | ---------- |
|
||||
| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384×384 |
|
||||
| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512×512 |
|
||||
| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576×576 |
|
||||
| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704×704 |
|
||||
| RF-DETR-XL | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700×700 |
|
||||
| RF-DETR-2XL | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880×880 |
|
||||
|
||||
### Segmentation
|
||||
|
||||
<img alt="Pareto front — segmentation accuracy vs latency: RF-DETR-Seg-2XL achieves 73.1 COCO AP50 (49.9 AP50:95) at 21.8 ms; RF-DETR-Seg-L achieves 70.5 AP50 at 8.8 ms" src="https://storage.googleapis.com/com-roboflow-marketing/rf-detr/rf_detr_1-4_latency_accuracy_instance_segmentation.png" width="840" height="630" style="max-width: 840px; height: auto;" />
|
||||
|
||||
| Architecture | COCO AP<sub>50</sub> | COCO AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution |
|
||||
| --------------- | -------------------- | ----------------------- | ------------ | ---------- | ---------- |
|
||||
| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312×312 |
|
||||
| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384×384 |
|
||||
| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432×432 |
|
||||
| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504×504 |
|
||||
| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624×624 |
|
||||
| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768×768 |
|
||||
|
||||
### Keypoints
|
||||
|
||||
<img alt="RF-DETR Keypoint mAP vs latency chart comparing against YOLO26-pose and YOLO11-pose on MS COCO" src="assets/keypoints/kp-map-latency.png" width="840" height="630" style="max-width: 840px; height: auto;" />
|
||||
|
||||
| Architecture | COCO AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution |
|
||||
| -------------------------- | ----------------------- | ------------ | ---------- | ---------- |
|
||||
| RF-DETR Keypoint (Preview) | 71.8 | 9.7 | 126.4 | 576×576 |
|
||||
|
||||
> Keypoint benchmarks report AP<sub>50:95</sub> (OKS-based); this is the standard COCO keypoint comparison metric.
|
||||
> For the full competitor comparison (YOLO11-pose, YOLO26-pose), see the [Benchmarks](learn/benchmarks.md#keypoints) page.
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
**What is RF-DETR?**
|
||||
RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy–latency trade-offs on COCO (60.1 AP50:95 for RF-DETR-2XL) and RF100-VL.
|
||||
|
||||
**How does RF-DETR compare to YOLOv11?**
|
||||
RF-DETR-L achieves 56.5 AP50:95 on COCO at 6.8 ms latency on an NVIDIA T4, outperforming YOLOv11x (50.9 AP) at lower latency. The DINOv2 backbone gives RF-DETR stronger performance on domain-shift benchmarks such as RF100-VL.
|
||||
|
||||
**What GPU is required to train RF-DETR?**
|
||||
A CUDA-capable GPU with at least 8 GB VRAM (e.g., NVIDIA RTX 3060, T4, A10) is recommended for fine-tuning. Smaller models (RF-DETR-N and RF-DETR-S) can fit in 6 GB VRAM with reduced batch size. CPU inference is supported for evaluation.
|
||||
|
||||
**Which dataset formats does RF-DETR support?**
|
||||
RF-DETR supports COCO JSON and YOLO-format datasets (with `dataset_file: "yolo"`). Roboflow datasets export directly to both formats. Detection and segmentation datasets use the same format — the model variant determines the task.
|
||||
|
||||
**Can RF-DETR run in real time?**
|
||||
Yes. RF-DETR-N runs at 2.3 ms per frame on a T4 GPU (TensorRT FP16, batch 1), and RF-DETR-L at 6.8 ms — both well within real-time thresholds. ONNX and TFLite exports are available for edge deployment.
|
||||
|
||||
**What is the difference between RF-DETR detection and segmentation models?**
|
||||
Detection models (e.g., `RFDETRLarge`) output bounding boxes. Segmentation models (e.g., `RFDETRSegLarge`) additionally output instance masks. Both share the same backbone and training API; segmentation adds a mask head and requires COCO-format segmentation annotations.
|
||||
|
||||
**Does RF-DETR support keypoint detection?**
|
||||
RF-DETR Keypoint (Preview) detects 17 body keypoints per person on COCO, achieving 71.8 AP50:95 at 9.7 ms on NVIDIA T4. It is available in the `rfdetr` package as `RFDETRKeypointPreview`. See [Run Keypoint Models](learn/run/keypoints.md) for usage.
|
||||
|
||||
**Is RF-DETR open source?**
|
||||
Yes. Core models (Nano through Large) and all training/inference code are released under the Apache 2.0 license. XLarge and 2XLarge models require the `rfdetr[plus]` package (PML 1.0 license).
|
||||
|
||||
**How do I fine-tune RF-DETR on a custom dataset?**
|
||||
Instantiate a model and call `model.train(...)` with your dataset directory in COCO JSON or YOLO format. Example: `model = RFDETRLarge(); model.train(dataset_dir='./dataset', epochs=50, batch_size=4)`. The model downloads pretrained weights automatically and saves best checkpoints automatically (use `resume=` to continue from one).
|
||||
|
||||
**How do I export RF-DETR to ONNX or TensorRT?**
|
||||
Call `model.export(format="onnx")` after training or loading a checkpoint. ONNX export works on CPU and produces a single `.onnx` file compatible with ONNX Runtime and OpenCV DNN. For TensorRT deployment, first export to ONNX and then convert the `.onnx` model with TensorRT tooling or helpers such as `trtexec`; this requires TensorRT and a CUDA GPU.
|
||||
|
||||
**Which RF-DETR model size should I use?**
|
||||
RF-DETR-Nano (2.3 ms, 67.6 AP50 on COCO) is best for edge and real-time applications. RF-DETR-Large (6.8 ms, 56.5 AP50:95) offers the best accuracy–latency trade-off for server deployment. RF-DETR-2XLarge (17.2 ms, 60.1 AP50:95) maximizes accuracy when latency allows.
|
||||
|
||||
> **Checkpoint note:** Current `RFDETRLarge` defaults to `rf-detr-large-2026.pth`. The older `rf-detr-large.pth` checkpoint is a legacy Large release kept for backward compatibility and has been superseded by the current release.
|
||||
@@ -0,0 +1,154 @@
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
|
||||
const palette = __md_get("__palette")
|
||||
const useDark = palette && typeof palette.color === "object" && palette.color.scheme === "slate"
|
||||
const theme = useDark ? "dark-theme" : "light-default";
|
||||
|
||||
const colorList = [
|
||||
"#22c55e",
|
||||
"#14b8a6",
|
||||
"#ef4444",
|
||||
"#eab308",
|
||||
"#8b5cf6",
|
||||
"#f97316",
|
||||
"#3b82f6",
|
||||
]
|
||||
|
||||
const logoSrc = (document.querySelector('link[rel="icon"]') || {}).href || '';
|
||||
const authorCache = {};
|
||||
|
||||
const repoCards = document.querySelectorAll(".repo-card");
|
||||
const labelsAll = Array
|
||||
.from(repoCards)
|
||||
.flatMap((element) => (element.getAttribute('data-labels') || '').split(','))
|
||||
.map(label => label.trim())
|
||||
.filter(label => label !== '');
|
||||
const uniqueLabels = [...new Set(labelsAll)];
|
||||
|
||||
const labelToColor = uniqueLabels.reduce((map, label, index) => {
|
||||
map[label] = colorList[index % colorList.length];
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
|
||||
async function renderCard(element, elementIndex) {
|
||||
const name = element.getAttribute('data-name') || '';
|
||||
const description = element.getAttribute('data-description') || '';
|
||||
const labels = element.getAttribute('data-labels') || '';
|
||||
const version = element.getAttribute('data-version') || '';
|
||||
const authors = element.getAttribute('data-author') || '';
|
||||
|
||||
const labelHTML = labels ? labels.split(',').filter(label => label !== '').map((label, index) => {
|
||||
const color = labelToColor[label.trim()];
|
||||
return `
|
||||
<span
|
||||
class="label non-selectable-text"
|
||||
style="background-color: ${color}"
|
||||
>
|
||||
${label.trim()}
|
||||
</span>
|
||||
`;
|
||||
}).join(' ') : '';
|
||||
|
||||
const authorArray = authors ? authors.split(',').filter(a => a.trim()) : [];
|
||||
const authorDataArray = await Promise.all(authorArray.map(async (author) => {
|
||||
const login = author.trim();
|
||||
if (authorCache[login]) return authorCache[login];
|
||||
try {
|
||||
const response = await fetch(`https://api.github.com/users/${login}`);
|
||||
if (!response.ok) return { login, avatar_url: `https://github.com/${login}.png` };
|
||||
const data = await response.json();
|
||||
authorCache[login] = data;
|
||||
return data;
|
||||
} catch {
|
||||
return { login, avatar_url: `https://github.com/${login}.png` };
|
||||
}
|
||||
}));
|
||||
|
||||
let authorAvatarsHTML = authorDataArray.map((authorData, index) => {
|
||||
const marginLeft = index === 0 ? '0' : '-10px';
|
||||
return `
|
||||
<div
|
||||
class="author-container"
|
||||
data-login="${authorData.login}-${elementIndex}"
|
||||
style="margin-left: ${marginLeft};"
|
||||
>
|
||||
<a
|
||||
href="https://github.com/${authorData.login}"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
style="line-height: 0;"
|
||||
>
|
||||
<img
|
||||
class="author-avatar"
|
||||
src="${authorData.avatar_url}"
|
||||
alt="${authorData.login}'s avatar"
|
||||
>
|
||||
</a>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
|
||||
let authorNamesHTML = authorDataArray.map(
|
||||
authorData => `
|
||||
<span
|
||||
class="author-name"
|
||||
data-login="${authorData.login}-${elementIndex}"
|
||||
>
|
||||
<a href="https://github.com/${authorData.login}" target="_blank" rel="noopener noreferrer">
|
||||
${authorData.login}
|
||||
</a>
|
||||
</span>`
|
||||
).join(', ');
|
||||
|
||||
let authorsHTML = `
|
||||
<div class="authors" style="margin: 0;">
|
||||
${authorAvatarsHTML}
|
||||
<div class="author-names">${authorNamesHTML}</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
const rawHTML = `
|
||||
<div style="
|
||||
display: grid !important;
|
||||
grid-template-rows: auto;
|
||||
height: 100%;
|
||||
font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji; font-size: 14px; line-height: 1.5;">
|
||||
<div style="display: flex; align-items: center;">
|
||||
<span style="font-weight: 700; font-size: 1rem;">
|
||||
${name}
|
||||
</span>
|
||||
</div>
|
||||
${description ? `<p style="font-size: 0.85rem; opacity: 0.75; margin: 0.5rem 0 0;">${description}</p>` : ''}
|
||||
${authorsHTML}
|
||||
<div style="font-size: 12px; display: grid; grid-template-columns: auto 3fr; justify-content: space-between; gap: 1rem;">
|
||||
<div style="display: flex; align-items: center;">
|
||||
<img src="${logoSrc}" aria-label="rf-detr" width="20" height="20" role="img" />
|
||||
|
||||
<span style="margin-left: 4px">${version}</span>
|
||||
</div>
|
||||
<div style="display: flex; align-items: center; flex-wrap: wrap; align-content: right; gap: 0.1rem;">
|
||||
${labelHTML}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
element.innerHTML = DOMPurify.sanitize(rawHTML);
|
||||
|
||||
element.querySelectorAll('.author-name').forEach(nameEl => {
|
||||
nameEl.addEventListener('mouseenter', function () {
|
||||
const login = this.getAttribute('data-login');
|
||||
element.querySelector(`.author-container[data-login="${login}"]`).classList.add('hover');
|
||||
});
|
||||
|
||||
nameEl.addEventListener('mouseleave', function () {
|
||||
const login = this.getAttribute('data-login');
|
||||
element.querySelector(`.author-container[data-login="${login}"]`).classList.remove('hover');
|
||||
});
|
||||
});
|
||||
}
|
||||
repoCards.forEach((element, index) => {
|
||||
renderCard(element, index);
|
||||
});
|
||||
})
|
||||
@@ -0,0 +1,112 @@
|
||||
---
|
||||
description: RF-DETR benchmark results on COCO and RF100-VL for detection, segmentation, and keypoint detection. Compare accuracy and latency against YOLO, D-FINE, and LW-DETR.
|
||||
---
|
||||
|
||||
# Benchmarks
|
||||
|
||||
!!! tip "Key Takeaways"
|
||||
|
||||
- RF-DETR-2XL achieves 60.1 AP50:95 on COCO detection at 17.2 ms latency (T4, TensorRT FP16)
|
||||
- RF-DETR-L outperforms YOLOv11x (56.5 vs 50.9 AP50:95) at lower latency (6.8 vs 10.5 ms)
|
||||
- Segmentation models range from 40.3 AP (Nano, 3.4 ms) to 49.9 AP (2XL, 21.8 ms) on COCO
|
||||
- All latency measured on NVIDIA T4 with TensorRT 10.4, CUDA 12.4, FP16, batch size 1
|
||||
- RF100-VL results demonstrate strong domain-shift generalization across 100 diverse datasets
|
||||
- RF-DETR Keypoint (Preview) achieves 71.8 AP50:95 on COCO person keypoints at 9.7 ms (T4, TensorRT FP16)
|
||||
|
||||
This page reports RF-DETR benchmark results for object detection, instance segmentation, and keypoint detection on Microsoft COCO and RF100-VL (detection only). All benchmark numbers and plots match the latest released checkpoints and tables shown below. Latency values are measured on an NVIDIA T4 with TensorRT in FP16 at batch size 1. For full methodology details and architectural context, see the RF-DETR paper.
|
||||
|
||||
## Methodology
|
||||
|
||||
Accuracy is reported using standard COCO metrics computed with pycocotools. For object detection, we report COCO AP50 and COCO AP50:95, and the same metrics are also reported for RF100-VL. COCO results are evaluated on the validation split, following common practice in detector benchmarking. RF100-VL results are averaged across all 100 datasets to reflect performance under diverse real-world data distributions.
|
||||
|
||||
Latency is measured as single-image inference latency rather than sustained throughput. All latency numbers are obtained on an NVIDIA T4 GPU using TensorRT 10.4 and CUDA 12.4 with FP16 inference and batch size 1. To reduce variance caused by GPU power throttling and thermal effects, a 200 ms buffer is inserted between consecutive forward passes. This procedure improves reproducibility of latency measurements but is not intended to measure maximum throughput.
|
||||
|
||||
Accuracy and latency are always measured using the same model artifact and the same numerical precision. This avoids reporting FP32 accuracy together with FP16 latency, which can lead to misleading comparisons because naive FP16 conversion can significantly degrade accuracy for some models.
|
||||
|
||||
!!! info "Metric definitions"
|
||||
|
||||
**AP50**: Detection accuracy at IoU threshold >= 0.50.
|
||||
**AP50:95**: Mean accuracy averaged over IoU thresholds 0.50 to 0.95 (step 0.05) — the primary COCO metric.
|
||||
Latency measured on NVIDIA T4, TensorRT 10.4, CUDA 12.4, FP16, batch size 1,
|
||||
with 200 ms thermal buffer between passes to reduce GPU thermal variance.
|
||||
|
||||
## Detection
|
||||
|
||||
<img alt="rf_detr_1-4_latency_accuracy_object_detection" src="https://storage.googleapis.com/com-roboflow-marketing/rf-detr/rf_detr_1-4_latency_accuracy_object_detection.png" />
|
||||
|
||||
| Architecture | COCO AP<sub>50</sub> | COCO AP<sub>50:95</sub> | RF100VL AP<sub>50</sub> | RF100VL AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution |
|
||||
| :----------: | :------------------: | :---------------------: | :---------------------: | :------------------------: | :----------: | :--------: | :--------: |
|
||||
| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384x384 |
|
||||
| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512x512 |
|
||||
| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576x576 |
|
||||
| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704x704 |
|
||||
| RF-DETR-XL | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700x700 |
|
||||
| RF-DETR-2XL | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880x880 |
|
||||
| YOLO11-N | 52.0 | 37.4 | 81.4 | 55.3 | 2.5 | 2.6 | 640x640 |
|
||||
| YOLO11-S | 59.7 | 44.4 | 82.3 | 56.2 | 3.2 | 9.4 | 640x640 |
|
||||
| YOLO11-M | 64.1 | 48.6 | 82.5 | 56.5 | 5.1 | 20.1 | 640x640 |
|
||||
| YOLO11-L | 64.9 | 49.9 | 82.2 | 56.5 | 6.5 | 25.3 | 640x640 |
|
||||
| YOLO11-X | 66.1 | 50.9 | 81.7 | 56.2 | 10.5 | 56.9 | 640x640 |
|
||||
| YOLO26-N | 55.8 | 40.3 | 76.7 | 52.0 | 1.7 | 2.6 | 640x640 |
|
||||
| YOLO26-S | 64.3 | 47.7 | 82.7 | 57.0 | 2.6 | 9.4 | 640x640 |
|
||||
| YOLO26-M | 69.7 | 52.5 | 84.4 | 58.7 | 4.4 | 20.1 | 640x640 |
|
||||
| YOLO26-L | 71.1 | 54.1 | 85.0 | 59.3 | 5.7 | 25.3 | 640x640 |
|
||||
| YOLO26-X | 74.0 | 56.9 | 85.6 | 60.0 | 9.6 | 56.9 | 640x640 |
|
||||
| LW-DETR-T | 60.7 | 42.9 | 84.7 | 57.1 | 1.9 | 12.1 | 640x640 |
|
||||
| LW-DETR-S | 66.8 | 48.0 | 85.0 | 57.4 | 2.6 | 14.6 | 640x640 |
|
||||
| LW-DETR-M | 72.0 | 52.6 | 86.8 | 59.8 | 4.4 | 28.2 | 640x640 |
|
||||
| LW-DETR-L | 74.6 | 56.1 | 87.4 | 61.5 | 6.9 | 46.8 | 640x640 |
|
||||
| LW-DETR-X | 76.9 | 58.3 | 87.9 | 62.1 | 13.0 | 118.0 | 640x640 |
|
||||
| D-FINE-N | 60.2 | 42.7 | 84.4 | 58.2 | 2.1 | 3.8 | 640x640 |
|
||||
| D-FINE-S | 67.6 | 50.6 | 85.3 | 60.3 | 3.5 | 10.2 | 640x640 |
|
||||
| D-FINE-M | 72.6 | 55.0 | 85.5 | 60.6 | 5.4 | 19.2 | 640x640 |
|
||||
| D-FINE-L | 74.9 | 57.2 | 86.4 | 61.6 | 7.5 | 31.0 | 640x640 |
|
||||
| D-FINE-X | 76.8 | 59.3 | 86.9 | 62.2 | 11.5 | 62.0 | 640x640 |
|
||||
|
||||
## Segmentation
|
||||
|
||||
<img alt="rf_detr_1-4_latency_accuracy_instance_segmentation" src="https://storage.googleapis.com/com-roboflow-marketing/rf-detr/rf_detr_1-4_latency_accuracy_instance_segmentation.png" />
|
||||
|
||||
| Architecture | COCO AP<sub>50</sub> | COCO AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution |
|
||||
| :-------------: | :------------------: | :---------------------: | :----------: | :--------: | :--------: |
|
||||
| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312x312 |
|
||||
| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384x384 |
|
||||
| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432x432 |
|
||||
| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504x504 |
|
||||
| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624x624 |
|
||||
| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768x768 |
|
||||
| YOLOv8-N-Seg | 45.6 | 28.3 | 3.5 | 3.4 | 640x640 |
|
||||
| YOLOv8-S-Seg | 53.8 | 34.0 | 4.2 | 11.8 | 640x640 |
|
||||
| YOLOv8-M-Seg | 58.2 | 37.3 | 7.0 | 27.3 | 640x640 |
|
||||
| YOLOv8-L-Seg | 60.5 | 39.0 | 9.7 | 46.0 | 640x640 |
|
||||
| YOLOv8-XL-Seg | 61.3 | 39.5 | 14.0 | 71.8 | 640x640 |
|
||||
| YOLOv11-N-Seg | 47.8 | 30.0 | 3.6 | 2.9 | 640x640 |
|
||||
| YOLOv11-S-Seg | 55.4 | 35.0 | 4.6 | 10.1 | 640x640 |
|
||||
| YOLOv11-M-Seg | 60.0 | 38.5 | 6.9 | 22.4 | 640x640 |
|
||||
| YOLOv11-L-Seg | 61.5 | 39.5 | 8.3 | 27.6 | 640x640 |
|
||||
| YOLOv11-XL-Seg | 62.4 | 40.1 | 13.7 | 62.1 | 640x640 |
|
||||
| YOLO26-N-Seg | 54.3 | 34.7 | 2.31 | 2.7 | 640x640 |
|
||||
| YOLO26-S-Seg | 62.4 | 40.2 | 3.47 | 10.4 | 640x640 |
|
||||
| YOLO26-M-Seg | 67.8 | 44.0 | 6.32 | 23.6 | 640x640 |
|
||||
| YOLO26-L-Seg | 69.8 | 45.5 | 7.58 | 28.0 | 640x640 |
|
||||
| YOLO26-X-Seg | 71.6 | 46.8 | 12.92 | 62.8 | 640x640 |
|
||||
|
||||
## Keypoints
|
||||
|
||||
<img alt="RF-DETR Keypoint mAP vs latency chart comparing against YOLO26-pose and YOLO11-pose on MS COCO" src="../assets/keypoints/kp-map-latency.png" />
|
||||
|
||||
| Architecture | COCO AP<sub>50:95</sub> | Latency (ms) |
|
||||
| :------------------------: | :---------------------: | :----------: |
|
||||
| RF-DETR Keypoint (Preview) | 71.8 | 9.7 |
|
||||
| YOLO11-pose N | 48.9 | 3.2 |
|
||||
| YOLO11-pose S | 57.5 | 3.4 |
|
||||
| YOLO11-pose M | 64.2 | 5.2 |
|
||||
| YOLO11-pose L | 65.2 | 6.6 |
|
||||
| YOLO11-pose X | 68.6 | 10.6 |
|
||||
| YOLO26-pose N | 55.9 | 1.9 |
|
||||
| YOLO26-pose S | 62.0 | 2.7 |
|
||||
| YOLO26-pose M | 68.0 | 4.6 |
|
||||
| YOLO26-pose L | 69.2 | 5.9 |
|
||||
| YOLO26-pose X | 71.0 | 9.8 |
|
||||
|
||||
> Keypoint benchmarks report AP<sub>50:95</sub> (OKS-based); this is the standard COCO keypoint comparison metric.
|
||||
@@ -0,0 +1,115 @@
|
||||
---
|
||||
description: Deploy fine-tuned RF-DETR detection and segmentation models to Roboflow for cloud inference, edge hardware, and multi-step vision workflows.
|
||||
---
|
||||
|
||||
# Deploy a Trained RF-DETR Model
|
||||
|
||||
!!! tip "Key Takeaways"
|
||||
|
||||
- Deploy fine-tuned RF-DETR models to Roboflow with a single `deploy_to_roboflow()` call
|
||||
- Supports both detection and segmentation model deployment
|
||||
- Run deployed models via Roboflow Inference on cloud, edge hardware, or NVIDIA Jetson
|
||||
- Model weights are cached locally after the first inference run for fast subsequent predictions
|
||||
|
||||
You can deploy a fine-tuned RF-DETR model to Roboflow.
|
||||
|
||||
Deploying to Roboflow allows you to create multi-step computer vision applications that run both in the cloud and your own hardware.
|
||||
|
||||
To deploy your model to Roboflow, run:
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRNano
|
||||
|
||||
x = RFDETRNano(pretrain_weights="<path/to/pretrain/weights/dir>")
|
||||
x.deploy_to_roboflow(
|
||||
workspace="<your-workspace>",
|
||||
project_id="<your-project-id>",
|
||||
version=1,
|
||||
api_key="<YOUR_API_KEY>",
|
||||
)
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegMedium
|
||||
|
||||
x = RFDETRSegMedium(pretrain_weights="<path/to/pretrain/weights/dir>")
|
||||
x.deploy_to_roboflow(
|
||||
workspace="<your-workspace>",
|
||||
project_id="<your-project-id>",
|
||||
version=1,
|
||||
api_key="<YOUR_API_KEY>",
|
||||
)
|
||||
```
|
||||
|
||||
Above, set your Roboflow Workspace ID, the ID of the project to which you want to upload your model, and your Roboflow API key.
|
||||
|
||||
- [Learn how to find your Workspace and Project ID.](https://docs.roboflow.com/developer/authentication/workspace-and-project-ids)
|
||||
- [Learn how to find your API key.](https://docs.roboflow.com/developer/authentication/find-your-roboflow-api-key)
|
||||
|
||||
You can then run your model with Roboflow Inference:
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
from inference import get_model
|
||||
from PIL import Image
|
||||
from io import BytesIO
|
||||
import requests
|
||||
|
||||
url = "https://media.roboflow.com/dog.jpeg"
|
||||
image = Image.open(BytesIO(requests.get(url).content))
|
||||
|
||||
model = get_model("rfdetr-large") # replace with your Roboflow model ID
|
||||
|
||||
predictions = model.infer(image, confidence=0.5)[0]
|
||||
|
||||
detections = sv.Detections.from_inference(predictions)
|
||||
|
||||
labels = [prediction.class_name for prediction in predictions.predictions]
|
||||
|
||||
annotated_image = image.copy()
|
||||
annotated_image = sv.BoxAnnotator().annotate(annotated_image, detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels)
|
||||
|
||||
sv.plot_image(annotated_image)
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
from inference import get_model
|
||||
from PIL import Image
|
||||
from io import BytesIO
|
||||
import requests
|
||||
|
||||
url = "https://media.roboflow.com/dog.jpeg"
|
||||
image = Image.open(BytesIO(requests.get(url).content))
|
||||
|
||||
model = get_model("rfdetr-seg-small") # replace with your Roboflow model ID
|
||||
|
||||
predictions = model.infer(image, confidence=0.5)[0]
|
||||
|
||||
detections = sv.Detections.from_inference(predictions)
|
||||
|
||||
labels = [prediction.class_name for prediction in predictions.predictions]
|
||||
|
||||
annotated_image = image.copy()
|
||||
annotated_image = sv.MaskAnnotator().annotate(annotated_image, detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels)
|
||||
|
||||
sv.plot_image(annotated_image)
|
||||
```
|
||||
|
||||
Above, replace `rfdetr-large` with the your Roboflow model ID. You can find this ID from the "Models" list in your Roboflow dashboard:
|
||||
|
||||
|
||||
|
||||
When you first run this model, your model weights will be cached for local use with Inference.
|
||||
|
||||
You will then see the results from your fine-tuned model.
|
||||
@@ -0,0 +1,396 @@
|
||||
---
|
||||
description: Export RF-DETR models to ONNX, TensorRT, and TFLite (FP32/FP16/INT8) for high-performance inference on GPUs, mobile, and edge devices.
|
||||
---
|
||||
|
||||
# Export RF-DETR Model
|
||||
|
||||
!!! tip "Key Takeaways"
|
||||
|
||||
- Export to ONNX for cross-platform inference with ONNX Runtime, OpenVINO, or TensorRT
|
||||
- Export to TFLite (FP32, FP16, INT8) for mobile and edge deployment
|
||||
- TensorRT conversion delivers lowest latency on NVIDIA GPUs (2.3 ms for Nano)
|
||||
- INT8 quantization requires calibration data from your dataset for accurate results
|
||||
- Custom input resolutions supported (must be divisible by `patch_size × num_windows`, which varies by model variant)
|
||||
|
||||
RF-DETR supports exporting models to ONNX and TFLite formats, enabling deployment across a wide range of inference frameworks, edge devices, and hardware accelerators.
|
||||
|
||||
## Installation
|
||||
|
||||
Install the export dependencies you need:
|
||||
|
||||
```bash
|
||||
# ONNX export only
|
||||
pip install "rfdetr[onnx]"
|
||||
|
||||
# TFLite export (includes ONNX dependency)
|
||||
pip install "rfdetr[onnx,tflite]"
|
||||
```
|
||||
|
||||
## Basic Export
|
||||
|
||||
Export your trained model to ONNX format:
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium(pretrain_weights="<path/to/checkpoint.pth>")
|
||||
|
||||
model.export()
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegMedium
|
||||
|
||||
model = RFDETRSegMedium(pretrain_weights="<path/to/checkpoint.pth>")
|
||||
|
||||
model.export()
|
||||
```
|
||||
|
||||
This command saves the ONNX model to the `output` directory by default.
|
||||
|
||||
## Export Parameters
|
||||
|
||||
The `export()` method accepts several parameters to customize the export process:
|
||||
|
||||
| Parameter | Default | Description |
|
||||
| ------------------ | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `output_dir` | `"output"` | Directory where the exported model will be saved. |
|
||||
| `format` | `"onnx"` | Export format: `"onnx"` or `"tflite"`. |
|
||||
| `quantization` | `None` | TFLite quantization mode: `None`/`"fp32"`, `"fp16"`, or `"int8"`. Only used when `format="tflite"`. |
|
||||
| `calibration_data` | `None` | Calibration data for TFLite export. Image directory, `.npy` file path, NumPy array, or `None`. See [TFLite Export](#tflite-export). |
|
||||
| `max_images` | `100` | Maximum number of images to load from a calibration directory for TFLite INT8 quantization. Ignored for other calibration data formats. |
|
||||
| `infer_dir` | `None` | Optional directory of sample images for inference validation during export tracing. If not provided, a random dummy image is generated. |
|
||||
| `backbone_only` | `False` | Export only the backbone feature extractor instead of the full model. |
|
||||
| `opset_version` | `17` | ONNX opset version to use for export. Higher versions support more operations. |
|
||||
| `verbose` | `True` | Whether to print verbose export information. |
|
||||
| `shape` | `None` | Input shape as tuple `(height, width)`. Each dimension must be divisible by the selected model's block size (`patch_size * num_windows`). If not provided, uses the model's default resolution. |
|
||||
| `batch_size` | `1` | Batch size for the exported model. |
|
||||
| `dynamic_batch` | `False` | If `True`, export with a dynamic batch dimension so the ONNX model accepts variable batch sizes at runtime. |
|
||||
| `patch_size` | `None` | Backbone patch size override. Defaults to the value from `model_config.patch_size`. Must match the instantiated model's patch size when provided. |
|
||||
| `notes` | `None` | Optional user-defined metadata (string, dict, list, or any JSON-serialisable value) to embed in the exported ONNX model under the `"rfdetr_notes"` metadata property. |
|
||||
|
||||
## Advanced Export Examples
|
||||
|
||||
### Export with Custom Output Directory
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium(pretrain_weights="<path/to/checkpoint.pth>")
|
||||
|
||||
model.export(output_dir="exports/my_model")
|
||||
```
|
||||
|
||||
### Export with Custom Resolution
|
||||
|
||||
Export the model with a specific input resolution. For example, `RFDETRMedium` expects dimensions divisible by `32` (`patch_size=16`, `num_windows=2`):
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium(pretrain_weights="<path/to/checkpoint.pth>")
|
||||
|
||||
model.export(shape=(608, 608))
|
||||
```
|
||||
|
||||
### Export Backbone Only
|
||||
|
||||
Export only the backbone feature extractor for use in custom pipelines:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium(pretrain_weights="<path/to/checkpoint.pth>")
|
||||
|
||||
model.export(backbone_only=True)
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
After running the export, you will find the following files in your output directory:
|
||||
|
||||
- `inference_model.onnx` - The exported ONNX model (or `backbone_model.onnx` if `backbone_only=True`)
|
||||
|
||||
## Optional: Convert ONNX to TensorRT
|
||||
|
||||
If you want lower latency on NVIDIA GPUs, you can convert the exported ONNX model to a TensorRT engine.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Run TensorRT conversion on the same machine and GPU family where you plan to deploy inference.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Install TensorRT (`trtexec` must be available in your `PATH`)
|
||||
- Export an ONNX model first (for example: `output/inference_model.onnx`)
|
||||
|
||||
### Python API Conversion
|
||||
|
||||
```python
|
||||
from argparse import Namespace
|
||||
|
||||
from rfdetr.export._tensorrt import trtexec
|
||||
|
||||
args = Namespace(
|
||||
verbose=True,
|
||||
profile=False,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
trtexec("output/inference_model.onnx", args)
|
||||
```
|
||||
|
||||
This produces `output/inference_model.engine`. If `profile=True`, it also writes an Nsight Systems report (`.nsys-rep`).
|
||||
|
||||
## TFLite Export
|
||||
|
||||
!!! warning "Experimental — Use with Caution"
|
||||
|
||||
TFLite export is **experimental and work-in-progress**. The pipeline depends on
|
||||
several upstream packages (`onnx2tf`, `ai_edge_litert`, `tflite-runtime`) that
|
||||
have experienced breaking API changes and installation instabilities across
|
||||
releases. You may encounter errors or unexpected results.
|
||||
|
||||
**Known instabilities:**
|
||||
|
||||
- `onnx2tf` output graph structure can change between minor versions, silently
|
||||
altering output tensor layout and breaking downstream inference code.
|
||||
- `ai_edge_litert` (Google's replacement for `tflite-runtime`) is still
|
||||
stabilising its public API; version pinning is strongly recommended.
|
||||
- INT8 quantization accuracy is sensitive to calibration data quality — poor
|
||||
calibration causes silent precision loss with no error at export time.
|
||||
- The ONNX → TF → TFLite conversion chain introduces numerical rounding that
|
||||
may produce slightly different predictions from the original PyTorch model.
|
||||
- Installation of the `[tflite]` extra may conflict with existing TensorFlow
|
||||
or NumPy versions in your environment.
|
||||
|
||||
**Recommendations:**
|
||||
|
||||
- Pin your dependency versions (e.g. `onnx2tf==X.Y.Z`) and test before each upgrade.
|
||||
- Validate exported `.tflite` files against a held-out evaluation set before deploying.
|
||||
- Prefer ONNX export when your target runtime supports it — it is more stable and
|
||||
better tested.
|
||||
- If export fails, check the [open issues](https://github.com/roboflow/rf-detr/issues)
|
||||
for known workarounds or report a new one with your environment details
|
||||
(`pip freeze`, Python version, OS).
|
||||
|
||||
Export your model to TFLite for deployment on mobile devices, microcontrollers, and edge hardware via TensorFlow Lite. The TFLite export pipeline converts ONNX → TensorFlow → TFLite using [onnx2tf](https://github.com/PINTO0309/onnx2tf).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
```bash
|
||||
pip install "rfdetr[onnx,tflite]"
|
||||
```
|
||||
|
||||
### Basic TFLite Export (FP32)
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSmall
|
||||
|
||||
model = RFDETRSmall()
|
||||
|
||||
model.export(format="tflite", output_dir="output")
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegNano
|
||||
|
||||
model = RFDETRSegNano()
|
||||
|
||||
model.export(format="tflite", output_dir="output")
|
||||
```
|
||||
|
||||
This produces both `output/inference_model_float32.tflite` and `output/inference_model_float16.tflite`.
|
||||
|
||||
### INT8 Quantization with Calibration Data
|
||||
|
||||
For INT8 quantization, provide representative images from your dataset as calibration data. This is **critical** for preserving model accuracy — without real calibration data, the quantizer uses random noise and accuracy will be poor.
|
||||
|
||||
#### Option 1: Point to an Image Directory (Recommended)
|
||||
|
||||
The simplest approach — just point `calibration_data` to a directory containing JPEG/PNG images. The converter automatically loads, resizes, and prepares the images:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRNano
|
||||
|
||||
model = RFDETRNano()
|
||||
model.export(
|
||||
format="tflite",
|
||||
quantization="int8",
|
||||
calibration_data="path/to/val2017/", # directory of images
|
||||
output_dir="output",
|
||||
)
|
||||
```
|
||||
|
||||
The converter loads up to 100 images from the directory by default, resizes them to the model's input resolution, and uses them for both output validation and INT8 calibration. Supported formats: JPEG, PNG, BMP, WebP.
|
||||
|
||||
You can control how many images are loaded with the `max_images` parameter:
|
||||
|
||||
```python
|
||||
model.export(
|
||||
format="tflite",
|
||||
quantization="int8",
|
||||
calibration_data="path/to/val2017/",
|
||||
max_images=200, # load up to 200 images (default: 100)
|
||||
output_dir="output",
|
||||
)
|
||||
```
|
||||
|
||||
#### Option 2: NumPy `.npy` File
|
||||
|
||||
Prepare calibration data as a NumPy array and save it to a `.npy` file:
|
||||
|
||||
- Shape: `(N, H, W, 3)` — NHWC format with 3 color channels
|
||||
- Data type: `float32`
|
||||
- Value range: `[0, 1]` (divide by 255, but do **not** apply ImageNet normalization — the converter handles that automatically)
|
||||
- Recommended: 20–100 representative images from your dataset
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from rfdetr import RFDETRSmall
|
||||
|
||||
model = RFDETRSmall()
|
||||
target_resolution = model.model_config.resolution
|
||||
|
||||
# Load representative images from your dataset
|
||||
images = []
|
||||
for path in image_paths[:50]: # 50 representative samples
|
||||
img = Image.open(path).convert("RGB").resize((target_resolution, target_resolution))
|
||||
images.append(np.array(img, dtype=np.float32) / 255.0)
|
||||
|
||||
calibration_data = np.stack(images) # shape: (50, H, W, 3)
|
||||
|
||||
# Save to .npy for reuse
|
||||
np.save("calibration_data.npy", calibration_data)
|
||||
|
||||
# Export with INT8 quantization
|
||||
model.export(
|
||||
format="tflite",
|
||||
quantization="int8",
|
||||
calibration_data="calibration_data.npy",
|
||||
output_dir="output",
|
||||
)
|
||||
```
|
||||
|
||||
#### Option 3: NumPy Array Directly
|
||||
|
||||
You can also pass the NumPy array directly without saving to disk:
|
||||
|
||||
```python
|
||||
model.export(
|
||||
format="tflite",
|
||||
quantization="int8",
|
||||
calibration_data=calibration_data, # np.ndarray
|
||||
output_dir="output",
|
||||
)
|
||||
```
|
||||
|
||||
### FP16 Export
|
||||
|
||||
FP16 models are always produced alongside FP32. You can explicitly request FP16 mode:
|
||||
|
||||
```python
|
||||
model.export(format="tflite", quantization="fp16", output_dir="output")
|
||||
```
|
||||
|
||||
### TFLite Output Files
|
||||
|
||||
The `onnx2tf` converter **always** produces both FP32 and FP16 TFLite files, regardless of the requested quantization mode. When `quantization="int8"` is specified, it additionally produces the INT8-quantized model.
|
||||
|
||||
| File | Description |
|
||||
| -------------------------------------- | --------------------------------------- |
|
||||
| `inference_model_float32.tflite` | FP32 model (always produced) |
|
||||
| `inference_model_float16.tflite` | FP16 model (always produced) |
|
||||
| `inference_model_integer_quant.tflite` | INT8 model (when `quantization="int8"`) |
|
||||
|
||||
!!! note
|
||||
|
||||
Segmentation models produce TFLite files with three outputs: `dets` (bounding boxes), `labels` (class scores), and `masks` (per-instance segmentation masks).
|
||||
|
||||
### TFLite Inference Example
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
# pip install tflite-runtime (or use tensorflow.lite)
|
||||
import tflite_runtime.interpreter as tflite
|
||||
|
||||
# Load model
|
||||
interpreter = tflite.Interpreter(model_path="output/inference_model_float32.tflite")
|
||||
interpreter.allocate_tensors()
|
||||
|
||||
input_details = interpreter.get_input_details()
|
||||
output_details = interpreter.get_output_details()
|
||||
|
||||
# Prepare input — TFLite model expects NHWC, ImageNet-normalized
|
||||
input_height, input_width = input_details[0]["shape"][1:3]
|
||||
image = Image.open("image.jpg").convert("RGB").resize((input_width, input_height))
|
||||
image_array = np.array(image, dtype=np.float32) / 255.0
|
||||
|
||||
# Apply ImageNet normalization
|
||||
mean = np.array([0.485, 0.456, 0.406])
|
||||
std = np.array([0.229, 0.224, 0.225])
|
||||
image_array = (image_array - mean) / std
|
||||
|
||||
# Add batch dimension: (1, H, W, 3)
|
||||
image_array = np.expand_dims(image_array, axis=0).astype(np.float32)
|
||||
|
||||
# Run inference
|
||||
interpreter.set_tensor(input_details[0]["index"], image_array)
|
||||
interpreter.invoke()
|
||||
|
||||
boxes = interpreter.get_tensor(output_details[0]["index"])
|
||||
labels = interpreter.get_tensor(output_details[1]["index"])
|
||||
```
|
||||
|
||||
## Using the Exported Model
|
||||
|
||||
Once exported, you can use the ONNX model with various inference frameworks:
|
||||
|
||||
### ONNX Runtime
|
||||
|
||||
```python
|
||||
import onnxruntime as ort
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
# Load the ONNX model
|
||||
session = ort.InferenceSession("output/inference_model.onnx")
|
||||
|
||||
# Prepare input image
|
||||
input_height, input_width = session.get_inputs()[0].shape[2:4]
|
||||
image = Image.open("image.jpg").convert("RGB")
|
||||
image = image.resize((input_width, input_height)) # Resize to the exported model shape
|
||||
image_array = np.array(image).astype(np.float32) / 255.0
|
||||
|
||||
# Normalize
|
||||
mean = np.array([0.485, 0.456, 0.406])
|
||||
std = np.array([0.229, 0.224, 0.225])
|
||||
image_array = (image_array - mean) / std
|
||||
|
||||
# Convert to NCHW format
|
||||
image_array = np.transpose(image_array, (2, 0, 1))
|
||||
image_array = np.expand_dims(image_array, axis=0)
|
||||
|
||||
# Run inference
|
||||
outputs = session.run(None, {"input": image_array})
|
||||
boxes, labels = outputs
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
After exporting your model, you may want to:
|
||||
|
||||
- [Deploy to Roboflow](deploy.md) for cloud-based inference and workflow integration
|
||||
- Use the ONNX model with TensorRT for optimized GPU inference
|
||||
- Deploy TFLite models on mobile/edge devices with TensorFlow Lite
|
||||
- Integrate with edge deployment frameworks like ONNX Runtime or OpenVINO
|
||||
@@ -0,0 +1,198 @@
|
||||
---
|
||||
description: Run pre-trained RF-DETR models (Nano to 2XLarge) on images, video, webcam, and RTSP streams. COCO-trained with real-time DINOv2 backbone.
|
||||
---
|
||||
|
||||
You can run any of the four supported RF-DETR base models -- Nano, Small, Medium, Large -- with [Inference](https://github.com/roboflow/inference), an open source computer vision inference server. The base models are trained on the [Microsoft COCO dataset](https://universe.roboflow.com/microsoft/coco). XLarge and 2XLarge detection models are also available via `pip install rfdetr[plus]` and are provided under the PML 1.0 license.
|
||||
|
||||
=== "Run on an Image"
|
||||
|
||||
To run RF-DETR on an image, use the following code:
|
||||
|
||||
```python
|
||||
import os
|
||||
import supervision as sv
|
||||
from inference import get_model
|
||||
from PIL import Image
|
||||
from io import BytesIO
|
||||
import requests
|
||||
|
||||
url = "https://media.roboflow.com/dog.jpeg"
|
||||
image = Image.open(BytesIO(requests.get(url).content))
|
||||
|
||||
model = get_model("rfdetr-large")
|
||||
|
||||
predictions = model.infer(image, confidence=0.5)[0]
|
||||
|
||||
detections = sv.Detections.from_inference(predictions)
|
||||
|
||||
labels = [prediction.class_name for prediction in predictions.predictions]
|
||||
|
||||
annotated_image = image.copy()
|
||||
annotated_image = sv.BoxAnnotator().annotate(annotated_image, detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels)
|
||||
|
||||
sv.plot_image(annotated_image)
|
||||
```
|
||||
|
||||
Above, replace the image URL with any image you want to use with the model.
|
||||
|
||||
Here are the results from the code above:
|
||||
|
||||
<figure markdown="span">
|
||||
{ width=300 }
|
||||
<figcaption>RF-DETR Base predictions</figcaption>
|
||||
</figure>
|
||||
|
||||
=== "Run on a Video File"
|
||||
|
||||
To run RF-DETR on a video file, use the following code:
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
|
||||
def callback(frame, index):
|
||||
detections = model.predict(frame[:, :, ::-1], threshold=0.5)
|
||||
|
||||
labels = [
|
||||
f"{COCO_CLASSES[class_id]} {confidence:.2f}"
|
||||
for class_id, confidence in zip(detections.class_id, detections.confidence)
|
||||
]
|
||||
|
||||
annotated_frame = frame.copy()
|
||||
annotated_frame = sv.BoxAnnotator().annotate(annotated_frame, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
return annotated_frame
|
||||
|
||||
|
||||
sv.process_video(
|
||||
source_path="<SOURCE_VIDEO_PATH>",
|
||||
target_path="<TARGET_VIDEO_PATH>",
|
||||
callback=callback,
|
||||
)
|
||||
```
|
||||
|
||||
Above, set your `SOURCE_VIDEO_PATH` and `TARGET_VIDEO_PATH` to the directories of the video you want to process and where you want to save the results from inference, respectively.
|
||||
|
||||
=== "Run on a Webcam Stream"
|
||||
|
||||
To run RF-DETR on a webcam input, use the following code:
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
cap = cv2.VideoCapture(0)
|
||||
while True:
|
||||
success, frame = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
detections = model.predict(frame[:, :, ::-1], threshold=0.5)
|
||||
|
||||
labels = [
|
||||
f"{COCO_CLASSES[class_id]} {confidence:.2f}"
|
||||
for class_id, confidence in zip(detections.class_id, detections.confidence)
|
||||
]
|
||||
|
||||
annotated_frame = frame.copy()
|
||||
annotated_frame = sv.BoxAnnotator().annotate(annotated_frame, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("Webcam", annotated_frame)
|
||||
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
cap.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "Run on an RTSP Stream"
|
||||
|
||||
To run RF-DETR on an RTSP (Real Time Streaming Protocol) stream, use the following code:
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
cap = cv2.VideoCapture("<RTSP_STREAM_URL>")
|
||||
while True:
|
||||
success, frame = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
detections = model.predict(frame[:, :, ::-1], threshold=0.5)
|
||||
|
||||
labels = [
|
||||
f"{COCO_CLASSES[class_id]} {confidence:.2f}"
|
||||
for class_id, confidence in zip(detections.class_id, detections.confidence)
|
||||
]
|
||||
|
||||
annotated_frame = frame.copy()
|
||||
annotated_frame = sv.BoxAnnotator().annotate(annotated_frame, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RTSP Stream", annotated_frame)
|
||||
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
cap.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
You can change the RF-DETR model that the code snippet above uses. To do so, update `rfdetr-base` to any of the following values:
|
||||
|
||||
- `rfdetr-nano`
|
||||
- `rfdetr-small`
|
||||
- `rfdetr-medium`
|
||||
- `rfdetr-large`
|
||||
|
||||
## Batch Inference
|
||||
|
||||
You can provide `.predict()` with either a single image or a list of images. When multiple images are supplied, they are processed together in a single forward pass, resulting in a corresponding list of detections.
|
||||
|
||||
```python
|
||||
import io
|
||||
import requests
|
||||
import supervision as sv
|
||||
from PIL import Image
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
urls = [
|
||||
"https://media.roboflow.com/notebooks/examples/dog-2.jpeg",
|
||||
"https://media.roboflow.com/notebooks/examples/dog-3.jpeg",
|
||||
]
|
||||
|
||||
images = [Image.open(io.BytesIO(requests.get(url).content)) for url in urls]
|
||||
|
||||
detections_list = model.predict(images, threshold=0.5)
|
||||
|
||||
for image, detections in zip(images, detections_list):
|
||||
labels = [
|
||||
f"{COCO_CLASSES[class_id]} {confidence:.2f}"
|
||||
for class_id, confidence in zip(detections.class_id, detections.confidence)
|
||||
]
|
||||
|
||||
annotated_image = image.copy()
|
||||
annotated_image = sv.BoxAnnotator().annotate(annotated_image, detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels)
|
||||
|
||||
sv.plot_image(annotated_image)
|
||||
```
|
||||
@@ -0,0 +1,185 @@
|
||||
---
|
||||
description: Run RF-DETR object detection on images, video, and streams. Nano to 2XLarge models with 2.3-17.2 ms latency and up to 60.1 AP on COCO.
|
||||
---
|
||||
|
||||
# Run an RF-DETR Object Detection Model
|
||||
|
||||
RF-DETR is a real-time transformer architecture for object detection, built on a DINOv2 vision transformer backbone. The base models are trained on the Microsoft COCO dataset and achieve state-of-the-art accuracy and latency trade-offs.
|
||||
|
||||
## Pre-trained Checkpoints
|
||||
|
||||
RF-DETR offers model sizes from Nano to 2XLarge, allowing trade-offs between accuracy, latency, and parameter count. All latency numbers were measured on an NVIDIA T4 using TensorRT, FP16, and batch size 1. Core models (Nano to Large) are licensed under Apache 2.0. XLarge and 2XLarge (marked with △) are provided by the [`rfdetr_plus`](https://github.com/roboflow/rf-detr-plus) extension (`pip install rfdetr[plus]`) under the Platform Model License 1.0 and require a Roboflow account.
|
||||
|
||||
| Size | RF-DETR package class | Inference package alias | COCO AP<sub>50</sub> | COCO AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution | License |
|
||||
| :--: | :-------------------: | :---------------------- | :------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: |
|
||||
| N | `RFDETRNano` | `rfdetr-nano` | 67.6 | 48.4 | 2.3 | 30.5 | 384x384 | Apache 2.0 |
|
||||
| S | `RFDETRSmall` | `rfdetr-small` | 72.1 | 53.0 | 3.5 | 32.1 | 512x512 | Apache 2.0 |
|
||||
| M | `RFDETRMedium` | `rfdetr-medium` | 73.6 | 54.7 | 4.4 | 33.7 | 576x576 | Apache 2.0 |
|
||||
| L | `RFDETRLarge` | `rfdetr-large` | 75.1 | 56.5 | 6.8 | 33.9 | 704x704 | Apache 2.0 |
|
||||
| XL | `RFDETRXLarge` △ | `rfdetr-xlarge` | 77.4 | 58.6 | 11.5 | 126.4 | 700x700 | PML 1.0 |
|
||||
| 2XL | `RFDETR2XLarge` △ | `rfdetr-2xlarge` | 78.5 | 60.1 | 17.2 | 126.9 | 880x880 | PML 1.0 |
|
||||
|
||||
> △ Requires the `rfdetr_plus` extension: `pip install rfdetr[plus]`
|
||||
|
||||
## Run on an Image
|
||||
|
||||
Perform inference on an image using either the `rfdetr` package or the `inference` package. To use a different model size, select the corresponding class or alias from the table above.
|
||||
|
||||
=== "rfdetr"
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
detections = model.predict("https://media.roboflow.com/dog.jpg", threshold=0.5)
|
||||
|
||||
labels = [f"{COCO_CLASSES[class_id]}" for class_id in detections.class_id]
|
||||
|
||||
annotated_image = sv.BoxAnnotator().annotate(detections.metadata["source_image"], detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels)
|
||||
```
|
||||
|
||||
=== "inference"
|
||||
|
||||
```python
|
||||
import requests
|
||||
import supervision as sv
|
||||
from PIL import Image
|
||||
from inference import get_model
|
||||
|
||||
model = get_model("rfdetr-medium")
|
||||
|
||||
image = Image.open(requests.get("https://media.roboflow.com/dog.jpg", stream=True).raw)
|
||||
predictions = model.infer(image, confidence=0.5)[0]
|
||||
detections = sv.Detections.from_inference(predictions)
|
||||
|
||||
annotated_image = sv.BoxAnnotator().annotate(image, detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections)
|
||||
```
|
||||
|
||||
!!! note "Using COCO classes vs. fine-tuned model classes"
|
||||
|
||||
`COCO_CLASSES` works for COCO-pretrained models (80 COCO classes, indexed 0-79).
|
||||
For fine-tuned models, use `detections.data["class_name"]` instead — it resolves
|
||||
class names from the checkpoint and works for both COCO and custom datasets.
|
||||
|
||||
For memory-constrained inference-only deployments with the `rfdetr` package, optimize the loaded model in place before
|
||||
calling `predict()`. Pass `dtype="float16"` to halve weight memory in addition to clearing the base model reference.
|
||||
This operation is irreversible — to restore the original model, create a new `RFDETR` instance:
|
||||
|
||||
```python
|
||||
model.optimize_for_inference(compile=False, inplace=True, dtype="float16")
|
||||
```
|
||||
|
||||
## Run on video, webcam, or RTSP stream
|
||||
|
||||
These examples use OpenCV for decoding and display. Replace `<SOURCE_VIDEO_PATH>`, `<WEBCAM_INDEX>`, and `<RTSP_STREAM_URL>` with your inputs. `<WEBCAM_INDEX>` is usually `0` for the default camera.
|
||||
|
||||
=== "video"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
video_capture = cv2.VideoCapture("<SOURCE_VIDEO_PATH>")
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError("Failed to open video source: <SOURCE_VIDEO_PATH>")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
detections = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]
|
||||
|
||||
annotated_frame = sv.BoxAnnotator().annotate(frame_bgr, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RF-DETR Video", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "webcam"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
WEBCAM_INDEX = 0
|
||||
video_capture = cv2.VideoCapture(WEBCAM_INDEX)
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError(f"Failed to open webcam: {WEBCAM_INDEX}")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
detections = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]
|
||||
|
||||
annotated_frame = sv.BoxAnnotator().annotate(frame_bgr, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RF-DETR Webcam", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "stream"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
video_capture = cv2.VideoCapture("<RTSP_STREAM_URL>")
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError("Failed to open RTSP stream: <RTSP_STREAM_URL>")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
detections = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]
|
||||
|
||||
annotated_frame = sv.BoxAnnotator().annotate(frame_bgr, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RF-DETR RTSP", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
@@ -0,0 +1,203 @@
|
||||
---
|
||||
description: Run RF-DETR keypoint detection on images, video, and streams. COCO-pretrained preview model predicts 17 person keypoints with 71.8 AP at 9.7 ms on NVIDIA T4.
|
||||
---
|
||||
|
||||
# Run an RF-DETR Keypoint Model
|
||||
|
||||
RF-DETR Keypoint is a real-time transformer architecture for keypoint detection, built on a DINOv2 vision transformer backbone. The preview model is pretrained on the Microsoft COCO dataset and predicts 17 body keypoints per detected person.
|
||||
|
||||
|
||||
|
||||
!!! note "Preview model"
|
||||
|
||||
`RFDETRKeypointPreview` is an early-access release. Fine-tuning on custom keypoint datasets is the primary intended use case. See [Keypoint Preview Parameters](../train/training-parameters.md#keypoint-preview-parameters) for training configuration. API surface and checkpoint weights may change before the stable release.
|
||||
|
||||
## Pre-trained Checkpoints
|
||||
|
||||
RF-DETR Keypoint outperforms YOLO26-pose X and YOLO11-pose X at comparable latency on MS COCO. Latency measured on NVIDIA T4, TensorRT FP16, batch size 1.
|
||||
|
||||
{ width=560 }
|
||||
|
||||
| Model | RF-DETR package class | COCO AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution | License |
|
||||
| :----------------: | :---------------------: | :---------------------: | :----------: | :--------: | :--------: | :--------: |
|
||||
| Keypoint (Preview) | `RFDETRKeypointPreview` | 71.8 | 9.7 | 126.4 | 576x576 | Apache 2.0 |
|
||||
|
||||
> The keypoint model is available in the `rfdetr` package only. It is not yet available via the `inference` package.
|
||||
|
||||
> Benchmark evaluated on COCO val2017 person keypoints (AP<sub>50:95</sub>) with the standard COCO 17-keypoint OKS sigmas; latency on NVIDIA T4, TensorRT FP16, batch size 1.
|
||||
|
||||
## Run on an Image
|
||||
|
||||
Perform inference on an image using the `rfdetr` package. `model.predict()` returns an [`sv.KeyPoints`](https://supervision.roboflow.com/latest/keypoint/core/) object containing skeleton coordinates and per-keypoint confidence scores for each detected person.
|
||||
|
||||
=== "rfdetr"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
|
||||
model = RFDETRKeypointPreview()
|
||||
|
||||
image_bgr = cv2.imread("/path/to/image.jpg")
|
||||
image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
|
||||
key_points = model.predict(image_rgb, threshold=0.5)
|
||||
|
||||
annotated_image = sv.VertexAnnotator().annotate(image_rgb, key_points)
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Understanding the Output
|
||||
|
||||
`model.predict()` returns an `sv.KeyPoints` object. The fields most commonly used downstream:
|
||||
|
||||
| Field | Shape | Description |
|
||||
| --------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `key_points.xy` | `(N, K, 2)` | Pixel coordinates of each keypoint per detected instance |
|
||||
| `key_points.keypoint_confidence` | `(N, K)` | Per-keypoint findability score; use to filter low-confidence points |
|
||||
| `key_points.detection_confidence` | `(N,)` | Per-instance detection score; this is what `threshold` filters on. For keypoint models it includes the default uncertainty fusion term normalized to `[0, 1)`. |
|
||||
| `key_points.class_id` | `(N,)` | Model label ID for each detection. COCO-pretrained checkpoints use sparse COCO category IDs (1–90). Fine-tuned active-first keypoint checkpoints use normal 0-based class IDs; in the one-class preview setup, `class_id=0` is the foreground class and `class_id=1` is `"__background__"`. Legacy background-first keypoint checkpoints use slot 0 as `"__background__"` and start foreground classes at slot 1. Use `key_points.data["class_name"]` for name resolution rather than indexing your class list by `class_id`. |
|
||||
| `key_points.data["class_name"]` | `(N,)` | Class names resolved from `class_id`; prefer this over indexing a class-name list directly. |
|
||||
| `key_points.data["xyxy"]` | `(N, 4)` | Bounding box for each detected instance in `[x1, y1, x2, y2]` format |
|
||||
| `key_points.data["source_image"]` | list of arrays | Source frame stored once per detection; all N entries are the same array — use `[0]` to access it |
|
||||
|
||||
`K=17` for the pretrained COCO person-keypoint preview checkpoint. Fine-tuned checkpoints use the keypoint count from their dataset schema, so custom keypoint datasets can return any `K` supported by their COCO keypoint annotations.
|
||||
|
||||
Keypoints with `visible=False` are skipped by supervision annotators. To hide low-confidence joints manually, threshold `key_points.keypoint_confidence` and set matching entries to `False` in `key_points.visible`.
|
||||
|
||||
For fine-tuning on a custom keypoint dataset, see [Keypoint preview custom datasets](../train/index.md#keypoint-preview-custom-datasets).
|
||||
|
||||
## Run on video, webcam, or RTSP stream
|
||||
|
||||
These examples use OpenCV for decoding and display. Replace `<SOURCE_VIDEO_PATH>`, `<WEBCAM_INDEX>`, and `<RTSP_STREAM_URL>` with your inputs. `<WEBCAM_INDEX>` is usually `0` for the default camera.
|
||||
|
||||
=== "video"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
|
||||
model = RFDETRKeypointPreview()
|
||||
|
||||
video_capture = cv2.VideoCapture("<SOURCE_VIDEO_PATH>")
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError("Failed to open video source: <SOURCE_VIDEO_PATH>")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
key_points = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
annotated_frame = sv.VertexAnnotator().annotate(frame_bgr, key_points)
|
||||
|
||||
cv2.imshow("RF-DETR Keypoint Video", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "webcam"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
|
||||
model = RFDETRKeypointPreview()
|
||||
|
||||
WEBCAM_INDEX = 0 # Change this to the desired webcam index (e.g., 1, 2, ...)
|
||||
video_capture = cv2.VideoCapture(WEBCAM_INDEX)
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError(f"Failed to open webcam: {WEBCAM_INDEX}")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
key_points = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
annotated_frame = sv.VertexAnnotator().annotate(frame_bgr, key_points)
|
||||
|
||||
cv2.imshow("RF-DETR Keypoint Webcam", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "stream"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
|
||||
model = RFDETRKeypointPreview()
|
||||
|
||||
video_capture = cv2.VideoCapture("<RTSP_STREAM_URL>")
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError("Failed to open RTSP stream: <RTSP_STREAM_URL>")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
key_points = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
annotated_frame = sv.VertexAnnotator().annotate(frame_bgr, key_points)
|
||||
|
||||
cv2.imshow("RF-DETR Keypoint RTSP", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
## Visualization
|
||||
|
||||
`supervision` provides several keypoint annotators. Choose based on what you want to draw.
|
||||
|
||||
=== "EdgeAnnotator"
|
||||
|
||||
Draws skeleton edges (lines between connected joints). Edges where either endpoint has `visible=False` are skipped automatically.
|
||||
|
||||
```python
|
||||
annotated = sv.EdgeAnnotator().annotate(image, key_points)
|
||||
```
|
||||
|
||||
=== "VertexAnnotator"
|
||||
|
||||
Draws a dot at each keypoint. Keypoints with `visible=False` are skipped automatically.
|
||||
|
||||
```python
|
||||
annotated = sv.VertexAnnotator().annotate(image, key_points)
|
||||
```
|
||||
|
||||
=== "VertexEllipseAnnotator"
|
||||
|
||||
Draws covariance ellipses from `key_points.data["covariance"]`, giving a visual footprint of per-keypoint uncertainty.
|
||||
|
||||
```python
|
||||
annotated = sv.VertexEllipseAnnotator().annotate(image, key_points)
|
||||
```
|
||||
|
||||
=== "VertexEllipseHaloAnnotator"
|
||||
|
||||
Draws the same covariance uncertainty with a soft halo for improved contrast on busy backgrounds.
|
||||
|
||||
```python
|
||||
annotated = sv.VertexEllipseHaloAnnotator().annotate(image, key_points)
|
||||
```
|
||||
@@ -0,0 +1,177 @@
|
||||
---
|
||||
description: Run RF-DETR instance segmentation on images, video, and streams. Mask predictions with 3.4-21.8 ms latency using DINOv2 backbone.
|
||||
---
|
||||
|
||||
# Run an RF-DETR Instance Segmentation Model
|
||||
|
||||
RF-DETR is a real-time transformer architecture for instance segmentation, built on a DINOv2 vision transformer backbone. The base models are trained on the Microsoft COCO dataset and achieve strong accuracy and latency trade-offs.
|
||||
|
||||
## Pre-trained Checkpoints
|
||||
|
||||
RF-DETR-Seg offers model sizes from Nano to 2XLarge, allowing trade-offs between accuracy, latency, and parameter count. All latency numbers were measured on an NVIDIA T4 using TensorRT, FP16, and batch size 1.
|
||||
|
||||
| Size | RF-DETR package class | Inference package alias | COCO AP<sub>50</sub> | COCO AP<sub>50:95</sub> | Latency (ms) | Params (M) | Resolution |
|
||||
| :--: | :-------------------: | :---------------------- | :------------------: | :---------------------: | :----------: | :--------: | :--------: |
|
||||
| N | `RFDETRSegNano` | `rfdetr-seg-nano` | 63.0 | 40.3 | 3.4 | 33.6 | 312x312 |
|
||||
| S | `RFDETRSegSmall` | `rfdetr-seg-small` | 66.2 | 43.1 | 4.4 | 33.7 | 384x384 |
|
||||
| M | `RFDETRSegMedium` | `rfdetr-seg-medium` | 68.4 | 45.3 | 5.9 | 35.7 | 432x432 |
|
||||
| L | `RFDETRSegLarge` | `rfdetr-seg-large` | 70.5 | 47.1 | 8.8 | 36.2 | 504x504 |
|
||||
| XL | `RFDETRSegXLarge` | `rfdetr-seg-xlarge` | 72.2 | 48.8 | 13.5 | 38.1 | 624x624 |
|
||||
| 2XL | `RFDETRSeg2XLarge` | `rfdetr-seg-2xlarge` | 73.1 | 49.9 | 21.8 | 38.6 | 768x768 |
|
||||
|
||||
## Run on an Image
|
||||
|
||||
Perform inference on an image using either the `rfdetr` package or the `inference` package. To use a different model size, select the corresponding class or alias from the table above.
|
||||
|
||||
=== "rfdetr"
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRSegMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
detections = model.predict("https://media.roboflow.com/dog.jpg", threshold=0.5)
|
||||
|
||||
labels = [f"{COCO_CLASSES[class_id]}" for class_id in detections.class_id]
|
||||
|
||||
annotated_image = sv.MaskAnnotator().annotate(detections.metadata["source_image"], detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections, labels)
|
||||
```
|
||||
|
||||
=== "inference"
|
||||
|
||||
```python
|
||||
import requests
|
||||
import supervision as sv
|
||||
from PIL import Image
|
||||
from inference import get_model
|
||||
|
||||
model = get_model("rfdetr-seg-medium")
|
||||
|
||||
image = Image.open(requests.get("https://media.roboflow.com/dog.jpg", stream=True).raw)
|
||||
predictions = model.infer(image, confidence=0.5)[0]
|
||||
detections = sv.Detections.from_inference(predictions)
|
||||
|
||||
annotated_image = sv.MaskAnnotator().annotate(image, detections)
|
||||
annotated_image = sv.LabelAnnotator().annotate(annotated_image, detections)
|
||||
```
|
||||
|
||||
For memory-constrained inference-only deployments with the `rfdetr` package, optimize the loaded model in place before
|
||||
calling `predict()`. Pass `dtype="float16"` to halve weight memory in addition to clearing the base model reference.
|
||||
This operation is irreversible — to restore the original model, create a new `RFDETR` instance:
|
||||
|
||||
```python
|
||||
model.optimize_for_inference(compile=False, inplace=True, dtype="float16")
|
||||
```
|
||||
|
||||
## Run on video, webcam, or RTSP stream
|
||||
|
||||
These examples use OpenCV for decoding and display. Replace `<SOURCE_VIDEO_PATH>`, `<WEBCAM_INDEX>`, and `<RTSP_STREAM_URL>` with your inputs. `<WEBCAM_INDEX>` is usually `0` for the default camera.
|
||||
|
||||
=== "video"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRSegMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
video_capture = cv2.VideoCapture("<SOURCE_VIDEO_PATH>")
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError("Failed to open video source: <SOURCE_VIDEO_PATH>")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
detections = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]
|
||||
|
||||
annotated_frame = sv.MaskAnnotator().annotate(frame_bgr, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RF-DETR-Seg Video", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "webcam"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRSegMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
WEBCAM_INDEX = 0 # Change this to the desired webcam index (e.g., 1, 2, ...)
|
||||
video_capture = cv2.VideoCapture(WEBCAM_INDEX)
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError(f"Failed to open webcam: {WEBCAM_INDEX}")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
detections = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]
|
||||
|
||||
annotated_frame = sv.MaskAnnotator().annotate(frame_bgr, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RF-DETR-Seg Webcam", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
|
||||
=== "stream"
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import supervision as sv
|
||||
from rfdetr import RFDETRSegMedium
|
||||
from rfdetr.assets.coco_classes import COCO_CLASSES
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
video_capture = cv2.VideoCapture("<RTSP_STREAM_URL>")
|
||||
if not video_capture.isOpened():
|
||||
raise RuntimeError("Failed to open RTSP stream: <RTSP_STREAM_URL>")
|
||||
|
||||
while True:
|
||||
success, frame_bgr = video_capture.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
detections = model.predict(frame_rgb, threshold=0.5)
|
||||
|
||||
labels = [COCO_CLASSES[class_id] for class_id in detections.class_id]
|
||||
|
||||
annotated_frame = sv.MaskAnnotator().annotate(frame_bgr, detections)
|
||||
annotated_frame = sv.LabelAnnotator().annotate(annotated_frame, detections, labels)
|
||||
|
||||
cv2.imshow("RF-DETR-Seg RTSP", annotated_frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
video_capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
```
|
||||
@@ -0,0 +1,381 @@
|
||||
---
|
||||
description: Advanced RF-DETR training with resume, early stopping, multi-GPU DDP, gradient checkpointing, and memory optimization for large models.
|
||||
---
|
||||
|
||||
# Advanced Training
|
||||
|
||||
This page covers advanced training topics including resuming training, early stopping, multi-GPU training, and memory optimization techniques.
|
||||
|
||||
!!! tip "PTL API for deeper customisation"
|
||||
|
||||
All examples on this page use the `RFDETR.train()` high-level API. For custom callbacks, non-default loggers, and fine-grained distributed training control, see the [Custom Training API](customization.md) guide.
|
||||
|
||||
## Resume Training
|
||||
|
||||
You can resume training from a previously saved checkpoint by passing the path to the `checkpoint.pth` file using the `resume` argument. This is useful when training is interrupted or you want to continue fine-tuning an already partially trained model.
|
||||
|
||||
The training loop will automatically load:
|
||||
|
||||
- Model weights
|
||||
- Optimizer state
|
||||
- Learning rate scheduler state
|
||||
- Training epoch number
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
resume="output/checkpoint.pth",
|
||||
)
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegMedium
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
resume="output/checkpoint.pth",
|
||||
)
|
||||
```
|
||||
|
||||
!!! tip "Resume vs Pretrain Weights"
|
||||
|
||||
- Use `resume="checkpoint.pth"` to continue training with optimizer state
|
||||
- Use `pretrain_weights="checkpoint_best_total.pth"` when initializing a model to start fresh training from those weights
|
||||
|
||||
---
|
||||
|
||||
## Early Stopping
|
||||
|
||||
Early stopping monitors the validation task metric and halts training if improvements remain below a threshold for a
|
||||
set number of epochs. Detection and segmentation models use box mAP; keypoint preview models use COCO keypoint AP.
|
||||
|
||||
### Basic Usage
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
early_stopping=True,
|
||||
)
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegMedium
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
early_stopping=True,
|
||||
)
|
||||
```
|
||||
|
||||
### Configuration Options
|
||||
|
||||
| Parameter | Default | Description |
|
||||
| -------------------------- | ------- | ---------------------------------------------------- |
|
||||
| `early_stopping_patience` | 10 | Number of epochs without improvement before stopping |
|
||||
| `early_stopping_min_delta` | 0.001 | Minimum metric change to count as improvement |
|
||||
| `early_stopping_use_ema` | False | Use EMA model metrics for comparisons |
|
||||
|
||||
### Advanced Example
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=200,
|
||||
early_stopping=True,
|
||||
early_stopping_patience=15, # Wait 15 epochs before stopping
|
||||
early_stopping_min_delta=0.005, # Require 0.5% validation metric improvement
|
||||
early_stopping_use_ema=True, # Track EMA model performance
|
||||
)
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. After each epoch, the validation task metric is computed
|
||||
2. If the metric improves by at least `min_delta`, the patience counter resets
|
||||
3. If the metric doesn't improve, the patience counter increments
|
||||
4. When patience counter reaches `patience`, training stops
|
||||
5. The best checkpoint is already saved as `checkpoint_best_total.pth`
|
||||
|
||||
```
|
||||
Epoch 10: mAP = 0.450 (best: 0.450) - counter: 0
|
||||
Epoch 11: mAP = 0.455 (best: 0.455) - counter: 0 (improved)
|
||||
Epoch 12: mAP = 0.454 (best: 0.455) - counter: 1 (no improvement)
|
||||
Epoch 13: mAP = 0.453 (best: 0.455) - counter: 2
|
||||
...
|
||||
Epoch 22: mAP = 0.452 (best: 0.455) - counter: 10 → STOP
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Multi-GPU Training
|
||||
|
||||
RF-DETR's training stack is built on PyTorch Lightning, so multi-GPU and multi-node training use the Lightning `Trainer` strategies directly. You can start multi-GPU runs through the high-level API or by using the Lightning primitives explicitly.
|
||||
|
||||
### Using RFDETR.train() with multiple GPUs
|
||||
|
||||
Create a training script and launch it with `torchrun`:
|
||||
|
||||
```python
|
||||
# train.py
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4, # per-GPU batch size
|
||||
grad_accum_steps=1,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
devices="auto", # required — see note below
|
||||
)
|
||||
```
|
||||
|
||||
```bash
|
||||
torchrun --nproc_per_node=4 train.py
|
||||
```
|
||||
|
||||
!!! warning "Pass `devices=` explicitly"
|
||||
|
||||
`build_trainer()` defaults to `devices=1`. Without overriding this, training silently
|
||||
runs on a single GPU even when `torchrun` launches multiple processes.
|
||||
|
||||
Pass `devices="auto"` to use all GPUs visible to the process, or pass an explicit
|
||||
integer (e.g. `devices=4`). These values are forwarded to `build_trainer` via
|
||||
`**trainer_kwargs`:
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=1,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
devices="auto", # or devices=4
|
||||
)
|
||||
```
|
||||
|
||||
### Batch Size with Multiple GPUs
|
||||
|
||||
When using multiple GPUs, your effective batch size is multiplied by the number of GPUs:
|
||||
|
||||
```
|
||||
effective_batch_size = batch_size × grad_accum_steps × num_gpus
|
||||
```
|
||||
|
||||
**Example configurations for effective batch size of 16:**
|
||||
|
||||
| GPUs | `batch_size` | `grad_accum_steps` | Effective |
|
||||
| ---- | ------------ | ------------------ | --------- |
|
||||
| 1 | 4 | 4 | 16 |
|
||||
| 2 | 4 | 2 | 16 |
|
||||
| 4 | 4 | 1 | 16 |
|
||||
| 8 | 2 | 1 | 16 |
|
||||
|
||||
!!! warning "Adjust for GPU count"
|
||||
|
||||
When switching between single and multi-GPU training, remember to adjust `batch_size` and `grad_accum_steps` to maintain the same effective batch size.
|
||||
|
||||
### Multi-Node Training
|
||||
|
||||
For training across multiple machines, pass the standard `torchrun` flags:
|
||||
|
||||
```bash
|
||||
torchrun \
|
||||
--nproc_per_node=8 \
|
||||
--nnodes=2 \
|
||||
--node_rank=0 \
|
||||
--master_addr="192.168.1.1" \
|
||||
--master_port=1234 \
|
||||
train.py
|
||||
```
|
||||
|
||||
Run this command on each node, changing `--node_rank` accordingly.
|
||||
|
||||
### Advanced multi-GPU options (PTL API)
|
||||
|
||||
For fine-grained control over strategy, sync batch norm, precision, and other distributed settings, use the Lightning API directly.
|
||||
|
||||
→ **[Multi-GPU with the PTL API](customization.md#multi-gpu-training)**
|
||||
|
||||
---
|
||||
|
||||
## Custom Augmentations
|
||||
|
||||
RF-DETR supports advanced data augmentations using the [Albumentations](https://albumentations.ai/) library, providing access to over 70 different image transformations optimized for object detection.
|
||||
|
||||
→ **[Complete Augmentation Guide](augmentations.md)** - Configuration examples, best practices, troubleshooting, and advanced topics.
|
||||
|
||||
### Quick Start
|
||||
|
||||
Pass an `aug_config` dictionary to `model.train()`. Each key is an Albumentations transform name; the value is a dict of keyword arguments for that transform:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
aug_config={
|
||||
"HorizontalFlip": {"p": 0.5},
|
||||
"VerticalFlip": {"p": 0.5},
|
||||
"Rotate": {"limit": 45, "p": 0.5},
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
Use a built-in preset by importing it from `rfdetr.datasets.aug_configs`:
|
||||
|
||||
```python
|
||||
from rfdetr.datasets.aug_configs import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL
|
||||
|
||||
model.train(dataset_dir="path/to/dataset", aug_config=AUG_AGGRESSIVE)
|
||||
```
|
||||
|
||||
To disable all augmentations, pass an empty dict:
|
||||
|
||||
```python
|
||||
model.train(dataset_dir="path/to/dataset", aug_config={})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Memory Optimization
|
||||
|
||||
### Gradient Checkpointing
|
||||
|
||||
For large models or high resolutions, enable gradient checkpointing to trade compute for memory.
|
||||
|
||||
!!! warning "Constructor parameter — not a `train()` parameter"
|
||||
|
||||
`gradient_checkpointing` is a `ModelConfig` field and must be passed to the **model constructor**, not to `train()`. Passing it to `train()` will raise a `ValidationError` because `TrainConfig` has `extra="forbid"`.
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium(gradient_checkpointing=True)
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
batch_size=2, # May be able to increase with checkpointing
|
||||
)
|
||||
```
|
||||
|
||||
This re-computes activations during the backward pass instead of storing them, reducing memory usage by ~30-40% at the cost of ~20% slower training.
|
||||
|
||||
### Memory-Efficient Configurations
|
||||
|
||||
| Memory Level | Configuration |
|
||||
| ----------------- | -------------------------------------------------------------------------------------- |
|
||||
| Very Low (8GB) | `batch_size=1`, `grad_accum_steps=16`, `gradient_checkpointing=True`, `resolution=576` |
|
||||
| Low (12GB) | `batch_size=2`, `grad_accum_steps=8`, `gradient_checkpointing=True` |
|
||||
| Medium (16GB) | `batch_size=4`, `grad_accum_steps=4` |
|
||||
| High (24GB) | `batch_size=8`, `grad_accum_steps=2` |
|
||||
| Very High (40GB+) | `batch_size=16`, `grad_accum_steps=1`, `resolution=768` |
|
||||
|
||||
---
|
||||
|
||||
## Training Tips
|
||||
|
||||
### Learning Rate Tuning
|
||||
|
||||
- **Fine-tuning from COCO weights (default):** Use default learning rates (`lr=1e-4`, `lr_encoder=1.5e-4`)
|
||||
- **Small dataset (\<1000 images):** Consider lower `lr` (e.g., `5e-5`) to prevent overfitting
|
||||
- **Large dataset (>10000 images):** May benefit from higher `lr` (e.g., `2e-4`)
|
||||
|
||||
### Epoch Count
|
||||
|
||||
| Dataset Size | Recommended Epochs |
|
||||
| ----------------- | ------------------ |
|
||||
| < 500 images | 100-200 |
|
||||
| 500-2000 images | 50-100 |
|
||||
| 2000-10000 images | 30-50 |
|
||||
| > 10000 images | 20-30 |
|
||||
|
||||
Use early stopping to automatically determine the optimal stopping point.
|
||||
|
||||
### Data Augmentation
|
||||
|
||||
RF-DETR applies built-in augmentations during training:
|
||||
|
||||
- Random resizing
|
||||
- Random cropping
|
||||
- Color jittering
|
||||
- Horizontal flipping
|
||||
|
||||
These are automatically configured and don't require manual setup.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Out of Memory (OOM)
|
||||
|
||||
If you encounter CUDA out of memory errors:
|
||||
|
||||
1. Reduce `batch_size`
|
||||
2. Enable `gradient_checkpointing=True` (pass to the model constructor, not `train()`)
|
||||
3. Reduce `resolution`
|
||||
4. Increase `grad_accum_steps` to maintain effective batch size
|
||||
|
||||
### Training Too Slow
|
||||
|
||||
1. Increase `batch_size` (if memory allows)
|
||||
2. Use multiple GPUs with DDP
|
||||
3. Ensure you're using GPU (check `device="cuda"`)
|
||||
4. Consider using a smaller model (e.g., `RFDETRSmall` instead of `RFDETRLarge`)
|
||||
|
||||
### Loss Not Decreasing
|
||||
|
||||
1. Check that your dataset is correctly formatted
|
||||
2. Verify annotations are correct (bounding boxes in correct format)
|
||||
3. Try reducing the learning rate
|
||||
4. Check for class imbalance in your dataset
|
||||
@@ -0,0 +1,156 @@
|
||||
---
|
||||
description: Configure RF-DETR data augmentations with Albumentations. Built-in presets for aerial, industrial, and small datasets plus custom transforms.
|
||||
---
|
||||
|
||||
# Augmentations
|
||||
|
||||
RF-DETR supports custom data augmentations via [Albumentations](https://albumentations.ai/), with automatic bounding box and mask handling for geometric transforms. Albumentations 1.4.24+ and 2.x are supported.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Pass `aug_config` to your training call. Import one of the built-in presets:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSmall
|
||||
from rfdetr.datasets.aug_configs import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL
|
||||
|
||||
model = RFDETRSmall()
|
||||
model.train(dataset_dir="path/to/dataset", epochs=100, aug_config=AUG_CONSERVATIVE)
|
||||
```
|
||||
|
||||
Or pass a custom dict directly — keys are Albumentations transform names:
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
aug_config={
|
||||
"HorizontalFlip": {"p": 0.5},
|
||||
"Rotate": {"limit": 15, "p": 0.3},
|
||||
"GaussianBlur": {"p": 0.2},
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
To disable augmentations: `aug_config={}`. Omitting it uses the default (horizontal flip at 50%).
|
||||
|
||||
## Built-in Presets
|
||||
|
||||
| Preset | Best for |
|
||||
| ------------------ | --------------------------------- |
|
||||
| `AUG_CONSERVATIVE` | Small datasets (under 500 images) |
|
||||
| `AUG_AGGRESSIVE` | Large datasets (2000+ images) |
|
||||
| `AUG_AERIAL` | Satellite / overhead imagery |
|
||||
| `AUG_INDUSTRIAL` | Manufacturing / inspection data |
|
||||
|
||||
All presets are plain dicts — inspect or extend them before passing:
|
||||
|
||||
```python
|
||||
from rfdetr.datasets.aug_configs import AUG_AGGRESSIVE
|
||||
|
||||
my_config = {**AUG_AGGRESSIVE, "VerticalFlip": {"p": 0.1}}
|
||||
model.train(dataset_dir="...", aug_config=my_config)
|
||||
```
|
||||
|
||||
### Recommendations by Dataset Size
|
||||
|
||||
| Dataset Size | Recommended preset |
|
||||
| ---------------- | --------------------------------------------------------------- |
|
||||
| Under 500 images | `AUG_CONSERVATIVE` — flip + mild brightness/contrast |
|
||||
| 500–2000 images | Default or `AUG_CONSERVATIVE` with a few extra transforms added |
|
||||
| 2000+ images | `AUG_AGGRESSIVE` — rotations, affine, color jitter |
|
||||
|
||||
## Nested Transforms
|
||||
|
||||
RF-DETR supports `OneOf`, `SomeOf`, and `Sequential` container transforms from Albumentations. The most common pattern is `OneOf`, which randomly picks one transform from a group:
|
||||
|
||||
```python
|
||||
aug_config = {
|
||||
"HorizontalFlip": {"p": 0.5},
|
||||
"OneOf": {
|
||||
"transforms": [
|
||||
{"Rotate": {"limit": 45, "p": 1.0}},
|
||||
{"Affine": {"scale": (0.8, 1.2), "p": 1.0}},
|
||||
],
|
||||
},
|
||||
"GaussianBlur": {"p": 0.2},
|
||||
}
|
||||
```
|
||||
|
||||
Each child's `p` controls its relative selection weight. The container itself always fires.
|
||||
|
||||
If you need the same transform twice, or want explicit ordering, pass a list instead of a dict:
|
||||
|
||||
```python
|
||||
aug_config = [
|
||||
{"HorizontalFlip": {"p": 0.5}},
|
||||
{"Rotate": {"limit": 45, "p": 0.3}},
|
||||
{"Rotate": {"limit": 5, "p": 0.5}}, # second Rotate — only possible with list format
|
||||
]
|
||||
```
|
||||
|
||||
Bounding boxes are updated automatically when a container holds any geometric transform — no extra configuration needed.
|
||||
|
||||
## Geometric vs. Pixel-Level Transforms
|
||||
|
||||
RF-DETR automatically handles bounding boxes for **geometric transforms** (flips, rotations, crops, affine, perspective). **Pixel-level transforms** (blur, noise, color) preserve coordinates unchanged. You don't need to handle this distinction — it's automatic based on the transform name.
|
||||
|
||||
## Best Practices
|
||||
|
||||
!!! tip "Start Conservative"
|
||||
|
||||
Begin with simple augmentations (horizontal flip, small brightness changes) and gradually add more as needed.
|
||||
|
||||
!!! warning "Geometric Transforms"
|
||||
|
||||
Be careful with aggressive rotations and crops on datasets where object orientation matters (e.g., text detection, oriented objects).
|
||||
|
||||
- **CPU-bound:** Augmentations run on CPU during data loading — more transforms means slower loading
|
||||
- **Use `num_workers`:** Parallelize augmentation across data loader workers
|
||||
- **Monitor training mAP vs validation mAP:** With strong augmentations it's normal for training mAP to be lower — validation uses original images while training uses augmented (harder) ones
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Training is slow** — reduce the number of transforms or increase `num_workers`.
|
||||
|
||||
**Boxes disappear after augmentation** — aggressive rotations or crops can push boxes outside the image boundary. Reduce rotation angles or avoid large crops.
|
||||
|
||||
**Model not improving** — augmentations may be too aggressive. Start with `AUG_CONSERVATIVE` and add transforms gradually. Try removing geometric transforms first to isolate the cause.
|
||||
|
||||
**Validation mAP is much higher than training mAP** — this is expected with strong augmentations and not a bug. See the monitoring tip above.
|
||||
|
||||
**Upgrading albumentations to 2.x with existing `RandomSizedCrop` configs?** RF-DETR automatically adapts `height`/`width` kwargs to the `size=(height, width)` format required by albumentations 2.x. No config changes needed.
|
||||
|
||||
## Advanced: Custom Transforms
|
||||
|
||||
Any Albumentations transform works by name. If your custom transform is geometric, register it in `rfdetr/datasets/transforms.py` so boxes are updated automatically:
|
||||
|
||||
```python
|
||||
GEOMETRIC_TRANSFORMS = {
|
||||
...,
|
||||
"YourCustomTransform",
|
||||
}
|
||||
```
|
||||
|
||||
Then use it like any other transform:
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="...",
|
||||
aug_config={
|
||||
"HorizontalFlip": {"p": 0.5},
|
||||
"YourCustomTransform": {"param": 1, "p": 0.3},
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
## Reference
|
||||
|
||||
- [Albumentations docs](https://albumentations.ai/docs/)
|
||||
- [All available transforms](https://albumentations.ai/docs/api_reference/augmentations/)
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Monitor training with TensorBoard](loggers.md#tensorboard)
|
||||
- [Use early stopping](advanced.md#early-stopping) to prevent overfitting
|
||||
- [Export your trained model](../export.md) for deployment
|
||||
@@ -0,0 +1,384 @@
|
||||
---
|
||||
description: Customize RF-DETR training with PyTorch Lightning primitives. Direct access to RFDETRModelModule, RFDETRDataModule, and build_trainer.
|
||||
---
|
||||
|
||||
# Custom Training API
|
||||
|
||||
The high-level `RFDETR.train()` method is the quickest path to fine-tuning, but the underlying training primitives are fully public and are the **recommended path for any customisation**: custom callbacks, alternative loggers, mixed-precision overrides, multi-GPU strategies, or integration with external training frameworks.
|
||||
|
||||
!!! tip "Quickstart vs. customisation"
|
||||
|
||||
If you want to start training with minimal code, use `model.train()` — it sets up and runs the full PTL stack automatically. Come here when you need to take direct control over any part of that stack.
|
||||
|
||||
## How `RFDETR.train()` relates to PTL
|
||||
|
||||
When you call `model.train(...)`, three things happen internally:
|
||||
|
||||
```python
|
||||
from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer
|
||||
|
||||
module = RFDETRModelModule(model_config, train_config)
|
||||
datamodule = RFDETRDataModule(model_config, train_config)
|
||||
trainer = build_trainer(train_config, model_config)
|
||||
trainer.fit(module, datamodule, ckpt_path=train_config.resume or None)
|
||||
```
|
||||
|
||||
Each of these objects is a standard PTL class. You can construct them directly, modify them, swap out callbacks, or replace the trainer entirely.
|
||||
|
||||
---
|
||||
|
||||
## RFDETRModelModule
|
||||
|
||||
`RFDETRModelModule` is a `pytorch_lightning.LightningModule`. It owns the model weights, the criterion, the postprocessor, and the optimizer/scheduler configuration.
|
||||
|
||||
```python
|
||||
from rfdetr.config import (
|
||||
RFDETRMediumConfig,
|
||||
TrainConfig,
|
||||
) # config classes live in rfdetr.config, not the top-level rfdetr namespace
|
||||
from rfdetr.training import RFDETRModelModule
|
||||
|
||||
model_config = RFDETRMediumConfig(num_classes=10)
|
||||
train_config = TrainConfig(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=50,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
)
|
||||
|
||||
module = RFDETRModelModule(model_config, train_config)
|
||||
```
|
||||
|
||||
### Lifecycle hooks
|
||||
|
||||
| Hook | Behaviour |
|
||||
| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `on_fit_start` | Seeds RNGs when `train_config.seed` is set. |
|
||||
| `on_train_batch_start` | Applies multi-scale random resize when `train_config.multi_scale=True`. |
|
||||
| `transfer_batch_to_device` | Moves `NestedTensor` batches to the target device. |
|
||||
| `training_step` | Computes loss and logs `train/loss` plus per-term losses. Keypoint models use manual optimization with box-normalized accumulation across microbatches; detection and segmentation use Lightning's automatic optimization path. |
|
||||
| `validation_step` | Runs forward pass and postprocessing; returns `{results, targets}` for `COCOEvalCallback`. |
|
||||
| `test_step` | Same as `validation_step`, logs under `test/`. |
|
||||
| `predict_step` | Runs inference-only forward pass and returns postprocessed detections. |
|
||||
| `configure_optimizers` | Builds AdamW with layer-wise LR decay and a LambdaLR scheduler (cosine or step). |
|
||||
| `on_load_checkpoint` | Auto-converts legacy `.pth` checkpoints to PTL format. |
|
||||
|
||||
### Accessing the underlying model
|
||||
|
||||
The raw `nn.Module` is `module.model`. After training completes, `RFDETR.train()` syncs it back onto `self.model.model` so `predict()` and `export()` continue to work.
|
||||
|
||||
---
|
||||
|
||||
## RFDETRDataModule
|
||||
|
||||
`RFDETRDataModule` is a `pytorch_lightning.LightningDataModule`. It builds train/val/test datasets and wraps them in `DataLoader` objects.
|
||||
|
||||
```python
|
||||
from rfdetr.training import RFDETRDataModule
|
||||
|
||||
datamodule = RFDETRDataModule(model_config, train_config)
|
||||
```
|
||||
|
||||
### Stages
|
||||
|
||||
| Stage | Datasets built |
|
||||
| ------------ | ------------------------------------------ |
|
||||
| `"fit"` | `train` + `val` |
|
||||
| `"validate"` | `val` only |
|
||||
| `"test"` | `test` (or `val` for COCO-format datasets) |
|
||||
|
||||
The `setup(stage)` method is lazy — each split is built at most once, even if called multiple times.
|
||||
|
||||
### class_names property
|
||||
|
||||
```python
|
||||
datamodule.setup("fit")
|
||||
print(datamodule.class_names) # e.g. ["cat", "dog", "person"]
|
||||
```
|
||||
|
||||
Returns sorted category names from the COCO annotation file of the first available split, or `None` if the dataset has not been set up yet.
|
||||
|
||||
---
|
||||
|
||||
## build_trainer
|
||||
|
||||
`build_trainer` assembles a `pytorch_lightning.Trainer` with the full RF-DETR callback and logger stack. All `TrainConfig` fields are wired automatically.
|
||||
|
||||
```python
|
||||
from rfdetr.training import build_trainer
|
||||
|
||||
trainer = build_trainer(train_config, model_config)
|
||||
```
|
||||
|
||||
### What build_trainer configures
|
||||
|
||||
| Concern | Source |
|
||||
| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Max epochs | `train_config.epochs` |
|
||||
| Gradient accumulation | Detection/segmentation: `train_config.grad_accum_steps` forwarded to Trainer. Keypoint models: owned by `RFDETRModelModule` manual optimization (Trainer always sees `1`). |
|
||||
| Gradient clipping | Detection/segmentation: `train_config.clip_max_norm` forwarded to Trainer. Keypoint models: owned by `RFDETRModelModule` manual optimization (Trainer always sees `None`). |
|
||||
| Mixed precision | `model_config.amp` enables AMP; dtype resolved from `train_config.amp_dtype` (`"auto"` selects `bf16-mixed` on Ampere+, `"bf16"` / `"fp16"` force a specific dtype) |
|
||||
| Accelerator | `train_config.accelerator` (default `"auto"`) |
|
||||
| Strategy | Set via `train_config.strategy` (default `"auto"`) or pass `strategy=` as a `**trainer_kwarg` to `build_trainer`. Common values: `"auto"`, `"ddp"`, `"ddp_spawn"`. `TrainConfig` also exposes `devices` and `num_nodes` for multi-GPU and multi-node training. |
|
||||
| Sync batch norm | `train_config.sync_bn` |
|
||||
| Progress bar | `train_config.progress_bar` |
|
||||
| Loggers | CSVLogger always; TensorBoard, WandB, MLflow when their `train_config` flags are `True` |
|
||||
| Callbacks | `RFDETREMACallback`, `DropPathCallback`, `COCOEvalCallback`, `BestModelCallback`, `RFDETREarlyStopping` (conditional) |
|
||||
|
||||
### Overriding PTL Trainer kwargs
|
||||
|
||||
Pass keyword arguments accepted by `pytorch_lightning.Trainer` via `**trainer_kwargs`. Most keys override the built configuration.
|
||||
|
||||
**Detection and segmentation models** forward `accumulate_grad_batches` and `gradient_clip_val` to the Trainer normally — you can override them via `trainer_kwargs` or configure them on `TrainConfig` (`grad_accum_steps`, `clip_max_norm`).
|
||||
|
||||
**Keypoint models** use manual optimization, so `RFDETRModelModule` owns accumulation and clipping internally. `build_trainer()` forces `accumulate_grad_batches=1` and `gradient_clip_val=None` regardless of what is passed, and emits a `UserWarning` if those keys appear in `trainer_kwargs` so the override is visible:
|
||||
|
||||
```python
|
||||
trainer = build_trainer(
|
||||
train_config,
|
||||
model_config,
|
||||
fast_dev_run=2, # run 2 batches per epoch for a smoke test
|
||||
log_every_n_steps=10,
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Running the training loop
|
||||
|
||||
### Full training run
|
||||
|
||||
```python
|
||||
from rfdetr.config import (
|
||||
RFDETRMediumConfig,
|
||||
TrainConfig,
|
||||
) # config classes live in rfdetr.config, not the top-level rfdetr namespace
|
||||
from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer
|
||||
|
||||
model_config = RFDETRMediumConfig(num_classes=10)
|
||||
train_config = TrainConfig(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
)
|
||||
|
||||
module = RFDETRModelModule(model_config, train_config)
|
||||
datamodule = RFDETRDataModule(model_config, train_config)
|
||||
trainer = build_trainer(train_config, model_config)
|
||||
|
||||
trainer.fit(module, datamodule)
|
||||
```
|
||||
|
||||
### Resume from checkpoint
|
||||
|
||||
Pass the checkpoint path to `trainer.fit` via `ckpt_path`. The path can be a PTL `.ckpt` file or a legacy RF-DETR `.pth` file — `RFDETRModelModule.on_load_checkpoint` converts either format automatically.
|
||||
|
||||
```python
|
||||
trainer.fit(module, datamodule, ckpt_path="output/last.ckpt")
|
||||
# or a legacy checkpoint:
|
||||
trainer.fit(module, datamodule, ckpt_path="output/checkpoint.pth")
|
||||
```
|
||||
|
||||
> **Note:** When `checkpoint_interval=1`, no `last.ckpt` is written. Use `checkpoint_{epoch}.ckpt` (e.g. `output/checkpoint_epoch=4.ckpt`) to resume instead.
|
||||
|
||||
If you need to persist a converted checkpoint on disk (for example to inspect it, share it, or use it outside of PTL), convert it explicitly before passing it to `trainer.fit`:
|
||||
|
||||
```python
|
||||
from rfdetr.training import convert_legacy_checkpoint
|
||||
|
||||
convert_legacy_checkpoint("old_checkpoint.pth", "new_checkpoint.ckpt")
|
||||
trainer.fit(module, datamodule, ckpt_path="new_checkpoint.ckpt")
|
||||
```
|
||||
|
||||
`convert_legacy_checkpoint` reads a pre-PTL `.pth` file produced by the legacy `engine.py` training loop and writes a PTL-compatible `.ckpt` file. Use it when migrating saved checkpoints to the PTL format rather than relying on on-the-fly conversion at load time.
|
||||
|
||||
### Validation only
|
||||
|
||||
```python
|
||||
trainer.validate(module, datamodule)
|
||||
```
|
||||
|
||||
Runs one full validation pass and logs `val/mAP_50_95`, `val/mAP_50`, `val/F1`, and per-class AP metrics to all active loggers.
|
||||
|
||||
### Inference with the data pipeline
|
||||
|
||||
```python
|
||||
predictions = trainer.predict(module, dataloaders=datamodule.val_dataloader())
|
||||
```
|
||||
|
||||
Calls `module.predict_step` on every batch and returns a list of postprocessed detection results. Pass any `DataLoader` instance — `datamodule.val_dataloader()`, `datamodule.test_dataloader()`, or a custom loader — as the `dataloaders` argument. This is useful for offline evaluation or generating submission files.
|
||||
|
||||
!!! note "predict_dataloader not implemented"
|
||||
|
||||
`RFDETRDataModule` does not define a `predict_dataloader()` method, so `trainer.predict(module, datamodule)` will raise an error. Always pass a dataloader explicitly via the `dataloaders=` argument.
|
||||
|
||||
---
|
||||
|
||||
## Multi-GPU training
|
||||
|
||||
`build_trainer` configures PyTorch Lightning's `Trainer` directly, so all PTL strategies work out of the box.
|
||||
|
||||
### Data Parallel (DDP) — recommended
|
||||
|
||||
Set `train_config.accelerator = "auto"` and pass `strategy="ddp"` to `build_trainer`, then launch with `torchrun`:
|
||||
|
||||
!!! note "`devices` must be overridden for multi-GPU runs"
|
||||
|
||||
`build_trainer` defaults to `devices=1`. To use all available GPUs, pass `devices="auto"` (or an explicit count) as a `**trainer_kwarg`:
|
||||
|
||||
```python
|
||||
trainer = build_trainer(train_config, model_config, strategy="ddp", devices="auto")
|
||||
```
|
||||
|
||||
Without this override, `torchrun` will spawn multiple processes but each process will only see one device, defeating the purpose of the multi-GPU launch.
|
||||
|
||||
```bash
|
||||
torchrun --nproc_per_node=4 train.py
|
||||
```
|
||||
|
||||
where `train.py` contains:
|
||||
|
||||
```python
|
||||
from rfdetr.config import (
|
||||
RFDETRMediumConfig,
|
||||
TrainConfig,
|
||||
) # config classes live in rfdetr.config, not the top-level rfdetr namespace
|
||||
from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer
|
||||
|
||||
model_config = RFDETRMediumConfig(num_classes=10)
|
||||
train_config = TrainConfig(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4, # per-GPU batch size
|
||||
grad_accum_steps=1, # reduce when using more GPUs
|
||||
output_dir="output",
|
||||
sync_bn=True, # sync batch norms across GPUs
|
||||
)
|
||||
|
||||
module = RFDETRModelModule(model_config, train_config)
|
||||
datamodule = RFDETRDataModule(model_config, train_config)
|
||||
trainer = build_trainer(train_config, model_config, strategy="ddp", devices="auto")
|
||||
|
||||
trainer.fit(module, datamodule)
|
||||
```
|
||||
|
||||
!!! warning "EMA is not compatible with FSDP or DeepSpeed"
|
||||
|
||||
`build_trainer` automatically disables `RFDETREMACallback` when `strategy` contains `"fsdp"` or `"deepspeed"`, and emits a `UserWarning`. Use `strategy="ddp"` or `strategy="auto"` to keep EMA active.
|
||||
|
||||
### Effective batch size
|
||||
|
||||
```
|
||||
effective_batch_size = batch_size × grad_accum_steps × num_gpus
|
||||
```
|
||||
|
||||
Maintain an effective batch size of 16 regardless of GPU count:
|
||||
|
||||
| GPUs | `batch_size` | `grad_accum_steps` | Effective |
|
||||
| ---- | ------------ | ------------------ | --------- |
|
||||
| 1 | 4 | 4 | 16 |
|
||||
| 2 | 4 | 2 | 16 |
|
||||
| 4 | 4 | 1 | 16 |
|
||||
| 8 | 2 | 1 | 16 |
|
||||
|
||||
---
|
||||
|
||||
## Custom callbacks
|
||||
|
||||
`build_trainer` builds the default callback stack. To add your own callbacks alongside the built-in ones, pass them via `trainer_kwargs`:
|
||||
|
||||
```python
|
||||
from pytorch_lightning.callbacks import LearningRateMonitor, ModelSummary
|
||||
from rfdetr.training import build_trainer
|
||||
|
||||
extra_callbacks = [
|
||||
LearningRateMonitor(logging_interval="step"),
|
||||
ModelSummary(max_depth=3),
|
||||
]
|
||||
|
||||
trainer = build_trainer(
|
||||
train_config,
|
||||
model_config,
|
||||
callbacks=extra_callbacks, # replaces the default callback list entirely
|
||||
)
|
||||
```
|
||||
|
||||
!!! warning "Replacing vs. extending callbacks"
|
||||
|
||||
Passing `callbacks=` to `build_trainer` via `trainer_kwargs` **replaces** the entire default callback list built inside `build_trainer` (EMA, COCO eval, best-model checkpointing, etc.). To extend rather than replace, build the extra callbacks separately and merge them after calling `build_trainer`:
|
||||
|
||||
```python
|
||||
trainer = build_trainer(train_config, model_config)
|
||||
trainer.callbacks.extend(
|
||||
[
|
||||
LearningRateMonitor(logging_interval="step"),
|
||||
]
|
||||
)
|
||||
trainer.fit(module, datamodule)
|
||||
```
|
||||
|
||||
### Built-in callbacks
|
||||
|
||||
| Class | Purpose | Enabled when |
|
||||
| --------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------- |
|
||||
| `RFDETREMACallback` | Maintains an EMA copy of model weights | `train_config.use_ema=True` and strategy is not sharded |
|
||||
| `DropPathCallback` | Anneals drop-path rate over training | `train_config.drop_path > 0` |
|
||||
| `COCOEvalCallback` | Computes task validation metrics after each validation epoch | Always |
|
||||
| `BestModelCallback` | Saves `checkpoint_best_regular.pth`, `checkpoint_best_ema.pth`, `checkpoint_best_total.pth` | Always |
|
||||
| `RFDETREarlyStopping` | Stops training when the validation task metric stops improving | `train_config.early_stopping=True` |
|
||||
|
||||
---
|
||||
|
||||
## Custom loggers
|
||||
|
||||
`build_trainer` adds loggers based on `TrainConfig` flags. To attach a logger not supported by `TrainConfig` (for example a custom Neptune or Comet logger), build it yourself and pass it alongside the defaults:
|
||||
|
||||
```python
|
||||
from pytorch_lightning.loggers import NeptuneLogger # hypothetical
|
||||
from rfdetr.training import build_trainer
|
||||
|
||||
trainer = build_trainer(train_config, model_config)
|
||||
trainer.loggers.append(NeptuneLogger(project="my-workspace/rf-detr"))
|
||||
trainer.fit(module, datamodule)
|
||||
```
|
||||
|
||||
All logged keys (`train/loss`, `val/mAP_50_95`, `val/keypoint_map_50_95`, `val/F1`, `val/ema_mAP_50_95`, etc.) are written to every active logger in the list.
|
||||
|
||||
---
|
||||
|
||||
## Logged metrics reference
|
||||
|
||||
| Key | When logged | Description |
|
||||
| ------------------------ | ---------------------- | --------------------------------------------------------- |
|
||||
| `train/loss` | Every step / epoch | Total weighted training loss |
|
||||
| `train/<term>` | Every step / epoch | Individual loss terms (e.g. `train/loss_bbox`) |
|
||||
| `val/loss` | Each epoch | Validation loss (if `train_config.compute_val_loss=True`) |
|
||||
| `val/mAP_50_95` | Each eval epoch | COCO box mAP@[.50:.05:.95] |
|
||||
| `val/mAP_50` | Each eval epoch | COCO box mAP@.50 |
|
||||
| `val/mAP_75` | Each eval epoch | COCO box mAP@.75 |
|
||||
| `val/mAR` | Each eval epoch | COCO mean average recall |
|
||||
| `val/ema_mAP_50_95` | Each eval epoch | EMA-model mAP@[.50:.05:.95] (if EMA active) |
|
||||
| `val/F1` | Each eval epoch | Macro F1 at best confidence threshold |
|
||||
| `val/precision` | Each eval epoch | Precision at best F1 threshold |
|
||||
| `val/recall` | Each eval epoch | Recall at best F1 threshold |
|
||||
| `val/AP/<class>` | Each eval epoch | Per-class AP (if `log_per_class_metrics=True`) |
|
||||
| `val/segm_mAP_50_95` | Each eval epoch | Segmentation mAP (segmentation models only) |
|
||||
| `val/segm_mAP_50` | Each eval epoch | Segmentation mAP@.50 (segmentation models only) |
|
||||
| `val/keypoint_map_50_95` | Each eval epoch | COCO keypoint AP@[.50:.05:.95] (keypoint preview only) |
|
||||
| `val/keypoint_map_50` | Each eval epoch | COCO keypoint AP@.50 (keypoint preview only) |
|
||||
| `test/*` | After `trainer.test()` | Mirror of `val/*` keys |
|
||||
|
||||
---
|
||||
|
||||
## See also
|
||||
|
||||
- [RFDETR.train() — high-level API](index.md#quick-start) — the one-liner training path
|
||||
- [Training parameters](training-parameters.md) — all `TrainConfig` fields
|
||||
- [Training loggers](loggers.md) — TensorBoard, WandB, MLflow setup
|
||||
- [Advanced training](advanced.md) — checkpointing, early stopping, memory optimisation
|
||||
- [PTL primitives API reference](../../reference/training.md) — full docstring reference
|
||||
@@ -0,0 +1,493 @@
|
||||
---
|
||||
description: RF-DETR dataset format guide for COCO JSON and YOLO. Auto-detection, directory structure, annotation schemas, and format conversion.
|
||||
---
|
||||
|
||||
# Dataset Formats
|
||||
|
||||
RF-DETR supports training on datasets in two popular formats: **COCO** and **YOLO**. The format is automatically detected based on your dataset's directory structure—simply pass your dataset directory to the `train()` method.
|
||||
|
||||
## Automatic Format Detection
|
||||
|
||||
When you call `model.train(dataset_dir=<path>)`, RF-DETR checks the following:
|
||||
|
||||
1. **COCO format**: Looks for `train/_annotations.coco.json`
|
||||
2. **YOLO format**: Looks for `data.yaml` (or `data.yml`) and `train/images/` directory
|
||||
|
||||
If neither format is detected, an error is raised with instructions on what's expected.
|
||||
|
||||
!!! tip "Roboflow Export"
|
||||
|
||||
[Roboflow](https://roboflow.com/annotate) can export datasets in both COCO and YOLO formats. When downloading from Roboflow, select the appropriate format based on your preference.
|
||||
|
||||
---
|
||||
|
||||
## COCO Format
|
||||
|
||||
COCO (Common Objects in Context) format uses JSON files to store annotations in a structured format with images, categories, and annotations.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
dataset/
|
||||
├── train/
|
||||
│ ├── _annotations.coco.json
|
||||
│ ├── image1.jpg
|
||||
│ ├── image2.jpg
|
||||
│ └── ... (other image files)
|
||||
├── valid/
|
||||
│ ├── _annotations.coco.json
|
||||
│ ├── image1.jpg
|
||||
│ ├── image2.jpg
|
||||
│ └── ... (other image files)
|
||||
└── test/
|
||||
├── _annotations.coco.json
|
||||
├── image1.jpg
|
||||
├── image2.jpg
|
||||
└── ... (other image files)
|
||||
```
|
||||
|
||||
### Annotation File Structure
|
||||
|
||||
Each `_annotations.coco.json` file contains:
|
||||
|
||||
```json
|
||||
{
|
||||
"info": {
|
||||
"description": "Dataset description",
|
||||
"version": "1.0"
|
||||
},
|
||||
"licenses": [],
|
||||
"images": [
|
||||
{
|
||||
"id": 1,
|
||||
"file_name": "image1.jpg",
|
||||
"width": 640,
|
||||
"height": 480
|
||||
}
|
||||
],
|
||||
"categories": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "cat",
|
||||
"supercategory": "animal"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "dog",
|
||||
"supercategory": "animal"
|
||||
}
|
||||
],
|
||||
"annotations": [
|
||||
{
|
||||
"id": 1,
|
||||
"image_id": 1,
|
||||
"category_id": 1,
|
||||
"bbox": [
|
||||
100,
|
||||
150,
|
||||
200,
|
||||
180
|
||||
],
|
||||
"area": 36000,
|
||||
"iscrowd": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Key Fields
|
||||
|
||||
| Field | Description |
|
||||
| ------------- | --------------------------------------------------------------------- |
|
||||
| `images` | List of image metadata including `id`, `file_name`, `width`, `height` |
|
||||
| `categories` | List of object categories with `id` and `name` |
|
||||
| `annotations` | List of object annotations linking images to categories |
|
||||
| `bbox` | Bounding box in `[x, y, width, height]` format (top-left corner) |
|
||||
| `area` | Area of the bounding box |
|
||||
| `iscrowd` | 0 for individual objects, 1 for crowd regions |
|
||||
|
||||
### Segmentation Annotations
|
||||
|
||||
For training segmentation models, your COCO annotations must include a `segmentation` key with polygon coordinates:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"image_id": 1,
|
||||
"category_id": 1,
|
||||
"bbox": [
|
||||
100,
|
||||
150,
|
||||
200,
|
||||
180
|
||||
],
|
||||
"area": 36000,
|
||||
"iscrowd": 0,
|
||||
"segmentation": [
|
||||
[
|
||||
100,
|
||||
150,
|
||||
150,
|
||||
150,
|
||||
200,
|
||||
200,
|
||||
150,
|
||||
250,
|
||||
100,
|
||||
200
|
||||
]
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `segmentation` field contains a list of polygons, where each polygon is a flat list of coordinates: `[x1, y1, x2, y2, x3, y3, ...]`.
|
||||
|
||||
---
|
||||
|
||||
### Keypoint Annotations
|
||||
|
||||
For training the keypoint preview model, use COCO JSON keypoint annotations. Roboflow-style COCO exports are supported
|
||||
when the split files are named `train/_annotations.coco.json` and `valid/_annotations.coco.json`.
|
||||
|
||||
Each keypoint annotation must include a bounding box plus COCO keypoint fields:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"image_id": 1,
|
||||
"category_id": 0,
|
||||
"bbox": [
|
||||
100,
|
||||
150,
|
||||
200,
|
||||
180
|
||||
],
|
||||
"area": 36000,
|
||||
"iscrowd": 0,
|
||||
"num_keypoints": 17,
|
||||
"keypoints": [
|
||||
110,
|
||||
160,
|
||||
2,
|
||||
125,
|
||||
158,
|
||||
2
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The category should declare the keypoint schema:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 0,
|
||||
"name": "person",
|
||||
"supercategory": "person",
|
||||
"keypoints": [
|
||||
"nose",
|
||||
"left_eye",
|
||||
"right_eye"
|
||||
],
|
||||
"skeleton": []
|
||||
}
|
||||
```
|
||||
|
||||
The `keypoints` array above is shortened for readability. In a valid COCO person-keypoint annotation it contains
|
||||
`17 * 3` values: `x`, `y`, and visibility for each keypoint.
|
||||
|
||||
The keypoint preview model is pretrained on COCO person-style keypoints. Its default COCO schema is `[17]`, so
|
||||
keypoint-bearing categories are mapped onto the active keypoint label slot during COCO loading. Legacy checkpoints may
|
||||
still report a background-first `[0, 17]` schema, which RF-DETR accepts for compatibility. Custom keypoint training can
|
||||
also use YOLO pose labels, described below.
|
||||
|
||||
---
|
||||
|
||||
## YOLO Format
|
||||
|
||||
YOLO format uses separate text files for each image's annotations and a `data.yaml` configuration file that defines class names.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
dataset/
|
||||
├── data.yaml
|
||||
├── train/
|
||||
│ ├── images/
|
||||
│ │ ├── image1.jpg
|
||||
│ │ ├── image2.jpg
|
||||
│ │ └── ...
|
||||
│ └── labels/
|
||||
│ ├── image1.txt
|
||||
│ ├── image2.txt
|
||||
│ └── ...
|
||||
├── valid/
|
||||
│ ├── images/
|
||||
│ │ ├── image1.jpg
|
||||
│ │ ├── image2.jpg
|
||||
│ │ └── ...
|
||||
│ └── labels/
|
||||
│ ├── image1.txt
|
||||
│ ├── image2.txt
|
||||
│ └── ...
|
||||
└── test/
|
||||
├── images/
|
||||
│ ├── image1.jpg
|
||||
│ └── ...
|
||||
└── labels/
|
||||
├── image1.txt
|
||||
└── ...
|
||||
```
|
||||
|
||||
### data.yaml Configuration
|
||||
|
||||
The `data.yaml` file at the root of your dataset directory defines the class names:
|
||||
|
||||
```yaml
|
||||
names:
|
||||
- cat
|
||||
- dog
|
||||
- bird
|
||||
|
||||
nc: 3
|
||||
|
||||
train: train/images
|
||||
val: valid/images
|
||||
test: test/images
|
||||
```
|
||||
|
||||
| Field | Description |
|
||||
| ---------------------- | -------------------------------------------------- |
|
||||
| `names` | List of class names (0-indexed) |
|
||||
| `nc` | Number of classes |
|
||||
| `train`, `val`, `test` | Paths to image directories (relative to data.yaml) |
|
||||
|
||||
!!! note "Alternative format"
|
||||
|
||||
Some YOLO datasets use a dictionary format for names:
|
||||
|
||||
```yaml
|
||||
names:
|
||||
0: cat
|
||||
1: dog
|
||||
2: bird
|
||||
```
|
||||
|
||||
Both formats are supported.
|
||||
|
||||
### Label File Format
|
||||
|
||||
Each image has a corresponding `.txt` file in the `labels/` directory with the same base name. Each line in the label file represents one object:
|
||||
|
||||
```
|
||||
<class_id> <x_center> <y_center> <width> <height>
|
||||
```
|
||||
|
||||
**Example** (`image1.txt`):
|
||||
|
||||
```
|
||||
0 0.5 0.4 0.3 0.2
|
||||
1 0.2 0.6 0.15 0.25
|
||||
```
|
||||
|
||||
#### Coordinate Format
|
||||
|
||||
| Field | Range | Description |
|
||||
| ---------- | ------------ | ----------------------------------------------- |
|
||||
| `class_id` | 0, 1, 2, ... | Zero-indexed class ID from `names` in data.yaml |
|
||||
| `x_center` | 0.0 - 1.0 | Normalized x-coordinate of bounding box center |
|
||||
| `y_center` | 0.0 - 1.0 | Normalized y-coordinate of bounding box center |
|
||||
| `width` | 0.0 - 1.0 | Normalized width of bounding box |
|
||||
| `height` | 0.0 - 1.0 | Normalized height of bounding box |
|
||||
|
||||
All coordinates are normalized relative to image dimensions. For example, if an image is 640×480 pixels and the bounding box center is at (320, 240):
|
||||
|
||||
- `x_center` = 320 / 640 = 0.5
|
||||
- `y_center` = 240 / 480 = 0.5
|
||||
|
||||
### Segmentation Labels (YOLO-Seg)
|
||||
|
||||
For segmentation, YOLO format extends the label format with polygon coordinates:
|
||||
|
||||
```
|
||||
<class_id> <x1> <y1> <x2> <y2> <x3> <y3> ...
|
||||
```
|
||||
|
||||
**Example** (`image1.txt` with segmentation):
|
||||
|
||||
```
|
||||
0 0.1 0.2 0.3 0.2 0.4 0.5 0.2 0.6 0.1 0.4
|
||||
```
|
||||
|
||||
The coordinates after the class ID represent the polygon vertices in normalized format.
|
||||
|
||||
---
|
||||
|
||||
### Pose Labels (YOLO Pose)
|
||||
|
||||
For keypoint preview training, RF-DETR supports Ultralytics YOLO pose labels in the same directory layout shown above.
|
||||
The `data.yaml` file must declare `kpt_shape`:
|
||||
|
||||
```yaml
|
||||
names:
|
||||
0: person
|
||||
|
||||
kpt_shape: [17, 3] # [number_of_keypoints, dimensions]; dimensions must be 2 or 3
|
||||
flip_idx: [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15]
|
||||
kpt_names:
|
||||
0:
|
||||
- nose
|
||||
- left_eye
|
||||
- right_eye
|
||||
```
|
||||
|
||||
`kpt_names` is optional. When omitted, RF-DETR creates placeholder names such as `keypoint_0`. `flip_idx` is an
|
||||
Ultralytics-style length-`K` permutation used to infer RF-DETR's flat `keypoint_flip_pairs` for horizontal-flip
|
||||
augmentation.
|
||||
|
||||
Each pose label row contains a bounding box followed by keypoints:
|
||||
|
||||
```text
|
||||
<class_id> <x_center> <y_center> <width> <height> <px1> <py1> <v1> ... <pxK> <pyK> <vK>
|
||||
```
|
||||
|
||||
For `kpt_shape: [K, 2]`, omit the visibility value:
|
||||
|
||||
```text
|
||||
<class_id> <x_center> <y_center> <width> <height> <px1> <py1> ... <pxK> <pyK>
|
||||
```
|
||||
|
||||
All box and keypoint coordinates are normalized to `[0, 1]`. RF-DETR converts keypoints to COCO-style `(x, y, visibility)` tensors internally. For `[K, 3]`, the visibility values are preserved. For `[K, 2]`, visibility is
|
||||
synthesized: nonzero points are marked visible (`2`) and `(0, 0)` points are marked absent (`0`).
|
||||
|
||||
Use the YOLO schema helper when you want to configure a model explicitly:
|
||||
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
from rfdetr.datasets._keypoint_schema import infer_yolo_keypoint_schema
|
||||
|
||||
DATASET_DIR = Path("/path/to/yolo-pose-dataset")
|
||||
schema = infer_yolo_keypoint_schema(DATASET_DIR / "data.yaml")
|
||||
|
||||
model = RFDETRKeypointPreview(
|
||||
num_classes=len(schema.class_names),
|
||||
num_keypoints_per_class=schema.num_keypoints_per_class,
|
||||
)
|
||||
|
||||
model.train(
|
||||
dataset_file="yolo",
|
||||
dataset_dir=str(DATASET_DIR),
|
||||
class_names=schema.class_names,
|
||||
keypoint_oks_sigmas=schema.keypoint_oks_sigmas,
|
||||
)
|
||||
```
|
||||
|
||||
!!! note "flip_idx and keypoint_flip_pairs"
|
||||
|
||||
`flip_idx` is a permutation, while `keypoint_flip_pairs` is a flat pair list. During `model.train()`, RF-DETR infers
|
||||
the pair list automatically from `flip_idx` when no explicit `keypoint_flip_pairs` is provided.
|
||||
|
||||
---
|
||||
|
||||
## Converting Between Formats
|
||||
|
||||
### YOLO to COCO
|
||||
|
||||
You can use the [supervision](https://github.com/roboflow/supervision) library to convert datasets:
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
|
||||
# Load YOLO dataset
|
||||
dataset = sv.DetectionDataset.from_yolo(
|
||||
images_directory_path="path/to/images",
|
||||
annotations_directory_path="path/to/labels",
|
||||
data_yaml_path="path/to/data.yaml",
|
||||
)
|
||||
|
||||
# Save as COCO
|
||||
dataset.as_coco(images_directory_path="output/images", annotations_path="output/annotations.json")
|
||||
```
|
||||
|
||||
### COCO to YOLO
|
||||
|
||||
```python
|
||||
import supervision as sv
|
||||
|
||||
# Load COCO dataset
|
||||
dataset = sv.DetectionDataset.from_coco(
|
||||
images_directory_path="path/to/images", annotations_path="path/to/annotations.json"
|
||||
)
|
||||
|
||||
# Save as YOLO
|
||||
dataset.as_yolo(
|
||||
images_directory_path="output/images", annotations_directory_path="output/labels", data_yaml_path="output/data.yaml"
|
||||
)
|
||||
```
|
||||
|
||||
### Using Roboflow
|
||||
|
||||
[Roboflow](https://roboflow.com) provides a web interface to:
|
||||
|
||||
1. Upload datasets in any format
|
||||
2. Annotate new images or edit existing annotations
|
||||
3. Export in COCO, YOLO, or other formats
|
||||
|
||||
This is often the easiest way to convert between formats while also having the option to augment your data.
|
||||
|
||||
---
|
||||
|
||||
## Which Format Should I Use?
|
||||
|
||||
Both formats work equally well with RF-DETR. Choose based on your workflow:
|
||||
|
||||
| Consideration | COCO | YOLO |
|
||||
| --------------------------------- | -------------------------- | ----------------------- |
|
||||
| **Annotation storage** | Single JSON file per split | One text file per image |
|
||||
| **Human readability** | JSON structure, verbose | Simple text, compact |
|
||||
| **Other framework compatibility** | DETR family, MMDetection | Ultralytics YOLO |
|
||||
| **Segmentation support** | Full polygon support | Full polygon support |
|
||||
| **Editing annotations** | Requires JSON parsing | Simple text editing |
|
||||
|
||||
!!! tip "Recommendation"
|
||||
|
||||
If you're exporting from Roboflow or already have a dataset in one format, simply use that format. RF-DETR handles both identically.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Format Detection Fails
|
||||
|
||||
If you see an error like:
|
||||
|
||||
```
|
||||
Could not detect dataset format in /path/to/dataset
|
||||
```
|
||||
|
||||
Check that:
|
||||
|
||||
**For COCO format:**
|
||||
|
||||
- `train/_annotations.coco.json` exists
|
||||
- The JSON file is valid
|
||||
|
||||
**For YOLO format:**
|
||||
|
||||
- `data.yaml` or `data.yml` exists at the root
|
||||
- `train/images/` directory exists with images
|
||||
|
||||
### Empty Annotations
|
||||
|
||||
If images have no objects, handle them as follows:
|
||||
|
||||
**COCO format:** Include the image in the `images` array but don't add any annotations for it.
|
||||
|
||||
**YOLO format:** Create an empty `.txt` file (0 bytes) for the image, or omit the label file entirely.
|
||||
|
||||
### Class ID Mismatch
|
||||
|
||||
**COCO format:** Category IDs in annotations must match IDs defined in the `categories` array.
|
||||
|
||||
**YOLO format:** Class IDs in label files must be valid indices (0 to `nc-1`) based on the `names` list in `data.yaml`.
|
||||
@@ -0,0 +1,247 @@
|
||||
---
|
||||
description: Train RF-DETR detection and segmentation models on custom datasets. Supports COCO and YOLO formats with one-line Python API and PyTorch Lightning.
|
||||
---
|
||||
|
||||
# Train an RF-DETR Model
|
||||
|
||||
!!! tip "Key Takeaways"
|
||||
|
||||
- Train detection, segmentation, or keypoint preview models with a single `model.train(dataset_dir=...)` call
|
||||
- Detection and segmentation support COCO JSON and YOLO dataset formats with automatic detection
|
||||
- Keypoint preview training supports COCO keypoint JSON and Ultralytics YOLO pose datasets
|
||||
- Fine-tune from COCO-pretrained checkpoints (Nano to 2XLarge) for fastest convergence
|
||||
- Built on PyTorch Lightning — use the high-level API or access PTL primitives directly for full control
|
||||
- EMA weights, early stopping, and best-model checkpointing are included by default
|
||||
|
||||
You can train RF-DETR object detection and segmentation models on a custom dataset using the `rfdetr` Python package, or in the cloud using Roboflow.
|
||||
|
||||
This guide describes how to train both an object detection and segmentation RF-DETR model.
|
||||
|
||||
## Training paths
|
||||
|
||||
RF-DETR provides two training paths:
|
||||
|
||||
| Path | When to use |
|
||||
| ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **`RFDETR.train()`** (this page) | Quickstart, fine-tuning with standard options, Colab notebooks. One call sets up and runs everything. |
|
||||
| **[Custom Training API](customization.md)** | Custom callbacks, alternative loggers, multi-GPU strategies, integration with external frameworks, or any other customisation of the training loop. |
|
||||
|
||||
Both paths run the same underlying PyTorch Lightning stack. `RFDETR.train()` constructs `RFDETRModelModule`, `RFDETRDataModule`, and a `Trainer` internally; the Lightning API page shows how to do the same thing explicitly so you can modify each component.
|
||||
|
||||
## Quick Start
|
||||
|
||||
!!! info "Training requires the `train` extra"
|
||||
|
||||
Training dependencies are not included in the base install. Install them with:
|
||||
|
||||
```bash
|
||||
pip install "rfdetr[train]"
|
||||
```
|
||||
|
||||
For experiment tracking, also add `pip install "rfdetr[train,loggers]"`.
|
||||
|
||||
RF-DETR supports training on datasets in both **COCO** and **YOLO** formats. The format is automatically detected based on the structure of your dataset directory.
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="<DATASET_PATH>",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="<OUTPUT_PATH>",
|
||||
)
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegMedium
|
||||
|
||||
model = RFDETRSegMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="<DATASET_PATH>",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="<OUTPUT_PATH>",
|
||||
)
|
||||
```
|
||||
|
||||
=== "Keypoint Preview"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
|
||||
model = RFDETRKeypointPreview()
|
||||
|
||||
model.train(
|
||||
dataset_dir="<KEYPOINT_DATASET_PATH>",
|
||||
epochs=50,
|
||||
batch_size=2,
|
||||
grad_accum_steps=8,
|
||||
lr=1e-5,
|
||||
output_dir="<OUTPUT_PATH>",
|
||||
)
|
||||
```
|
||||
|
||||
Different GPUs have different VRAM capacities, so adjust batch_size and grad_accum_steps to maintain a total batch size of 16. For example, on a powerful GPU like the A100, use `batch_size=16` and `grad_accum_steps=1`; on smaller GPUs like the T4, use `batch_size=4` and `grad_accum_steps=4`. This gradient accumulation strategy helps train effectively even with limited memory.
|
||||
|
||||
Each model class downloads its COCO-pretrained checkpoint automatically when instantiated. To get started quickly with training an object detection model, please refer to our fine-tuning Google Colab [notebook](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/how-to-finetune-rf-detr-on-detection-dataset.ipynb).
|
||||
|
||||
## Keypoint preview custom datasets
|
||||
|
||||
The pretrained keypoint preview checkpoint predicts 17 COCO person keypoints. Fine-tuned keypoint preview models can use the keypoint schema from your own COCO or YOLO pose dataset, so the output keypoint count is not limited to 17.
|
||||
|
||||
Use COCO keypoint JSON or Ultralytics YOLO pose labels for custom keypoint training. Roboflow COCO exports are supported when split annotations are named `train/_annotations.coco.json`, `valid/_annotations.coco.json`, and optionally `test/_annotations.coco.json`. YOLO pose datasets use the existing RF-DETR YOLO directory layout with `data.yaml`, `train/images`, `train/labels`, `valid/images`, and `valid/labels`.
|
||||
|
||||
The keypoint fine-tuning demo infers the class names and keypoint schema from the training annotation file, then passes those values into `RFDETRKeypointPreview` and `model.train()`:
|
||||
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
from rfdetr import RFDETRKeypointPreview
|
||||
from rfdetr.datasets._keypoint_schema import infer_coco_keypoint_schema
|
||||
|
||||
DATASET_DIR = Path("/path/to/coco-keypoint-dataset")
|
||||
schema = infer_coco_keypoint_schema(DATASET_DIR / "train" / "_annotations.coco.json")
|
||||
|
||||
model = RFDETRKeypointPreview(
|
||||
num_classes=len(schema.class_names),
|
||||
num_keypoints_per_class=schema.num_keypoints_per_class,
|
||||
)
|
||||
|
||||
model.train(
|
||||
dataset_file="roboflow",
|
||||
dataset_dir=str(DATASET_DIR),
|
||||
class_names=schema.class_names,
|
||||
keypoint_oks_sigmas=schema.keypoint_oks_sigmas,
|
||||
epochs=50,
|
||||
batch_size=8,
|
||||
grad_accum_steps=2,
|
||||
lr=2e-5,
|
||||
lr_encoder=2e-5,
|
||||
output_dir="output/keypoint_custom",
|
||||
use_ema=False,
|
||||
run_test=False,
|
||||
)
|
||||
```
|
||||
|
||||
Set `keypoint_flip_pairs` if horizontal flips should swap left/right keypoints for your schema.
|
||||
|
||||
For YOLO pose datasets, use `infer_yolo_keypoint_schema(DATASET_DIR / "data.yaml")` instead. RF-DETR also infers YOLO pose schema automatically during `model.train()` when `data.yaml` declares `kpt_shape`.
|
||||
|
||||
## Dataset Format
|
||||
|
||||
RF-DETR **automatically detects** whether your dataset is in COCO or YOLO format. Simply pass your dataset directory to the `train()` method and the appropriate data loader will be used.
|
||||
|
||||
| Format | Detection Method | Learn More |
|
||||
| -------- | ---------------------------------------- | --------------------------------------------------- |
|
||||
| **COCO** | Looks for `train/_annotations.coco.json` | [COCO Format Guide](dataset-formats.md#coco-format) |
|
||||
| **YOLO** | Looks for `data.yaml` + `train/images/` | [YOLO Format Guide](dataset-formats.md#yolo-format) |
|
||||
|
||||
For keypoint preview training, use COCO keypoint JSON or YOLO pose labels. YOLO pose datasets must declare
|
||||
`kpt_shape` in `data.yaml`; detection-only YOLO datasets still fail clearly in keypoint mode instead of being treated as
|
||||
pose labels.
|
||||
|
||||
[Roboflow](https://roboflow.com/annotate) allows you to create object detection datasets from scratch and export them in either COCO JSON or YOLO format for training. You can also explore [Roboflow Universe](https://universe.roboflow.com/) to find pre-labeled datasets for a range of use cases.
|
||||
|
||||
→ **[Learn more about dataset formats](dataset-formats.md)**
|
||||
|
||||
## Training Configuration
|
||||
|
||||
RF-DETR provides many configuration options to customize your training run. See the complete reference for all available parameters.
|
||||
|
||||
→ **[View all training parameters](training-parameters.md)**
|
||||
|
||||
## Advanced Topics
|
||||
|
||||
- [Resume training](advanced.md#resume-training) from a checkpoint
|
||||
- [Early stopping](advanced.md#early-stopping) to prevent overfitting
|
||||
- [Multi-GPU training](advanced.md#multi-gpu-training) with PyTorch Lightning DDP
|
||||
- [Custom augmentations with Albumentations](augmentations.md) - Dedicated guide
|
||||
- [Memory optimization](advanced.md#memory-optimization) with gradient checkpointing
|
||||
|
||||
→ **[Learn more about advanced training](advanced.md)**
|
||||
|
||||
## Custom Training API
|
||||
|
||||
RF-DETR's training stack is built on PyTorch Lightning. The `RFDETR.train()` call above constructs and runs PTL primitives internally. Use them directly when you need custom callbacks, non-default loggers, multi-GPU strategies, or full control over the training loop.
|
||||
|
||||
→ **[Custom Training API guide](customization.md)**
|
||||
|
||||
## Training Loggers
|
||||
|
||||
Track your experiments with popular logging platforms:
|
||||
|
||||
- [TensorBoard](loggers.md#tensorboard) for local visualization
|
||||
- [Weights and Biases](loggers.md#weights-and-biases) for cloud-based tracking
|
||||
- [ClearML](loggers.md#clearml) workaround for SDK auto-binding
|
||||
- [MLflow](loggers.md#mlflow) for experiment lifecycle management
|
||||
|
||||
→ **[Learn more about training loggers](loggers.md)**
|
||||
|
||||
## Result Checkpoints
|
||||
|
||||
During training, multiple model checkpoints are saved to the output directory:
|
||||
|
||||
- `checkpoint.pth` – the most recent checkpoint, saved at the end of the latest epoch.
|
||||
|
||||
- `checkpoint_<number>.pth` – periodic checkpoints saved every N epochs (default is every 10).
|
||||
|
||||
- `checkpoint_best_ema.pth` – best checkpoint based on validation score, using the EMA (Exponential Moving Average) weights. EMA weights are a smoothed version of the model's parameters across training steps, often yielding better generalization.
|
||||
|
||||
- `checkpoint_best_regular.pth` – best checkpoint based on validation score, using the raw (non-EMA) model weights.
|
||||
|
||||
- `checkpoint_best_total.pth` – final checkpoint selected for inference and benchmarking. It contains only the model weights (no optimizer state or scheduler) and is chosen as the better of the EMA and non-EMA models based on validation performance.
|
||||
|
||||
For detection and segmentation models, the validation score is box mAP (`val/mAP_50_95`). For keypoint preview models,
|
||||
best-checkpoint selection uses COCO keypoint AP (`val/keypoint_map_50_95`) and checkpoints persist the model keypoint
|
||||
schema so `RFDETR.from_checkpoint()` can reconstruct the same label/keypoint slots.
|
||||
|
||||
??? note "Checkpoint file sizes"
|
||||
|
||||
Checkpoint sizes vary based on what they contain:
|
||||
|
||||
- **Training checkpoints** (e.g. `checkpoint.pth`, `checkpoint_<number>.pth`) include model weights, optimizer state, scheduler state, and training metadata. Use these to resume training.
|
||||
|
||||
- **Evaluation checkpoints** (e.g. `checkpoint_best_ema.pth`, `checkpoint_best_regular.pth`) store only the model weights — either EMA or raw — and are used to track the best-performing models. These may come from different epochs depending on which version achieved the highest validation score.
|
||||
|
||||
- **Stripped checkpoint** (e.g. `checkpoint_best_total.pth`) contains only the final model weights and is optimized for inference and deployment.
|
||||
|
||||
## Load and Run Fine-Tuned Model
|
||||
|
||||
=== "Object Detection"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium(pretrain_weights="<CHECKPOINT_PATH>")
|
||||
|
||||
detections = model.predict("<IMAGE_PATH>")
|
||||
```
|
||||
|
||||
=== "Image Segmentation"
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegMedium
|
||||
|
||||
model = RFDETRSegMedium(pretrain_weights="<CHECKPOINT_PATH>")
|
||||
|
||||
detections = model.predict("<IMAGE_PATH>")
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
After training your model, you can:
|
||||
|
||||
- [Export your model to ONNX](../export.md) for deployment with various inference frameworks
|
||||
- [Deploy to Roboflow](../deploy.md) for cloud-based inference and workflow integration
|
||||
@@ -0,0 +1,316 @@
|
||||
---
|
||||
description: Track RF-DETR training with TensorBoard, Weights and Biases, and MLflow. Configure multiple experiment loggers simultaneously.
|
||||
---
|
||||
|
||||
# Training Loggers
|
||||
|
||||
RF-DETR supports integration with popular experiment tracking and visualization platforms. You can enable one or more supported loggers to monitor your training runs, compare experiments, and track metrics over time.
|
||||
|
||||
## CSV (always active)
|
||||
|
||||
A `CSVLogger` is always active regardless of any flags. It requires no extra packages and writes all metrics to `{output_dir}/metrics.csv` on every validation step.
|
||||
|
||||
---
|
||||
|
||||
## TensorBoard
|
||||
|
||||
[TensorBoard](https://www.tensorflow.org/tensorboard) is a powerful toolkit for visualizing and tracking training metrics.
|
||||
|
||||
TensorBoard logging is enabled by default. Pass `tensorboard=False` to disable it.
|
||||
|
||||
!!! note "Missing package behaviour"
|
||||
|
||||
If the `tensorboard` package is not installed, training continues without error — a
|
||||
`UserWarning` is emitted and TensorBoard logging is silently suppressed. Install
|
||||
`rfdetr[loggers]` to avoid this.
|
||||
|
||||
### Setup
|
||||
|
||||
Install the required packages:
|
||||
|
||||
```bash
|
||||
pip install "rfdetr[loggers]"
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
TensorBoard is active unless you explicitly disable it:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
# tensorboard=True is the default; pass tensorboard=False to disable
|
||||
)
|
||||
```
|
||||
|
||||
### Viewing Logs
|
||||
|
||||
**Local environment:**
|
||||
|
||||
```bash
|
||||
tensorboard --logdir output
|
||||
```
|
||||
|
||||
Then open `http://localhost:6006/` in your browser.
|
||||
|
||||
**Google Colab:**
|
||||
|
||||
```ipython
|
||||
%load_ext tensorboard
|
||||
%tensorboard --logdir output
|
||||
```
|
||||
|
||||
### Logged Metrics
|
||||
|
||||
All logged metric keys are listed in the [Logged Metrics Reference](customization.md#logged-metrics-reference).
|
||||
|
||||
---
|
||||
|
||||
## Weights and Biases
|
||||
|
||||
[Weights and Biases (W&B)](https://www.wandb.ai) is a cloud-based platform for experiment tracking and visualization.
|
||||
|
||||
### Setup
|
||||
|
||||
Install the required packages:
|
||||
|
||||
```bash
|
||||
pip install "rfdetr[loggers]"
|
||||
```
|
||||
|
||||
Log in to W&B:
|
||||
|
||||
```bash
|
||||
wandb login
|
||||
```
|
||||
|
||||
You can retrieve your API key at [wandb.ai/authorize](https://wandb.ai/authorize).
|
||||
|
||||
### Usage
|
||||
|
||||
Enable W&B logging in your training:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
wandb=True,
|
||||
project="my-detection-project",
|
||||
run="experiment-001",
|
||||
)
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
| Parameter | Description |
|
||||
| --------- | --------------------------------------- |
|
||||
| `project` | Groups related experiments together |
|
||||
| `run` | Identifies individual training sessions |
|
||||
|
||||
If you don't specify a run name, W&B assigns a random one automatically.
|
||||
|
||||
### Features
|
||||
|
||||
Access your runs at [wandb.ai](https://wandb.ai). W&B provides:
|
||||
|
||||
- Real-time metric visualization
|
||||
- Experiment comparison
|
||||
- Hyperparameter tracking
|
||||
- System metrics (GPU usage, memory)
|
||||
- Training config logging
|
||||
|
||||
### Logged Metrics
|
||||
|
||||
All logged metric keys are listed in the [Logged Metrics Reference](customization.md#logged-metrics-reference).
|
||||
|
||||
---
|
||||
|
||||
## ClearML
|
||||
|
||||
[ClearML](https://clear.ml) is an open-source platform for managing, tracking, and automating machine learning experiments.
|
||||
|
||||
**ClearML is not yet integrated as a native PTL logger.** Passing `clearml=True` to `model.train()` raises `NotImplementedError`; metrics are not logged to ClearML through RF-DETR's built-in logger wiring.
|
||||
|
||||
### Workaround: ClearML SDK auto-binding
|
||||
|
||||
ClearML's SDK captures PyTorch Lightning metrics automatically when a `Task` is initialised before training begins:
|
||||
|
||||
```python
|
||||
from clearml import Task
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
# Initialise before model.train() — ClearML auto-binds to PTL logging
|
||||
task = Task.init(project_name="my-detection-project", task_name="experiment-001")
|
||||
|
||||
model = RFDETRMedium()
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
# Do NOT pass clearml=True — RF-DETR raises NotImplementedError for that flag
|
||||
)
|
||||
```
|
||||
|
||||
Alternatively, attach a ClearML callback directly using the [Custom Training API](#attaching-loggers-via-the-custom-training-api).
|
||||
|
||||
---
|
||||
|
||||
## MLflow
|
||||
|
||||
[MLflow](https://mlflow.org/) is an open-source platform for the machine learning lifecycle that helps track experiments, package code into reproducible runs, and share and deploy models.
|
||||
|
||||
### Setup
|
||||
|
||||
Install the required packages:
|
||||
|
||||
```bash
|
||||
pip install "rfdetr[loggers]"
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Enable MLflow logging in your training:
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
mlflow=True,
|
||||
project="my-detection-project",
|
||||
run="experiment-001",
|
||||
)
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
| Parameter | Description |
|
||||
| --------- | --------------------------------------------------- |
|
||||
| `project` | Sets the experiment name in MLflow |
|
||||
| `run` | Sets the run name (auto-generated if not specified) |
|
||||
|
||||
### Custom Tracking Server
|
||||
|
||||
To use a custom MLflow tracking server, set environment variables:
|
||||
|
||||
```python
|
||||
import os
|
||||
|
||||
# Set MLflow tracking URI
|
||||
os.environ["MLFLOW_TRACKING_URI"] = "https://your-mlflow-server.com"
|
||||
|
||||
# For authentication with tracking servers that require it
|
||||
os.environ["MLFLOW_TRACKING_TOKEN"] = "your-auth-token"
|
||||
|
||||
# Then initialize and train your model
|
||||
model = RFDETRMedium()
|
||||
model.train(..., mlflow=True)
|
||||
```
|
||||
|
||||
For teams using a hosted MLflow service (like Databricks), you'll typically need to set:
|
||||
|
||||
- `MLFLOW_TRACKING_URI`: The URL of your MLflow tracking server
|
||||
- `MLFLOW_TRACKING_TOKEN`: Authentication token for your MLflow server
|
||||
|
||||
### Viewing Logs
|
||||
|
||||
Start the MLflow UI:
|
||||
|
||||
```bash
|
||||
mlflow ui --backend-store-uri <OUTPUT_PATH>
|
||||
```
|
||||
|
||||
Then open `http://localhost:5000` in your browser to access the MLflow dashboard.
|
||||
|
||||
### Logged Metrics
|
||||
|
||||
All logged metric keys are listed in the [Logged Metrics Reference](customization.md#logged-metrics-reference).
|
||||
|
||||
---
|
||||
|
||||
## Using Multiple Loggers
|
||||
|
||||
You can enable multiple logging systems simultaneously:
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
tensorboard=True,
|
||||
wandb=True,
|
||||
mlflow=True,
|
||||
project="my-project",
|
||||
run="experiment-001",
|
||||
)
|
||||
```
|
||||
|
||||
This allows you to leverage the strengths of different platforms:
|
||||
|
||||
- **TensorBoard**: Local visualization and debugging
|
||||
- **W&B**: Cloud-based collaboration and experiment comparison
|
||||
- **MLflow**: Model registry and deployment tracking
|
||||
|
||||
Note: `clearml=True` is accepted by the config schema but raises `NotImplementedError` when the trainer is built. Use the [ClearML SDK workaround](#clearml) instead.
|
||||
|
||||
---
|
||||
|
||||
## Attaching loggers via the Custom Training API
|
||||
|
||||
`build_trainer` automatically creates loggers from `TrainConfig` flags. To attach a logger not listed above (for example Neptune, Comet, or a fully custom logger), build it separately and append it to `trainer.loggers` before calling `trainer.fit`:
|
||||
|
||||
```python
|
||||
from rfdetr.config import RFDETRMediumConfig, TrainConfig
|
||||
from rfdetr.training import RFDETRModelModule, RFDETRDataModule, build_trainer
|
||||
|
||||
model_config = RFDETRMediumConfig(num_classes=10)
|
||||
train_config = TrainConfig(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
output_dir="output",
|
||||
tensorboard=True, # built-in loggers still work
|
||||
)
|
||||
|
||||
module = RFDETRModelModule(model_config, train_config)
|
||||
datamodule = RFDETRDataModule(model_config, train_config)
|
||||
trainer = build_trainer(train_config, model_config)
|
||||
|
||||
# Attach any additional PTL-compatible logger
|
||||
from pytorch_lightning.loggers import CSVLogger # example — use any PTL logger
|
||||
|
||||
trainer.loggers.append(CSVLogger(save_dir="output", name="extra"))
|
||||
|
||||
trainer.fit(module, datamodule)
|
||||
```
|
||||
|
||||
CSVLogger is always active (it requires no extra packages). All logged metric keys — `train/loss`, `val/mAP_50_95`,
|
||||
`val/keypoint_map_50_95`, `val/F1`, `val/ema_mAP_50_95`, `val/AP/<class>`, etc. — are written to every logger in the
|
||||
list.
|
||||
|
||||
→ **[Full list of logged metrics](customization.md#logged-metrics-reference)**
|
||||
@@ -0,0 +1,288 @@
|
||||
---
|
||||
description: Complete RF-DETR training parameter reference. Learning rate, batch size, EMA, early stopping, resolution, and hardware configuration.
|
||||
---
|
||||
|
||||
# Training Parameters
|
||||
|
||||
This page provides a complete reference of all parameters available when training RF-DETR models.
|
||||
|
||||
## Basic Example
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRMedium
|
||||
|
||||
model = RFDETRMedium()
|
||||
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4,
|
||||
lr=1e-4,
|
||||
output_dir="output",
|
||||
)
|
||||
```
|
||||
|
||||
## Core Parameters
|
||||
|
||||
These are the essential parameters for training:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ------------------ | --------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `dataset_dir` | `str` | Required | Path to your dataset directory. RF-DETR auto-detects if it's in COCO or YOLO format. See [Dataset Formats](dataset-formats.md). |
|
||||
| `output_dir` | `str` | `"output"` | Directory where training artifacts (checkpoints, logs) are saved. |
|
||||
| `epochs` | `int` | `100` | Number of full passes over the training dataset. |
|
||||
| `batch_size` | `int or "auto"` | `4` | Number of samples processed per iteration. Higher values require more GPU memory. Set to `"auto"` to probe the GPU for the largest safe batch size automatically. |
|
||||
| `grad_accum_steps` | `int` | `4` | Accumulates gradients over multiple mini-batches. Use with `batch_size` to achieve effective batch size. |
|
||||
| `resume` | `str` | `None` | Path to a saved checkpoint to continue training. Restores model weights, optimizer state, and scheduler. |
|
||||
|
||||
### Understanding Batch Size
|
||||
|
||||
The **effective batch size** is calculated as:
|
||||
|
||||
```
|
||||
effective_batch_size = batch_size × grad_accum_steps × num_gpus
|
||||
```
|
||||
|
||||
Recommended configurations for different GPUs (targeting effective batch size of 16):
|
||||
|
||||
| GPU | VRAM | `batch_size` | `grad_accum_steps` |
|
||||
| -------- | ------- | ------------ | ------------------ |
|
||||
| A100 | 40-80GB | 16 | 1 |
|
||||
| RTX 4090 | 24GB | 8 | 2 |
|
||||
| RTX 3090 | 24GB | 8 | 2 |
|
||||
| T4 | 16GB | 4 | 4 |
|
||||
| RTX 3070 | 8GB | 2 | 8 |
|
||||
|
||||
## Learning Rate Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ------------ | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `lr` | `float` | `1e-4` | Learning rate for most parts of the model. |
|
||||
| `lr_encoder` | `float` | `1.5e-4` | Learning rate specifically for the backbone encoder. Can be set lower than `lr` if you want to fine-tune the encoder more conservatively than the rest of the model. |
|
||||
|
||||
!!! tip "Learning rate tips"
|
||||
|
||||
- Start with the default values for fine-tuning
|
||||
- If the model doesn't converge, try reducing `lr` by half
|
||||
- For training from scratch (not recommended), you may need higher learning rates
|
||||
|
||||
## Resolution Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ------------ | ----- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `resolution` | `int` | Model-dependent | Input image resolution. Higher values can improve accuracy but require more memory. Each model has its own valid block size: current standard detection checkpoints use multiples of 32, current segmentation checkpoints use multiples of 24 (most variants) or 12 (`RFDETRSegNano`), and the definitive rule is that the resolution must be divisible by `patch_size * num_windows` for the selected model. |
|
||||
|
||||
Common resolution values for currently documented checkpoints:
|
||||
|
||||
- Detection: `384`, `512`, `576`, `704`
|
||||
- Segmentation: `312`, `384`, `432`, `504`, `624`, `768`
|
||||
|
||||
For example, `RFDETRSegXLarge` uses `624x624`, which is valid because `624` is divisible by `24`.
|
||||
|
||||
## Regularization Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| -------------- | ------- | ------- | ------------------------------------------------------------------------------------- |
|
||||
| `weight_decay` | `float` | `1e-4` | L2 regularization coefficient. Helps prevent overfitting by penalizing large weights. |
|
||||
|
||||
## Hardware Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ------------------------ | ------ | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `device` | `str` | `None` | Device to run training on. `None` means auto-detected by PyTorch Lightning. Options: `"cuda"`, `"cpu"`, `"mps"` (Apple Silicon). |
|
||||
| `gradient_checkpointing` | `bool` | `False` | **Constructor-only parameter** — pass to the model constructor (`RFDETRMedium(gradient_checkpointing=True)`), not to `train()`. Re-computes activations during backprop to reduce memory usage by ~30-40% at the cost of ~20% slower training. |
|
||||
|
||||
## EMA (Exponential Moving Average)
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| --------- | ------ | ------- | -------------------------------------------------------------------------------------------------------------------- |
|
||||
| `use_ema` | `bool` | `True` | Enables Exponential Moving Average of weights. Produces a smoothed checkpoint that often improves final performance. |
|
||||
|
||||
!!! info "What is EMA?"
|
||||
|
||||
EMA maintains a moving average of the model weights throughout training. This smoothed version often generalizes better than the raw weights and is commonly used for the final model.
|
||||
|
||||
## Checkpoint Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| --------------------- | ----- | ------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `checkpoint_interval` | `int` | `10` | Frequency (in epochs) at which model checkpoints are saved. More frequent saves provide better coverage but consume more storage. |
|
||||
| `skip_best_epochs` | `int` | `0` | Ignore the first N epochs when tracking best checkpoints and early-stopping patience. Useful when fine-tuning from a prior checkpoint. |
|
||||
|
||||
### Checkpoint Files
|
||||
|
||||
During training, multiple checkpoints are saved:
|
||||
|
||||
| File | Description |
|
||||
| ----------------------------- | ----------------------------------------- |
|
||||
| `checkpoint.pth` | Most recent checkpoint (for resuming) |
|
||||
| `checkpoint_<N>.pth` | Periodic checkpoint at epoch N |
|
||||
| `checkpoint_best_ema.pth` | Best validation performance (EMA weights) |
|
||||
| `checkpoint_best_regular.pth` | Best validation performance (raw weights) |
|
||||
| `checkpoint_best_total.pth` | Final best model for inference |
|
||||
|
||||
Best validation performance uses the task metric for the model family: box mAP for detection/segmentation and COCO
|
||||
keypoint AP for keypoint preview.
|
||||
|
||||
## Early Stopping Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| -------------------------- | ------- | ------- | ---------------------------------------------------------------------------------------- |
|
||||
| `early_stopping` | `bool` | `False` | Enable early stopping based on the validation task metric. |
|
||||
| `early_stopping_patience` | `int` | `10` | Number of epochs without improvement before stopping. |
|
||||
| `early_stopping_min_delta` | `float` | `0.001` | Minimum metric change to qualify as an improvement. |
|
||||
| `early_stopping_use_ema` | `bool` | `False` | Whether to track improvements using EMA model metrics. |
|
||||
| `skip_best_epochs` | `int` | `0` | Ignore the first N epochs (0..N-1) for best-model selection and early-stopping patience. |
|
||||
|
||||
### Early Stopping Example
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=200,
|
||||
batch_size=4,
|
||||
early_stopping=True,
|
||||
early_stopping_patience=15,
|
||||
early_stopping_min_delta=0.005,
|
||||
skip_best_epochs=3,
|
||||
)
|
||||
```
|
||||
|
||||
This configuration will:
|
||||
|
||||
- Train for up to 200 epochs
|
||||
- Ignore epochs 0-2 for best-checkpoint tracking and patience counting
|
||||
- Stop early if the validation metric doesn't improve by at least 0.005 for 15 consecutive epochs
|
||||
|
||||
!!! note "Transfer learning with `pretrain_weights`"
|
||||
|
||||
When fine-tuning from `pretrain_weights`, the pretrained model's epoch-0 validation metric can be artificially high
|
||||
relative to the training trajectory on the new dataset. This causes `checkpoint_best_total.pth` to always contain
|
||||
the untrained pretrained weights and may trigger early stopping prematurely. Use `skip_best_epochs` to defer
|
||||
best-checkpoint selection and patience counting until the model has had time to adapt.
|
||||
|
||||
## Logging Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ------------- | ------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `tensorboard` | `bool` | `True` | Enable TensorBoard logging. Requires `pip install "rfdetr[loggers]"`. If the `tensorboard` package is not installed, training continues with a `UserWarning` and TensorBoard output is silently suppressed. |
|
||||
| `wandb` | `bool` | `False` | Enable Weights & Biases logging. Requires `pip install "rfdetr[loggers]"`. |
|
||||
| `project` | `str` | `None` | Project name for W&B logging. |
|
||||
| `run` | `str` | `None` | Run name for W&B logging. If not specified, W&B assigns a random name. |
|
||||
|
||||
### Logging Example
|
||||
|
||||
```python
|
||||
model.train(
|
||||
dataset_dir="path/to/dataset",
|
||||
epochs=100,
|
||||
tensorboard=True,
|
||||
wandb=True,
|
||||
project="my-detection-project",
|
||||
run="experiment-001",
|
||||
)
|
||||
```
|
||||
|
||||
## Evaluation Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ----------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------ |
|
||||
| `eval_max_dets` | `int` | `500` | Maximum number of detections per image considered during COCO evaluation. Lower values speed up evaluation. |
|
||||
| `eval_interval` | `int` | `1` | Run COCO evaluation every N epochs. Set to a higher value to reduce evaluation overhead during long training runs. |
|
||||
| `log_per_class_metrics` | `bool` | `True` | Log per-class AP metrics to the console and loggers. Disable to reduce log verbosity when there are many classes. |
|
||||
| `progress_bar` | str \| bool \| None | `None` | Progress bar style: `"tqdm"`, `"rich"`, or `None`. Legacy booleans are still accepted. |
|
||||
|
||||
## Keypoint Preview Parameters
|
||||
|
||||
These parameters apply when training `RFDETRKeypointPreview` on COCO keypoint annotations or Ultralytics YOLO pose labels.
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ----------------------------- | --------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `num_keypoints_per_class` | `list[int]` | `[17]` | **Constructor parameter** — pass to `RFDETRKeypointPreview(num_keypoints_per_class=...)`. Keypoint schema by model label slot. A zero entry marks a detection-only class slot; legacy checkpoints may use a background-first `[0, 17]` schema. |
|
||||
| `keypoint_flip_pairs` | `list[int]` | `[]` | Flat left/right keypoint index pairs used to swap joints after horizontal-flip augmentation. YOLO `flip_idx` metadata is a permutation; RF-DETR converts it to this pair-list form during automatic schema inference when possible — it extracts only symmetric mutual pairs where `flip_idx[i] == j` and `flip_idx[j] == i`. Asymmetric entries and self-mapped keypoints (`flip_idx[i] == i`) are silently omitted; supply `keypoint_flip_pairs` explicitly when your `flip_idx` includes such entries. |
|
||||
| `keypoint_l1_loss_coef` | `float` | `1.0` | Weight for keypoint coordinate L1 loss in keypoint preview training. |
|
||||
| `keypoint_findable_loss_coef` | `float` | `1.0` | Weight for keypoint findable/objectness loss. |
|
||||
| `keypoint_visible_loss_coef` | `float` | `1.0` | Weight for keypoint visibility loss. |
|
||||
| `keypoint_nll_loss_coef` | `float` | `1.0` | Weight for keypoint negative-log-likelihood loss. |
|
||||
| `keypoint_oks_sigmas` | `list[float] \| None` | `None` | Per-keypoint OKS sigma values used for COCO AP evaluation. When `None`, 17-keypoint person datasets use the evaluator's standard COCO sigmas and custom keypoint counts use RF-DETR's uniform custom fallback. Pass explicit values, such as schema-inferred sigmas, when you need a specific custom OKS policy. |
|
||||
|
||||
!!! note "OKS sigma values: flat vs per-keypoint"
|
||||
|
||||
`infer_coco_keypoint_schema` and `infer_yolo_keypoint_schema` return a flat sigma of 0.1 for all inferred keypoints, and the keypoint demos pass those values explicitly for custom datasets. If `keypoint_oks_sigmas=None`, COCO person-keypoint evaluation uses the standard 17-keypoint COCO sigmas, while non-17 custom keypoint counts use RF-DETR's uniform custom fallback. Flat custom sigmas are not directly comparable to official COCO benchmark numbers.
|
||||
|
||||
## Advanced Parameters
|
||||
|
||||
The parameters below are available for fine-grained control over training behaviour. Most users can leave these at their defaults.
|
||||
|
||||
### Scheduler and Regularization
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| --------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------------- |
|
||||
| `lr_scheduler` | `str` | `"step"` | Learning rate scheduler type. Options: `"step"` (step decay at `lr_drop`) or `"cosine"` (cosine annealing). |
|
||||
| `lr_min_factor` | `float` | `0.0` | Floor for the cosine scheduler, expressed as a fraction of the initial LR. Ignored when using `"step"`. |
|
||||
| `warmup_epochs` | `float` | `0.0` | Number of epochs for linear learning rate warmup at the start of training. |
|
||||
| `drop_path` | `float` | `0.0` | Stochastic depth drop-path rate applied to the backbone. Higher values add more regularization. |
|
||||
|
||||
### Runtime and Accelerator
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| ------------------- | ------ | -------- | ------------------------------------------------------------------------------------------------ |
|
||||
| `accelerator` | `str` | `"auto"` | PyTorch Lightning accelerator selection. `"auto"` picks GPU if available, then MPS, then CPU. |
|
||||
| `seed` | `int` | `None` | Global random seed for reproducibility. `None` means no fixed seed is set. |
|
||||
| `fp16_eval` | `bool` | `False` | Run evaluation passes in FP16 precision. Reduces memory usage but may lower numerical precision. |
|
||||
| `compute_val_loss` | `bool` | `True` | Compute and log the detection loss on the validation set each epoch. |
|
||||
| `compute_test_loss` | `bool` | `True` | Compute and log the detection loss during the final test run. |
|
||||
|
||||
### DataLoader Tuning
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| -------------------- | ------ | ------- | --------------------------------------------------------------------------------------------------------- |
|
||||
| `pin_memory` | `bool` | `None` | Pin host memory in the DataLoader for faster GPU transfers. `None` defers to PyTorch Lightning's default. |
|
||||
| `persistent_workers` | `bool` | `None` | Keep DataLoader worker processes alive between epochs. `None` defers to PyTorch Lightning's default. |
|
||||
| `prefetch_factor` | `int` | `None` | Number of batches to prefetch per DataLoader worker. `None` uses PyTorch's built-in default. |
|
||||
|
||||
## Complete Parameter Reference
|
||||
|
||||
Below is a summary table of all training parameters:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
| -------------------------- | ------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `dataset_dir` | str | Required | Path to COCO or YOLO formatted dataset with train/valid/test splits. |
|
||||
| `output_dir` | str | "output" | Directory for checkpoints, logs, and other training artifacts. |
|
||||
| `epochs` | int | 100 | Number of full passes over the dataset. |
|
||||
| `batch_size` | int or "auto" | 4 | Samples per iteration. Set to `"auto"` to let RF-DETR probe the GPU for the largest safe batch size. Balance with `grad_accum_steps`. |
|
||||
| `grad_accum_steps` | int | 4 | Gradient accumulation steps for effective larger batch sizes. |
|
||||
| `lr` | float | 1e-4 | Learning rate for the model (excluding encoder). |
|
||||
| `lr_encoder` | float | 1.5e-4 | Learning rate for the backbone encoder. |
|
||||
| `resolution` | int | Model-specific | Input image size (must be divisible by the selected model's `patch_size * num_windows`). |
|
||||
| `weight_decay` | float | 1e-4 | L2 regularization coefficient. |
|
||||
| `device` | str | "cuda" | Training device: cuda, cpu, or mps. |
|
||||
| `use_ema` | bool | True | Enable Exponential Moving Average of weights. |
|
||||
| `gradient_checkpointing` | bool | False | Trade compute for memory during backprop. |
|
||||
| `checkpoint_interval` | int | 10 | Save checkpoint every N epochs. |
|
||||
| `resume` | str | None | Path to checkpoint for resuming training. |
|
||||
| `tensorboard` | bool | True | Enable TensorBoard logging. |
|
||||
| `wandb` | bool | False | Enable Weights & Biases logging. |
|
||||
| `project` | str | None | W&B project name. |
|
||||
| `run` | str | None | W&B run name. |
|
||||
| `early_stopping` | bool | False | Enable early stopping. |
|
||||
| `early_stopping_patience` | int | 10 | Epochs without improvement before stopping. |
|
||||
| `early_stopping_min_delta` | float | 0.001 | Minimum validation metric change to qualify as improvement. |
|
||||
| `early_stopping_use_ema` | bool | False | Use EMA model for early stopping metrics. |
|
||||
| `eval_max_dets` | int | 500 | Maximum detections per image considered during COCO evaluation. |
|
||||
| `eval_interval` | int | 1 | Run COCO evaluation every N epochs. |
|
||||
| `log_per_class_metrics` | bool | True | Log per-class AP metrics to the console and loggers. |
|
||||
| `progress_bar` | str \| bool \| None | None | Progress bar style: `"tqdm"`, `"rich"`, or `None`. Legacy booleans are still accepted. |
|
||||
| `accelerator` | str | "auto" | PyTorch Lightning accelerator. "auto" selects GPU/MPS/CPU automatically. |
|
||||
| `seed` | int | None | Random seed for reproducibility. None means no fixed seed. |
|
||||
| `lr_scheduler` | str | "step" | Learning rate scheduler type: "step" or "cosine". |
|
||||
| `lr_min_factor` | float | 0.0 | Minimum LR as a fraction of the initial LR (cosine scheduler floor). |
|
||||
| `warmup_epochs` | float | 0.0 | Number of linear warmup epochs at the start of training. |
|
||||
| `drop_path` | float | 0.0 | Stochastic depth drop-path rate for the backbone. |
|
||||
| `compute_val_loss` | bool | True | Compute and log loss during validation. |
|
||||
| `compute_test_loss` | bool | True | Compute and log loss during the test run. |
|
||||
| `fp16_eval` | bool | False | Run evaluation in FP16 precision to reduce memory usage. |
|
||||
| `pin_memory` | bool | None | Pin DataLoader memory. None defers to PyTorch Lightning's default. |
|
||||
| `persistent_workers` | bool | None | Keep DataLoader workers alive between epochs. None uses PTL default. |
|
||||
| `prefetch_factor` | int | None | Number of batches prefetched per worker. None uses PyTorch default. |
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
description: RF-DETR 2XLarge API reference for the highest-accuracy detection model. Requires rfdetr[plus] and the Platform Model License.
|
||||
---
|
||||
|
||||
!!! warning "License Notice"
|
||||
This model is licensed under the Platform Model License (PML-1.0) and requires `pip install rfdetr[plus]`. See the [rfdetr_plus repository](https://github.com/roboflow/rf-detr-plus) for license details.
|
||||
|
||||
:::rfdetr.platform.models.RFDETR2XLarge
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: KeypointTrainConfig API reference for configuring RF-DETR keypoint training parameters with keypoint loss coefficients and optimization settings.
|
||||
---
|
||||
|
||||
:::rfdetr.config.KeypointTrainConfig
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: API reference for the RF-DETR keypoint preview model — keypoint prediction and fine-tuning with inherited training and inference methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRKeypointPreview
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Large API reference for high-accuracy real-time object detection with inherited training, prediction, export, and deployment methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRLarge
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Medium API reference for balanced real-time object detection with inherited training, prediction, export, and deployment methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRMedium
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Nano API reference for the fastest real-time object detection model with inherited training, prediction, export, and deployment methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRNano
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
description: RFDETR base class API reference covering common train, predict, export, optimize, and deploy methods for RF-DETR models.
|
||||
---
|
||||
|
||||
:::rfdetr.detr.RFDETR
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Seg 2XLarge API reference for the highest-accuracy instance segmentation model with inherited training and inference methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSeg2XLarge
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Seg Large API reference for high-accuracy real-time instance segmentation with inherited training and inference methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSegLarge
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Seg Medium API reference for balanced real-time instance segmentation with inherited training and inference methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSegMedium
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Seg Nano API reference for the fastest instance segmentation model with inherited training, prediction, and export methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSegNano
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Seg Small API reference for efficient real-time instance segmentation with inherited training and inference methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSegSmall
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Seg XLarge API reference for high-capacity instance segmentation with inherited training, prediction, and export methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSegXLarge
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: SegmentationTrainConfig API reference for configuring RF-DETR segmentation training parameters, dataset options, and optimization settings.
|
||||
---
|
||||
|
||||
:::rfdetr.config.SegmentationTrainConfig
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: RF-DETR Small API reference for efficient real-time object detection with inherited training, prediction, export, and deployment methods.
|
||||
---
|
||||
|
||||
:::rfdetr.variants.RFDETRSmall
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: TrainConfig API reference for RF-DETR detection training parameters, dataset configuration, optimization settings, and checkpoint behavior.
|
||||
---
|
||||
|
||||
:::rfdetr.config.TrainConfig
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,126 @@
|
||||
---
|
||||
description: RF-DETR Lightning training API reference for RFDETRModelModule, RFDETRDataModule, build_trainer, callbacks, and training primitives.
|
||||
---
|
||||
|
||||
# Training API Reference
|
||||
|
||||
This page documents the training primitives that power RF-DETR. For a narrative guide with runnable examples, see [Custom Training API](../learn/train/customization.md).
|
||||
|
||||
## RFDETRModelModule
|
||||
|
||||
::: rfdetr.training.module_model.RFDETRModelModule
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
- on_fit_start
|
||||
- on_train_batch_start
|
||||
- transfer_batch_to_device
|
||||
- training_step
|
||||
- validation_step
|
||||
- test_step
|
||||
- predict_step
|
||||
- configure_optimizers
|
||||
- clip_gradients
|
||||
- on_load_checkpoint
|
||||
- reinitialize_detection_head
|
||||
|
||||
---
|
||||
|
||||
## RFDETRDataModule
|
||||
|
||||
::: rfdetr.training.module_data.RFDETRDataModule
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
- setup
|
||||
- train_dataloader
|
||||
- val_dataloader
|
||||
- test_dataloader
|
||||
- class_names
|
||||
|
||||
---
|
||||
|
||||
## build_trainer
|
||||
|
||||
::: rfdetr.training.trainer.build_trainer
|
||||
options:
|
||||
show_source: false
|
||||
|
||||
---
|
||||
|
||||
## Callbacks
|
||||
|
||||
### RFDETREMACallback
|
||||
|
||||
::: rfdetr.training.callbacks.ema.RFDETREMACallback
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
|
||||
### BestModelCallback
|
||||
|
||||
::: rfdetr.training.callbacks.best_model.BestModelCallback
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
|
||||
### RFDETREarlyStopping
|
||||
|
||||
::: rfdetr.training.callbacks.best_model.RFDETREarlyStopping
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
|
||||
### DropPathCallback
|
||||
|
||||
::: rfdetr.training.callbacks.drop_schedule.DropPathCallback
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
|
||||
### COCOEvalCallback
|
||||
|
||||
::: rfdetr.training.callbacks.coco_eval.COCOEvalCallback
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
|
||||
---
|
||||
|
||||
## RFDETRCli
|
||||
|
||||
!!! info "CLI requires the `train` and `cli` extras"
|
||||
|
||||
```bash
|
||||
pip install "rfdetr[train,cli]"
|
||||
```
|
||||
|
||||
The `rfdetr` console script and its `--config` / `--print_config` flags are provided by `jsonargparse`, which is included in the `cli` extra.
|
||||
|
||||
`RFDETRCli` is the command-line entry point for RF-DETR. It wraps
|
||||
`RFDETRModelModule` and `RFDETRDataModule` under a single `rfdetr` command and
|
||||
auto-generates four subcommands from the PyTorch Lightning CLI machinery:
|
||||
|
||||
```bash
|
||||
rfdetr fit --config configs/rfdetr_base.yaml
|
||||
rfdetr validate --ckpt_path output/best.ckpt
|
||||
rfdetr test --ckpt_path output/best.ckpt
|
||||
rfdetr predict --ckpt_path output/best.ckpt
|
||||
```
|
||||
|
||||
Both `model_config` and `train_config` are specified once; `RFDETRCli`
|
||||
automatically links them to the datamodule so you do not need to repeat the
|
||||
same arguments under `--data.*`.
|
||||
|
||||
::: rfdetr.training.cli.RFDETRCli
|
||||
options:
|
||||
show_source: false
|
||||
members:
|
||||
- __init__
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
description: RF-DETR XLarge API reference for high-capacity detection. Requires rfdetr[plus] and the Platform Model License.
|
||||
---
|
||||
|
||||
!!! warning "License Notice"
|
||||
This model is licensed under the Platform Model License (PML-1.0) and requires `pip install rfdetr[plus]`. See the [rfdetr_plus repository](https://github.com/roboflow/rf-detr-plus) for license details.
|
||||
|
||||
:::rfdetr.platform.models.RFDETRXLarge
|
||||
options:
|
||||
inherited_members: true
|
||||
@@ -0,0 +1,103 @@
|
||||
---
|
||||
description: RF-DETR resources — blog posts, videos, and guides for training, deploying, and running detection and segmentation models across cloud, edge, iOS, Jetson, and Lightning workflows.
|
||||
hide:
|
||||
- toc
|
||||
- navigation
|
||||
---
|
||||
|
||||
# Resources
|
||||
|
||||
Use the resources below to learn how to train, build with, and deploy RF-DETR models both in the cloud and on your own hardware.
|
||||
|
||||
<div class="grid cards" markdown>
|
||||
|
||||
- **SOTA Instance Segmentation with RF-DETR Seg (Preview) [October 2025]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Read more about the RF-DETR Segmentation architecture and how to train an RF-DETR Seg (Preview) model.
|
||||
|
||||
[:octicons-arrow-right-24: Learn more](https://blog.roboflow.com/rf-detr-segmentation-preview/)
|
||||
|
||||
- **Announcing RF-DETR Nano, Small, and Medium [July 2025]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Read more about the state-of-the-art RF-DETR Nano, Small, and Medium models we released in July 2025.
|
||||
|
||||
[:octicons-arrow-right-24: Learn more](https://blog.roboflow.com/rf-detr-nano-small-medium/)
|
||||
|
||||
- **RF-DETR: How to Train SOTA for Object Detection on a Custom Dataset [video]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Learn how to train an RF-DETR model on a custom dataset.
|
||||
|
||||
[:octicons-arrow-right-24: Watch the video](https://www.youtube.com/watch?v=-OvpdLAElFA)
|
||||
|
||||
- **How to Train RF-DETR on a Custom Dataset [article]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Learn how to train an RF-DETR model on a custom dataset.
|
||||
|
||||
[:octicons-arrow-right-24: Read the guide](https://blog.roboflow.com/train-rf-detr-on-a-custom-dataset/)
|
||||
|
||||
- **Deploy RF-DETR on iOS [tutorial & example application]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Read this guide to learn how to run RF-DETR models on an iPhone using the Roboflow iOS SDK.
|
||||
|
||||
[:octicons-arrow-right-24: Get started](https://blog.roboflow.com/ios-rf-detr-nano/)
|
||||
|
||||
- **How to Deploy RF-DETR to an NVIDIA Jetson [article]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Learn how to deploy an RF-DETR model to an NVIDIA Jetson using Roboflow Inference.
|
||||
|
||||
[:octicons-arrow-right-24: Read the tutorial](https://blog.roboflow.com/how-to-deploy-rf-detr-to-an-nvidia-jetson/)
|
||||
|
||||
- **Deploy RF-DETR with LitServe [Lightning AI Studio]**
|
||||
|
||||
---
|
||||
|
||||
|
||||
Learn how to deploy RF-DETR as a scalable inference server using LitServe, the AI model serving framework from Lightning AI.
|
||||
|
||||
[:octicons-arrow-right-24: Open the Studio](https://lightning.ai/bhimrajyadav/studios/deploy-rf-detr-a-sota-real-time-object-detection-model-using-litserve)
|
||||
|
||||
- **Train and Deploy RF-DETR Models with Roboflow**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Learn how to train RF-DETR models in the cloud with Roboflow and deploy your models on your own hardware with Roboflow Inference and Workflows.
|
||||
|
||||
[:octicons-arrow-right-24: Get started](https://blog.roboflow.com/train-deploy-rf-detr/)
|
||||
|
||||
- **RF-DETR: A SOTA Real-Time Object Detection Model**
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
Read our announcement for RF-DETR, the first real-time model to achieve 60+ mean Average Precision when benchmarked on the COCO dataset.
|
||||
|
||||
[:octicons-arrow-right-24: Read the announcement](https://blog.roboflow.com/rf-detr/)
|
||||
|
||||
</div>
|
||||
@@ -0,0 +1 @@
|
||||
cace9ef287cb8d641db90d4295fcb1e1
|
||||
@@ -0,0 +1,361 @@
|
||||
# RF-DETR — Full Documentation
|
||||
|
||||
> RF-DETR is a real-time transformer architecture for object detection and instance segmentation by Roboflow.
|
||||
> DINOv2 vision transformer backbone. ICLR 2026. SOTA on COCO (60.1 AP50:95, RF-DETR-2XL).
|
||||
> Apache 2.0 license for core models (Nano through Large). Python 3.10+.
|
||||
> Source: https://github.com/roboflow/rf-detr | Paper: https://arxiv.org/abs/2511.09554
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy–latency trade-offs on Microsoft COCO and RF100-VL.
|
||||
|
||||
RF-DETR is the first real-time model to exceed 60 mean Average Precision (mAP) when benchmarked on the COCO dataset. RF-DETR-2XL achieves 60.1 AP50:95 at 17.2 ms latency on NVIDIA T4 (TensorRT FP16, batch 1).
|
||||
|
||||
**Licensing:**
|
||||
- Core models (Nano, Small, Medium, Large) and all code: Apache 2.0
|
||||
- XLarge and 2XLarge detection models: PML 1.0 (requires rfdetr[plus])
|
||||
- Segmentation models follow the same tier structure
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
**Requirements:** Python 3.10 or higher, pip or uv.
|
||||
|
||||
```bash
|
||||
pip install rfdetr
|
||||
```
|
||||
|
||||
For uv projects:
|
||||
```bash
|
||||
uv add rfdetr
|
||||
```
|
||||
|
||||
For XLarge/2XLarge models (PML 1.0):
|
||||
```bash
|
||||
pip install "rfdetr[plus]"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Run Detection
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRLarge
|
||||
|
||||
model = RFDETRLarge()
|
||||
detections = model.predict("image.jpg", threshold=0.5)
|
||||
print(detections)
|
||||
```
|
||||
|
||||
**Available detection model classes:**
|
||||
- `RFDETRNano` — 2.3 ms, 67.6 AP50, 30.5M params, 384×384
|
||||
- `RFDETRSmall` — 3.5 ms, 72.1 AP50, 32.1M params, 512×512
|
||||
- `RFDETRMedium` — 4.4 ms, 73.6 AP50, 33.7M params, 576×576
|
||||
- `RFDETRLarge` — 6.8 ms, 75.1 AP50 (56.5 AP50:95), 33.9M params, 704×704
|
||||
- `RFDETRXLarge` — 11.5 ms, 77.4 AP50, 126.4M params, 700×700 (requires rfdetr[plus])
|
||||
- `RFDETR2XLarge` — 17.2 ms, 78.5 AP50 (60.1 AP50:95), 126.9M params, 880×880 (requires rfdetr[plus])
|
||||
|
||||
All latency figures: NVIDIA T4, TensorRT FP16, batch size 1.
|
||||
|
||||
**Load fine-tuned checkpoint:**
|
||||
```python
|
||||
model = RFDETRLarge(pretrain_weights="path/to/checkpoint_best_total.pth")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Run Segmentation
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegLarge
|
||||
|
||||
model = RFDETRSegLarge()
|
||||
detections = model.predict("image.jpg", threshold=0.5)
|
||||
```
|
||||
|
||||
**Available segmentation model classes:**
|
||||
- `RFDETRSegNano` — 3.4 ms, 63.0 AP50, 33.6M params, 312×312
|
||||
- `RFDETRSegSmall` — 4.4 ms, 66.2 AP50, 33.7M params, 384×384
|
||||
- `RFDETRSegMedium` — 5.9 ms, 68.4 AP50, 35.7M params, 432×432
|
||||
- `RFDETRSegLarge` — 8.8 ms, 70.5 AP50 (47.1 AP50:95), 36.2M params, 504×504
|
||||
- `RFDETRSegXLarge` — 13.5 ms, 72.2 AP50, 38.1M params, 624×624
|
||||
- `RFDETRSeg2XLarge` — 21.8 ms, 73.1 AP50 (49.9 AP50:95), 38.6M params, 768×768
|
||||
|
||||
Segmentation models output instance masks in addition to bounding boxes. Masks are returned as a `torch.Tensor` or dict with `spatial_features`, `query_features`, and `bias` keys.
|
||||
|
||||
---
|
||||
|
||||
## Train a Custom Model
|
||||
|
||||
### Quick Start (Detection)
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRLarge
|
||||
|
||||
model = RFDETRLarge()
|
||||
model.train(
|
||||
dataset_dir="./dataset",
|
||||
epochs=50,
|
||||
batch_size=4,
|
||||
grad_accum_steps=4, # effective batch = batch_size * grad_accum_steps = 16
|
||||
lr=1e-4,
|
||||
output_dir="./output"
|
||||
)
|
||||
```
|
||||
|
||||
### Quick Start (Segmentation)
|
||||
|
||||
```python
|
||||
from rfdetr import RFDETRSegLarge
|
||||
|
||||
model = RFDETRSegLarge()
|
||||
model.train(dataset_dir="./dataset", epochs=50, batch_size=4)
|
||||
```
|
||||
|
||||
### Dataset Formats
|
||||
|
||||
**COCO JSON format** (auto-detected):
|
||||
```
|
||||
dataset/
|
||||
train/
|
||||
_annotations.coco.json
|
||||
image1.jpg
|
||||
...
|
||||
valid/
|
||||
_annotations.coco.json
|
||||
image1.jpg
|
||||
...
|
||||
```
|
||||
|
||||
**YOLO format** (set `dataset_file="yolo"`):
|
||||
```
|
||||
dataset/
|
||||
data.yaml
|
||||
train/
|
||||
images/
|
||||
labels/
|
||||
valid/
|
||||
images/
|
||||
labels/
|
||||
```
|
||||
|
||||
### Key Training Parameters
|
||||
|
||||
| Parameter | Default | Description |
|
||||
|-----------|---------|-------------|
|
||||
| `epochs` | 50 | Training epochs |
|
||||
| `batch_size` | 4 | Images per GPU step |
|
||||
| `grad_accum_steps` | 4 | Gradient accumulation steps (effective batch = batch_size × grad_accum_steps) |
|
||||
| `lr` | 1e-4 | Peak learning rate |
|
||||
| `output_dir` | `"output"` | Checkpoint directory |
|
||||
| `checkpoint_interval` | 1 | Save checkpoint every N epochs |
|
||||
|
||||
**Recommended effective batch size: 16** (e.g., batch_size=4, grad_accum_steps=4 on 8GB VRAM GPU).
|
||||
|
||||
### Checkpoint Types
|
||||
|
||||
After training, the `output_dir` contains:
|
||||
- `checkpoint_best_total.pth` — best checkpoint by total loss (use for production)
|
||||
- `checkpoint_best_ap.pth` — best checkpoint by COCO AP
|
||||
- `checkpoint_epoch_N.pth` — periodic snapshots
|
||||
|
||||
**Load fine-tuned model:**
|
||||
```python
|
||||
model = RFDETRLarge(pretrain_weights="output/checkpoint_best_total.pth")
|
||||
```
|
||||
|
||||
### Advanced Training
|
||||
|
||||
**Resume from checkpoint:**
|
||||
```python
|
||||
model.train(dataset_dir="./dataset", resume="output/checkpoint_epoch_20.pth")
|
||||
```
|
||||
|
||||
**Multi-GPU DDP training:**
|
||||
```bash
|
||||
torchrun --nproc_per_node=4 train.py
|
||||
```
|
||||
|
||||
Here, `train.py` refers to your own PyTorch distributed training entrypoint. In this standalone reference, it should be a script that initializes RF-DETR and calls `model.train(...)` with your dataset and training arguments, so `torchrun` can launch one process per GPU.
|
||||
**Gradient checkpointing** (reduces VRAM at cost of speed):
|
||||
```python
|
||||
model.train(dataset_dir="./dataset", gradient_checkpointing=True)
|
||||
```
|
||||
|
||||
### Training Loggers
|
||||
|
||||
```python
|
||||
# TensorBoard
|
||||
model.train(dataset_dir="./dataset", tensorboard=True)
|
||||
|
||||
# Weights and Biases
|
||||
model.train(dataset_dir="./dataset", wandb=True)
|
||||
|
||||
# ClearML is not yet integrated as a native RF-DETR logger.
|
||||
# Do not pass clearml=True; that flag raises NotImplementedError
|
||||
# when the trainer is built.
|
||||
|
||||
# MLflow
|
||||
model.train(dataset_dir="./dataset", mlflow=True)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Export and Deploy
|
||||
|
||||
### Export to ONNX
|
||||
|
||||
```python
|
||||
model.export(format="onnx")
|
||||
```
|
||||
|
||||
Produces a single `.onnx` file compatible with ONNX Runtime and OpenCV DNN. Export works on CPU.
|
||||
|
||||
### Export to TFLite
|
||||
|
||||
```python
|
||||
model.export(format="tflite")
|
||||
```
|
||||
|
||||
### TensorRT Deployment
|
||||
|
||||
Export to ONNX first, then convert with TensorRT tooling:
|
||||
```bash
|
||||
trtexec --onnx=model.onnx --saveEngine=model.trt --fp16
|
||||
```
|
||||
|
||||
### Deploy to Roboflow
|
||||
|
||||
```python
|
||||
model.deploy_to_roboflow(
|
||||
workspace="your-workspace",
|
||||
project_id="your-project-id",
|
||||
version=1,
|
||||
api_key="YOUR_API_KEY"
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Benchmarks
|
||||
|
||||
**Benchmark methodology:** Accuracy measured with standard COCO metrics (pycocotools) on COCO val2017 split. Latency measured on NVIDIA T4 GPU, TensorRT 10.4, CUDA 12.4, FP16 precision, batch size 1, with a 200ms buffer between passes to reduce thermal variance. All accuracy and latency measurements use the same model artifact and numerical precision.
|
||||
|
||||
### Detection Results
|
||||
|
||||
| Architecture | COCO AP50 | COCO AP50:95 | RF100VL AP50 | RF100VL AP50:95 | Latency (ms) | Params (M) | Resolution |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| RF-DETR-N | 67.6 | 48.4 | 85.0 | 57.7 | 2.3 | 30.5 | 384×384 |
|
||||
| RF-DETR-S | 72.1 | 53.0 | 86.7 | 60.2 | 3.5 | 32.1 | 512×512 |
|
||||
| RF-DETR-M | 73.6 | 54.7 | 87.4 | 61.2 | 4.4 | 33.7 | 576×576 |
|
||||
| RF-DETR-L | 75.1 | 56.5 | 88.2 | 62.2 | 6.8 | 33.9 | 704×704 |
|
||||
| RF-DETR-XL | 77.4 | 58.6 | 88.5 | 62.9 | 11.5 | 126.4 | 700×700 |
|
||||
| RF-DETR-2XL | 78.5 | 60.1 | 89.0 | 63.2 | 17.2 | 126.9 | 880×880 |
|
||||
|
||||
### Segmentation Results
|
||||
|
||||
| Architecture | COCO AP50 | COCO AP50:95 | Latency (ms) | Params (M) | Resolution |
|
||||
|---|---|---|---|---|---|
|
||||
| RF-DETR-Seg-N | 63.0 | 40.3 | 3.4 | 33.6 | 312×312 |
|
||||
| RF-DETR-Seg-S | 66.2 | 43.1 | 4.4 | 33.7 | 384×384 |
|
||||
| RF-DETR-Seg-M | 68.4 | 45.3 | 5.9 | 35.7 | 432×432 |
|
||||
| RF-DETR-Seg-L | 70.5 | 47.1 | 8.8 | 36.2 | 504×504 |
|
||||
| RF-DETR-Seg-XL | 72.2 | 48.8 | 13.5 | 38.1 | 624×624 |
|
||||
| RF-DETR-Seg-2XL | 73.1 | 49.9 | 21.8 | 38.6 | 768×768 |
|
||||
|
||||
---
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
**What is RF-DETR?**
|
||||
RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy–latency trade-offs on COCO (60.1 AP50:95 for RF-DETR-2XL) and RF100-VL.
|
||||
|
||||
**How does RF-DETR compare to YOLO11?**
|
||||
RF-DETR-L achieves 56.5 AP50:95 on COCO at 6.8 ms latency on an NVIDIA T4, outperforming YOLO11x (54.7 AP) at lower latency. The DINOv2 backbone gives RF-DETR stronger performance on domain-shift benchmarks such as RF100-VL.
|
||||
|
||||
**What GPU is required to train RF-DETR?**
|
||||
A CUDA-capable GPU with at least 8 GB VRAM (e.g., NVIDIA RTX 3060, T4, A10) is recommended for fine-tuning. Smaller models (RF-DETR-N and RF-DETR-S) can fit in 6 GB VRAM with reduced batch size. CPU inference is supported for evaluation.
|
||||
|
||||
**Which dataset formats does RF-DETR support?**
|
||||
RF-DETR supports COCO JSON and YOLO-format datasets. Roboflow datasets export directly to both formats. Detection and segmentation datasets use the same format — the model variant determines the task.
|
||||
|
||||
**Can RF-DETR run in real time?**
|
||||
Yes. RF-DETR-N runs at 2.3 ms per frame on a T4 GPU (TensorRT FP16, batch 1), and RF-DETR-L at 6.8 ms — both well within real-time thresholds. ONNX and TFLite exports are available for edge deployment.
|
||||
|
||||
**What is the difference between RF-DETR detection and segmentation models?**
|
||||
Detection models output bounding boxes. Segmentation models additionally output instance masks. Both share the same backbone and training API; segmentation adds a mask head and requires COCO-format segmentation annotations.
|
||||
|
||||
**Is RF-DETR open source?**
|
||||
Yes. Core models (Nano through Large) and all training/inference code are released under the Apache 2.0 license. XLarge and 2XLarge models require the rfdetr[plus] package (PML 1.0 license).
|
||||
|
||||
**How do I fine-tune RF-DETR on a custom dataset?**
|
||||
Instantiate a model and call model.train() with your dataset directory in COCO JSON or YOLO format. Example: model = RFDETRLarge(); model.train(dataset_dir='./dataset', epochs=50, batch_size=4). The model downloads pretrained weights automatically and resumes from the best checkpoint.
|
||||
|
||||
**How do I export RF-DETR to ONNX or TensorRT?**
|
||||
Call model.export(format='onnx') after training or loading a checkpoint. ONNX export works on CPU. For TensorRT, export to ONNX first, then convert with trtexec --onnx=model.onnx --saveEngine=model.trt --fp16.
|
||||
|
||||
**Which RF-DETR model size should I use?**
|
||||
RF-DETR-Nano (2.3 ms, 67.6 AP50) is best for edge/real-time. RF-DETR-Large (6.8 ms, 56.5 AP50:95) is best accuracy–latency trade-off for server deployment. RF-DETR-2XLarge (17.2 ms, 60.1 AP50:95) maximizes accuracy when latency budget allows.
|
||||
|
||||
Checkpoint note: current RFDETRLarge defaults to rf-detr-large-2026.pth. The older rf-detr-large.pth checkpoint is a legacy Large release kept for backward compatibility and has been superseded by the current release.
|
||||
|
||||
---
|
||||
|
||||
## API Reference
|
||||
|
||||
### RFDETR base class
|
||||
|
||||
All model classes inherit from `RFDETR` and share these methods:
|
||||
|
||||
- `predict(image, threshold=0.5)` — run inference on a single image (path, URL, numpy array, or PIL Image)
|
||||
- `train(dataset_dir, epochs, batch_size, ...)` — fine-tune on custom dataset
|
||||
- `export(format)` — export to ONNX, TFLite, or TensorRT
|
||||
- `deploy_to_roboflow(workspace, project_id, version)` — deploy model to Roboflow hosted inference
|
||||
|
||||
### TrainConfig
|
||||
|
||||
Key fields for detection training configuration:
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `epochs` | int | Training epochs |
|
||||
| `batch_size` | int | Images per GPU step |
|
||||
| `grad_accum_steps` | int | Gradient accumulation steps |
|
||||
| `lr` | float | Peak learning rate |
|
||||
| `output_dir` | str | Checkpoint output directory |
|
||||
| `resume` | str | Path to checkpoint to resume from |
|
||||
| `gradient_checkpointing` | bool | Reduce VRAM at cost of speed |
|
||||
| `tensorboard` | bool | Enable TensorBoard logging |
|
||||
| `wandb` | bool | Enable Weights & Biases logging |
|
||||
|
||||
---
|
||||
|
||||
## Migration Guide
|
||||
|
||||
If upgrading from rfdetr < 1.4.0, update these imports:
|
||||
|
||||
```python
|
||||
# Old model class (deprecated)
|
||||
from rfdetr import RFDETRBase
|
||||
# New
|
||||
from rfdetr import RFDETRLarge
|
||||
|
||||
# Old
|
||||
from rfdetr.util.misc import get_rank
|
||||
# New (unchanged — still works)
|
||||
from rfdetr.util.misc import get_rank
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## External Links
|
||||
|
||||
- [GitHub](https://github.com/roboflow/rf-detr) — Source code, issues, pull requests (Apache 2.0)
|
||||
- [arXiv Paper](https://arxiv.org/abs/2511.09554) — RF-DETR: Real-Time Detection Transformers, ICLR 2026
|
||||
- [PyPI](https://pypi.org/project/rfdetr/) — pip install rfdetr
|
||||
- [Hugging Face Demo](https://huggingface.co/spaces/Roboflow/RF-DETR) — Interactive demo
|
||||
- [Discord](https://discord.gg/GbfgXGJ8Bk) — Community support
|
||||
- [Documentation](https://rfdetr.roboflow.com/) — Full docs
|
||||
@@ -0,0 +1,67 @@
|
||||
# RF-DETR
|
||||
> RF-DETR is a real-time object detection and instance segmentation transformer by Roboflow.
|
||||
> DINOv2 backbone. ICLR 2026. SOTA on COCO (60.1 AP50:95, RF-DETR-2XL).
|
||||
> Apache 2.0 for base models (Nano-Large). Install: pip install rfdetr.
|
||||
|
||||
## Canonical Facts
|
||||
- Product: RF-DETR (Roboflow Detection Transformer)
|
||||
- Python package: `rfdetr`
|
||||
- Maintainer: Roboflow
|
||||
- Canonical docs: https://rfdetr.roboflow.com/
|
||||
- Source repository: https://github.com/roboflow/rf-detr
|
||||
- Paper: https://arxiv.org/abs/2511.09554
|
||||
- Runtime: Python 3.10+, torch >=2.2.0, torchvision >=0.17.0, transformers >=5.1.0 and <6.0.0
|
||||
- Tasks: object detection and instance segmentation
|
||||
- Backbone: DINOv2 vision transformer
|
||||
- Dataset formats: COCO JSON and YOLO
|
||||
- Export targets: ONNX, TensorRT, and TFLite
|
||||
- License: Apache 2.0 for code and core Nano through Large models; XLarge and 2XLarge detection models require `rfdetr[plus]` and PML 1.0
|
||||
|
||||
## Getting Started
|
||||
- [Install](https://rfdetr.roboflow.com/latest/getting-started/install/): pip install rfdetr
|
||||
- [Run Detection](https://rfdetr.roboflow.com/latest/learn/run/detection/): Run RF-DETR detection on images, video, webcam, and streams
|
||||
- [Run Segmentation](https://rfdetr.roboflow.com/latest/learn/run/segmentation/): Run RF-DETR Seg instance segmentation on images and video
|
||||
- [Pretrained Models](https://rfdetr.roboflow.com/latest/learn/pretrained/): Nano, Small, Medium, Large, XLarge, and 2XLarge checkpoints
|
||||
|
||||
## Train
|
||||
- [Train Overview](https://rfdetr.roboflow.com/latest/learn/train/): Fine-tune RF-DETR detection and segmentation models
|
||||
- [Dataset Formats](https://rfdetr.roboflow.com/latest/learn/train/dataset-formats/): COCO JSON and YOLO dataset formats
|
||||
- [Training Parameters](https://rfdetr.roboflow.com/latest/learn/train/training-parameters/): RF-DETR hyperparameters and TrainConfig options
|
||||
- [Advanced Training](https://rfdetr.roboflow.com/latest/learn/train/advanced/): Resume, early stopping, multi-GPU DDP, and memory optimization
|
||||
- [Custom Training API](https://rfdetr.roboflow.com/latest/learn/train/customization/): PyTorch Lightning training primitives
|
||||
- [Augmentations](https://rfdetr.roboflow.com/latest/learn/train/augmentations/): Albumentations presets and custom transforms
|
||||
- [Training Loggers](https://rfdetr.roboflow.com/latest/learn/train/loggers/): TensorBoard, Weights and Biases, MLflow, and a ClearML workaround
|
||||
|
||||
## Export and Deploy
|
||||
- [Export](https://rfdetr.roboflow.com/latest/learn/export/): ONNX, TFLite, TensorRT
|
||||
- [Deploy](https://rfdetr.roboflow.com/latest/learn/deploy/): deploy_to_roboflow() API
|
||||
|
||||
## Benchmarks
|
||||
- [COCO and RF100-VL Results](https://rfdetr.roboflow.com/latest/learn/benchmarks/): T4 TensorRT FP16, batch 1
|
||||
- Detection model range: RF-DETR-Nano 2.3 ms and 48.4 COCO AP50:95 through RF-DETR-2XLarge 17.2 ms and 60.1 COCO AP50:95
|
||||
- Segmentation model range: RF-DETR-Seg-Nano 3.4 ms and 40.3 COCO AP50:95 through RF-DETR-Seg-2XLarge 21.8 ms and 49.9 COCO AP50:95
|
||||
|
||||
## Model Selection Answers
|
||||
- Use RF-DETR-Nano for lowest latency edge detection: 2.3 ms on T4 TensorRT FP16, 48.4 AP50:95 on COCO, 30.5 M params.
|
||||
- Use RF-DETR-Small (3.5 ms, 53.0 AP50:95) or RF-DETR-Medium (4.4 ms, 54.7 AP50:95) when latency is tight but accuracy must improve over Nano.
|
||||
- Use RF-DETR-Large for the best open Apache 2.0 accuracy-latency trade-off: 6.8 ms, 56.5 AP50:95 on COCO.
|
||||
- Use RF-DETR-XLarge (11.5 ms, 58.6 AP50:95) or RF-DETR-2XLarge (17.2 ms, 60.1 AP50:95) when maximum accuracy matters; requires rfdetr[plus] and PML 1.0 license.
|
||||
- Use RF-DETR-Seg-Large (8.8 ms, 47.1 AP50:95) or RF-DETR-Seg-2XLarge (21.8 ms, 49.9 AP50:95) when instance masks are required; detection-only models return bounding boxes only.
|
||||
|
||||
## API Reference
|
||||
- [RFDETR base class](https://rfdetr.roboflow.com/latest/reference/rfdetr/): train(), predict(), export(), deploy_to_roboflow()
|
||||
- [Detection models](https://rfdetr.roboflow.com/latest/reference/nano/): RFDETRNano through RFDETR2XLarge
|
||||
- [Segmentation models](https://rfdetr.roboflow.com/latest/reference/seg_nano/): RFDETRSegNano through RFDETRSeg2XLarge
|
||||
- [TrainConfig](https://rfdetr.roboflow.com/latest/reference/train_config/): Detection training configuration schema
|
||||
- [SegmentationTrainConfig](https://rfdetr.roboflow.com/latest/reference/segmentation_train_config/): Segmentation training configuration schema
|
||||
- [Lightning Training](https://rfdetr.roboflow.com/latest/reference/training/): PyTorch Lightning training module, datamodule, callbacks, and trainer builder
|
||||
|
||||
## Migration
|
||||
- [Migration Guide](https://rfdetr.roboflow.com/latest/learn/migration/): Update imports from deprecated module paths
|
||||
|
||||
## Optional
|
||||
- [GitHub](https://github.com/roboflow/rf-detr): 6.4k stars, Apache 2.0
|
||||
- [arXiv Paper](https://arxiv.org/abs/2511.09554): ICLR 2026
|
||||
- [PyPI](https://pypi.org/project/rfdetr/): rfdetr package
|
||||
- [Hugging Face Demo](https://huggingface.co/spaces/Roboflow/RF-DETR): Interactive RF-DETR demo
|
||||
- [Discord](https://discord.gg/GbfgXGJ8Bk): RF-DETR community
|
||||
@@ -0,0 +1,34 @@
|
||||
User-agent: *
|
||||
Allow: /
|
||||
# Block numeric-versioned paths (/0.x/, /1.x/, etc.) — canonical URL is /latest/
|
||||
# robots.txt has no character classes, so one Disallow per leading digit is required.
|
||||
Disallow: /0.*/
|
||||
Disallow: /1.*/
|
||||
Disallow: /2.*/
|
||||
Disallow: /3.*/
|
||||
Disallow: /4.*/
|
||||
Disallow: /5.*/
|
||||
Disallow: /6.*/
|
||||
Disallow: /7.*/
|
||||
Disallow: /8.*/
|
||||
Disallow: /9.*/
|
||||
|
||||
# AI crawlers — explicitly allow all paths including numeric-versioned ones.
|
||||
# Named groups are independent of User-agent: * — they do NOT inherit its Disallow rules.
|
||||
# Listing Allow: / here is sufficient; no Disallow needed.
|
||||
User-agent: GPTBot
|
||||
User-agent: ClaudeBot
|
||||
User-agent: PerplexityBot
|
||||
User-agent: CCBot
|
||||
User-agent: Google-Extended
|
||||
User-agent: Bytespider
|
||||
User-agent: GoogleOther
|
||||
User-agent: OAI-SearchBot
|
||||
User-agent: ChatGPT-User
|
||||
User-agent: Amazonbot
|
||||
User-agent: FacebookBot
|
||||
User-agent: Cohere-ai
|
||||
User-agent: Applebot-Extended
|
||||
Allow: /
|
||||
|
||||
Sitemap: https://rfdetr.roboflow.com/sitemap.xml
|
||||
@@ -0,0 +1,123 @@
|
||||
.custom-grid {
|
||||
display: grid;
|
||||
grid-gap: 1rem;
|
||||
/* Start with a single column layout */
|
||||
grid-template-columns: repeat(1, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
/* Medium screens (640px and up) */
|
||||
@media (min-width: 640px) {
|
||||
.custom-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
/* Large screens (1024px and up) */
|
||||
@media (min-width: 1024px) {
|
||||
.custom-grid {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
.custom-grid a {
|
||||
color: inherit;
|
||||
text-decoration: none;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.custom-grid a:hover {
|
||||
color: inherit; /* Ensure color does not change on hover */
|
||||
}
|
||||
|
||||
.cookbooks-subtitle {
|
||||
color: var(--md-default-fg-color--light);
|
||||
font-size: 1rem;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.repo-card {
|
||||
background: radial-gradient(at right top, #A351FB66, #A5F9EA66);
|
||||
border-radius: 0.5rem;
|
||||
padding: 1rem;
|
||||
transition: transform 0.2s ease-in-out; /* Smooth transition for the transform */
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.repo-card:hover {
|
||||
transform: translateY(-0.125rem); /* Move up by -0.125rem on hover */
|
||||
}
|
||||
|
||||
.authors {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-start;
|
||||
margin-bottom: 1rem;
|
||||
margin-top: 1rem;
|
||||
font-size: 0.75rem;
|
||||
line-height: 1rem;
|
||||
}
|
||||
|
||||
.author-names {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-start;
|
||||
margin-bottom: 1rem;
|
||||
margin-top: 1rem;
|
||||
margin-left: 0.5rem;
|
||||
}
|
||||
|
||||
.author-container {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
transition: transform 0.2s;
|
||||
}
|
||||
|
||||
.author-container.hover {
|
||||
transform: translateY(-0.125rem);
|
||||
}
|
||||
|
||||
.author-container:hover {
|
||||
transform: translateY(-0.125rem);
|
||||
}
|
||||
|
||||
.author-container a {
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.author-avatar {
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.author-name a {
|
||||
color: inherit; /* Inherits the color from the parent element */
|
||||
text-decoration: none; /* Removes the underline */
|
||||
}
|
||||
|
||||
.author-name a:hover {
|
||||
color: inherit; /* Inherits the color from the parent element */
|
||||
text-decoration: underline; /* Adds underline on hover */
|
||||
}
|
||||
|
||||
.label {
|
||||
color: #fff;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
margin-right: 4px;
|
||||
}
|
||||
|
||||
.non-selectable-text {
|
||||
-webkit-user-select: none; /* Safari */
|
||||
-moz-user-select: none; /* Firefox */
|
||||
-ms-user-select: none; /* Internet Explorer/Edge */
|
||||
user-select: none; /* Non-prefixed version, currently supported by Chrome, Opera, and Edge */
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
:root, body {
|
||||
/* Default to light theme */
|
||||
--md-primary-fg-color: #8315F9;
|
||||
--md-code-hl-color: #8315F9 !important;
|
||||
--md-accent-fg-color: #8315F9 !important;
|
||||
--md-code-hl-color--light: #e8d2ff89 !important;
|
||||
--md-footer-fg-color--light: rgb(111, 108, 121) !important;
|
||||
}
|
||||
|
||||
body.light {
|
||||
/* Light theme */
|
||||
--md-text-color: #000000;
|
||||
--md-h2-color: #000000;
|
||||
}
|
||||
.md-grid {
|
||||
max-width: 85%;
|
||||
margin: auto;
|
||||
}
|
||||
.sublist {
|
||||
display: none;
|
||||
list-style: none;
|
||||
padding-left: 0;
|
||||
background: white;
|
||||
position: absolute;
|
||||
border-radius: 8px;
|
||||
margin-top: 0.25rem;
|
||||
}
|
||||
.sublist {
|
||||
transition: opacity 0.5s ease-in-out;
|
||||
display: none;
|
||||
position: absolute; /* Ensure it overlaps and doesn't break flow */
|
||||
background: white; /* So it's visible */
|
||||
z-index: 1000;
|
||||
}
|
||||
.sublist li {
|
||||
padding: 0.5rem;
|
||||
}
|
||||
#products-list *:hover .products-sublist {
|
||||
display: block;
|
||||
}
|
||||
#resources-list *, #products-list * {
|
||||
cursor: pointer;
|
||||
}
|
||||
.products-sublist, .resources-sublist {
|
||||
padding: 0.25rem;
|
||||
}
|
||||
.products-sublist li:hover, .resources-sublist li:hover, .md-nav__link[href]:hover {
|
||||
background: rgb(242, 241, 247) !important;
|
||||
border-radius: 6px;
|
||||
color: initial !important;
|
||||
}
|
||||
.md-search {
|
||||
flex-grow: 2;
|
||||
}
|
||||
.portfolio-section .md-grid {
|
||||
max-width: 100%;
|
||||
}
|
||||
.md-header__inner {
|
||||
align-items: center;
|
||||
display: grid;
|
||||
grid-template-columns: 0.1fr 1.4fr 2fr 2fr;
|
||||
padding-right: 1rem;
|
||||
}
|
||||
.md-search__inner {
|
||||
max-width: 600px;
|
||||
width: 100%;
|
||||
min-width: 100%;
|
||||
}
|
||||
.md-search__input {
|
||||
background: white;
|
||||
border: 1px solid rgb(229, 231, 235);
|
||||
border-radius: 8px;
|
||||
color: rgb(111, 108, 121);
|
||||
}
|
||||
.md-search__form *, .md-search__icon, .md-search__input {
|
||||
color: rgb(111, 108, 121);
|
||||
}
|
||||
.md-search__input::placeholder {
|
||||
color: rgb(156, 163, 175);
|
||||
}
|
||||
.md-search__form {
|
||||
background: none !important;
|
||||
}
|
||||
.md-footer, .md-footer-meta {
|
||||
background-color: transparent;
|
||||
color: rgb(111, 108, 121);
|
||||
}
|
||||
.md-typeset .tabbed-set > input:first-child:checked ~ .tabbed-labels > :first-child, .md-typeset .tabbed-set > input:nth-child(10):checked ~ .tabbed-labels > :nth-child(10), .md-typeset .tabbed-set > input:nth-child(11):checked ~ .tabbed-labels > :nth-child(11), .md-typeset .tabbed-set > input:nth-child(12):checked ~ .tabbed-labels > :nth-child(12), .md-typeset .tabbed-set > input:nth-child(13):checked ~ .tabbed-labels > :nth-child(13), .md-typeset .tabbed-set > input:nth-child(14):checked ~ .tabbed-labels > :nth-child(14), .md-typeset .tabbed-set > input:nth-child(15):checked ~ .tabbed-labels > :nth-child(15), .md-typeset .tabbed-set > input:nth-child(16):checked ~ .tabbed-labels > :nth-child(16), .md-typeset .tabbed-set > input:nth-child(17):checked ~ .tabbed-labels > :nth-child(17), .md-typeset .tabbed-set > input:nth-child(18):checked ~ .tabbed-labels > :nth-child(18), .md-typeset .tabbed-set > input:nth-child(19):checked ~ .tabbed-labels > :nth-child(19), .md-typeset .tabbed-set > input:nth-child(2):checked ~ .tabbed-labels > :nth-child(2), .md-typeset .tabbed-set > input:nth-child(20):checked ~ .tabbed-labels > :nth-child(20), .md-typeset .tabbed-set > input:nth-child(3):checked ~ .tabbed-labels > :nth-child(3), .md-typeset .tabbed-set > input:nth-child(4):checked ~ .tabbed-labels > :nth-child(4), .md-typeset .tabbed-set > input:nth-child(5):checked ~ .tabbed-labels > :nth-child(5), .md-typeset .tabbed-set > input:nth-child(6):checked ~ .tabbed-labels > :nth-child(6), .md-typeset .tabbed-set > input:nth-child(7):checked ~ .tabbed-labels > :nth-child(7), .md-typeset .tabbed-set > input:nth-child(8):checked ~ .tabbed-labels > :nth-child(8), .md-typeset .tabbed-set > input:nth-child(9):checked ~ .tabbed-labels > :nth-child(9) {
|
||||
color: #8315F9;
|
||||
border-bottom: 1px solid #8315F9;
|
||||
}
|
||||
.md-footer *, html .md-footer-meta.md-typeset a {
|
||||
color: rgb(111, 108, 121);
|
||||
}
|
||||
.repo-card {
|
||||
height: 100%;
|
||||
}
|
||||
.header-btn {
|
||||
text-align: center;
|
||||
}
|
||||
.header-btn, .sublist {
|
||||
box-shadow: rgb(255, 255, 255) 0px 0px 0px 0px, rgb(217, 215, 226) 0px 0px 0px 1px, rgb(217, 215, 226) 0px 1px 2px 0px;
|
||||
}
|
||||
.header-btn:hover {
|
||||
box-shadow: rgb(255, 255, 255) 0px 0px 0px 0px, rgb(217, 215, 226) 0px 0px 0px 1px, rgb(217, 215, 226) 0px 1.0001px 2.00013px -0.0000327245px, rgba(0, 0, 0, 0) 0px 0.000065449px 0.000130898px -0.000065449px;
|
||||
}
|
||||
.md-typeset .headerlink:hover, .md-typeset .headerlink:target {
|
||||
color: #8315F9;
|
||||
}
|
||||
.md-typeset h1, .md-header__title {
|
||||
color: black;
|
||||
font-weight: 800;
|
||||
}
|
||||
.md-typeset h1 {
|
||||
font-weight: normal;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
body {
|
||||
background: linear-gradient(to left bottom, rgb(243, 238, 255), rgb(255, 255, 255) 60%) no-repeat;
|
||||
}
|
||||
|
||||
/* .md-nav__link:has([tabindex=""]) {
|
||||
text-transform: uppercase;
|
||||
} */
|
||||
|
||||
.header-list {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
list-style: none;
|
||||
font-size: 0.75rem;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
.header-chevron {
|
||||
border: solid currentcolor;
|
||||
border-width: 0 1px 1px 0;
|
||||
display: inline-block;
|
||||
height: 0.4rem;
|
||||
margin-left: 0.35rem;
|
||||
transform: rotate(45deg) translateY(-0.1rem);
|
||||
width: 0.4rem;
|
||||
}
|
||||
#dropdown-resources:checked ~ .resources-sublist,
|
||||
#dropdown-products:checked ~ .products-sublist {
|
||||
display: block;
|
||||
}
|
||||
body:not(:has(#dropdown-resources:checked)) .resources-sublist,
|
||||
body:not(:has(#dropdown-products:checked)) .products-sublist {
|
||||
display: none;
|
||||
}
|
||||
.md-typeset code .na {
|
||||
color: #4a176f !important;
|
||||
}
|
||||
.md-nav__list label, .md-nav--secondary label {
|
||||
/* text-transform: uppercase; */
|
||||
color: rgb(29, 29, 31) !important;
|
||||
font-size: 0.7rem;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
.md-nav--secondary label {
|
||||
margin-left: 0.5rem;
|
||||
}
|
||||
|
||||
.md-nav__link {
|
||||
padding: 0.25rem;
|
||||
padding-left: 0.5rem;
|
||||
padding-right: 0.5rem;
|
||||
}
|
||||
|
||||
.md-nav__link--active {
|
||||
background: rgb(243, 238, 255);
|
||||
border-radius: 6px;
|
||||
padding-top: 0.25rem;
|
||||
}
|
||||
|
||||
.md-tabs__item--active {
|
||||
color: var(--md-primary-fg-color);
|
||||
border-bottom: 2px solid var(--md-primary-fg-color);
|
||||
}
|
||||
|
||||
.md-nav--secondary .md-nav__title {
|
||||
background: transparent;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.md-header, .md-tabs {
|
||||
color: rgb(111, 108, 121);
|
||||
background-color: transparent;
|
||||
}
|
||||
.md-header--shadow {
|
||||
background: linear-gradient(to left bottom, rgb(243, 238, 255), rgb(255, 255, 255) 60%);
|
||||
box-shadow: none;
|
||||
border-bottom: 1px solid rgb(229, 231, 235);
|
||||
}
|
||||
|
||||
#item-logo {
|
||||
display: none;
|
||||
}
|
||||
.md-main__inner, .md-header__inner, .md-grid {
|
||||
max-width: 100%;
|
||||
}
|
||||
@media (max-width: 1200px) {
|
||||
.md-header__inner {
|
||||
display: flex;
|
||||
}
|
||||
.header-list {
|
||||
display: none;
|
||||
}
|
||||
#item-logo {
|
||||
display: block;
|
||||
}
|
||||
}
|
||||
.md-content {
|
||||
max-width: 40rem;
|
||||
margin: auto;
|
||||
}
|
||||
/* // if no md-sidebar--primary, make .md-content full width */
|
||||
.md-main__inner:has(.md-sidebar--primary[hidden]) .md-content {
|
||||
max-width: 100%;
|
||||
}
|
||||
.md-sidebar--primary {
|
||||
flex: 0 20%;
|
||||
}
|
||||
.md-tabs {
|
||||
border-bottom: 1px solid rgb(229, 231, 235);
|
||||
}
|
||||
.md-main__inner {
|
||||
padding-top: 1rem;
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
body.dark {
|
||||
/* Dark theme */
|
||||
--md-text-color: #FFFFFF;
|
||||
--md-h2-color: #add8e6;
|
||||
}
|
||||
|
||||
body.light .md-content *, body.dark .md-content * {
|
||||
color: var(--md-text-color) !important;
|
||||
}
|
||||
|
||||
body[data-md-url$="/cookbooks/"] .md-sidebar--primary,
|
||||
body[data-md-url$="/cookbooks/"] .md-sidebar--secondary {
|
||||
display: none;
|
||||
}
|
||||
|
||||
body[data-md-url$="/cookbooks/"] .md-content {
|
||||
margin-left: 0;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.md-main, nav .md-grid, .md-header__inner {
|
||||
max-width: 1600px;
|
||||
width: 100%;
|
||||
margin: auto;
|
||||
}
|
||||
.md-search__scrollwrap {
|
||||
width: 100% !important;
|
||||
}
|
||||
.md-nav--secondary .md-nav__title {
|
||||
position: initial !important;
|
||||
}
|
||||
|
||||
.md-header__title .md-ellipsis {
|
||||
overflow: initial !important;
|
||||
text-overflow: initial !important;
|
||||
}
|
||||
.md-search {
|
||||
flex-grow: 0;
|
||||
}
|
||||
Vendored
+283
@@ -0,0 +1,283 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block htmltitle %}
|
||||
{% if page and page.is_homepage %}
|
||||
<title>RF-DETR — Real-Time Object Detection & Segmentation by Roboflow</title>
|
||||
{% else %}
|
||||
{{ super() }}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
{% block extrahead %}
|
||||
{{ super() }}
|
||||
<link rel="preconnect" href="https://storage.googleapis.com">
|
||||
|
||||
{% if page %}
|
||||
{% set og_image_url = page.meta.image | default(config.extra.og_image | default('')) %}
|
||||
{# Open Graph — all pages #}
|
||||
<meta property="og:title" content="{% if page.is_homepage %}RF-DETR — Real-Time Object Detection & Segmentation by Roboflow{% else %}{{ page.title }} - {{ config.site_name }}{% endif %}">
|
||||
<meta property="og:description" content="{{ page.meta.description | default(config.site_description) }}">
|
||||
<meta property="og:type" content="{% if page.is_homepage %}website{% else %}article{% endif %}">
|
||||
<meta property="og:url" content="{{ page.canonical_url }}">
|
||||
{% if og_image_url %}<meta property="og:image" content="{{ og_image_url }}">
|
||||
{% if og_image_url == config.extra.og_image %}<meta property="og:image:width" content="840">
|
||||
<meta property="og:image:height" content="630">{% endif %}{% endif %}
|
||||
|
||||
{# Twitter Card — all pages #}
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:site" content="@roboflow">
|
||||
<meta name="twitter:title" content="{% if page.is_homepage %}RF-DETR — Real-Time Object Detection & Segmentation by Roboflow{% else %}{{ page.title }} - {{ config.site_name }}{% endif %}">
|
||||
<meta name="twitter:description" content="{{ page.meta.description | default(config.site_description) }}">
|
||||
{% if og_image_url %}<meta name="twitter:image" content="{{ og_image_url }}">{% endif %}
|
||||
|
||||
{# JSON-LD: Organization + SoftwareApplication + WebSite — homepage only #}
|
||||
{% if page.is_homepage %}
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@graph": [
|
||||
{
|
||||
"@type": "Organization",
|
||||
"@id": "https://roboflow.com/#organization",
|
||||
"name": "Roboflow",
|
||||
"url": "https://roboflow.com",
|
||||
"logo": {
|
||||
"@type": "ImageObject",
|
||||
"url": "https://rfdetr.roboflow.com/latest/assets/roboflow-logo.svg",
|
||||
"width": 400,
|
||||
"height": 400
|
||||
},
|
||||
"description": "Roboflow is a computer vision platform for dataset management, model training, and deployment. Creator of RF-DETR, Supervision, and Roboflow Inference.",
|
||||
"sameAs": [
|
||||
"https://github.com/roboflow",
|
||||
"https://www.linkedin.com/company/roboflow-ai/",
|
||||
"https://twitter.com/roboflow",
|
||||
"https://www.youtube.com/@Roboflow",
|
||||
"https://en.wikipedia.org/wiki/Roboflow",
|
||||
"https://huggingface.co/roboflow",
|
||||
"https://www.crunchbase.com/organization/roboflow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"@type": "SoftwareApplication",
|
||||
"@id": "{{ config.site_url }}#software",
|
||||
"name": "RF-DETR",
|
||||
"alternateName": "rfdetr",
|
||||
"applicationCategory": "DeveloperApplication",
|
||||
"applicationSubCategory": "Computer vision model",
|
||||
"operatingSystem": "Linux, macOS, Windows",
|
||||
"url": "{{ config.site_url }}",
|
||||
"downloadUrl": "https://pypi.org/project/rfdetr/",
|
||||
"softwareVersion": {{ config.extra.software_version | tojson }},
|
||||
"softwareRequirements": "Python >=3.10; torch >=2.2.0; torchvision >=0.17.0; transformers >=5.1.0,<6.0.0",
|
||||
"identifier": [
|
||||
{"@type": "PropertyValue", "propertyID": "Python package", "value": "rfdetr"},
|
||||
{"@type": "PropertyValue", "propertyID": "GitHub repository", "value": "roboflow/rf-detr"},
|
||||
{"@type": "PropertyValue", "propertyID": "arXiv", "value": "2511.09554"}
|
||||
],
|
||||
"releaseNotes": "https://github.com/roboflow/rf-detr/blob/main/CHANGELOG.md",
|
||||
"description": "RF-DETR is a real-time transformer architecture for object detection and instance segmentation by Roboflow. DINOv2 backbone, SOTA on COCO (60.1 AP50:95). ICLR 2026. Apache 2.0.",
|
||||
"keywords": "RF-DETR, rfdetr, object detection, instance segmentation, real-time detection transformer, DINOv2, COCO, RF100-VL, ONNX, TensorRT",
|
||||
"featureList": [
|
||||
"Real-time object detection",
|
||||
"Instance segmentation",
|
||||
"DINOv2 vision transformer backbone",
|
||||
"ONNX and TensorRT FP16/INT8 export",
|
||||
"Fine-tuning on custom COCO and YOLO datasets",
|
||||
"Multi-GPU distributed training",
|
||||
"EMA model averaging"
|
||||
],
|
||||
"author": {"@type": "Organization", "name": "Roboflow", "url": "https://roboflow.com"},
|
||||
"license": "https://www.apache.org/licenses/LICENSE-2.0",
|
||||
"codeRepository": "https://github.com/roboflow/rf-detr",
|
||||
"programmingLanguage": "Python",
|
||||
"runtimePlatform": "Python 3.10+",
|
||||
"isAccessibleForFree": true,
|
||||
"citation": "https://arxiv.org/abs/2511.09554",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": "0",
|
||||
"priceCurrency": "USD",
|
||||
"availability": "https://schema.org/InStock",
|
||||
"url": "https://pypi.org/project/rfdetr/"
|
||||
},
|
||||
"sameAs": [
|
||||
"https://github.com/roboflow/rf-detr",
|
||||
"https://pypi.org/project/rfdetr/",
|
||||
"https://arxiv.org/abs/2511.09554"
|
||||
]
|
||||
},
|
||||
{
|
||||
"@type": "WebSite",
|
||||
"@id": "{{ config.site_url }}#website",
|
||||
"name": "RF-DETR Documentation",
|
||||
"url": "{{ config.site_url }}",
|
||||
"description": {{ config.site_description | tojson }},
|
||||
"publisher": {"@type": "Organization", "name": "Roboflow", "url": "https://roboflow.com"},
|
||||
"potentialAction": {
|
||||
"@type": "SearchAction",
|
||||
"target": {
|
||||
"@type": "EntryPoint",
|
||||
"urlTemplate": "https://rfdetr.roboflow.com/latest/?q={search_term_string}"
|
||||
},
|
||||
"query-input": "required name=search_term_string"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
</script>
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "FAQPage",
|
||||
"mainEntity": [
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "What is RF-DETR?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "RF-DETR (Roboflow Detection Transformer) is a real-time object detection and instance segmentation model from Roboflow, accepted at ICLR 2026. It uses a DINOv2 vision transformer backbone and achieves state-of-the-art accuracy\u2013latency trade-offs on COCO (60.1 AP50:95 for RF-DETR-2XL) and RF100-VL."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "How does RF-DETR compare to YOLOv11?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "RF-DETR-L achieves 56.5 AP50:95 on COCO at 6.8\u00a0ms latency on an NVIDIA T4, outperforming YOLOv11x (54.7 AP) at lower latency. The DINOv2 backbone gives RF-DETR stronger performance on domain-shift benchmarks such as RF100-VL."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "What GPU is required to train RF-DETR?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "A CUDA-capable GPU with at least 8\u00a0GB VRAM is recommended for fine-tuning. Smaller models (RF-DETR-N and RF-DETR-S) can fit in 6\u00a0GB VRAM with a reduced batch size. CPU inference is supported for evaluation."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "Which dataset formats does RF-DETR support?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "RF-DETR supports COCO JSON and YOLO-format datasets. Roboflow datasets export directly to both formats. Detection and segmentation datasets use the same format \u2014 the model variant determines the task."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "Can RF-DETR run in real time?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Yes. RF-DETR-N runs at 2.3\u00a0ms per frame on a T4 GPU (TensorRT FP16, batch 1), and RF-DETR-L at 6.8\u00a0ms \u2014 both well within real-time thresholds. ONNX and TFLite exports are available for edge deployment."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "What is the difference between RF-DETR detection and segmentation models?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Detection models output bounding boxes. Segmentation models additionally output instance masks. Both share the same backbone and training API; segmentation adds a mask head and requires COCO-format segmentation annotations."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "Is RF-DETR open source?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Yes. Core models (Nano through Large) and all training/inference code are released under the Apache 2.0 license. XLarge and 2XLarge models require the rfdetr[plus] package (PML 1.0 license)."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "How do I fine-tune RF-DETR on a custom dataset?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Instantiate a model and call model.train(...) with your dataset directory in COCO JSON or YOLO format. Example: model = RFDETRLarge(); model.train(dataset_dir='./dataset', epochs=50, batch_size=4). The model downloads pretrained weights automatically and resumes from the best checkpoint."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "How do I export RF-DETR to ONNX or TensorRT?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Call model.export(format='onnx') after training or loading a checkpoint. ONNX export works on CPU and produces a single .onnx file compatible with ONNX Runtime and OpenCV DNN. For TensorRT deployment, first export to ONNX and then convert the .onnx model with TensorRT tooling such as trtexec; this requires TensorRT and a CUDA GPU."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "Which RF-DETR model size should I use?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "RF-DETR-Nano (2.3\u00a0ms, 67.6\u00a0AP50 on COCO) is best for edge and real-time applications. RF-DETR-Large (6.8\u00a0ms, 56.5\u00a0AP50:95) offers the best accuracy-latency trade-off for server deployment. RF-DETR-2XLarge (17.2\u00a0ms, 60.1\u00a0AP50:95) maximizes accuracy when latency allows."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
</script>
|
||||
{% else %}
|
||||
{# article:modified_time — content pages only; emit only when a full ISO-8601 date-time is available #}
|
||||
{# striptags: plugin may return a <time> HTML element (Markup-safe); strip tags before embedding in content attr #}
|
||||
{% if page.meta.git_revision_date_localized %}<meta property="article:modified_time" content="{{ page.meta.git_revision_date_localized | string | striptags | replace(' ', 'T') | e }}">{% endif %}
|
||||
{# JSON-LD: TechArticle — content pages #}
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "TechArticle",
|
||||
"headline": {{ page.title | tojson }},
|
||||
"description": {{ (page.meta.description | default(config.site_description)) | tojson }},
|
||||
"url": "{{ page.canonical_url }}",
|
||||
"isPartOf": {"@id": "{{ config.site_url }}#website"},
|
||||
"about": {"@id": "{{ config.site_url }}#software"},
|
||||
"proficiencyLevel": "Beginner to advanced",
|
||||
"inLanguage": "en",
|
||||
"keywords": "RF-DETR, rfdetr, object detection, instance segmentation, computer vision, Roboflow",
|
||||
"datePublished": "2025-03-01",
|
||||
{% if page.meta.git_revision_date_localized %}"dateModified": {{ page.meta.git_revision_date_localized | string | striptags | replace(' ', 'T') | tojson }},
|
||||
{% endif %}"image": {{ (page.meta.image | default(config.extra.og_image | default('https://rfdetr.roboflow.com/latest/assets/roboflow-logo.svg'))) | tojson }},
|
||||
"author": {"@type": "Organization", "name": "Roboflow", "url": "https://roboflow.com"},
|
||||
"publisher": {"@type": "Organization", "name": "Roboflow", "url": "https://roboflow.com"},
|
||||
"mainEntityOfPage": {"@type": "WebPage", "@id": "{{ page.canonical_url }}"},
|
||||
"speakable": {
|
||||
"@type": "SpeakableSpecification",
|
||||
"cssSelector": [".md-content h1", ".md-typeset > p:first-child"]
|
||||
}
|
||||
}
|
||||
</script>
|
||||
{# JSON-LD: BreadcrumbList — content pages; 3-level for pages inside a section #}
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "BreadcrumbList",
|
||||
"itemListElement": [
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 1,
|
||||
"name": "RF-DETR",
|
||||
"item": "{{ config.site_url }}"
|
||||
}{% if page.parent %},
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 2,
|
||||
"name": {{ page.parent.title | tojson }},
|
||||
"item": "{% if page.parent.children and page.parent.children[0].canonical_url %}{{ page.parent.children[0].canonical_url }}{% else %}{{ config.site_url }}{% endif %}"
|
||||
},
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 3,
|
||||
"name": {{ page.title | tojson }},
|
||||
"item": "{{ page.canonical_url }}"
|
||||
}
|
||||
{% else %},
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 2,
|
||||
"name": {{ page.title | tojson }},
|
||||
"item": "{{ page.canonical_url }}"
|
||||
}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
</script>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
Vendored
+29
@@ -0,0 +1,29 @@
|
||||
{% extends "main.html" %}
|
||||
{% block libs %}
|
||||
{{ super() }}
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/dompurify/3.0.8/purify.min.js"></script>
|
||||
<link rel="stylesheet" href="{{ 'stylesheets/cookbooks_card.css' | url }}">
|
||||
<script src="{{ 'javascripts/cookbooks-card.js' | url }}"></script>
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<section class="mdx-container portfolio-section">
|
||||
<div class="md-grid md-typeset">
|
||||
<div class="text-center">
|
||||
<h1>RF-DETR Cookbooks</h1>
|
||||
<p class="cookbooks-subtitle">Runnable notebooks for training, fine-tuning, exporting, and deploying RF-DETR models.</p>
|
||||
</div>
|
||||
<div class="custom-grid">
|
||||
{% for card in config.extra.cookbooks_cards %}
|
||||
<a href="{{ card.href }}">
|
||||
<div class="card repo-card"
|
||||
data-name="{{ card.name }}"
|
||||
data-labels="{{ card.labels | join(',') }}"
|
||||
data-version="{{ card.version }}"
|
||||
data-author="{{ card.author }}"
|
||||
data-description="{{ card.description }}"></div>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
Vendored
+175
@@ -0,0 +1,175 @@
|
||||
{% set class = "md-header" %}
|
||||
{% if "navigation.tabs.sticky" in features %}
|
||||
{% set class = class ~ " md-header--shadow md-header--lifted" %}
|
||||
{% elif "navigation.tabs" not in features %}
|
||||
{% set class = class ~ " md-header--shadow" %}
|
||||
{% endif %}
|
||||
|
||||
<!-- Header -->
|
||||
<header class="{{ class }}" data-md-component="header">
|
||||
<nav
|
||||
class="md-header__inner md-grid"
|
||||
aria-label="{{ lang.t('header') }}"
|
||||
>
|
||||
|
||||
<!-- Link to home -->
|
||||
<a
|
||||
href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
|
||||
title="{{ config.site_name | e }}"
|
||||
class="md-header__button md-logo"
|
||||
aria-label="{{ config.site_name }}"
|
||||
data-md-component="logo"
|
||||
>
|
||||
{% include "partials/logo.html" %}
|
||||
</a>
|
||||
|
||||
<!-- Button to open drawer -->
|
||||
<label class="md-header__button md-icon" for="__drawer">
|
||||
{% set icon = config.theme.icon.menu or "material/menu" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
</label>
|
||||
|
||||
<!-- Header title -->
|
||||
<div class="md-header__title" data-md-component="header-title">
|
||||
<div class="md-header__ellipsis">
|
||||
<div class="md-header__topic">
|
||||
<span class="md-ellipsis">
|
||||
{{ config.site_name }} Docs
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-header__topic" data-md-component="header-topic">
|
||||
<span class="md-ellipsis" style="display: flex; align-items: center; gap: 0.5rem;">
|
||||
<a
|
||||
href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
|
||||
title="{{ config.site_name | e }}"
|
||||
class="md-header__button md-logo"
|
||||
aria-label="{{ config.site_name }}"
|
||||
data-md-component="logo"
|
||||
id="item-logo"
|
||||
>
|
||||
{% include "partials/logo.html" %}
|
||||
</a>
|
||||
{% if page.meta and page.meta.title %}
|
||||
{{ page.meta.title }}
|
||||
{% else %}
|
||||
{{ page.title }}
|
||||
{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Button to open search modal -->
|
||||
{% if "material/search" in config.plugins %}
|
||||
{% set search = config.plugins["material/search"] | attr("config") %}
|
||||
|
||||
<!-- Check if search is actually enabled - see https://t.ly/DT_0V -->
|
||||
{% if search.enabled %}
|
||||
<label class="md-header__button md-icon" for="__search">
|
||||
{% set icon = config.theme.icon.search or "material/magnify" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
</label>
|
||||
|
||||
<!-- Search interface -->
|
||||
{% include "partials/search.html" %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
<ul class="header-list">
|
||||
<li style="align-items: center;" id="resources-list">
|
||||
<label for="dropdown-resources"><span>Resources <span class="header-chevron" aria-hidden="true"></span></span></label>
|
||||
<input type="radio" name="dropdown" id="dropdown-resources" style="display: none;" />
|
||||
<ul class="resources-sublist sublist">
|
||||
<li><a href="https://blog.roboflow.com">Blog</a></li>
|
||||
<li><a href="https://discuss.roboflow.com">Community Forum</a></li>
|
||||
<li><a href="https://roboflow.com/sales">Contact Sales</a></li>
|
||||
<li><a href="https://universe.roboflow.com">Universe</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li style="align-items: center;" id="products-list">
|
||||
<label for="dropdown-products"><span>Docs <span class="header-chevron" aria-hidden="true"></span></span></label>
|
||||
<input type="radio" name="dropdown" id="dropdown-products" style="display: none;" />
|
||||
<ul class="products-sublist sublist">
|
||||
<li><a href="https://inference.roboflow.com">Inference</a></li>
|
||||
<li><a href="https://supervision.roboflow.com">Supervision</a></li>
|
||||
<li><a href="https://trackers.roboflow.com">Trackers</a></li>
|
||||
<li><a href="https://rfdetr.roboflow.com">RF-DETR</a></li>
|
||||
<li><a href="https://maestro.roboflow.com">Maestro</a></li>
|
||||
<li><a href="https://docs.roboflow.com">Roboflow</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
<a href="https://github.com/roboflow/rf-detr" class="header-btn" style="border-radius: 5px; color: white; background: var(--md-typeset-a-color); padding-top: 0.25rem; padding-left: 0.5rem; padding-bottom: 0.25rem; padding-right: 0.5rem; border: 1px solid #8315F9;">Go to GitHub</a>
|
||||
</li>
|
||||
</ul>
|
||||
<script>
|
||||
document.addEventListener('click', function(event) {
|
||||
const resourcesList = document.getElementById('resources-list');
|
||||
const productsList = document.getElementById('products-list');
|
||||
const dropdownResources = document.getElementById('dropdown-resources');
|
||||
const dropdownProducts = document.getElementById('dropdown-products');
|
||||
|
||||
if (!resourcesList.contains(event.target)) {
|
||||
dropdownResources.checked = false;
|
||||
}
|
||||
if (!productsList.contains(event.target)) {
|
||||
dropdownProducts.checked = false;
|
||||
}
|
||||
});
|
||||
document.getElementById('resources-list').addEventListener('mouseover', function() {
|
||||
document.getElementById('dropdown-resources').checked = true;
|
||||
});
|
||||
document.getElementById('products-list').addEventListener('mouseover', function() {
|
||||
document.getElementById('dropdown-products').checked = true;
|
||||
});
|
||||
document.getElementById('resources-list').addEventListener('mouseout', function() {
|
||||
setTimeout(function() {
|
||||
if (!document.querySelector('.resources-sublist:hover') && !document.querySelector('#resources-list:hover')) {
|
||||
document.getElementById('dropdown-resources').checked = false;
|
||||
}
|
||||
}, 350);
|
||||
});
|
||||
document.querySelector('.resources-sublist').addEventListener('mouseout', function() {
|
||||
setTimeout(function() {
|
||||
if (!document.querySelector('.resources-sublist:hover') && !document.querySelector('#resources-list:hover')) {
|
||||
document.getElementById('dropdown-resources').checked = false;
|
||||
}
|
||||
}, 450);
|
||||
});
|
||||
document.getElementById('products-list').addEventListener('mouseout', function() {
|
||||
setTimeout(function() {
|
||||
if (!document.querySelector('.products-sublist:hover') && !document.querySelector('#products-list:hover')) {
|
||||
document.getElementById('dropdown-products').checked = false;
|
||||
}
|
||||
}, 500);
|
||||
});
|
||||
document.querySelector('.products-sublist').addEventListener('mouseout', function() {
|
||||
setTimeout(function() {
|
||||
if (!document.querySelector('.products-sublist:hover') && !document.querySelector('#products-list:hover')) {
|
||||
document.getElementById('dropdown-products').checked = false;
|
||||
}
|
||||
}, 500);
|
||||
});
|
||||
function labelCurrentVersion() {
|
||||
document.querySelectorAll('.md-version__current').forEach(function(button) {
|
||||
const label = button.textContent.trim();
|
||||
if (label) {
|
||||
button.setAttribute('aria-label', label);
|
||||
}
|
||||
});
|
||||
}
|
||||
labelCurrentVersion();
|
||||
const versionMenu = document.querySelector('.md-version');
|
||||
if (versionMenu) {
|
||||
new MutationObserver(labelCurrentVersion).observe(versionMenu, { childList: true, subtree: true });
|
||||
}
|
||||
</script>
|
||||
</nav>
|
||||
|
||||
<!-- Navigation tabs (sticky) -->
|
||||
{% if "navigation.tabs.sticky" in features %}
|
||||
{% if "navigation.tabs" in features %}
|
||||
{% include "partials/tabs.html" %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</header>
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
{% if config.theme.logo %}
|
||||
<img src="{{ config.theme.logo | url }}" alt="{{ config.site_name }}" width="24" height="24">
|
||||
{% else %}
|
||||
{% set icon = config.theme.icon.logo or "material/library" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
{% endif %}
|
||||
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
<div class="md-search" data-md-component="search" role="dialog" aria-modal="true" aria-label="{{ lang.t('search') }}">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" aria-label="{{ lang.t('search.placeholder') }}" placeholder="{{ lang.t('search.placeholder') }}" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||
<label class="md-search__icon md-icon" for="__search">
|
||||
{% set icon = config.theme.icon.search or "material/magnify" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
{% set icon = config.theme.icon.previous or "material/arrow-left" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
</label>
|
||||
<nav class="md-search__options" aria-label="{{ lang.t('search') }}">
|
||||
{% if "search.share" in features %}
|
||||
<a href="javascript:void(0)" class="md-search__icon md-icon" title="{{ lang.t('search.share') }}" aria-label="{{ lang.t('search.share') }}" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
|
||||
{% set icon = config.theme.icon.share or "material/share-variant" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
</a>
|
||||
{% endif %}
|
||||
<button type="reset" class="md-search__icon md-icon" title="{{ lang.t('search.reset') }}" aria-label="{{ lang.t('search.reset') }}" tabindex="-1">
|
||||
{% set icon = config.theme.icon.close or "material/close" %}
|
||||
{% include ".icons/" ~ icon ~ ".svg" %}
|
||||
</button>
|
||||
</nav>
|
||||
{% if "search.suggest" in features %}
|
||||
<div class="md-search__suggest" data-md-component="search-suggest"></div>
|
||||
{% endif %}
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="search-result">
|
||||
<div class="md-search-result__meta">
|
||||
{{ lang.t("search.result.initializer") }}
|
||||
</div>
|
||||
<ol class="md-search-result__list" role="presentation"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
Reference in New Issue
Block a user