import numpy as np import pytest from supervision.config import ORIENTED_BOX_COORDINATES from supervision.detection.core import Detections from supervision.metrics.core import MetricTarget from supervision.metrics.mean_average_precision import ( EvaluationDataset, MeanAveragePrecision, ) def _mask_detections( row_slice: slice, confidence: bool = False, mask_shape: tuple[int, int] = (32, 32) ) -> Detections: """Build single-detection `Detections` with a mask filling the given rows.""" mask = np.zeros((1, *mask_shape), dtype=bool) mask[0, row_slice, :] = True return Detections( xyxy=np.array([[0, 0, 10, 10]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]) if confidence else None, mask=mask, ) def _obb_detections(corners: list[list[int]], confidence: bool = False) -> Detections: """Build single-detection `Detections` with the given oriented box corners.""" return Detections( xyxy=np.array([[0, 0, 30, 30]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]) if confidence else None, data={ORIENTED_BOX_COORDINATES: np.array([corners], dtype=np.float32)}, ) class TestMeanAveragePrecision: def test_single_perfect_detection(self, detections_50_50, targets_50_50): """Test that single perfect detection gets 1.0 mAP (not 0.0 due to ID=0 bug)""" metric = MeanAveragePrecision() metric.update([detections_50_50], [targets_50_50]) result = metric.compute() # Should be perfect 1.0 mAP, not 0.0 due to ID=0 bug assert abs(result.map50_95 - 1.0) < 1e-6 def test_multiple_perfect_detections(self): """Test that multiple perfect detections get 1.0 mAP""" # Multiple perfect detections in one image detections = Detections( xyxy=np.array( [[10, 10, 50, 50], [100, 100, 140, 140], [200, 200, 240, 240]], dtype=np.float64, ), class_id=np.array([0, 0, 0]), confidence=np.array([0.9, 0.9, 0.9]), ) metric = MeanAveragePrecision() metric.update([detections], [detections]) result = metric.compute() # Should be perfect 1.0 mAP assert abs(result.map50_95 - 1.0) < 1e-6 def test_perfect_non_square_oriented_boxes_get_full_map(self): """Perfect non-square OBB predictions score full mAP via OBB IoU.""" obb = np.array( [[[10, 0], [0, 1], [30, 4], [40, 3]]], dtype=np.float32, ) detections = Detections( xyxy=np.array([[0, 0, 40, 4]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), data={ORIENTED_BOX_COORDINATES: obb}, ) targets = Detections( xyxy=np.array([[0, 0, 40, 4]], dtype=np.float64), class_id=np.array([0]), data={ORIENTED_BOX_COORDINATES: obb}, ) metric = MeanAveragePrecision( metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES ) metric.update([detections], [targets]) result = metric.compute() assert abs(result.map50_95 - 1.0) < 1e-6 def test_batch_updates_perfect_detections(self, detections_50_50, targets_50_50): """Test that batch updates with perfect detections get 1.0 mAP""" metric = MeanAveragePrecision() # Add 3 batch updates metric.update([detections_50_50], [targets_50_50]) metric.update([detections_50_50], [targets_50_50]) metric.update([detections_50_50], [targets_50_50]) result = metric.compute() # Should be perfect 1.0 mAP across all batches assert abs(result.map50_95 - 1.0) < 1e-6 def test_scenario_1_success_case_imperfect_match(self): """Scenario 1: Success Case with imperfect match""" # Small object (class 0) - area = 30*30 = 900 < 1024 small_perfect = Detections( xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.95]), data={"area": np.array([900])}, ) # Medium object (class 1) - area = 50*50 = 2500 (between 1024 and 9216) medium_target = Detections( xyxy=np.array([[10, 10, 60, 60]], dtype=np.float64), class_id=np.array([1]), data={"area": np.array([2500])}, ) medium_pred = Detections( xyxy=np.array([[12, 12, 60, 60]], dtype=np.float64), # Slightly off class_id=np.array([1]), confidence=np.array([0.9]), data={"area": np.array([2304])}, # 48*48 ) # Large objects (classes 0, 1, 2) - area = 100*100 = 10000 > 9216 large_targets = Detections( xyxy=np.array( [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], dtype=np.float64, ), class_id=np.array([2, 0, 1]), data={"area": np.array([10000, 10000, 10000])}, ) large_preds = Detections( xyxy=np.array( [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], dtype=np.float64, ), class_id=np.array([2, 0, 1]), confidence=np.array([0.9, 0.9, 0.9]), data={"area": np.array([10000, 10000, 10000])}, ) metric = MeanAveragePrecision() metric.update([small_perfect], [small_perfect]) metric.update([medium_pred], [medium_target]) metric.update([large_preds], [large_targets]) result = metric.compute() # Should be close to 0.9 (slightly less than perfect due to medium object) assert 0.85 < result.map50_95 < 0.98 # Adjusted upper bound assert ( result.medium_objects.map50_95 < 1.0 ) # Medium should be less than perfect def test_scenario_2_missed_detection(self): """Scenario 2: GT Present, No Prediction (Missed Detection)""" # Small object - area = 30*30 = 900 < 1024 small_detection = Detections( xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.95]), data={"area": np.array([900])}, ) # Medium object - area = 50*50 = 2500 (between 1024 and 9216) - missed medium_target = Detections( xyxy=np.array([[10, 10, 60, 60]], dtype=np.float64), class_id=np.array([1]), data={"area": np.array([2500])}, ) no_medium_pred = Detections.empty() # Large objects - area = 100*100 = 10000 > 9216 large_detections = Detections( xyxy=np.array( [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], dtype=np.float64, ), class_id=np.array([2, 0, 1]), confidence=np.array([0.9, 0.9, 0.9]), data={"area": np.array([10000, 10000, 10000])}, ) metric = MeanAveragePrecision() metric.update([small_detection], [small_detection]) metric.update([no_medium_pred], [medium_target]) metric.update([large_detections], [large_detections]) result = metric.compute() # Medium objects should have 0.0 mAP (missed detection) assert abs(result.medium_objects.map50_95 - 0.0) < 1e-6 def test_scenario_3_false_positive(self): """Scenario 3: No GT, Prediction Present (False Positive)""" # Small object - area = 30*30 = 900 < 1024 small_detection = Detections( xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.95]), data={"area": np.array([900])}, ) # Medium object - area = 50*50 = 2500 - false positive (no GT) medium_pred = Detections( xyxy=np.array([[12, 12, 62, 62]], dtype=np.float64), class_id=np.array([1]), confidence=np.array([0.9]), data={"area": np.array([2500])}, ) no_medium_target = Detections.empty() # Large objects - area = 100*100 = 10000 > 9216 large_detections = Detections( xyxy=np.array( [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], dtype=np.float64, ), class_id=np.array([2, 0, 1]), confidence=np.array([0.9, 0.9, 0.9]), data={"area": np.array([10000, 10000, 10000])}, ) metric = MeanAveragePrecision() metric.update([small_detection], [small_detection]) metric.update([medium_pred], [no_medium_target]) metric.update([large_detections], [large_detections]) result = metric.compute() # Medium objects should have -1 mAP (false positive, matching pycocotools) assert result.medium_objects.map50_95 == -1 def test_scenario_4_no_data(self): """Scenario 4: No GT, No Prediction (Category has no data)""" # Small object - area = 30*30 = 900 < 1024 small_detection = Detections( xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.95]), data={"area": np.array([900])}, ) # Medium object - no data at all no_medium = Detections.empty() # Large objects - area = 100*100 = 10000 > 9216 # only classes 0 and 2 (no class 1) large_targets = Detections( xyxy=np.array( [ [10, 10, 110, 110], [120, 120, 220, 220], ], dtype=np.float64, ), class_id=np.array([2, 0]), data={"area": np.array([10000, 10000])}, ) large_preds = Detections( xyxy=np.array( [ [10, 10, 110, 110], [120, 120, 220, 220], ], dtype=np.float64, ), class_id=np.array([2, 0]), confidence=np.array([0.9, 0.9]), data={"area": np.array([10000, 10000])}, ) metric = MeanAveragePrecision() metric.update([small_detection], [small_detection]) metric.update([no_medium], [no_medium]) metric.update([large_preds], [large_targets]) result = metric.compute() # Should NOT have negative mAP values for overall assert result.map50_95 >= 0.0 # Medium objects should have -1 mAP (no data, matching pycocotools) assert result.medium_objects.map50_95 == -1 def test_scenario_5_only_one_class_present(self): """Scenario 5: Only 1 of 3 Classes Present (Perfect Match)""" # Only class 0 objects with perfect matches detections_class_0 = [ Detections( xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.95]), ), Detections( xyxy=np.array([[20, 20, 230, 130]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), ), ] metric = MeanAveragePrecision() for det in detections_class_0: metric.update([det], [det]) result = metric.compute() # Should be 1.0 mAP (perfect match for the only class present) assert abs(result.map50_95 - 1.0) < 1e-6 assert abs(result.map50 - 1.0) < 1e-6 assert abs(result.map75 - 1.0) < 1e-6 def test_mixed_classes_with_missing_detections( self, detections_50_50, targets_50_50 ): """Test mixed scenario with some classes having no detections""" # Class 1: GT exists but no prediction class_1_target = Detections( xyxy=np.array([[60, 60, 100, 100]], dtype=np.float64), class_id=np.array([1]), ) class_1_pred = Detections.empty() # Class 2: Prediction exists but no GT (false positive) class_2_pred = Detections( xyxy=np.array([[110, 110, 150, 150]], dtype=np.float64), class_id=np.array([2]), confidence=np.array([0.8]), ) class_2_target = Detections.empty() metric = MeanAveragePrecision() metric.update([detections_50_50], [targets_50_50]) metric.update([class_1_pred], [class_1_target]) metric.update([class_2_pred], [class_2_target]) result = metric.compute() # Should not have negative mAP assert result.map50_95 >= 0.0 # Should be less than 1.0 due to missed detection and false positive assert result.map50_95 < 1.0 def test_empty_predictions_and_targets(self): """Test completely empty predictions and targets""" metric = MeanAveragePrecision() metric.update([Detections.empty()], [Detections.empty()]) result = metric.compute() # Should return -1 for no data (matching pycocotools behavior) assert result.map50_95 == -1 assert result.map50 == -1 assert result.map75 == -1 # All object size categories should also be -1 assert result.small_objects.map50_95 == -1 assert result.medium_objects.map50_95 == -1 assert result.large_objects.map50_95 == -1 class TestMeanAveragePrecisionMasks: @pytest.mark.parametrize( ("prediction_rows", "target_rows", "expected_map50"), [ pytest.param(slice(0, 16), slice(0, 16), 1.0, id="matching-masks"), pytest.param(slice(0, 16), slice(16, 32), 0.0, id="disjoint-masks"), ], ) def test_map50_follows_mask_overlap( self, prediction_rows: slice, target_rows: slice, expected_map50: float ) -> None: """With MASKS target, map50 must reflect mask IoU, not identical boxes.""" predictions = _mask_detections(prediction_rows, confidence=True) targets = _mask_detections(target_rows) metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(expected_map50, abs=1e-6) def test_missing_masks_raise(self) -> None: """With MASKS target, detections without masks must raise ValueError.""" predictions = Detections( xyxy=np.array([[0, 0, 10, 10]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), ) targets = _mask_detections(slice(0, 16)) metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) metric.update([predictions], [targets]) with pytest.raises(ValueError, match="MASKS"): metric.compute() def test_mask_pixel_count_drives_size_buckets(self) -> None: """With MASKS target, object size buckets use mask area, not bbox area.""" # bbox area is 100*100 = 10000 (large), mask area is 30*30 = 900 (small) mask = np.zeros((1, 120, 120), dtype=bool) mask[0, 10:40, 10:40] = True predictions = Detections( xyxy=np.array([[0, 0, 100, 100]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), mask=mask, ) targets = Detections( xyxy=np.array([[0, 0, 100, 100]], dtype=np.float64), class_id=np.array([0]), mask=mask.copy(), ) metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) result = metric.update([predictions], [targets]).compute() assert result.small_objects.map50 == pytest.approx(1.0, abs=1e-6) assert result.large_objects.map50 == -1 def test_boxes_target_ignores_masks(self) -> None: """With default BOXES target, disjoint masks must not affect the score.""" predictions = _mask_detections(slice(0, 16), confidence=True) targets = _mask_detections(slice(16, 32)) metric = MeanAveragePrecision() result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(1.0, abs=1e-6) class TestMeanAveragePrecisionOrientedBoundingBoxes: @pytest.mark.parametrize( ("prediction_corners", "target_corners", "expected_map50"), [ pytest.param( [[0, 0], [10, 0], [10, 10], [0, 10]], [[0, 0], [10, 0], [10, 10], [0, 10]], 1.0, id="matching-obb", ), pytest.param( [[0, 0], [10, 0], [10, 10], [0, 10]], [[20, 20], [30, 20], [30, 30], [20, 30]], 0.0, id="disjoint-obb", ), ], ) def test_map50_follows_oriented_box_overlap( self, prediction_corners: list[list[int]], target_corners: list[list[int]], expected_map50: float, ) -> None: """With OBB target, map50 must reflect OBB IoU, not identical boxes.""" predictions = _obb_detections(prediction_corners, confidence=True) targets = _obb_detections(target_corners) metric = MeanAveragePrecision( metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES ) result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(expected_map50, abs=1e-6) def test_missing_oriented_boxes_raise(self) -> None: """With OBB target, detections without OBB data must raise ValueError.""" predictions = Detections( xyxy=np.array([[0, 0, 30, 30]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), ) targets = _obb_detections([[0, 0], [10, 0], [10, 10], [0, 10]]) metric = MeanAveragePrecision( metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES ) metric.update([predictions], [targets]) with pytest.raises(ValueError, match=ORIENTED_BOX_COORDINATES): metric.compute() def test_cross_matched_obb_orients_iou_correctly(self) -> None: """2x2 cross-match: pred0->target1, pred1->target0 must both score as TP. A transposed (gt, dt) matrix would yield 0 IoU for every pair; map50=0. Passing asserts the (dt, gt) orientation is correct end-to-end. """ box_tl = np.array([[0, 0], [10, 0], [10, 10], [0, 10]], dtype=np.float32) box_br = np.array([[20, 20], [30, 20], [30, 30], [20, 30]], dtype=np.float32) targets = Detections( xyxy=np.array([[0, 0, 10, 10], [20, 20, 30, 30]], dtype=np.float64), class_id=np.array([0, 0]), data={ORIENTED_BOX_COORDINATES: np.stack([box_tl, box_br])}, ) # Predictions deliberately swapped: pred0 matches target1, pred1 matches target0 predictions = Detections( xyxy=np.array([[20, 20, 30, 30], [0, 0, 10, 10]], dtype=np.float64), class_id=np.array([0, 0]), confidence=np.array([0.9, 0.8]), data={ORIENTED_BOX_COORDINATES: np.stack([box_br, box_tl])}, ) metric = MeanAveragePrecision( metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES ) result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(1.0, abs=1e-6) class TestMeanAveragePrecisionMasksCrowdBranch: """Tests for the crowd-aware Jaccard path in _mask_iou_with_jaccard.""" def test_crowd_gt_ignores_contained_detection(self) -> None: """Detection inside a crowd GT is ignored (not FP) with Jaccard crowd IoU. Without Jaccard: small pred's standard IoU with crowd GT is 0.25 < 0.5, so pred0 is a FP, which reduces map50. With Jaccard: IoU = 1.0, pred0 is matched to crowd and ignored, so only pred1 (perfect TP) is scored -> map50=1.0. """ mask_normal = np.zeros((1, 32, 32), dtype=bool) mask_normal[0, :16, :] = True # normal GT: top half mask_crowd = np.ones((1, 32, 32), dtype=bool) # crowd GT: full image targets = Detections( xyxy=np.array([[0, 0, 32, 16], [0, 0, 32, 32]], dtype=np.float64), class_id=np.array([0, 0]), mask=np.concatenate([mask_normal, mask_crowd]), data={"iscrowd": np.array([0, 1], dtype=np.int64)}, ) # pred0 (conf=0.9): bottom quarter - inside crowd, no overlap with normal GT mask_pred0 = np.zeros((1, 32, 32), dtype=bool) mask_pred0[0, 16:24, :] = True # pred1 (conf=0.8): exact match with normal GT mask_pred1 = np.zeros((1, 32, 32), dtype=bool) mask_pred1[0, :16, :] = True predictions = Detections( xyxy=np.array([[0, 16, 32, 24], [0, 0, 32, 16]], dtype=np.float64), class_id=np.array([0, 0]), confidence=np.array([0.9, 0.8]), mask=np.concatenate([mask_pred0, mask_pred1]), ) metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(1.0, abs=1e-6) class TestMeanAveragePrecisionIgnoreFlag: """Tests for explicit target ignore flags in COCO-style evaluation.""" def test_user_ignore_flag_excludes_target_from_scoring(self) -> None: """Targets marked ignored by the user must not count as normal GT.""" targets = Detections( xyxy=np.array([[0, 0, 10, 10]], dtype=np.float64), class_id=np.array([0]), data={"ignore": np.array([1], dtype=np.int64)}, ) predictions = Detections( xyxy=np.array([[0, 0, 10, 10]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), ) metric = MeanAveragePrecision() result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(-1.0, abs=1e-6) def test_normal_gt_matched_correctly_alongside_crowd_gt(self) -> None: """Normal GT is matched and scored when a crowd GT is also present.""" mask_normal = np.zeros((1, 32, 32), dtype=bool) mask_normal[0, :16, :] = True mask_crowd = np.ones((1, 32, 32), dtype=bool) targets = Detections( xyxy=np.array([[0, 0, 32, 16], [0, 0, 32, 32]], dtype=np.float64), class_id=np.array([0, 0]), mask=np.concatenate([mask_normal, mask_crowd]), data={"iscrowd": np.array([0, 1], dtype=np.int64)}, ) mask_pred = np.zeros((1, 32, 32), dtype=bool) mask_pred[0, :16, :] = True # exact match with normal GT predictions = Detections( xyxy=np.array([[0, 0, 32, 16]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.9]), mask=mask_pred, ) metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(1.0, abs=1e-6) class TestMeanAveragePrecisionMasksOrientation: """Tests that the (dt, gt) IoU-matrix orientation is correct end-to-end.""" def test_cross_matched_masks_orient_iou_correctly(self) -> None: """2x2 cross-match: pred0->target1, pred1->target0 must both score as TP. A transposed (gt, dt) matrix would yield 0 IoU for every pair; map50=0. Passing asserts the (dt, gt) orientation is correct end-to-end. """ top_mask = np.zeros((1, 32, 32), dtype=bool) top_mask[0, :16, :] = True bottom_mask = np.zeros((1, 32, 32), dtype=bool) bottom_mask[0, 16:, :] = True # target0 = top half, target1 = bottom half targets = Detections( xyxy=np.array([[0, 0, 32, 16], [0, 16, 32, 32]], dtype=np.float64), class_id=np.array([0, 0]), mask=np.concatenate([top_mask, bottom_mask]), ) # Predictions deliberately swapped: pred0=bottom, pred1=top predictions = Detections( xyxy=np.array([[0, 16, 32, 32], [0, 0, 32, 16]], dtype=np.float64), class_id=np.array([0, 0]), confidence=np.array([0.9, 0.8]), mask=np.concatenate([bottom_mask, top_mask]), ) metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) result = metric.update([predictions], [targets]).compute() assert result.map50 == pytest.approx(1.0, abs=1e-6) class TestEvaluationDatasetLoadPredictions: """Tests for `EvaluationDataset.load_predictions` input validation.""" @pytest.mark.parametrize( ("known_image_ids", "prediction_image_ids"), [ pytest.param([1], [999], id="all-unknown-ids"), pytest.param([1, 2], [1, 999], id="mixed-known-and-unknown-ids"), pytest.param([], [1], id="empty-dataset-with-nonempty-predictions"), ], ) def test_unknown_image_id_raises_value_error( self, known_image_ids: list[int], prediction_image_ids: list[int] ) -> None: """Predictions referencing any unknown image id raise ValueError.""" dataset = EvaluationDataset( targets={ "images": [{"id": image_id} for image_id in known_image_ids], "annotations": [], "categories": [{"id": 1}], } ) predictions = [ {"image_id": image_id, "category_id": 1, "bbox": [0, 0, 1, 1]} for image_id in prediction_image_ids ] with pytest.raises(ValueError, match="current coco set"): dataset.load_predictions(predictions) def test_predictions_subset_of_known_ids_does_not_raise(self) -> None: """Predictions referencing only a subset of known image ids are accepted.""" dataset = EvaluationDataset( targets={ "images": [{"id": 1}, {"id": 2}], "annotations": [], "categories": [{"id": 1}], } ) predictions = [{"image_id": 1, "category_id": 1, "bbox": [0, 0, 1, 1]}] result = dataset.load_predictions(predictions) loaded_annotations = result.get_annotations([1]) assert len(loaded_annotations) == 1 assert loaded_annotations[0]["image_id"] == 1 assert loaded_annotations[0]["category_id"] == 1 assert loaded_annotations[0]["bbox"] == [0, 0, 1, 1]