learningcircuit--local-deep-research/tests/benchmarks/test_metrics_calculation.py

"""
Tests for benchmarks/metrics/calculation.py

Tests cover:
- calculate_metrics function
- calculate_combined_score function
- calculate_resource_metrics function
"""

import json


class TestCalculateMetrics:
    """Tests for the calculate_metrics function."""

    def test_calculates_basic_metrics(self, tmp_path):
        """Test calculation of basic metrics from results file."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        # Create a test results file
        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": True, "processing_time": 1.5, "confidence": "90"},
            {"is_correct": True, "processing_time": 2.0, "confidence": "85"},
            {"is_correct": False, "processing_time": 1.0, "confidence": "70"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))

        assert metrics["total_examples"] == 3
        assert metrics["graded_examples"] == 3
        assert metrics["correct"] == 2
        assert metrics["accuracy"] == 2 / 3
        assert metrics["average_processing_time"] == 1.5
        assert metrics["average_confidence"] == (90 + 85 + 70) / 3

    def test_handles_empty_file(self, tmp_path):
        """Test handling of empty results file."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "empty.jsonl"
        results_file.write_text("")

        metrics = calculate_metrics(str(results_file))

        assert "error" in metrics
        assert metrics["error"] == "No results found"

    def test_handles_missing_file(self, tmp_path):
        """Test handling of missing results file."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        metrics = calculate_metrics(str(tmp_path / "nonexistent.jsonl"))

        assert "error" in metrics

    def test_calculates_error_rate(self, tmp_path):
        """Test calculation of error rate."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": True},
            {"error": "Some error occurred"},
            {"is_correct": False},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))

        assert metrics["error_count"] == 1
        assert metrics["error_rate"] == 1 / 3

    def test_calculates_per_category_metrics(self, tmp_path):
        """Test calculation of per-category metrics."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": True, "category": "science"},
            {"is_correct": False, "category": "science"},
            {"is_correct": True, "category": "history"},
            {"is_correct": True, "category": "history"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))

        assert "categories" in metrics
        assert metrics["categories"]["science"]["total"] == 2
        assert metrics["categories"]["science"]["correct"] == 1
        assert metrics["categories"]["science"]["accuracy"] == 0.5
        assert metrics["categories"]["history"]["total"] == 2
        assert metrics["categories"]["history"]["correct"] == 2
        assert metrics["categories"]["history"]["accuracy"] == 1.0

    def test_handles_missing_confidence(self, tmp_path):
        """Test handling of results without confidence values."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": True},
            {"is_correct": False},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))

        assert metrics["average_confidence"] == 0

    def test_handles_invalid_confidence(self, tmp_path):
        """Test handling of invalid confidence values."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": True, "confidence": "invalid"},
            {"is_correct": True, "confidence": "80"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))

        # Should skip invalid and only use valid confidence
        assert metrics["average_confidence"] == 80


class TestCalculateCombinedScore:
    """Tests for the calculate_combined_score function."""

    def test_calculates_with_default_weights(self):
        """Test combined score with default weights."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 0.8},
            "speed": {"speed_score": 0.6},
            "resource": {"resource_score": 0.9},
        }

        score = calculate_combined_score(metrics)

        # Default weights: quality=0.6, speed=0.3, resource=0.1
        expected = 0.8 * 0.6 + 0.6 * 0.3 + 0.9 * 0.1
        assert abs(score - expected) < 0.001

    def test_calculates_with_custom_weights(self):
        """Test combined score with custom weights."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 1.0},
            "speed": {"speed_score": 0.5},
            "resource": {"resource_score": 0.0},
        }
        weights = {"quality": 0.5, "speed": 0.5, "resource": 0.0}

        score = calculate_combined_score(metrics, weights)

        expected = 1.0 * 0.5 + 0.5 * 0.5 + 0.0 * 0.0
        assert abs(score - expected) < 0.001

    def test_handles_missing_metrics(self):
        """Test handling of missing metric categories."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 0.8},
            # Missing speed and resource
        }

        score = calculate_combined_score(metrics)

        # Should only count quality component
        assert score > 0
        assert score <= 1.0

    def test_handles_zero_weights(self):
        """Test handling of zero total weight."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 0.8},
        }
        weights = {"quality": 0, "speed": 0, "resource": 0}

        score = calculate_combined_score(metrics, weights)

        assert score == 0.0

    def test_normalizes_weights(self):
        """Test that weights are normalized."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 1.0},
            "speed": {"speed_score": 1.0},
            "resource": {"resource_score": 1.0},
        }
        # Weights that don't sum to 1
        weights = {"quality": 2, "speed": 1, "resource": 1}

        score = calculate_combined_score(metrics, weights)

        # All scores are 1.0, so normalized result should be 1.0
        assert abs(score - 1.0) < 0.001


class TestCalculateResourceMetrics:
    """Tests for the calculate_resource_metrics function."""

    def test_calculates_resource_score(self):
        """Test calculation of resource score."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        config = {
            "iterations": 2,
            "questions_per_iteration": 2,
            "max_results": 50,
        }

        metrics = calculate_resource_metrics(config)

        assert "resource_score" in metrics
        assert "estimated_complexity" in metrics
        assert 0 <= metrics["resource_score"] <= 1

    def test_higher_complexity_lower_score(self):
        """Test that higher complexity results in lower resource score."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        low_complexity_config = {
            "iterations": 1,
            "questions_per_iteration": 1,
            "max_results": 25,
        }
        high_complexity_config = {
            "iterations": 5,
            "questions_per_iteration": 5,
            "max_results": 100,
        }

        low_metrics = calculate_resource_metrics(low_complexity_config)
        high_metrics = calculate_resource_metrics(high_complexity_config)

        assert low_metrics["resource_score"] > high_metrics["resource_score"]
        assert (
            low_metrics["estimated_complexity"]
            < high_metrics["estimated_complexity"]
        )

    def test_uses_default_values(self):
        """Test that default values are used for missing config."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        empty_config = {}

        metrics = calculate_resource_metrics(empty_config)

        assert "resource_score" in metrics
        assert metrics["resource_score"] > 0