learningcircuit--local-deep-research/tests/benchmarks/test_metrics_calculation_extended.py

"""Extended tests for benchmarks/metrics/calculation.py - covering edge cases
in calculate_metrics and calculate_combined_score."""

import json
import tempfile

import pytest


class TestCalculateMetricsEdgeCases:
    """Tests for calculate_metrics edge cases."""

    def test_nonexistent_file_returns_error(self):
        """Non-existent file should return error dict."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        result = calculate_metrics("/nonexistent/path/results.jsonl")

        assert "error" in result

    def test_empty_file_returns_error(self):
        """Empty file should return error about no results."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write("")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["error"] == "No results found"

    def test_malformed_json_lines_returns_error(self):
        """File with malformed JSON should return error."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write("not valid json\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert "error" in result

    def test_file_with_only_whitespace_lines(self):
        """File with only whitespace lines should return no results error."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write("\n\n\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["error"] == "No results found"

    def test_all_correct_results(self):
        """All correct results should give accuracy of 1.0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            for i in range(5):
                f.write(
                    json.dumps({"is_correct": True, "processing_time": 1.0})
                    + "\n"
                )
            f.flush()

            result = calculate_metrics(f.name)

        assert result["accuracy"] == 1.0
        assert result["correct"] == 5
        assert result["graded_examples"] == 5

    def test_all_incorrect_results(self):
        """All incorrect results should give accuracy of 0.0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            for i in range(3):
                f.write(json.dumps({"is_correct": False}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["accuracy"] == 0.0
        assert result["correct"] == 0

    def test_mixed_graded_and_ungraded(self):
        """Mixed graded and ungraded results should only use graded for accuracy."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"is_correct": True}) + "\n")
            f.write(json.dumps({"is_correct": False}) + "\n")
            f.write(json.dumps({"answer": "no grading"}) + "\n")  # Ungraded
            f.flush()

            result = calculate_metrics(f.name)

        assert result["total_examples"] == 3
        assert result["graded_examples"] == 2
        assert result["accuracy"] == 0.5

    def test_results_with_errors(self):
        """Results with errors should be counted in error_rate."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"is_correct": True}) + "\n")
            f.write(json.dumps({"error": "timeout"}) + "\n")
            f.write(json.dumps({"error": "API error"}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["error_count"] == 2
        assert result["error_rate"] == pytest.approx(2 / 3)

    def test_processing_time_calculation(self):
        """Average processing time should be calculated correctly."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(
                json.dumps({"processing_time": 10.0, "is_correct": True}) + "\n"
            )
            f.write(
                json.dumps({"processing_time": 20.0, "is_correct": True}) + "\n"
            )
            f.write(
                json.dumps({"processing_time": 30.0, "is_correct": True}) + "\n"
            )
            f.flush()

            result = calculate_metrics(f.name)

        assert result["average_processing_time"] == pytest.approx(20.0)

    def test_confidence_parsing(self):
        """Confidence values should be parsed and averaged."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"confidence": "80", "is_correct": True}) + "\n")
            f.write(json.dumps({"confidence": "60", "is_correct": True}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["average_confidence"] == 70.0

    def test_invalid_confidence_values_skipped(self):
        """Invalid confidence values should be skipped."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(
                json.dumps({"confidence": "not_a_number", "is_correct": True})
                + "\n"
            )
            f.write(json.dumps({"confidence": "90", "is_correct": True}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["average_confidence"] == 90.0

    def test_per_category_metrics(self):
        """Per-category metrics should be calculated correctly."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"is_correct": True, "category": "math"}) + "\n")
            f.write(
                json.dumps({"is_correct": False, "category": "math"}) + "\n"
            )
            f.write(
                json.dumps({"is_correct": True, "category": "science"}) + "\n"
            )
            f.flush()

            result = calculate_metrics(f.name)

        assert "categories" in result
        assert result["categories"]["math"]["accuracy"] == 0.5
        assert result["categories"]["math"]["total"] == 2
        assert result["categories"]["science"]["accuracy"] == 1.0

    def test_no_processing_times(self):
        """Results without processing_time should give avg_time of 0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"is_correct": True}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["average_processing_time"] == 0

    def test_no_graded_results_accuracy_zero(self):
        """Results with no is_correct field should give accuracy 0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"answer": "something"}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert result["accuracy"] == 0
        assert result["graded_examples"] == 0

    def test_timestamp_included(self):
        """Result should include a timestamp."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(json.dumps({"is_correct": True}) + "\n")
            f.flush()

            result = calculate_metrics(f.name)

        assert "timestamp" in result
        assert isinstance(result["timestamp"], str)


class TestCalculateCombinedScoreEdgeCases:
    """Tests for calculate_combined_score edge cases."""

    def test_default_weights(self):
        """Default weights should be quality=0.6, speed=0.3, resource=0.1."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 1.0},
            "speed": {"speed_score": 1.0},
            "resource": {"resource_score": 1.0},
        }

        score = calculate_combined_score(metrics)
        assert score == pytest.approx(1.0)

    def test_zero_weights_returns_zero(self):
        """All zero weights should return 0.0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {"quality": {"quality_score": 1.0}}
        weights = {"quality": 0.0, "speed": 0.0, "resource": 0.0}

        score = calculate_combined_score(metrics, weights)
        assert score == 0.0

    def test_missing_metric_categories(self):
        """Missing metric categories should be skipped."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 0.8},
            # No speed or resource
        }

        score = calculate_combined_score(metrics)
        # Only quality contributes: 0.8 * (0.6/1.0) = 0.48
        assert score == pytest.approx(0.8 * 0.6)

    def test_custom_weights(self):
        """Custom weights should be used instead of defaults."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 1.0},
            "speed": {"speed_score": 0.5},
        }
        weights = {"quality": 1.0, "speed": 1.0}

        score = calculate_combined_score(metrics, weights)
        # Normalized: quality=0.5, speed=0.5
        # Score: 1.0*0.5 + 0.5*0.5 = 0.75
        assert score == pytest.approx(0.75)

    def test_empty_metrics(self):
        """Empty metrics should return 0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        score = calculate_combined_score({})
        assert score == 0.0

    def test_missing_score_keys_in_metrics(self):
        """Missing score keys within metric dicts should default to 0."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {},  # No quality_score key
            "speed": {"speed_score": 0.8},
        }

        score = calculate_combined_score(metrics)
        # quality_score defaults to 0.0, speed_score=0.8
        assert score == pytest.approx(0.0 * 0.6 + 0.8 * 0.3)


class TestCalculateResourceMetrics:
    """Tests for calculate_resource_metrics."""

    def test_basic_resource_calculation(self):
        """Basic resource metrics with default config."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        config = {
            "iterations": 2,
            "questions_per_iteration": 2,
            "max_results": 50,
        }
        result = calculate_resource_metrics(config)

        assert "resource_score" in result
        assert "estimated_complexity" in result
        assert 0 <= result["resource_score"] <= 1

    def test_higher_config_means_more_resources(self):
        """Higher config values should give lower resource score (more resources used)."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        low_config = {
            "iterations": 1,
            "questions_per_iteration": 1,
            "max_results": 10,
        }
        high_config = {
            "iterations": 10,
            "questions_per_iteration": 10,
            "max_results": 100,
        }

        low_result = calculate_resource_metrics(low_config)
        high_result = calculate_resource_metrics(high_config)

        assert low_result["resource_score"] > high_result["resource_score"]
        assert (
            low_result["estimated_complexity"]
            < high_result["estimated_complexity"]
        )

    def test_default_values_used_when_missing(self):
        """Default values should be used when config keys are missing."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        result = calculate_resource_metrics({})  # Empty config

        assert "resource_score" in result
        assert result["estimated_complexity"] > 0