learningcircuit--local-deep-research/tests/benchmarks/test_calculation_coverage.py

"""
Coverage-focused tests for benchmarks/metrics/calculation.py.

Targets the ~37% uncovered code, specifically:
- evaluate_benchmark_quality (with mocked runner)
- measure_execution_time (with mocked SearchSystem)
- calculate_quality_metrics (delegates to evaluate_benchmark_quality)
- calculate_speed_metrics (delegates to measure_execution_time)
- calculate_resource_metrics edge cases not yet covered
- calculate_combined_score with weights that need normalization
- calculate_metrics: malformed JSON mixed with valid lines, integer confidence,
  confidence as None, confidence as a list (TypeError path)
"""

import json
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest


# ---------------------------------------------------------------------------
# Helper to write JSONL files
# ---------------------------------------------------------------------------


def _write_jsonl(path, records):
    """Write a list of dicts (or raw strings) to a JSONL file."""
    with open(path, "w") as f:
        for r in records:
            if isinstance(r, str):
                f.write(r + "\n")
            else:
                f.write(json.dumps(r) + "\n")


# ===========================================================================
# calculate_metrics – coverage-gap tests
# ===========================================================================


class TestCalculateMetricsCoverageGaps:
    """Tests targeting lines not yet exercised by existing suites."""

    def test_malformed_json_among_valid_lines_returns_error(self, tmp_path):
        """A single malformed JSON line causes json.loads to raise,
        which is caught by the broad except and returns an error dict."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "mixed.jsonl"
        _write_jsonl(
            results_file,
            [
                '{"is_correct": true}',
                "NOT JSON AT ALL",
            ],
        )
        result = calculate_metrics(str(results_file))
        # The function catches *any* exception while iterating and returns error
        assert "error" in result

    def test_confidence_as_integer_value(self, tmp_path):
        """Integer confidence (not string) is truthy when non-zero,
        and int() on an int succeeds."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "int_conf.jsonl"
        _write_jsonl(
            results_file,
            [
                {"confidence": 75},
                {"confidence": 25},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["average_confidence"] == 50

    def test_confidence_none_is_skipped(self, tmp_path):
        """confidence=None is falsy, so it should be skipped."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "none_conf.jsonl"
        _write_jsonl(
            results_file,
            [
                {"confidence": None},
                {"confidence": "60"},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["average_confidence"] == 60

    def test_confidence_as_list_triggers_type_error(self, tmp_path):
        """confidence=[1,2] is truthy, but int([1,2]) raises TypeError,
        which is caught and skipped."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "list_conf.jsonl"
        _write_jsonl(
            results_file,
            [
                {"confidence": [1, 2]},
                {"confidence": "50"},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["average_confidence"] == 50

    def test_confidence_as_float_string(self, tmp_path):
        """confidence='85.5' – int('85.5') raises ValueError, skipped."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "float_conf.jsonl"
        _write_jsonl(
            results_file,
            [
                {"confidence": "85.5"},
                {"confidence": "100"},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        # "85.5" is skipped (ValueError on int()), only "100" counted
        assert metrics["average_confidence"] == 100

    def test_no_categories_key_absent_from_metrics(self, tmp_path):
        """When no result has 'category', metrics should not contain 'categories'."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "no_cat.jsonl"
        _write_jsonl(
            results_file,
            [
                {"is_correct": True},
                {"is_correct": False},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert "categories" not in metrics

    def test_processing_time_zero_included(self, tmp_path):
        """processing_time=0 is in the result (key exists), so it counts."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "zero_time.jsonl"
        _write_jsonl(
            results_file,
            [
                {"processing_time": 0},
                {"processing_time": 10.0},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["average_processing_time"] == 5.0

    def test_multiple_categories(self, tmp_path):
        """Three categories, each with different accuracy."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "multi_cat.jsonl"
        _write_jsonl(
            results_file,
            [
                {"is_correct": True, "category": "A"},
                {"is_correct": True, "category": "A"},
                {"is_correct": False, "category": "B"},
                {"is_correct": True, "category": "C"},
                {"is_correct": False, "category": "C"},
                {"is_correct": False, "category": "C"},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["categories"]["A"]["accuracy"] == 1.0
        assert metrics["categories"]["B"]["accuracy"] == 0.0
        assert metrics["categories"]["C"]["accuracy"] == pytest.approx(1 / 3)

    def test_large_number_of_results(self, tmp_path):
        """Ensure it handles a larger dataset correctly."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "large.jsonl"
        records = [
            {"is_correct": i % 2 == 0, "processing_time": float(i)}
            for i in range(100)
        ]
        _write_jsonl(results_file, records)
        metrics = calculate_metrics(str(results_file))
        assert metrics["total_examples"] == 100
        assert metrics["graded_examples"] == 100
        assert metrics["correct"] == 50
        assert metrics["accuracy"] == 0.5

    def test_error_field_with_is_correct(self, tmp_path):
        """A result can have both 'error' and 'is_correct'."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_metrics,
        )

        results_file = tmp_path / "both.jsonl"
        _write_jsonl(
            results_file,
            [
                {"is_correct": True, "error": "partial failure"},
            ],
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["error_count"] == 1
        assert metrics["graded_examples"] == 1
        assert metrics["correct"] == 1


# ===========================================================================
# evaluate_benchmark_quality – mocked runner
# ===========================================================================


class TestEvaluateBenchmarkQuality:
    """Tests for evaluate_benchmark_quality with mocked run_simpleqa_benchmark."""

    def test_returns_accuracy_and_quality_score(self, tmp_path):
        from local_deep_research.benchmarks.metrics.calculation import (
            evaluate_benchmark_quality,
        )

        mock_results = {"metrics": {"accuracy": 0.75}}
        with patch(
            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
            return_value=mock_results,
        ) as mock_run:
            # Provide an output_dir so no tempdir is created
            result = evaluate_benchmark_quality(
                system_config={"iterations": 1},
                num_examples=5,
                output_dir=str(tmp_path),
            )

        assert result["accuracy"] == 0.75
        assert result["quality_score"] == 0.75
        mock_run.assert_called_once()

    def test_uses_temp_dir_when_output_dir_none(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            evaluate_benchmark_quality,
        )

        mock_results = {"metrics": {"accuracy": 0.5}}
        with patch(
            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
            return_value=mock_results,
        ):
            result = evaluate_benchmark_quality(
                system_config={},
                num_examples=2,
                output_dir=None,
            )

        assert result["accuracy"] == 0.5

    def test_exception_returns_zero_scores(self, tmp_path):
        from local_deep_research.benchmarks.metrics.calculation import (
            evaluate_benchmark_quality,
        )

        with patch(
            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
            side_effect=RuntimeError("boom"),
        ):
            result = evaluate_benchmark_quality(
                system_config={},
                num_examples=2,
                output_dir=str(tmp_path),
            )

        assert result["accuracy"] == 0.0
        assert result["quality_score"] == 0.0
        assert "error" in result

    def test_missing_metrics_key_defaults_to_zero(self, tmp_path):
        from local_deep_research.benchmarks.metrics.calculation import (
            evaluate_benchmark_quality,
        )

        # run_simpleqa_benchmark returns dict without "metrics"
        with patch(
            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
            return_value={},
        ):
            result = evaluate_benchmark_quality(
                system_config={},
                num_examples=2,
                output_dir=str(tmp_path),
            )

        assert result["accuracy"] == 0.0
        assert result["quality_score"] == 0.0

    def test_config_values_passed_to_runner(self, tmp_path):
        from local_deep_research.benchmarks.metrics.calculation import (
            evaluate_benchmark_quality,
        )

        with patch(
            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
            return_value={"metrics": {"accuracy": 0.9}},
        ) as mock_run:
            evaluate_benchmark_quality(
                system_config={
                    "iterations": 5,
                    "questions_per_iteration": 3,
                    "search_strategy": "custom",
                    "search_tool": "google",
                    "model_name": "gpt-4",
                    "provider": "openai",
                },
                num_examples=10,
                output_dir=str(tmp_path),
            )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["num_examples"] == 10
        search_config = call_kwargs["search_config"]
        assert search_config["iterations"] == 5
        assert search_config["questions_per_iteration"] == 3
        assert search_config["search_strategy"] == "custom"
        assert search_config["search_tool"] == "google"
        assert search_config["model_name"] == "gpt-4"
        assert search_config["provider"] == "openai"

    def test_temp_dir_cleaned_up_on_success(self):
        """When output_dir is None, a temp dir is created and cleaned up."""
        from local_deep_research.benchmarks.metrics.calculation import (
            evaluate_benchmark_quality,
        )

        created_dirs = []

        original_mkdtemp = __import__("tempfile").mkdtemp

        def tracking_mkdtemp(**kwargs):
            d = original_mkdtemp(**kwargs)
            created_dirs.append(d)
            return d

        with (
            patch(
                "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
                return_value={"metrics": {"accuracy": 0.5}},
            ),
            patch(
                "tempfile.mkdtemp",
                side_effect=tracking_mkdtemp,
            ),
        ):
            evaluate_benchmark_quality(
                system_config={},
                num_examples=1,
                output_dir=None,
            )

        assert len(created_dirs) == 1
        # The temp dir should have been cleaned up
        assert not Path(created_dirs[0]).exists()


# ===========================================================================
# measure_execution_time – mocked SearchSystem
# ===========================================================================


class TestMeasureExecutionTime:
    """Tests for measure_execution_time with mocked SearchSystem."""

    def _patch_search_system(self, search_time=0.1):
        """Return a context manager that patches AdvancedSearchSystem and its deps."""
        from contextlib import ExitStack

        mock_system = MagicMock()
        mock_system.search.return_value = "result"

        mock_cls = MagicMock(return_value=mock_system)

        class CombinedPatcher:
            def __enter__(self2):
                self2.stack = ExitStack().__enter__()
                self2.stack.enter_context(
                    patch(
                        "local_deep_research.config.llm_config.get_llm",
                        return_value=MagicMock(),
                    )
                )
                self2.stack.enter_context(
                    patch(
                        "local_deep_research.config.search_config.get_search",
                        return_value=MagicMock(),
                    )
                )
                self2.stack.enter_context(
                    patch(
                        "local_deep_research.search_system.AdvancedSearchSystem",
                        mock_cls,
                    )
                )
                return self2

            def __exit__(self2, *args):
                self2.stack.__exit__(*args)

        return (
            CombinedPatcher(),
            mock_cls,
            mock_system,
        )

    def test_basic_speed_measurement(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            measure_execution_time,
        )

        patcher, mock_cls, mock_system = self._patch_search_system()
        with patcher:
            result = measure_execution_time(
                system_config={"iterations": 1},
                query="test",
                num_runs=1,
            )

        assert "average_time" in result
        assert "speed_score" in result
        assert "min_time" in result
        assert "max_time" in result
        assert result["average_time"] >= 0
        assert 0 < result["speed_score"] <= 1.0
        mock_system.search.assert_called_once_with("test", full_response=False)

    def test_multiple_runs_averaged(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            measure_execution_time,
        )

        patcher, mock_cls, mock_system = self._patch_search_system()
        with patcher:
            result = measure_execution_time(
                system_config={},
                query="multi",
                num_runs=3,
            )

        assert mock_system.search.call_count == 3
        assert result["average_time"] >= 0
        assert result["min_time"] <= result["max_time"]

    def test_search_tool_override(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            measure_execution_time,
        )

        config = {"iterations": 2}
        patcher, mock_cls, mock_system = self._patch_search_system()
        with patcher:
            measure_execution_time(
                system_config=config,
                search_tool="duckduckgo",
                num_runs=1,
            )

        # search_tool should have been set on the config
        assert config["search_tool"] == "duckduckgo"

    def test_exception_returns_zero_scores(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            measure_execution_time,
        )

        mock_system = MagicMock()
        mock_system.search.side_effect = RuntimeError("connection failed")
        mock_cls = MagicMock(return_value=mock_system)

        patcher, _, _ = self._patch_search_system()
        # Override the mock_cls with our exception-raising one
        with patcher:
            with patch(
                "local_deep_research.search_system.AdvancedSearchSystem",
                mock_cls,
            ):
                result = measure_execution_time(
                    system_config={},
                    num_runs=1,
                )

        assert result["average_time"] == 0.0
        assert result["speed_score"] == 0.0
        assert "error" in result

    def test_speed_score_formula(self):
        """Verify speed_score = 1/(1 + avg_time/30)."""
        from local_deep_research.benchmarks.metrics.calculation import (
            measure_execution_time,
        )

        patcher, mock_cls, mock_system = self._patch_search_system()
        mock_system.search.return_value = "r"

        with patcher:
            result = measure_execution_time(
                system_config={},
                num_runs=1,
            )

        avg = result["average_time"]
        expected_score = 1.0 / (1.0 + avg / 30.0)
        assert result["speed_score"] == pytest.approx(expected_score, abs=0.01)


# ===========================================================================
# calculate_quality_metrics – delegates to evaluate_benchmark_quality
# ===========================================================================


class TestCalculateQualityMetrics:
    """Tests for calculate_quality_metrics."""

    def test_returns_quality_and_accuracy(self, tmp_path):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_quality_metrics,
        )

        with patch(
            "local_deep_research.benchmarks.metrics.calculation.evaluate_benchmark_quality",
            return_value={"quality_score": 0.85, "accuracy": 0.85},
        ):
            result = calculate_quality_metrics(
                system_config={"iterations": 1},
                num_examples=5,
                output_dir=str(tmp_path),
            )

        assert result["quality_score"] == 0.85
        assert result["accuracy"] == 0.85

    def test_defaults_to_zero_on_missing_keys(self, tmp_path):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_quality_metrics,
        )

        with patch(
            "local_deep_research.benchmarks.metrics.calculation.evaluate_benchmark_quality",
            return_value={},
        ):
            result = calculate_quality_metrics(
                system_config={},
                output_dir=str(tmp_path),
            )

        assert result["quality_score"] == 0.0
        assert result["accuracy"] == 0.0


# ===========================================================================
# calculate_speed_metrics – delegates to measure_execution_time
# ===========================================================================


class TestCalculateSpeedMetrics:
    """Tests for calculate_speed_metrics."""

    def test_returns_speed_score_and_time(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_speed_metrics,
        )

        with patch(
            "local_deep_research.benchmarks.metrics.calculation.measure_execution_time",
            return_value={"speed_score": 0.7, "average_time": 12.0},
        ):
            result = calculate_speed_metrics(
                system_config={},
                query="hello",
                num_runs=2,
            )

        assert result["speed_score"] == 0.7
        assert result["average_time"] == 12.0

    def test_defaults_to_zero_on_missing_keys(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_speed_metrics,
        )

        with patch(
            "local_deep_research.benchmarks.metrics.calculation.measure_execution_time",
            return_value={},
        ):
            result = calculate_speed_metrics(system_config={})

        assert result["speed_score"] == 0.0
        assert result["average_time"] == 0.0

    def test_passes_search_tool_and_query(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_speed_metrics,
        )

        with patch(
            "local_deep_research.benchmarks.metrics.calculation.measure_execution_time",
            return_value={"speed_score": 0.5, "average_time": 20.0},
        ) as mock_measure:
            calculate_speed_metrics(
                system_config={"iterations": 3},
                query="deep query",
                search_tool="brave",
                num_runs=5,
            )

        mock_measure.assert_called_once_with(
            system_config={"iterations": 3},
            query="deep query",
            search_tool="brave",
            num_runs=5,
        )


# ===========================================================================
# calculate_resource_metrics – additional coverage
# ===========================================================================


class TestCalculateResourceMetricsAdditional:
    """Additional resource metric tests for coverage gaps."""

    def test_search_tool_parameter_ignored_in_heuristic(self):
        """search_tool and query params exist but don't affect the heuristic."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        config = {
            "iterations": 2,
            "questions_per_iteration": 2,
            "max_results": 50,
        }
        r1 = calculate_resource_metrics(
            config, query="query A", search_tool="brave"
        )
        r2 = calculate_resource_metrics(
            config, query="query B", search_tool="google"
        )
        assert r1["resource_score"] == r2["resource_score"]
        assert r1["estimated_complexity"] == r2["estimated_complexity"]

    def test_fractional_max_results(self):
        """max_results can be a float; formula still works."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_resource_metrics,
        )

        config = {
            "iterations": 1,
            "questions_per_iteration": 1,
            "max_results": 25,
        }
        metrics = calculate_resource_metrics(config)
        expected_complexity = 1 * 1 * (25 / 50)
        assert metrics["estimated_complexity"] == pytest.approx(
            expected_complexity
        )


# ===========================================================================
# calculate_combined_score – additional coverage
# ===========================================================================


class TestCalculateCombinedScoreAdditional:
    """Additional combined score tests for coverage gaps."""

    def test_weights_with_only_some_matching_metrics(self):
        """Weights for categories not in metrics are harmless."""
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {"quality": {"quality_score": 1.0}}
        weights = {"quality": 0.5, "speed": 0.3, "resource": 0.2}
        score = calculate_combined_score(metrics, weights)
        # Only quality matches: 1.0 * (0.5/1.0) = 0.5
        assert score == pytest.approx(0.5)

    def test_all_weights_equal(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 0.9},
            "speed": {"speed_score": 0.6},
            "resource": {"resource_score": 0.3},
        }
        weights = {"quality": 1, "speed": 1, "resource": 1}
        score = calculate_combined_score(metrics, weights)
        expected = (0.9 + 0.6 + 0.3) / 3.0
        assert score == pytest.approx(expected)

    def test_very_large_weights_still_normalize(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {
            "quality": {"quality_score": 0.8},
            "speed": {"speed_score": 0.4},
            "resource": {"resource_score": 0.2},
        }
        weights = {"quality": 1000, "speed": 500, "resource": 500}
        score = calculate_combined_score(metrics, weights)
        # norm: quality=0.5, speed=0.25, resource=0.25
        expected = 0.8 * 0.5 + 0.4 * 0.25 + 0.2 * 0.25
        assert score == pytest.approx(expected)

    def test_empty_weights_dict_returns_zero(self):
        from local_deep_research.benchmarks.metrics.calculation import (
            calculate_combined_score,
        )

        metrics = {"quality": {"quality_score": 1.0}}
        score = calculate_combined_score(metrics, weights={})
        assert score == 0.0