learningcircuit--local-deep-research/tests/benchmarks/comparison/test_evaluator_coverage.py

"""
Coverage tests for benchmarks/comparison/evaluator.py.

Targets the 44 missing lines:
- compare_configurations() with empty configurations
- compare_configurations() with all-failed runs
- compare_configurations() with successful runs
- compare_configurations() with mixed success/fail
- _evaluate_single_configuration() success and error paths
- _calculate_average_metrics() with empty/single/multi results
- _create_comparison_visualizations() with no successful results
"""

import pytest
from unittest.mock import MagicMock, patch

MODULE = "local_deep_research.benchmarks.comparison.evaluator"


# ---------------------------------------------------------------------------
# compare_configurations – guard clauses
# ---------------------------------------------------------------------------


class TestCompareConfigurationsEmpty:
    @patch(f"{MODULE}.os.makedirs")
    def test_empty_configurations_returns_error(self, mock_makedirs):
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(query="test query", configurations=[])
        assert "error" in result
        assert result["error"] == "No configurations provided"

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_single_config_all_failed(
        self,
        mock_write_json,
        mock_viz,
        mock_evaluate,
        mock_makedirs,
        tmp_path,
    ):
        mock_evaluate.return_value = {
            "success": False,
            "error": "LLM unreachable",
        }
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Cfg1"}],
            output_dir=str(tmp_path),
        )
        assert result["successful_configurations"] == 0
        assert result["failed_configurations"] == 1


class TestCompareConfigurationsSuccess:
    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._calculate_average_metrics")
    @patch(f"{MODULE}.calculate_combined_score")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_successful_run_included_in_results(
        self,
        mock_write_json,
        mock_viz,
        mock_combined_score,
        mock_avg_metrics,
        mock_evaluate,
        mock_makedirs,
        tmp_path,
    ):
        mock_evaluate.return_value = {
            "success": True,
            "quality_metrics": {"overall_quality": 0.8},
            "speed_metrics": {"total_duration": 30.0},
            "resource_metrics": {},
        }
        mock_avg_metrics.return_value = {
            "quality_metrics": {"overall_quality": 0.8},
            "speed_metrics": {"total_duration": 30.0},
            "resource_metrics": {},
        }
        mock_combined_score.return_value = 0.75
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="research q",
            configurations=[{"name": "Cfg1"}],
            output_dir=str(tmp_path),
        )
        assert result["successful_configurations"] == 1
        assert result["failed_configurations"] == 0
        assert len(result["results"]) == 1
        assert result["results"][0]["overall_score"] == 0.75

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._calculate_average_metrics")
    @patch(f"{MODULE}.calculate_combined_score")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_results_sorted_by_score_descending(
        self,
        mock_write_json,
        mock_viz,
        mock_combined_score,
        mock_avg_metrics,
        mock_evaluate,
        mock_makedirs,
        tmp_path,
    ):
        mock_evaluate.return_value = {
            "success": True,
            "quality_metrics": {},
            "speed_metrics": {},
            "resource_metrics": {},
        }
        mock_avg_metrics.return_value = {
            "quality_metrics": {},
            "speed_metrics": {},
            "resource_metrics": {},
        }
        scores = [0.3, 0.9, 0.6]
        mock_combined_score.side_effect = scores
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        cfgs = [{"name": f"Cfg{i}"} for i in range(3)]
        result = compare_configurations(
            query="q", configurations=cfgs, output_dir=str(tmp_path)
        )
        returned_scores = [r["overall_score"] for r in result["results"]]
        assert returned_scores == sorted(returned_scores, reverse=True)

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._calculate_average_metrics")
    @patch(f"{MODULE}.calculate_combined_score")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_repetitions_aggregated(
        self,
        mock_write_json,
        mock_viz,
        mock_combined_score,
        mock_avg_metrics,
        mock_evaluate,
        mock_makedirs,
        tmp_path,
    ):
        mock_evaluate.return_value = {
            "success": True,
            "quality_metrics": {},
            "speed_metrics": {},
            "resource_metrics": {},
        }
        mock_avg_metrics.return_value = {
            "quality_metrics": {},
            "speed_metrics": {},
            "resource_metrics": {},
        }
        mock_combined_score.return_value = 0.5
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Cfg1"}],
            output_dir=str(tmp_path),
            repetitions=3,
        )
        assert result["results"][0]["runs_completed"] == 3


# ---------------------------------------------------------------------------
# compare_configurations – mixed success/failure
# ---------------------------------------------------------------------------


class TestCompareConfigurationsMixed:
    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._calculate_average_metrics")
    @patch(f"{MODULE}.calculate_combined_score")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_mixed_run_counts_correct(
        self,
        mock_write_json,
        mock_viz,
        mock_combined_score,
        mock_avg_metrics,
        mock_evaluate,
        mock_makedirs,
        tmp_path,
    ):
        # First config succeeds, second fails
        mock_evaluate.side_effect = [
            {
                "success": True,
                "quality_metrics": {},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {"success": False, "error": "fail"},
        ]
        mock_avg_metrics.return_value = {
            "quality_metrics": {},
            "speed_metrics": {},
            "resource_metrics": {},
        }
        mock_combined_score.return_value = 0.6
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Cfg1"}, {"name": "Cfg2"}],
            output_dir=str(tmp_path),
        )
        assert result["successful_configurations"] == 1
        assert result["failed_configurations"] == 1

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._calculate_average_metrics")
    @patch(f"{MODULE}.calculate_combined_score")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_exception_during_run_marked_as_failed(
        self,
        mock_write_json,
        mock_viz,
        mock_combined_score,
        mock_avg_metrics,
        mock_evaluate,
        mock_makedirs,
        tmp_path,
    ):
        mock_evaluate.side_effect = RuntimeError("connection error")
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "CfgBad"}],
            output_dir=str(tmp_path),
        )
        assert result["failed_configurations"] == 1


# ---------------------------------------------------------------------------
# _calculate_average_metrics
# ---------------------------------------------------------------------------


class TestCalculateAverageMetrics:
    def test_empty_list_returns_empty_dict(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        result = _calculate_average_metrics([])
        assert result == {}

    def test_single_result_averaged_correctly(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"overall_quality": 0.8},
                "speed_metrics": {"total_duration": 30.0},
                "resource_metrics": {"process_memory_max_mb": 256.0},
            }
        ]
        avg = _calculate_average_metrics(results)
        assert avg["quality_metrics"]["overall_quality"] == 0.8
        assert avg["speed_metrics"]["total_duration"] == 30.0
        assert avg["resource_metrics"]["process_memory_max_mb"] == 256.0

    def test_multiple_results_averaged(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"score": 0.6},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {"score": 0.8},
                "speed_metrics": {},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        assert avg["quality_metrics"]["score"] == pytest.approx(0.7, abs=1e-9)

    def test_missing_keys_in_some_results_handled(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"a": 0.5},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {},
                "speed_metrics": {"b": 1.0},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        # "a" only in first result → average of [0.5] = 0.5
        assert avg["quality_metrics"]["a"] == 0.5
        # "b" only in second result → average of [1.0] = 1.0
        assert avg["speed_metrics"]["b"] == 1.0


# ---------------------------------------------------------------------------
# _create_comparison_visualizations – no successful results
# ---------------------------------------------------------------------------


class TestCreateComparisonVisualizationsNoResults:
    @patch(f"{MODULE}.plt")
    def test_no_successful_results_returns_early(self, mock_plt, tmp_path):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _create_comparison_visualizations,
        )

        report = {
            "results": [{"success": False, "error": "failed"}],
        }
        _create_comparison_visualizations(
            report, output_dir=str(tmp_path), timestamp="20240101_000000"
        )
        mock_plt.figure.assert_not_called()


# ---------------------------------------------------------------------------
# _evaluate_single_configuration – error path
# ---------------------------------------------------------------------------


class TestEvaluateSingleConfigurationError:
    @patch(f"{MODULE}.SpeedProfiler")
    @patch(f"{MODULE}.ResourceMonitor")
    @patch(f"{MODULE}.get_llm")
    def test_llm_error_returns_failure_dict(
        self, mock_get_llm, mock_monitor, mock_profiler
    ):
        mock_get_llm.side_effect = RuntimeError("no LLM available")
        mock_profiler_instance = MagicMock()
        mock_profiler.return_value = mock_profiler_instance
        mock_monitor_instance = MagicMock()
        mock_monitor.return_value = mock_monitor_instance
        from local_deep_research.benchmarks.comparison.evaluator import (
            _evaluate_single_configuration,
        )

        result = _evaluate_single_configuration(
            query="test query",
            config={"name": "bad_config"},
        )
        assert result["success"] is False
        assert "no LLM available" in result["error"]