learningcircuit--local-deep-research/tests/benchmarks/test_benchmark_functions_extended.py

"""Extended tests for benchmarks/benchmark_functions.py.

Covers get_available_benchmarks() and compare_configurations().
"""

import tempfile
from copy import deepcopy
from unittest.mock import patch

import pytest

from local_deep_research.benchmarks.benchmark_functions import (
    compare_configurations,
    get_available_benchmarks,
)


# ── get_available_benchmarks ───────────────────────────────────────


class TestGetAvailableBenchmarks:
    """Tests for get_available_benchmarks."""

    def test_returns_list(self):
        """Return type is a list."""
        result = get_available_benchmarks()
        assert isinstance(result, list)
        assert len(result) > 0

    def test_has_required_fields(self):
        """Each benchmark dict has id, name, description, recommended_examples."""
        for b in get_available_benchmarks():
            assert "id" in b
            assert "name" in b
            assert "description" in b
            assert "recommended_examples" in b
            assert isinstance(b["recommended_examples"], int)

    def test_includes_all_types(self):
        """All three benchmark types are present."""
        ids = {b["id"] for b in get_available_benchmarks()}
        assert "simpleqa" in ids
        assert "browsecomp" in ids
        assert "xbench_deepsearch" in ids


# ── compare_configurations ─────────────────────────────────────────


class TestCompareConfigurations:
    """Tests for compare_configurations."""

    def _mock_benchmark_result(self, config_name="test"):
        return {
            "metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
            "total_examples": 20,
        }

    def test_default_configurations(self):
        """Default 3 configurations are used when none provided."""
        with (
            patch(
                "local_deep_research.benchmarks.benchmark_functions.run_benchmark"
            ) as mock_run,
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = self._mock_benchmark_result()

            with tempfile.TemporaryDirectory() as tmpdir:
                result = compare_configurations(
                    num_examples=5, output_dir=tmpdir
                )

                # 3 default configs → 3 calls
                assert mock_run.call_count == 3
                assert result["status"] == "complete"
                assert result["configurations_tested"] == 3

    def test_custom_configurations(self):
        """Custom configurations list is used."""
        custom_configs = [
            {
                "name": "Custom A",
                "search_tool": "brave",
                "iterations": 2,
                "questions_per_iteration": 4,
            },
            {
                "name": "Custom B",
                "search_tool": "duckduckgo",
                "iterations": 1,
                "questions_per_iteration": 2,
            },
        ]

        with (
            patch(
                "local_deep_research.benchmarks.benchmark_functions.run_benchmark"
            ) as mock_run,
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = self._mock_benchmark_result()

            with tempfile.TemporaryDirectory() as tmpdir:
                result = compare_configurations(
                    configurations=deepcopy(custom_configs),
                    num_examples=5,
                    output_dir=tmpdir,
                )

                assert mock_run.call_count == 2
                assert result["configurations_tested"] == 2

    def test_pops_name_from_config(self):
        """config.pop('name') mutates the caller's dicts (documenting the bug)."""
        configs = [
            {
                "name": "Foo",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
        ]

        with (
            patch(
                "local_deep_research.benchmarks.benchmark_functions.run_benchmark"
            ) as mock_run,
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = self._mock_benchmark_result()

            with tempfile.TemporaryDirectory() as tmpdir:
                compare_configurations(
                    configurations=configs,
                    num_examples=5,
                    output_dir=tmpdir,
                )

                # The original dict has been mutated: "name" was popped
                assert "name" not in configs[0]

    def test_generates_report(self):
        """Comparison report is written via write_file_verified."""
        with (
            patch(
                "local_deep_research.benchmarks.benchmark_functions.run_benchmark"
            ) as mock_run,
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ) as mock_write,
        ):
            mock_run.return_value = self._mock_benchmark_result()

            with tempfile.TemporaryDirectory() as tmpdir:
                result = compare_configurations(
                    num_examples=5, output_dir=tmpdir
                )

                mock_write.assert_called_once()
                # Report path is returned
                assert "report_path" in result

    def test_handles_benchmark_error(self):
        """Error in run_benchmark for one config propagates (no swallowing)."""
        configs = [
            {
                "name": "Good",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
            {
                "name": "Bad",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
        ]

        call_count = 0

        def side_effect(**kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 2:
                raise RuntimeError("benchmark failed")
            return self._mock_benchmark_result()

        with (
            patch(
                "local_deep_research.benchmarks.benchmark_functions.run_benchmark",
                side_effect=side_effect,
            ),
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            with tempfile.TemporaryDirectory() as tmpdir:
                with pytest.raises(RuntimeError, match="benchmark failed"):
                    compare_configurations(
                        configurations=configs,
                        num_examples=5,
                        output_dir=tmpdir,
                    )