learningcircuit--local-deep-research/tests/benchmarks/test_benchmark_functions.py

"""
Tests for benchmarks/benchmark_functions.py

Tests cover:
- evaluate_simpleqa function
- evaluate_browsecomp function
- evaluate_xbench function
- Configuration handling
"""

from unittest.mock import patch
import tempfile


class TestEvaluateSimpleqa:
    """Tests for the evaluate_simpleqa function."""

    def test_evaluate_simpleqa_default_params(self):
        """evaluate_simpleqa works with default parameters."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    result = evaluate_simpleqa(
                        num_examples=1,
                        output_dir=tmpdir,
                    )

                    mock_run.assert_called_once()
                    assert isinstance(result, dict)

    def test_evaluate_simpleqa_custom_search_config(self):
        """evaluate_simpleqa passes custom search config."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_simpleqa(
                        num_examples=1,
                        search_iterations=5,
                        questions_per_iteration=2,
                        search_tool="wikipedia",
                        output_dir=tmpdir,
                    )

                    # Check that search config was passed correctly
                    call_kwargs = mock_run.call_args[1]
                    assert (
                        call_kwargs.get("search_config", {}).get("iterations")
                        == 5
                    )

    def test_evaluate_simpleqa_with_evaluation_model(self):
        """evaluate_simpleqa accepts evaluation model config."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    result = evaluate_simpleqa(
                        num_examples=1,
                        evaluation_model="gpt-4",
                        evaluation_provider="openai",
                        output_dir=tmpdir,
                    )

                    assert isinstance(result, dict)

    def test_evaluate_simpleqa_human_evaluation(self):
        """evaluate_simpleqa accepts human_evaluation flag."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_simpleqa(
                        num_examples=1,
                        human_evaluation=True,
                        output_dir=tmpdir,
                    )

                    call_kwargs = mock_run.call_args[1]
                    assert call_kwargs.get("human_evaluation") is True


class TestEvaluateBrowsecomp:
    """Tests for the evaluate_browsecomp function."""

    def test_evaluate_browsecomp_default_params(self):
        """evaluate_browsecomp works with default parameters."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_browsecomp,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_browsecomp_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    result = evaluate_browsecomp(
                        num_examples=1,
                        output_dir=tmpdir,
                    )

                    mock_run.assert_called_once()
                    assert isinstance(result, dict)

    def test_evaluate_browsecomp_custom_strategy(self):
        """evaluate_browsecomp accepts custom search strategy."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_browsecomp,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_browsecomp_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_browsecomp(
                        num_examples=1,
                        search_strategy="iterdrag",
                        output_dir=tmpdir,
                    )

                    call_kwargs = mock_run.call_args[1]
                    search_config = call_kwargs.get("search_config", {})
                    assert search_config.get("search_strategy") == "iterdrag"


class TestEvaluateXbenchDeepsearch:
    """Tests for the evaluate_xbench_deepsearch function."""

    def test_evaluate_xbench_deepsearch_default_params(self):
        """evaluate_xbench_deepsearch works with default parameters."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_xbench_deepsearch,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_xbench_deepsearch_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    result = evaluate_xbench_deepsearch(
                        num_examples=1,
                        output_dir=tmpdir,
                    )

                    mock_run.assert_called_once()
                    assert isinstance(result, dict)


class TestSettingsIntegration:
    """Tests for settings integration in benchmark functions."""

    def test_uses_settings_for_model(self):
        """Benchmark functions use settings for model configuration."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                # Return values for different setting keys
                def get_setting(key, *args, **kwargs):
                    settings = {
                        "llm.model": "custom-model",
                        "llm.provider": "custom-provider",
                        "llm.openai_endpoint.url": "http://custom-endpoint",
                    }
                    return settings.get(key)

                mock_settings.side_effect = get_setting

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_simpleqa(
                        num_examples=1,
                        output_dir=tmpdir,
                    )

                    # Should have used the settings
                    call_kwargs = mock_run.call_args[1]
                    search_config = call_kwargs.get("search_config", {})
                    assert search_config.get("model_name") == "custom-model"


class TestBenchmarkOutputDir:
    """Tests for output directory handling."""

    def test_output_dir_default(self):
        """Default output directory is benchmark_results."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                evaluate_simpleqa(num_examples=1)

                call_kwargs = mock_run.call_args[1]
                assert "output_dir" in call_kwargs
                assert call_kwargs["output_dir"] == "benchmark_results"

    def test_output_dir_custom(self):
        """Custom output directory is used."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_simpleqa(
                        num_examples=1,
                        output_dir=tmpdir,
                    )

                    call_kwargs = mock_run.call_args[1]
                    assert call_kwargs["output_dir"] == tmpdir


class TestSearchToolConfiguration:
    """Tests for search tool configuration."""

    def test_default_search_tool_is_searxng(self):
        """Default search tool is searxng."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_simpleqa(
                        num_examples=1,
                        output_dir=tmpdir,
                    )

                    call_kwargs = mock_run.call_args[1]
                    search_config = call_kwargs.get("search_config", {})
                    assert search_config.get("search_tool") == "searxng"

    def test_custom_search_tool(self):
        """Custom search tool is used."""
        from local_deep_research.benchmarks.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
        ) as mock_run:
            mock_run.return_value = {"metrics": {}, "results": []}

            with patch(
                "local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
            ) as mock_settings:
                mock_settings.return_value = None

                with tempfile.TemporaryDirectory() as tmpdir:
                    evaluate_simpleqa(
                        num_examples=1,
                        search_tool="wikipedia",
                        output_dir=tmpdir,
                    )

                    call_kwargs = mock_run.call_args[1]
                    search_config = call_kwargs.get("search_config", {})
                    assert search_config.get("search_tool") == "wikipedia"