learningcircuit--local-deep-research/tests/api/test_benchmark_functions.py

"""Tests for benchmark API functions."""

from unittest.mock import patch


class TestGetAvailableBenchmarks:
    def test_returns_list(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        assert isinstance(result, list)

    def test_contains_simpleqa(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        ids = [b["id"] for b in result]
        assert "simpleqa" in ids

    def test_contains_browsecomp(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        ids = [b["id"] for b in result]
        assert "browsecomp" in ids

    def test_contains_xbench(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        ids = [b["id"] for b in result]
        assert "xbench_deepsearch" in ids

    def test_benchmarks_have_required_fields(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        for benchmark in result:
            assert "id" in benchmark
            assert "name" in benchmark
            assert "description" in benchmark
            assert "recommended_examples" in benchmark


class TestEvaluateSimpleqa:
    def test_calls_run_benchmark(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
        ) as mock:
            mock.return_value = {"status": "complete"}
            result = evaluate_simpleqa(num_examples=10)
            mock.assert_called_once()
            assert result["status"] == "complete"

    def test_passes_search_config(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
        ) as mock:
            mock.return_value = {}
            evaluate_simpleqa(search_iterations=5, questions_per_iteration=7)
            call_args = mock.call_args
            assert call_args.kwargs["search_config"]["iterations"] == 5
            assert (
                call_args.kwargs["search_config"]["questions_per_iteration"]
                == 7
            )

    def test_passes_evaluation_config(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
        ) as mock:
            mock.return_value = {}
            evaluate_simpleqa(
                evaluation_model="gpt-4", evaluation_provider="openai"
            )
            call_args = mock.call_args
            assert (
                call_args.kwargs["evaluation_config"]["model_name"] == "gpt-4"
            )
            assert call_args.kwargs["evaluation_config"]["provider"] == "openai"


class TestEvaluateBrowsecomp:
    def test_calls_run_benchmark(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_browsecomp,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_browsecomp_benchmark"
        ) as mock:
            mock.return_value = {"status": "complete"}
            result = evaluate_browsecomp(num_examples=10)
            mock.assert_called_once()
            assert result["status"] == "complete"


class TestEvaluateXbenchDeepsearch:
    def test_calls_run_benchmark(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_xbench_deepsearch,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_xbench_deepsearch_benchmark"
        ) as mock:
            mock.return_value = {"status": "complete"}
            result = evaluate_xbench_deepsearch(num_examples=10)
            mock.assert_called_once()
            assert result["status"] == "complete"


class TestCompareConfigurations:
    def test_returns_dict(self):
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_benchmark"
        ) as mock_run:
            with patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ):
                mock_run.return_value = {"metrics": {"accuracy": 0.8}}
                result = compare_configurations(num_examples=5)
                assert isinstance(result, dict)
                assert "status" in result

    def test_uses_default_configurations(self):
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_benchmark"
        ) as mock_run:
            with patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ):
                mock_run.return_value = {"metrics": {}}
                result = compare_configurations(num_examples=5)
                assert result["configurations_tested"] == 3

    def test_custom_configurations(self):
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        custom_configs = [
            {"name": "Custom", "search_tool": "wikipedia", "iterations": 2}
        ]
        with patch(
            "local_deep_research.api.benchmark_functions.run_benchmark"
        ) as mock_run:
            with patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ):
                mock_run.return_value = {"metrics": {}}
                result = compare_configurations(
                    num_examples=5, configurations=custom_configs
                )
                assert result["configurations_tested"] == 1