learningcircuit--local-deep-research/tests/benchmarks/test_benchmark_functions_coverage.py

"""
Coverage tests for benchmarks/benchmark_functions.py.

Focuses on logic paths not covered by existing test files:
- search_model / search_provider / endpoint_url propagation into search_config
- openai_endpoint evaluation provider with url param vs env fallback
- Settings snapshot overriding explicit params
- temperature=0 in evaluation_config
- evaluate_xbench_deepsearch default iterations=4
"""

from unittest.mock import patch

MODULE = "local_deep_research.benchmarks.benchmark_functions"


def _no_settings(key, *args, **kwargs):
    return None


def _settings_with_url(key, *args, **kwargs):
    if key == "llm.openai_endpoint.url":
        return "http://env-endpoint.example.com"
    return None


def _full_settings(key, *args, **kwargs):
    return {
        "llm.model": "env-model",
        "llm.provider": "env-provider",
        "llm.openai_endpoint.url": "http://env-url.example.com",
    }.get(key)


class TestSimpleqaSearchModelConfig:
    def test_search_model_added_to_config(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(num_examples=1, search_model="my-model")
            sc = mock_run.call_args[1]["search_config"]
            assert sc["model_name"] == "my-model"

    def test_search_provider_added_to_config(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(num_examples=1, search_provider="ollama")
            sc = mock_run.call_args[1]["search_config"]
            assert sc["provider"] == "ollama"

    def test_endpoint_url_added_to_config(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(
                num_examples=1, endpoint_url="http://my-endpoint.com"
            )
            sc = mock_run.call_args[1]["search_config"]
            assert sc["openai_endpoint_url"] == "http://my-endpoint.com"


class TestSimpleqaOpenaiEndpointEvaluation:
    def test_openai_endpoint_uses_explicit_url(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(
                num_examples=1,
                evaluation_model="gpt-4",
                evaluation_provider="openai_endpoint",
                endpoint_url="http://explicit.example.com",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert ec["temperature"] == 0
            assert ec["provider"] == "openai_endpoint"
            assert ec["openai_endpoint_url"] == "http://explicit.example.com"

    def test_openai_endpoint_falls_back_to_env_url(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot",
                side_effect=_settings_with_url,
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(
                num_examples=1,
                evaluation_model="gpt-4",
                evaluation_provider="openai_endpoint",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert (
                ec["openai_endpoint_url"] == "http://env-endpoint.example.com"
            )

    def test_non_openai_endpoint_provider_no_url(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot",
                side_effect=_settings_with_url,
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(
                num_examples=1,
                evaluation_model="gpt-4",
                evaluation_provider="openai",
                endpoint_url="http://explicit.example.com",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert "openai_endpoint_url" not in ec


class TestSimpleqaSettingsOverride:
    def test_settings_override_explicit_params(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot",
                side_effect=_full_settings,
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(
                num_examples=1,
                search_model="param-model",
                search_provider="param-provider",
                endpoint_url="http://param-url.com",
            )
            sc = mock_run.call_args[1]["search_config"]
            assert sc["model_name"] == "env-model"
            assert sc["provider"] == "env-provider"
            assert sc["openai_endpoint_url"] == "http://env-url.example.com"


class TestSimpleqaEvaluationTemperature:
    def test_temperature_zero_with_model_only(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(num_examples=1, evaluation_model="gpt-4")
            ec = mock_run.call_args[1]["evaluation_config"]
            assert ec["temperature"] == 0
            assert ec["model_name"] == "gpt-4"
            assert "provider" not in ec

    def test_no_evaluation_config_when_no_eval_params(self):
        with (
            patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_simpleqa,
            )

            evaluate_simpleqa(num_examples=1)
            ec = mock_run.call_args[1]["evaluation_config"]
            assert ec is None


class TestBrowsecompSearchModelConfig:
    def test_search_model_and_provider_in_config(self):
        with (
            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_browsecomp,
            )

            evaluate_browsecomp(
                num_examples=1,
                search_model="bc-model",
                search_provider="bc-provider",
                endpoint_url="http://bc-url.com",
            )
            sc = mock_run.call_args[1]["search_config"]
            assert sc["model_name"] == "bc-model"
            assert sc["provider"] == "bc-provider"
            assert sc["openai_endpoint_url"] == "http://bc-url.com"

    def test_evaluation_openai_endpoint_with_explicit_url(self):
        with (
            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_browsecomp,
            )

            evaluate_browsecomp(
                num_examples=1,
                evaluation_model="eval-m",
                evaluation_provider="openai_endpoint",
                endpoint_url="http://bc-eval-url.com",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert ec["temperature"] == 0
            assert ec["openai_endpoint_url"] == "http://bc-eval-url.com"

    def test_evaluation_openai_endpoint_env_fallback(self):
        with (
            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot",
                side_effect=_settings_with_url,
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_browsecomp,
            )

            evaluate_browsecomp(
                num_examples=1,
                evaluation_provider="openai_endpoint",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert (
                ec["openai_endpoint_url"] == "http://env-endpoint.example.com"
            )


class TestXbenchDeepsearchDefaults:
    def test_default_iterations_is_4(self):
        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(num_examples=1)
            sc = mock_run.call_args[1]["search_config"]
            assert sc["iterations"] == 4

    def test_search_model_and_endpoint_in_config(self):
        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(
                num_examples=1,
                search_model="xb-model",
                search_provider="xb-prov",
                endpoint_url="http://xb-url.com",
            )
            sc = mock_run.call_args[1]["search_config"]
            assert sc["model_name"] == "xb-model"
            assert sc["provider"] == "xb-prov"
            assert sc["openai_endpoint_url"] == "http://xb-url.com"

    def test_evaluation_openai_endpoint_env_fallback(self):
        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot",
                side_effect=_settings_with_url,
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(
                num_examples=1,
                evaluation_model="xb-eval",
                evaluation_provider="openai_endpoint",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert ec["temperature"] == 0
            assert (
                ec["openai_endpoint_url"] == "http://env-endpoint.example.com"
            )

    def test_settings_override_search_config(self):
        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot",
                side_effect=_full_settings,
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(
                num_examples=1,
                search_model="param-m",
                endpoint_url="http://param-url.com",
            )
            sc = mock_run.call_args[1]["search_config"]
            assert sc["model_name"] == "env-model"
            assert sc["provider"] == "env-provider"
            assert sc["openai_endpoint_url"] == "http://env-url.example.com"