learningcircuit--local-deep-research/tests/benchmarks/test_benchmark_functions_extra_coverage.py

"""
Extra coverage tests for benchmarks/benchmark_functions.py.

Targets the 36 missing lines not covered by test_benchmark_functions_coverage.py:
- compare_configurations() with default configurations
- compare_configurations() with custom configurations
- compare_configurations() with various dataset_type values
- get_available_benchmarks() return structure
- evaluate_xbench_deepsearch() with non-openai_endpoint provider
- evaluate_browsecomp() with non-openai_endpoint provider
- compare_configurations() report content and path
"""

from unittest.mock import patch

MODULE = "local_deep_research.benchmarks.benchmark_functions"


def _no_settings(key, *args, **kwargs):
    return None


# ---------------------------------------------------------------------------
# get_available_benchmarks
# ---------------------------------------------------------------------------


class TestGetAvailableBenchmarks:
    def test_returns_three_benchmarks(self):
        from local_deep_research.benchmarks.benchmark_functions import (
            get_available_benchmarks,
        )

        benchmarks = get_available_benchmarks()
        assert len(benchmarks) == 3

    def test_each_benchmark_has_required_fields(self):
        from local_deep_research.benchmarks.benchmark_functions import (
            get_available_benchmarks,
        )

        benchmarks = get_available_benchmarks()
        for b in benchmarks:
            assert "id" in b
            assert "name" in b
            assert "description" in b

    def test_simpleqa_is_present(self):
        from local_deep_research.benchmarks.benchmark_functions import (
            get_available_benchmarks,
        )

        ids = [b["id"] for b in get_available_benchmarks()]
        assert "simpleqa" in ids

    def test_xbench_deepsearch_is_present(self):
        from local_deep_research.benchmarks.benchmark_functions import (
            get_available_benchmarks,
        )

        ids = [b["id"] for b in get_available_benchmarks()]
        assert "xbench_deepsearch" in ids


# ---------------------------------------------------------------------------
# compare_configurations – default configs used when none provided
# ---------------------------------------------------------------------------


class TestCompareConfigurationsDefaults:
    def test_default_configurations_used_when_none_provided(self):
        with (
            patch(f"{MODULE}.run_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = {
                "metrics": {"accuracy": 0.5, "average_processing_time": 1.0},
                "total_examples": 10,
                "configuration_name": "test",
                "search_config": {},
            }
            from local_deep_research.benchmarks.benchmark_functions import (
                compare_configurations,
            )

            result = compare_configurations(
                dataset_type="simpleqa",
                num_examples=1,
            )
        # Default has 3 configurations
        assert result["configurations_tested"] == 3

    def test_status_is_complete(self):
        with (
            patch(f"{MODULE}.run_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = {
                "metrics": {"accuracy": 0.5, "average_processing_time": 1.0},
                "total_examples": 10,
                "configuration_name": "test",
                "search_config": {},
            }
            from local_deep_research.benchmarks.benchmark_functions import (
                compare_configurations,
            )

            result = compare_configurations(num_examples=1)
        assert result["status"] == "complete"


# ---------------------------------------------------------------------------
# compare_configurations – custom configurations
# ---------------------------------------------------------------------------


class TestCompareConfigurationsCustom:
    def test_custom_configs_run_for_each(self):
        with (
            patch(f"{MODULE}.run_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = {
                "metrics": {"accuracy": 0.6, "average_processing_time": 2.0},
                "total_examples": 5,
                "configuration_name": "custom",
                "search_config": {},
            }
            from local_deep_research.benchmarks.benchmark_functions import (
                compare_configurations,
            )

            cfgs = [
                {
                    "name": "Cfg A",
                    "search_tool": "searxng",
                    "iterations": 1,
                    "questions_per_iteration": 2,
                },
                {
                    "name": "Cfg B",
                    "search_tool": "wikipedia",
                    "iterations": 2,
                    "questions_per_iteration": 3,
                },
            ]
            result = compare_configurations(configurations=cfgs, num_examples=1)
        assert mock_run.call_count == 2
        assert result["configurations_tested"] == 2

    def test_report_path_in_result(self):
        with (
            patch(f"{MODULE}.run_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = {
                "metrics": {"accuracy": 0.7, "average_processing_time": 1.5},
                "total_examples": 3,
                "configuration_name": "rep",
                "search_config": {},
            }
            from local_deep_research.benchmarks.benchmark_functions import (
                compare_configurations,
            )

            result = compare_configurations(
                configurations=[
                    {
                        "name": "Only",
                        "search_tool": "searxng",
                        "iterations": 1,
                        "questions_per_iteration": 2,
                    }
                ],
                num_examples=1,
            )
        assert "report_path" in result

    def test_dataset_type_reflected_in_result(self):
        with (
            patch(f"{MODULE}.run_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
            patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ),
        ):
            mock_run.return_value = {
                "metrics": {"accuracy": 0.8, "average_processing_time": 1.0},
                "total_examples": 2,
                "configuration_name": "t",
                "search_config": {},
            }
            from local_deep_research.benchmarks.benchmark_functions import (
                compare_configurations,
            )

            result = compare_configurations(
                dataset_type="browsecomp",
                configurations=[
                    {
                        "name": "X",
                        "search_tool": "searxng",
                        "iterations": 1,
                        "questions_per_iteration": 2,
                    }
                ],
                num_examples=1,
            )
        assert result["dataset_type"] == "browsecomp"


# ---------------------------------------------------------------------------
# evaluate_browsecomp – non-openai_endpoint provider
# ---------------------------------------------------------------------------


class TestEvaluateBrowsecompNonOpenaiEndpoint:
    def test_non_openai_endpoint_provider_has_no_url(self):
        with (
            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_browsecomp,
            )

            evaluate_browsecomp(
                num_examples=1,
                evaluation_model="my-model",
                evaluation_provider="ollama",
                endpoint_url="http://ignored.com",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert "openai_endpoint_url" not in ec

    def test_evaluation_config_none_when_no_eval_params(self):
        with (
            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_browsecomp,
            )

            evaluate_browsecomp(num_examples=1)
            ec = mock_run.call_args[1]["evaluation_config"]
            assert ec is None


# ---------------------------------------------------------------------------
# evaluate_xbench_deepsearch – non-openai_endpoint provider
# ---------------------------------------------------------------------------


class TestEvaluateXbenchNonOpenaiEndpoint:
    def test_non_openai_endpoint_provider_has_no_url(self):
        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(
                num_examples=1,
                evaluation_model="m",
                evaluation_provider="anthropic",
                endpoint_url="http://ignored.com",
            )
            ec = mock_run.call_args[1]["evaluation_config"]
            assert "openai_endpoint_url" not in ec

    def test_full_settings_override_xbench_search_config(self):
        def full_settings(key, *args, **kwargs):
            return {
                "llm.model": "env-model",
                "llm.provider": "env-provider",
                "llm.openai_endpoint.url": "http://env-xb.com",
            }.get(key)

        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=full_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(num_examples=1)
            sc = mock_run.call_args[1]["search_config"]
            assert sc["model_name"] == "env-model"
            assert sc["provider"] == "env-provider"
            assert sc["openai_endpoint_url"] == "http://env-xb.com"

    def test_human_evaluation_passed_through(self):
        with (
            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
            patch(
                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
            ),
        ):
            mock_run.return_value = {}
            from local_deep_research.benchmarks.benchmark_functions import (
                evaluate_xbench_deepsearch,
            )

            evaluate_xbench_deepsearch(num_examples=1, human_evaluation=True)
            assert mock_run.call_args[1]["human_evaluation"] is True