chore: import upstream snapshot with attribution

tests/benchmarks/test_benchmark_functions_extra_coverage.py

@@ -0,0 +1,343 @@
+"""
+Extra coverage tests for benchmarks/benchmark_functions.py.
+
+Targets the 36 missing lines not covered by test_benchmark_functions_coverage.py:
+- compare_configurations() with default configurations
+- compare_configurations() with custom configurations
+- compare_configurations() with various dataset_type values
+- get_available_benchmarks() return structure
+- evaluate_xbench_deepsearch() with non-openai_endpoint provider
+- evaluate_browsecomp() with non-openai_endpoint provider
+- compare_configurations() report content and path
+"""
+
+from unittest.mock import patch
+
+MODULE = "local_deep_research.benchmarks.benchmark_functions"
+
+
+def _no_settings(key, *args, **kwargs):
+    return None
+
+
+# ---------------------------------------------------------------------------
+# get_available_benchmarks
+# ---------------------------------------------------------------------------
+
+
+class TestGetAvailableBenchmarks:
+    def test_returns_three_benchmarks(self):
+        from local_deep_research.benchmarks.benchmark_functions import (
+            get_available_benchmarks,
+        )
+
+        benchmarks = get_available_benchmarks()
+        assert len(benchmarks) == 3
+
+    def test_each_benchmark_has_required_fields(self):
+        from local_deep_research.benchmarks.benchmark_functions import (
+            get_available_benchmarks,
+        )
+
+        benchmarks = get_available_benchmarks()
+        for b in benchmarks:
+            assert "id" in b
+            assert "name" in b
+            assert "description" in b
+
+    def test_simpleqa_is_present(self):
+        from local_deep_research.benchmarks.benchmark_functions import (
+            get_available_benchmarks,
+        )
+
+        ids = [b["id"] for b in get_available_benchmarks()]
+        assert "simpleqa" in ids
+
+    def test_xbench_deepsearch_is_present(self):
+        from local_deep_research.benchmarks.benchmark_functions import (
+            get_available_benchmarks,
+        )
+
+        ids = [b["id"] for b in get_available_benchmarks()]
+        assert "xbench_deepsearch" in ids
+
+
+# ---------------------------------------------------------------------------
+# compare_configurations – default configs used when none provided
+# ---------------------------------------------------------------------------
+
+
+class TestCompareConfigurationsDefaults:
+    def test_default_configurations_used_when_none_provided(self):
+        with (
+            patch(f"{MODULE}.run_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+            patch(
+                "local_deep_research.security.file_write_verifier.write_file_verified"
+            ),
+        ):
+            mock_run.return_value = {
+                "metrics": {"accuracy": 0.5, "average_processing_time": 1.0},
+                "total_examples": 10,
+                "configuration_name": "test",
+                "search_config": {},
+            }
+            from local_deep_research.benchmarks.benchmark_functions import (
+                compare_configurations,
+            )
+
+            result = compare_configurations(
+                dataset_type="simpleqa",
+                num_examples=1,
+            )
+        # Default has 3 configurations
+        assert result["configurations_tested"] == 3
+
+    def test_status_is_complete(self):
+        with (
+            patch(f"{MODULE}.run_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+            patch(
+                "local_deep_research.security.file_write_verifier.write_file_verified"
+            ),
+        ):
+            mock_run.return_value = {
+                "metrics": {"accuracy": 0.5, "average_processing_time": 1.0},
+                "total_examples": 10,
+                "configuration_name": "test",
+                "search_config": {},
+            }
+            from local_deep_research.benchmarks.benchmark_functions import (
+                compare_configurations,
+            )
+
+            result = compare_configurations(num_examples=1)
+        assert result["status"] == "complete"
+
+
+# ---------------------------------------------------------------------------
+# compare_configurations – custom configurations
+# ---------------------------------------------------------------------------
+
+
+class TestCompareConfigurationsCustom:
+    def test_custom_configs_run_for_each(self):
+        with (
+            patch(f"{MODULE}.run_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+            patch(
+                "local_deep_research.security.file_write_verifier.write_file_verified"
+            ),
+        ):
+            mock_run.return_value = {
+                "metrics": {"accuracy": 0.6, "average_processing_time": 2.0},
+                "total_examples": 5,
+                "configuration_name": "custom",
+                "search_config": {},
+            }
+            from local_deep_research.benchmarks.benchmark_functions import (
+                compare_configurations,
+            )
+
+            cfgs = [
+                {
+                    "name": "Cfg A",
+                    "search_tool": "searxng",
+                    "iterations": 1,
+                    "questions_per_iteration": 2,
+                },
+                {
+                    "name": "Cfg B",
+                    "search_tool": "wikipedia",
+                    "iterations": 2,
+                    "questions_per_iteration": 3,
+                },
+            ]
+            result = compare_configurations(configurations=cfgs, num_examples=1)
+        assert mock_run.call_count == 2
+        assert result["configurations_tested"] == 2
+
+    def test_report_path_in_result(self):
+        with (
+            patch(f"{MODULE}.run_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+            patch(
+                "local_deep_research.security.file_write_verifier.write_file_verified"
+            ),
+        ):
+            mock_run.return_value = {
+                "metrics": {"accuracy": 0.7, "average_processing_time": 1.5},
+                "total_examples": 3,
+                "configuration_name": "rep",
+                "search_config": {},
+            }
+            from local_deep_research.benchmarks.benchmark_functions import (
+                compare_configurations,
+            )
+
+            result = compare_configurations(
+                configurations=[
+                    {
+                        "name": "Only",
+                        "search_tool": "searxng",
+                        "iterations": 1,
+                        "questions_per_iteration": 2,
+                    }
+                ],
+                num_examples=1,
+            )
+        assert "report_path" in result
+
+    def test_dataset_type_reflected_in_result(self):
+        with (
+            patch(f"{MODULE}.run_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+            patch(
+                "local_deep_research.security.file_write_verifier.write_file_verified"
+            ),
+        ):
+            mock_run.return_value = {
+                "metrics": {"accuracy": 0.8, "average_processing_time": 1.0},
+                "total_examples": 2,
+                "configuration_name": "t",
+                "search_config": {},
+            }
+            from local_deep_research.benchmarks.benchmark_functions import (
+                compare_configurations,
+            )
+
+            result = compare_configurations(
+                dataset_type="browsecomp",
+                configurations=[
+                    {
+                        "name": "X",
+                        "search_tool": "searxng",
+                        "iterations": 1,
+                        "questions_per_iteration": 2,
+                    }
+                ],
+                num_examples=1,
+            )
+        assert result["dataset_type"] == "browsecomp"
+
+
+# ---------------------------------------------------------------------------
+# evaluate_browsecomp – non-openai_endpoint provider
+# ---------------------------------------------------------------------------
+
+
+class TestEvaluateBrowsecompNonOpenaiEndpoint:
+    def test_non_openai_endpoint_provider_has_no_url(self):
+        with (
+            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+        ):
+            mock_run.return_value = {}
+            from local_deep_research.benchmarks.benchmark_functions import (
+                evaluate_browsecomp,
+            )
+
+            evaluate_browsecomp(
+                num_examples=1,
+                evaluation_model="my-model",
+                evaluation_provider="ollama",
+                endpoint_url="http://ignored.com",
+            )
+            ec = mock_run.call_args[1]["evaluation_config"]
+            assert "openai_endpoint_url" not in ec
+
+    def test_evaluation_config_none_when_no_eval_params(self):
+        with (
+            patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+        ):
+            mock_run.return_value = {}
+            from local_deep_research.benchmarks.benchmark_functions import (
+                evaluate_browsecomp,
+            )
+
+            evaluate_browsecomp(num_examples=1)
+            ec = mock_run.call_args[1]["evaluation_config"]
+            assert ec is None
+
+
+# ---------------------------------------------------------------------------
+# evaluate_xbench_deepsearch – non-openai_endpoint provider
+# ---------------------------------------------------------------------------
+
+
+class TestEvaluateXbenchNonOpenaiEndpoint:
+    def test_non_openai_endpoint_provider_has_no_url(self):
+        with (
+            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+        ):
+            mock_run.return_value = {}
+            from local_deep_research.benchmarks.benchmark_functions import (
+                evaluate_xbench_deepsearch,
+            )
+
+            evaluate_xbench_deepsearch(
+                num_examples=1,
+                evaluation_model="m",
+                evaluation_provider="anthropic",
+                endpoint_url="http://ignored.com",
+            )
+            ec = mock_run.call_args[1]["evaluation_config"]
+            assert "openai_endpoint_url" not in ec
+
+    def test_full_settings_override_xbench_search_config(self):
+        def full_settings(key, *args, **kwargs):
+            return {
+                "llm.model": "env-model",
+                "llm.provider": "env-provider",
+                "llm.openai_endpoint.url": "http://env-xb.com",
+            }.get(key)
+
+        with (
+            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=full_settings
+            ),
+        ):
+            mock_run.return_value = {}
+            from local_deep_research.benchmarks.benchmark_functions import (
+                evaluate_xbench_deepsearch,
+            )
+
+            evaluate_xbench_deepsearch(num_examples=1)
+            sc = mock_run.call_args[1]["search_config"]
+            assert sc["model_name"] == "env-model"
+            assert sc["provider"] == "env-provider"
+            assert sc["openai_endpoint_url"] == "http://env-xb.com"
+
+    def test_human_evaluation_passed_through(self):
+        with (
+            patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
+            patch(
+                f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
+            ),
+        ):
+            mock_run.return_value = {}
+            from local_deep_research.benchmarks.benchmark_functions import (
+                evaluate_xbench_deepsearch,
+            )
+
+            evaluate_xbench_deepsearch(num_examples=1, human_evaluation=True)
+            assert mock_run.call_args[1]["human_evaluation"] is True