chore: import upstream snapshot with attribution

tests/api/test_benchmark_functions_deep_coverage.py

@@ -0,0 +1,275 @@
+"""
+Tests for uncovered code paths in api/benchmark_functions.py.
+
+Targets:
+- evaluate_simpleqa: evaluation_config with model only, provider only
+- evaluate_browsecomp: evaluation_config branches
+- evaluate_xbench_deepsearch: evaluation_config branches
+- compare_configurations: default configs, report generation, metrics formatting
+"""
+
+from unittest.mock import patch
+
+
+MODULE = "local_deep_research.api.benchmark_functions"
+
+
+class TestEvaluateSimpleqa:
+    @patch(f"{MODULE}.run_simpleqa_benchmark")
+    def test_evaluation_config_with_model_only(self, mock_run):
+        """evaluation_config is set when only model is provided."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_simpleqa,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_simpleqa(
+            num_examples=5,
+            evaluation_model="gpt-4",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"] == {"model_name": "gpt-4"}
+
+    @patch(f"{MODULE}.run_simpleqa_benchmark")
+    def test_evaluation_config_with_provider_only(self, mock_run):
+        """evaluation_config is set when only provider is provided."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_simpleqa,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_simpleqa(
+            num_examples=5,
+            evaluation_provider="openai",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"] == {"provider": "openai"}
+
+    @patch(f"{MODULE}.run_simpleqa_benchmark")
+    def test_evaluation_config_with_both(self, mock_run):
+        """evaluation_config includes both model and provider."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_simpleqa,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_simpleqa(
+            num_examples=5,
+            evaluation_model="gpt-4",
+            evaluation_provider="openai",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4"
+        assert call_kwargs["evaluation_config"]["provider"] == "openai"
+
+    @patch(f"{MODULE}.run_simpleqa_benchmark")
+    def test_no_evaluation_config(self, mock_run):
+        """evaluation_config is None when neither model nor provider."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_simpleqa,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_simpleqa(num_examples=5)
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"] is None
+
+
+class TestEvaluateBrowsecomp:
+    @patch(f"{MODULE}.run_browsecomp_benchmark")
+    def test_evaluation_config_with_model(self, mock_run):
+        """evaluation_config with model for browsecomp."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_browsecomp,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_browsecomp(
+            num_examples=5,
+            evaluation_model="claude-3",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"]["model_name"] == "claude-3"
+
+    @patch(f"{MODULE}.run_browsecomp_benchmark")
+    def test_evaluation_config_with_provider(self, mock_run):
+        """evaluation_config with provider for browsecomp."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_browsecomp,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_browsecomp(
+            num_examples=5,
+            evaluation_provider="anthropic",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"]["provider"] == "anthropic"
+
+
+class TestEvaluateXbenchDeepsearch:
+    @patch(f"{MODULE}.run_xbench_deepsearch_benchmark")
+    def test_evaluation_config_with_model(self, mock_run):
+        """evaluation_config with model for xbench."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_xbench_deepsearch,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_xbench_deepsearch(
+            num_examples=5,
+            evaluation_model="gpt-4o",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4o"
+
+    @patch(f"{MODULE}.run_xbench_deepsearch_benchmark")
+    def test_evaluation_config_with_provider(self, mock_run):
+        """evaluation_config with provider for xbench."""
+        from local_deep_research.api.benchmark_functions import (
+            evaluate_xbench_deepsearch,
+        )
+
+        mock_run.return_value = {"status": "complete"}
+
+        evaluate_xbench_deepsearch(
+            num_examples=5,
+            evaluation_provider="openai",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["evaluation_config"]["provider"] == "openai"
+
+
+class TestCompareConfigurations:
+    @patch(
+        "local_deep_research.security.file_write_verifier.write_file_verified"
+    )
+    @patch(f"{MODULE}.run_benchmark")
+    def test_default_configurations(self, mock_run, mock_write):
+        """Uses default configurations when none provided."""
+        from local_deep_research.api.benchmark_functions import (
+            compare_configurations,
+        )
+
+        mock_run.return_value = {
+            "metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
+            "total_examples": 20,
+        }
+
+        result = compare_configurations(
+            dataset_type="simpleqa",
+            num_examples=5,
+            output_dir="/tmp/test_bench",
+        )
+
+        assert result["status"] == "complete"
+        assert result["configurations_tested"] == 3  # 3 default configs
+        assert mock_run.call_count == 3
+
+    @patch(
+        "local_deep_research.security.file_write_verifier.write_file_verified"
+    )
+    @patch(f"{MODULE}.run_benchmark")
+    def test_custom_configurations(self, mock_run, mock_write):
+        """Passes custom configurations correctly."""
+        from local_deep_research.api.benchmark_functions import (
+            compare_configurations,
+        )
+
+        mock_run.return_value = {
+            "metrics": {"accuracy": 0.9, "average_processing_time": 2.0},
+            "total_examples": 10,
+        }
+
+        configs = [
+            {
+                "name": "Fast",
+                "search_tool": "wikipedia",
+                "iterations": 1,
+                "questions_per_iteration": 2,
+            },
+        ]
+
+        result = compare_configurations(
+            configurations=configs,
+            num_examples=10,
+            output_dir="/tmp/test_bench",
+        )
+
+        assert result["configurations_tested"] == 1
+
+    @patch(
+        "local_deep_research.security.file_write_verifier.write_file_verified"
+    )
+    @patch(f"{MODULE}.run_benchmark")
+    def test_report_written(self, mock_run, mock_write):
+        """Comparison report is written to file."""
+        from local_deep_research.api.benchmark_functions import (
+            compare_configurations,
+        )
+
+        mock_run.return_value = {
+            "metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
+            "total_examples": 20,
+        }
+
+        result = compare_configurations(
+            num_examples=5,
+            output_dir="/tmp/test_bench",
+        )
+
+        mock_write.assert_called_once()
+        # Report content should contain markdown table
+        report_content = mock_write.call_args[0][1]
+        assert "Configuration" in report_content
+        assert "Accuracy" in report_content
+        assert result["report_path"].endswith(".md")
+
+    @patch(
+        "local_deep_research.security.file_write_verifier.write_file_verified"
+    )
+    @patch(f"{MODULE}.run_benchmark")
+    def test_extra_config_items_passed(self, mock_run, mock_write):
+        """Extra config items beyond standard ones are passed through."""
+        from local_deep_research.api.benchmark_functions import (
+            compare_configurations,
+        )
+
+        mock_run.return_value = {
+            "metrics": {"accuracy": 0.5},
+            "total_examples": 5,
+        }
+
+        configs = [
+            {
+                "name": "Custom",
+                "search_tool": "searxng",
+                "iterations": 1,
+                "questions_per_iteration": 1,
+                "custom_param": "value",
+            },
+        ]
+
+        compare_configurations(
+            configurations=configs,
+            num_examples=5,
+            output_dir="/tmp/test_bench",
+        )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["search_config"]["custom_param"] == "value"