learningcircuit--local-deep-research/tests/api/test_benchmark_functions_deep_coverage.py

"""
Tests for uncovered code paths in api/benchmark_functions.py.

Targets:
- evaluate_simpleqa: evaluation_config with model only, provider only
- evaluate_browsecomp: evaluation_config branches
- evaluate_xbench_deepsearch: evaluation_config branches
- compare_configurations: default configs, report generation, metrics formatting
"""

from unittest.mock import patch


MODULE = "local_deep_research.api.benchmark_functions"


class TestEvaluateSimpleqa:
    @patch(f"{MODULE}.run_simpleqa_benchmark")
    def test_evaluation_config_with_model_only(self, mock_run):
        """evaluation_config is set when only model is provided."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_simpleqa(
            num_examples=5,
            evaluation_model="gpt-4",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"] == {"model_name": "gpt-4"}

    @patch(f"{MODULE}.run_simpleqa_benchmark")
    def test_evaluation_config_with_provider_only(self, mock_run):
        """evaluation_config is set when only provider is provided."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_simpleqa(
            num_examples=5,
            evaluation_provider="openai",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"] == {"provider": "openai"}

    @patch(f"{MODULE}.run_simpleqa_benchmark")
    def test_evaluation_config_with_both(self, mock_run):
        """evaluation_config includes both model and provider."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_simpleqa(
            num_examples=5,
            evaluation_model="gpt-4",
            evaluation_provider="openai",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4"
        assert call_kwargs["evaluation_config"]["provider"] == "openai"

    @patch(f"{MODULE}.run_simpleqa_benchmark")
    def test_no_evaluation_config(self, mock_run):
        """evaluation_config is None when neither model nor provider."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_simpleqa(num_examples=5)

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"] is None


class TestEvaluateBrowsecomp:
    @patch(f"{MODULE}.run_browsecomp_benchmark")
    def test_evaluation_config_with_model(self, mock_run):
        """evaluation_config with model for browsecomp."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_browsecomp,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_browsecomp(
            num_examples=5,
            evaluation_model="claude-3",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"]["model_name"] == "claude-3"

    @patch(f"{MODULE}.run_browsecomp_benchmark")
    def test_evaluation_config_with_provider(self, mock_run):
        """evaluation_config with provider for browsecomp."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_browsecomp,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_browsecomp(
            num_examples=5,
            evaluation_provider="anthropic",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"]["provider"] == "anthropic"


class TestEvaluateXbenchDeepsearch:
    @patch(f"{MODULE}.run_xbench_deepsearch_benchmark")
    def test_evaluation_config_with_model(self, mock_run):
        """evaluation_config with model for xbench."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_xbench_deepsearch,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_xbench_deepsearch(
            num_examples=5,
            evaluation_model="gpt-4o",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4o"

    @patch(f"{MODULE}.run_xbench_deepsearch_benchmark")
    def test_evaluation_config_with_provider(self, mock_run):
        """evaluation_config with provider for xbench."""
        from local_deep_research.api.benchmark_functions import (
            evaluate_xbench_deepsearch,
        )

        mock_run.return_value = {"status": "complete"}

        evaluate_xbench_deepsearch(
            num_examples=5,
            evaluation_provider="openai",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["evaluation_config"]["provider"] == "openai"


class TestCompareConfigurations:
    @patch(
        "local_deep_research.security.file_write_verifier.write_file_verified"
    )
    @patch(f"{MODULE}.run_benchmark")
    def test_default_configurations(self, mock_run, mock_write):
        """Uses default configurations when none provided."""
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        mock_run.return_value = {
            "metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
            "total_examples": 20,
        }

        result = compare_configurations(
            dataset_type="simpleqa",
            num_examples=5,
            output_dir="/tmp/test_bench",
        )

        assert result["status"] == "complete"
        assert result["configurations_tested"] == 3  # 3 default configs
        assert mock_run.call_count == 3

    @patch(
        "local_deep_research.security.file_write_verifier.write_file_verified"
    )
    @patch(f"{MODULE}.run_benchmark")
    def test_custom_configurations(self, mock_run, mock_write):
        """Passes custom configurations correctly."""
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        mock_run.return_value = {
            "metrics": {"accuracy": 0.9, "average_processing_time": 2.0},
            "total_examples": 10,
        }

        configs = [
            {
                "name": "Fast",
                "search_tool": "wikipedia",
                "iterations": 1,
                "questions_per_iteration": 2,
            },
        ]

        result = compare_configurations(
            configurations=configs,
            num_examples=10,
            output_dir="/tmp/test_bench",
        )

        assert result["configurations_tested"] == 1

    @patch(
        "local_deep_research.security.file_write_verifier.write_file_verified"
    )
    @patch(f"{MODULE}.run_benchmark")
    def test_report_written(self, mock_run, mock_write):
        """Comparison report is written to file."""
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        mock_run.return_value = {
            "metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
            "total_examples": 20,
        }

        result = compare_configurations(
            num_examples=5,
            output_dir="/tmp/test_bench",
        )

        mock_write.assert_called_once()
        # Report content should contain markdown table
        report_content = mock_write.call_args[0][1]
        assert "Configuration" in report_content
        assert "Accuracy" in report_content
        assert result["report_path"].endswith(".md")

    @patch(
        "local_deep_research.security.file_write_verifier.write_file_verified"
    )
    @patch(f"{MODULE}.run_benchmark")
    def test_extra_config_items_passed(self, mock_run, mock_write):
        """Extra config items beyond standard ones are passed through."""
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        mock_run.return_value = {
            "metrics": {"accuracy": 0.5},
            "total_examples": 5,
        }

        configs = [
            {
                "name": "Custom",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 1,
                "custom_param": "value",
            },
        ]

        compare_configurations(
            configurations=configs,
            num_examples=5,
            output_dir="/tmp/test_bench",
        )

        call_kwargs = mock_run.call_args[1]
        assert call_kwargs["search_config"]["custom_param"] == "value"