learningcircuit--local-deep-research/tests/benchmarks/test_benchmark_commands_high_value.py

"""High-value tests for benchmarks/cli/benchmark_commands.py.

Covers CLI output formatting, compare_configs_cli, main() verbose logging,
argument parsing edge cases, and search/eval config building.
"""

import argparse
import sys
from unittest.mock import patch

import pytest


@pytest.fixture
def mock_data_directory(tmp_path):
    """Mock get_data_directory so module-level default paths use tmp_path."""
    with patch(
        "local_deep_research.benchmarks.cli.benchmark_commands.get_data_directory"
    ) as mock:
        mock.return_value = tmp_path
        yield tmp_path


def _make_args(**overrides):
    """Build a minimal argparse.Namespace with sane defaults."""
    defaults = dict(
        examples=10,
        iterations=3,
        questions=2,
        search_tool="searxng",
        output_dir="/tmp/test_output",
        human_eval=False,
        no_eval=False,
        custom_dataset=None,
        eval_model=None,
        eval_provider=None,
        search_model=None,
        search_provider=None,
        endpoint_url=None,
        search_strategy="source_based",
    )
    defaults.update(overrides)
    return argparse.Namespace(**defaults)


class TestSimpleqaCliOutput:
    """Tests for run_simpleqa_cli output formatting paths."""

    def test_prints_metrics_when_present(self, mock_data_directory, capsys):
        """When result contains 'metrics', summary stats are printed."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args()
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {
                "metrics": {
                    "accuracy": 0.85,
                    "correct": 17,
                    "average_processing_time": 4.321,
                },
                "total_examples": 20,
                "report_path": "/tmp/report.html",
            }
            run_simpleqa_cli(args)

        out = capsys.readouterr().out
        assert "SimpleQA Benchmark Results:" in out
        assert "0.850" in out
        assert "17" in out

    def test_prints_no_eval_message_when_no_metrics(
        self, mock_data_directory, capsys
    ):
        """When result has no 'metrics' key, no-eval summary is printed."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args()
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {
                "total_examples": 20,
                "results_path": "/tmp/results.json",
            }
            run_simpleqa_cli(args)

        out = capsys.readouterr().out
        assert "Completed (no evaluation)" in out
        assert "20" in out

    def test_report_path_na_when_missing(self, mock_data_directory, capsys):
        """report_path defaults to N/A when not in result dict."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args()
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {
                "metrics": {
                    "accuracy": 0,
                    "correct": 0,
                    "average_processing_time": 0,
                },
                "total_examples": 0,
            }
            run_simpleqa_cli(args)

        out = capsys.readouterr().out
        assert "N/A" in out


class TestBrowsecompCliOutput:
    """Tests for run_browsecomp_cli output formatting paths."""

    def test_prints_metrics_when_present(self, mock_data_directory, capsys):
        """BrowseComp prints metrics summary when result has 'metrics'."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_browsecomp_cli,
        )

        args = _make_args()
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_browsecomp_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {
                "metrics": {
                    "accuracy": 0.6,
                    "correct": 6,
                    "average_processing_time": 12.5,
                },
                "total_examples": 10,
                "report_path": "/tmp/browsecomp_report.html",
            }
            run_browsecomp_cli(args)

        out = capsys.readouterr().out
        assert "BrowseComp Benchmark Results:" in out
        assert "0.600" in out

    def test_prints_no_eval_message_when_no_metrics(
        self, mock_data_directory, capsys
    ):
        """BrowseComp prints no-eval message when result has no 'metrics'."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_browsecomp_cli,
        )

        args = _make_args()
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_browsecomp_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {
                "total_examples": 10,
                "results_path": "/tmp/browse_results.json",
            }
            run_browsecomp_cli(args)

        out = capsys.readouterr().out
        assert "BrowseComp Benchmark Completed (no evaluation)" in out

    def test_browsecomp_endpoint_url_in_search_config(
        self, mock_data_directory
    ):
        """endpoint_url is mapped to openai_endpoint_url in search_config."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_browsecomp_cli,
        )

        args = _make_args(endpoint_url="https://api.example.com/v1")
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_browsecomp_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {"total_examples": 1}
            run_browsecomp_cli(args)
            kw = mock_bench.call_args[1]
            assert (
                kw["search_config"]["openai_endpoint_url"]
                == "https://api.example.com/v1"
            )


class TestSearchConfigEdgeCases:
    """Test search config building for uncommon attribute combinations."""

    def test_endpoint_url_added_to_simpleqa_search_config(
        self, mock_data_directory
    ):
        """endpoint_url maps to openai_endpoint_url in SimpleQA search config."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args(endpoint_url="https://openrouter.ai/api/v1")
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {"total_examples": 1}
            run_simpleqa_cli(args)
            kw = mock_bench.call_args[1]
            assert (
                kw["search_config"]["openai_endpoint_url"]
                == "https://openrouter.ai/api/v1"
            )

    def test_no_eval_flag_passed_as_run_evaluation_false(
        self, mock_data_directory
    ):
        """--no-eval sets run_evaluation=False in benchmark call."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args(no_eval=True)
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {"total_examples": 1}
            run_simpleqa_cli(args)
            kw = mock_bench.call_args[1]
            assert kw["run_evaluation"] is False

    def test_human_eval_flag_passed_through(self, mock_data_directory):
        """--human-eval flag is forwarded as human_evaluation=True."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args(human_eval=True)
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {"total_examples": 1}
            run_simpleqa_cli(args)
            kw = mock_bench.call_args[1]
            assert kw["human_evaluation"] is True

    def test_eval_config_only_model_no_provider(self, mock_data_directory):
        """eval_model alone creates evaluation_config with only model_name."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            run_simpleqa_cli,
        )

        args = _make_args(eval_model="claude-3")
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
        ) as mock_bench:
            mock_bench.return_value = {"total_examples": 1}
            run_simpleqa_cli(args)
            kw = mock_bench.call_args[1]
            assert kw["evaluation_config"] == {"model_name": "claude-3"}


class TestCompareConfigsCli:
    """Tests for compare_configs_cli."""

    def test_compare_calls_compare_configurations(self, mock_data_directory):
        """compare_configs_cli delegates to compare_configurations."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            compare_configs_cli,
        )

        args = argparse.Namespace(
            dataset="simpleqa", examples=20, output_dir="/tmp/compare"
        )
        with patch(
            "local_deep_research.api.benchmark_functions.compare_configurations"
        ) as mock_cmp:
            mock_cmp.return_value = {
                "configurations_tested": 2,
                "report_path": "/tmp/compare/report.html",
                "results": [
                    {
                        "configuration_name": "baseline",
                        "metrics": {
                            "accuracy": 0.8,
                            "average_processing_time": 3.0,
                        },
                    },
                ],
            }
            compare_configs_cli(args)
            mock_cmp.assert_called_once_with(
                dataset_type="simpleqa",
                num_examples=20,
                output_dir="/tmp/compare",
            )

    def test_compare_prints_summary_table(self, mock_data_directory, capsys):
        """compare_configs_cli prints a comparison table to stdout."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            compare_configs_cli,
        )

        args = argparse.Namespace(
            dataset="browsecomp", examples=5, output_dir="/tmp/compare"
        )
        with patch(
            "local_deep_research.api.benchmark_functions.compare_configurations"
        ) as mock_cmp:
            mock_cmp.return_value = {
                "configurations_tested": 1,
                "report_path": "/tmp/compare/report.html",
                "results": [
                    {
                        "configuration_name": "default",
                        "metrics": {
                            "accuracy": 0.75,
                            "average_processing_time": 5.55,
                        },
                    },
                ],
            }
            compare_configs_cli(args)

        out = capsys.readouterr().out
        assert "Configuration Comparison Results:" in out
        assert "default" in out
        assert "0.750" in out


class TestMainEntryPoint:
    """Tests for the main() entry point."""

    def test_main_verbose_flag_sets_debug(self, mock_data_directory):
        """--verbose adds DEBUG-level loguru handler."""
        from local_deep_research.benchmarks.cli.benchmark_commands import main

        with patch.object(sys, "argv", ["ldr-benchmark", "--verbose", "list"]):
            with patch(
                "local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
            ) as mock_ds:
                mock_ds.return_value = []
                with patch(
                    "local_deep_research.benchmarks.cli.benchmark_commands.logger"
                ) as mock_logger:
                    main()
                    mock_logger.add.assert_called_once()
                    _, kwargs = mock_logger.add.call_args
                    assert kwargs.get("level") == "DEBUG"

    def test_main_no_verbose_sets_info(self, mock_data_directory):
        """Without --verbose, loguru handler uses INFO level."""
        from local_deep_research.benchmarks.cli.benchmark_commands import main

        with patch.object(sys, "argv", ["ldr-benchmark", "list"]):
            with patch(
                "local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
            ) as mock_ds:
                mock_ds.return_value = []
                with patch(
                    "local_deep_research.benchmarks.cli.benchmark_commands.logger"
                ) as mock_logger:
                    main()
                    mock_logger.add.assert_called_once()
                    _, kwargs = mock_logger.add.call_args
                    assert kwargs.get("level") == "INFO"


class TestArgumentParsingEdgeCases:
    """Test edge cases in argument parsing."""

    def test_all_search_strategies_accepted(self, mock_data_directory):
        """All five strategy choices parse without error."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            setup_benchmark_parser,
        )

        parser = argparse.ArgumentParser()
        subparsers = parser.add_subparsers(dest="command")
        setup_benchmark_parser(subparsers)

        for strategy in [
            "source_based",
            "standard",
            "rapid",
            "parallel",
            "iterdrag",
        ]:
            args = parser.parse_args(
                ["simpleqa", "--search-strategy", strategy]
            )
            assert args.search_strategy == strategy

    def test_eval_model_and_provider_options(self, mock_data_directory):
        """--eval-model and --eval-provider parse correctly."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            setup_benchmark_parser,
        )

        parser = argparse.ArgumentParser()
        subparsers = parser.add_subparsers(dest="command")
        setup_benchmark_parser(subparsers)

        args = parser.parse_args(
            [
                "browsecomp",
                "--eval-model",
                "gpt-4o",
                "--eval-provider",
                "openai",
            ]
        )
        assert args.eval_model == "gpt-4o"
        assert args.eval_provider == "openai"

    def test_custom_dataset_path_parsed(self, mock_data_directory):
        """--custom-dataset stores the given path."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            setup_benchmark_parser,
        )

        parser = argparse.ArgumentParser()
        subparsers = parser.add_subparsers(dest="command")
        setup_benchmark_parser(subparsers)

        args = parser.parse_args(
            ["simpleqa", "--custom-dataset", "/data/my_dataset.jsonl"]
        )
        assert args.custom_dataset == "/data/my_dataset.jsonl"


class TestListBenchmarksEdgeCases:
    """Edge cases for list_benchmarks_cli."""

    def test_empty_datasets_list(self, mock_data_directory, capsys):
        """Empty datasets list prints header but no dataset entries."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            list_benchmarks_cli,
        )

        args = argparse.Namespace()
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
        ) as mock_ds:
            mock_ds.return_value = []
            list_benchmarks_cli(args)

        out = capsys.readouterr().out
        assert "Available Benchmarks:" in out

    def test_multiple_datasets_all_printed(self, mock_data_directory, capsys):
        """Multiple datasets are each printed with id, name, description, url."""
        from local_deep_research.benchmarks.cli.benchmark_commands import (
            list_benchmarks_cli,
        )

        args = argparse.Namespace()
        datasets = [
            {
                "id": "simpleqa",
                "name": "SimpleQA",
                "description": "Basic QA",
                "url": "https://example.com/simpleqa",
            },
            {
                "id": "browsecomp",
                "name": "BrowseComp",
                "description": "Browsing comprehension",
                "url": "https://example.com/browsecomp",
            },
        ]
        with patch(
            "local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
        ) as mock_ds:
            mock_ds.return_value = datasets
            list_benchmarks_cli(args)

        out = capsys.readouterr().out
        for ds in datasets:
            assert ds["id"] in out
            assert ds["name"] in out