learningcircuit--local-deep-research/tests/benchmarks/evaluators/test_browsecomp.py

"""
Tests for BrowseCompEvaluator class.

Tests the BrowseComp benchmark evaluator implementation.
"""

import tempfile
from pathlib import Path
from unittest.mock import patch


from local_deep_research.benchmarks.evaluators.browsecomp import (
    BrowseCompEvaluator,
)
from local_deep_research.benchmarks.evaluators.base import (
    BaseBenchmarkEvaluator,
)


class TestBrowseCompEvaluatorInit:
    """Test initialization of BrowseCompEvaluator."""

    def test_init_sets_name(self):
        """Test that initialization sets the benchmark name to 'browsecomp'."""
        evaluator = BrowseCompEvaluator()
        assert evaluator.name == "browsecomp"

    def test_inherits_from_base(self):
        """Test that BrowseCompEvaluator inherits from BaseBenchmarkEvaluator."""
        evaluator = BrowseCompEvaluator()
        assert isinstance(evaluator, BaseBenchmarkEvaluator)

    def test_get_name_returns_browsecomp(self):
        """Test that get_name returns 'browsecomp'."""
        evaluator = BrowseCompEvaluator()
        assert evaluator.get_name() == "browsecomp"


class TestBrowseCompEvaluate:
    """Test evaluate method."""

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_calls_runner(self, mock_runner):
        """Test that evaluate calls run_browsecomp_benchmark."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.75},
            "report_path": "/tmp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            evaluator.evaluate(
                system_config={"key": "value"},
                num_examples=10,
                output_dir=tmpdir,
            )

            mock_runner.assert_called_once()
            call_kwargs = mock_runner.call_args[1]
            assert call_kwargs["num_examples"] == 10
            assert call_kwargs["run_evaluation"] is True

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_returns_accuracy(self, mock_runner):
        """Test that evaluate returns accuracy from results."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.85},
            "report_path": "/tmp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["accuracy"] == 0.85
            assert result["quality_score"] == 0.85

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_returns_benchmark_type(self, mock_runner):
        """Test that evaluate returns correct benchmark_type."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": "/tmp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["benchmark_type"] == "browsecomp"

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_includes_raw_results(self, mock_runner):
        """Test that evaluate includes raw_results from runner."""
        raw_data = {
            "metrics": {"accuracy": 0.6},
            "report_path": "/tmp/report.md",
            "extra_data": "test",
        }
        mock_runner.return_value = raw_data

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["raw_results"] == raw_data

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_includes_report_path(self, mock_runner):
        """Test that evaluate includes report_path from results."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": "/output/browsecomp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["report_path"] == "/output/browsecomp/report.md"

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_creates_subdirectory(self, mock_runner):
        """Test that evaluate creates benchmark-specific subdirectory."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": None,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            # Check that subdirectory was created
            expected_subdir = Path(tmpdir) / "browsecomp"
            assert expected_subdir.exists()

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_passes_search_config(self, mock_runner):
        """Test that evaluate passes search_config to runner."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": None,
        }

        config = {"iterations": 5, "search_tool": "google"}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            evaluator.evaluate(
                system_config=config,
                num_examples=10,
                output_dir=tmpdir,
            )

            call_kwargs = mock_runner.call_args[1]
            assert call_kwargs["search_config"] == config


class TestBrowseCompEvaluateErrors:
    """Test error handling in evaluate method."""

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_handles_runner_exception(self, mock_runner):
        """Test that evaluate handles exceptions from runner."""
        mock_runner.side_effect = RuntimeError("Benchmark failed")

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["benchmark_type"] == "browsecomp"
            assert result["quality_score"] == 0.0
            assert result["accuracy"] == 0.0
            assert "error" in result
            assert "Benchmark failed" in result["error"]

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_handles_missing_metrics(self, mock_runner):
        """Test that evaluate handles missing metrics in results."""
        mock_runner.return_value = {}  # No metrics key

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["accuracy"] == 0.0
            assert result["quality_score"] == 0.0

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_evaluate_handles_missing_accuracy(self, mock_runner):
        """Test that evaluate handles missing accuracy in metrics."""
        mock_runner.return_value = {"metrics": {}}  # No accuracy key

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["accuracy"] == 0.0
            assert result["quality_score"] == 0.0


class TestBrowseCompQualityScore:
    """Test quality_score mapping."""

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_quality_score_equals_accuracy(self, mock_runner):
        """Test that quality_score is mapped directly from accuracy."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.923},
            "report_path": None,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["quality_score"] == result["accuracy"]
            assert result["quality_score"] == 0.923

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_quality_score_zero_on_zero_accuracy(self, mock_runner):
        """Test that quality_score is 0 when accuracy is 0."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.0},
            "report_path": None,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["quality_score"] == 0.0

    @patch(
        "local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
    )
    def test_quality_score_one_on_perfect_accuracy(self, mock_runner):
        """Test that quality_score is 1.0 when accuracy is 1.0."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 1.0},
            "report_path": None,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = BrowseCompEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert result["quality_score"] == 1.0