learningcircuit--local-deep-research/tests/benchmarks/evaluators/test_simpleqa.py

"""
Tests for SimpleQAEvaluator class.

Tests the SimpleQA benchmark evaluator implementation.
"""

import tempfile
from pathlib import Path
from unittest.mock import patch


from local_deep_research.benchmarks.evaluators.simpleqa import SimpleQAEvaluator
from local_deep_research.benchmarks.evaluators.base import (
    BaseBenchmarkEvaluator,
)


class TestSimpleQAEvaluatorInit:
    """Test initialization of SimpleQAEvaluator."""

    def test_init_sets_name(self):
        """Test that initialization sets the benchmark name to 'simpleqa'."""
        evaluator = SimpleQAEvaluator()
        assert evaluator.name == "simpleqa"

    def test_inherits_from_base(self):
        """Test that SimpleQAEvaluator inherits from BaseBenchmarkEvaluator."""
        evaluator = SimpleQAEvaluator()
        assert isinstance(evaluator, BaseBenchmarkEvaluator)

    def test_get_name_returns_simpleqa(self):
        """Test that get_name returns 'simpleqa'."""
        evaluator = SimpleQAEvaluator()
        assert evaluator.get_name() == "simpleqa"


class TestSimpleQAEvaluateWithRunner:
    """Test evaluate method with legacy runner."""

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_calls_runner_when_not_direct(self, mock_runner):
        """Test that evaluate calls run_simpleqa_benchmark when use_direct_dataset=False."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.75},
            "report_path": "/tmp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            evaluator.evaluate(
                system_config={"key": "value"},
                num_examples=10,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            mock_runner.assert_called_once()
            call_kwargs = mock_runner.call_args[1]
            assert call_kwargs["num_examples"] == 10
            assert call_kwargs["run_evaluation"] is True

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_returns_accuracy_from_runner(self, mock_runner):
        """Test that evaluate returns accuracy from runner results."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.85},
            "report_path": "/tmp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["accuracy"] == 0.85
            assert result["quality_score"] == 0.85

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_returns_benchmark_type(self, mock_runner):
        """Test that evaluate returns correct benchmark_type."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": "/tmp/report.md",
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["benchmark_type"] == "simpleqa"

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_includes_raw_results(self, mock_runner):
        """Test that evaluate includes raw_results from runner."""
        raw_data = {
            "metrics": {"accuracy": 0.6},
            "report_path": "/tmp/report.md",
            "extra_data": "test",
        }
        mock_runner.return_value = raw_data

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["raw_results"] == raw_data

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_passes_search_config(self, mock_runner):
        """Test that evaluate passes search_config to runner."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": None,
        }

        config = {"iterations": 5, "search_tool": "google"}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            evaluator.evaluate(
                system_config=config,
                num_examples=10,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            call_kwargs = mock_runner.call_args[1]
            assert call_kwargs["search_config"] == config


class TestSimpleQAEvaluateWithDirectDataset:
    """Test evaluate method with direct dataset class."""

    @patch.object(SimpleQAEvaluator, "_run_with_dataset_class")
    def test_evaluate_uses_direct_dataset_by_default(self, mock_method):
        """Test that evaluate uses direct dataset by default."""
        mock_method.return_value = {
            "status": "complete",
            "metrics": {"accuracy": 0.8},
            "accuracy": 0.8,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            mock_method.assert_called_once()

    @patch.object(SimpleQAEvaluator, "_run_with_dataset_class")
    def test_evaluate_passes_params_to_direct_method(self, mock_method):
        """Test that evaluate passes correct params to direct method."""
        mock_method.return_value = {
            "status": "complete",
            "metrics": {"accuracy": 0.7},
            "accuracy": 0.7,
        }

        config = {"seed": 123, "iterations": 3}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            evaluator.evaluate(
                system_config=config,
                num_examples=15,
                output_dir=tmpdir,
                use_direct_dataset=True,
            )

            call_kwargs = mock_method.call_args[1]
            assert call_kwargs["system_config"] == config
            assert call_kwargs["num_examples"] == 15


class TestSimpleQAEvaluateErrors:
    """Test error handling in evaluate method."""

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_handles_runner_exception(self, mock_runner):
        """Test that evaluate handles exceptions from runner."""
        mock_runner.side_effect = RuntimeError("Benchmark failed")

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["benchmark_type"] == "simpleqa"
            assert result["quality_score"] == 0.0
            assert result["accuracy"] == 0.0
            assert "error" in result
            assert "Benchmark failed" in result["error"]

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_handles_missing_metrics(self, mock_runner):
        """Test that evaluate handles missing metrics in results."""
        mock_runner.return_value = {}  # No metrics key

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["accuracy"] == 0.0
            assert result["quality_score"] == 0.0

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_handles_missing_accuracy(self, mock_runner):
        """Test that evaluate handles missing accuracy in metrics."""
        mock_runner.return_value = {"metrics": {}}  # No accuracy key

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["accuracy"] == 0.0
            assert result["quality_score"] == 0.0


class TestSimpleQACreateSubdirectory:
    """Test subdirectory creation."""

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_evaluate_creates_subdirectory(self, mock_runner):
        """Test that evaluate creates benchmark-specific subdirectory."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.5},
            "report_path": None,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            # Check that subdirectory was created
            expected_subdir = Path(tmpdir) / "simpleqa"
            assert expected_subdir.exists()


class TestSimpleQAQualityScore:
    """Test quality_score mapping."""

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_quality_score_equals_accuracy(self, mock_runner):
        """Test that quality_score is mapped directly from accuracy."""
        mock_runner.return_value = {
            "metrics": {"accuracy": 0.923},
            "report_path": None,
        }

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["quality_score"] == result["accuracy"]
            assert result["quality_score"] == 0.923

    @patch(
        "local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
    )
    def test_quality_score_zero_on_error(self, mock_runner):
        """Test that quality_score is 0 on error."""
        mock_runner.side_effect = Exception("Test error")

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = SimpleQAEvaluator()
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=False,
            )

            assert result["quality_score"] == 0.0