learningcircuit--local-deep-research/tests/benchmarks/evaluators/test_simpleqa_coverage.py

"""
Coverage tests for SimpleQAEvaluator._run_with_dataset_class and evaluate error paths.

These tests focus on the internal logic of _run_with_dataset_class that is not
covered by the existing test_simpleqa.py, including dataset loading, example
processing, error handling within examples, file writing, metrics calculation,
report generation, and search parameter extraction.
"""

import json
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch

from local_deep_research.benchmarks.evaluators.simpleqa import SimpleQAEvaluator


def _make_evaluator():
    return SimpleQAEvaluator()


def _mock_dataset_instance(examples=None):
    """Create a mock dataset instance with configurable examples."""
    ds = MagicMock()
    if examples is None:
        examples = [
            {
                "id": "q1",
                "question": "What is the capital of France?",
                "answer": "Paris",
            },
        ]
    ds.load.return_value = examples
    ds.get_question.side_effect = lambda ex: ex.get("question", "")
    ds.get_answer.side_effect = lambda ex: ex.get("answer", "")
    return ds


def _standard_patches(
    examples=None,
    quick_summary_return=None,
    extract_return=None,
    metrics_return=None,
    report_return="/tmp/report.md",
):
    """Return a dict of patch targets and their mock values for a full happy-path run."""
    ds = _mock_dataset_instance(examples)
    if quick_summary_return is None:
        quick_summary_return = {
            "summary": "Paris is the capital.",
            "sources": [],
        }
    if extract_return is None:
        extract_return = {"extracted_answer": "Paris", "confidence": 0.95}
    if metrics_return is None:
        metrics_return = {"accuracy": 0.9, "total": 1, "correct": 1}
    return {
        "dataset": ds,
        "quick_summary_return": quick_summary_return,
        "extract_return": extract_return,
        "metrics_return": metrics_return,
        "report_return": report_return,
    }


_QUICK_SUMMARY = "local_deep_research.api.quick_summary"
_EXTRACT_ANSWER = (
    "local_deep_research.benchmarks.graders.extract_answer_from_response"
)
_GRADE_RESULTS = "local_deep_research.benchmarks.graders.grade_results"
_DATASET_REGISTRY = (
    "local_deep_research.benchmarks.evaluators.simpleqa.DatasetRegistry"
)
_CALC_METRICS = (
    "local_deep_research.benchmarks.evaluators.simpleqa.calculate_metrics"
)
_GEN_REPORT = (
    "local_deep_research.benchmarks.evaluators.simpleqa.generate_report"
)


class TestRunWithDatasetClassLoadAndProcess:
    """Test dataset loading and processing in _run_with_dataset_class."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_dataset_load_and_process(
        self, mock_registry, mock_calc, mock_report
    ):
        """Test that dataset is created with correct params and load() is called."""
        helpers = _standard_patches()
        mock_registry.create_dataset.return_value = helpers["dataset"]
        mock_calc.return_value = helpers["metrics_return"]
        mock_report.return_value = helpers["report_return"]
        evaluator = _make_evaluator()
        config = {
            "seed": 99,
            "iterations": 2,
            "questions_per_iteration": 5,
            "search_tool": "google",
        }
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY, return_value=helpers["quick_summary_return"]
                ),
                patch(_EXTRACT_ANSWER, return_value=helpers["extract_return"]),
                patch(_GRADE_RESULTS),
            ):
                result = evaluator._run_with_dataset_class(
                    system_config=config, num_examples=10, output_dir=tmpdir
                )
        mock_registry.create_dataset.assert_called_once_with(
            dataset_id="simpleqa", num_examples=10, seed=99
        )
        helpers["dataset"].load.assert_called_once()
        assert result["status"] == "complete"

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_default_seed_when_not_in_config(
        self, mock_registry, mock_calc, mock_report
    ):
        """Test that seed defaults to 42 when not provided in system_config."""
        helpers = _standard_patches()
        mock_registry.create_dataset.return_value = helpers["dataset"]
        mock_calc.return_value = helpers["metrics_return"]
        mock_report.return_value = helpers["report_return"]
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY, return_value=helpers["quick_summary_return"]
                ),
                patch(_EXTRACT_ANSWER, return_value=helpers["extract_return"]),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={}, num_examples=5, output_dir=tmpdir
                )
        _, kwargs = mock_registry.create_dataset.call_args
        assert kwargs["seed"] == 42


class TestRunWithDatasetClassExampleErrorHandling:
    """Test error handling for individual examples in _run_with_dataset_class."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_example_error_creates_error_result(
        self, mock_registry, mock_calc, mock_report
    ):
        """When quick_summary raises, an error result is appended and processing continues."""
        examples = [
            {"id": "err1", "question": "Bad question?", "answer": "N/A"}
        ]
        ds = _mock_dataset_instance(examples)
        mock_registry.create_dataset.return_value = ds
        mock_calc.return_value = {"accuracy": 0.0}
        mock_report.return_value = "/tmp/report.md"
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(_QUICK_SUMMARY, side_effect=RuntimeError("API timeout")),
                patch(_GRADE_RESULTS),
            ):
                result = evaluator._run_with_dataset_class(
                    system_config={}, num_examples=1, output_dir=tmpdir
                )
        assert result["status"] == "complete"
        assert result["total_examples"] == 1

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_error_result_contains_expected_fields(
        self, mock_registry, mock_calc, mock_report
    ):
        """Error results written to file contain id, problem, correct_answer, error, processing_time."""
        examples = [{"id": "e1", "question": "Failing Q", "answer": "A"}]
        ds = _mock_dataset_instance(examples)
        mock_registry.create_dataset.return_value = ds
        mock_calc.return_value = {"accuracy": 0.0}
        mock_report.return_value = "/tmp/report.md"
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(_QUICK_SUMMARY, side_effect=ValueError("bad input")),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={}, num_examples=1, output_dir=tmpdir
                )
            results_files = list(Path(tmpdir).glob("simpleqa_*_results.jsonl"))
            assert len(results_files) == 1
            with open(results_files[0]) as f:
                line = json.loads(f.readline())
            assert line["id"] == "e1"
            assert line["problem"] == "Failing Q"
            assert line["correct_answer"] == "A"
            assert "bad input" in line["error"]
            assert line["processing_time"] == 0


class TestRunWithDatasetClassCompleteStatus:
    """Test that _run_with_dataset_class returns complete status on success."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_complete_status_fields(
        self, mock_registry, mock_calc, mock_report
    ):
        """Complete result contains all expected keys."""
        helpers = _standard_patches()
        mock_registry.create_dataset.return_value = helpers["dataset"]
        mock_calc.return_value = helpers["metrics_return"]
        mock_report.return_value = "/tmp/report.md"
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY, return_value=helpers["quick_summary_return"]
                ),
                patch(_EXTRACT_ANSWER, return_value=helpers["extract_return"]),
                patch(_GRADE_RESULTS),
            ):
                result = evaluator._run_with_dataset_class(
                    system_config={}, num_examples=1, output_dir=tmpdir
                )
        assert result["status"] == "complete"
        assert result["dataset_type"] == "simpleqa"
        assert result["metrics"] == helpers["metrics_return"]
        assert result["total_examples"] == 1
        assert result["accuracy"] == 0.9
        assert result["report_path"] == "/tmp/report.md"
        assert "results_path" in result
        assert "evaluation_path" in result


class TestRunWithDatasetClassErrorStatus:
    """Test that _run_with_dataset_class returns error status on dataset failure."""

    @patch(_DATASET_REGISTRY)
    def test_dataset_creation_failure_returns_error(self, mock_registry):
        """When DatasetRegistry.create_dataset raises, return error status."""
        mock_registry.create_dataset.side_effect = ImportError(
            "Dataset not found"
        )
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            result = evaluator._run_with_dataset_class(
                system_config={}, num_examples=5, output_dir=tmpdir
            )
        assert result["status"] == "error"
        assert result["dataset_type"] == "simpleqa"
        assert "Dataset not found" in result["error"]

    @patch(_DATASET_REGISTRY)
    def test_dataset_load_failure_returns_error(self, mock_registry):
        """When dataset.load() raises, return error status."""
        ds = MagicMock()
        ds.load.side_effect = RuntimeError("Download failed")
        mock_registry.create_dataset.return_value = ds
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            result = evaluator._run_with_dataset_class(
                system_config={}, num_examples=5, output_dir=tmpdir
            )
        assert result["status"] == "error"
        assert "Download failed" in result["error"]


class TestRunWithDatasetClassQuestionAnswerExtraction:
    """Test question and answer extraction from examples."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_get_question_and_get_answer_called(
        self, mock_registry, mock_calc, mock_report
    ):
        """Verify dataset_instance.get_question and get_answer are called for each example."""
        examples = [
            {"id": "q1", "question": "Q1?", "answer": "A1"},
            {"id": "q2", "question": "Q2?", "answer": "A2"},
        ]
        ds = _mock_dataset_instance(examples)
        mock_registry.create_dataset.return_value = ds
        mock_calc.return_value = {"accuracy": 1.0}
        mock_report.return_value = "/tmp/report.md"
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY,
                    return_value={"summary": "answer", "sources": []},
                ),
                patch(
                    _EXTRACT_ANSWER,
                    return_value={"extracted_answer": "ans", "confidence": 0.9},
                ),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={}, num_examples=2, output_dir=tmpdir
                )
        assert ds.get_question.call_count == 2
        assert ds.get_answer.call_count == 2
        ds.get_question.assert_any_call(examples[0])
        ds.get_question.assert_any_call(examples[1])
        ds.get_answer.assert_any_call(examples[0])
        ds.get_answer.assert_any_call(examples[1])


class TestRunWithDatasetClassResultsFileWriting:
    """Test that results are written to JSONL files correctly."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_successful_result_written_to_file(
        self, mock_registry, mock_calc, mock_report
    ):
        """A successful example result is written as a JSON line to the results file."""
        examples = [{"id": "s1", "question": "What is 2+2?", "answer": "4"}]
        ds = _mock_dataset_instance(examples)
        mock_registry.create_dataset.return_value = ds
        mock_calc.return_value = {"accuracy": 1.0}
        mock_report.return_value = "/tmp/report.md"
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY,
                    return_value={"summary": "4", "sources": ["src1"]},
                ),
                patch(
                    _EXTRACT_ANSWER,
                    return_value={"extracted_answer": "4", "confidence": 0.99},
                ),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={
                        "iterations": 2,
                        "questions_per_iteration": 4,
                        "search_tool": "duckduckgo",
                    },
                    num_examples=1,
                    output_dir=tmpdir,
                )
            results_files = list(Path(tmpdir).glob("simpleqa_*_results.jsonl"))
            assert len(results_files) == 1
            with open(results_files[0]) as f:
                line = json.loads(f.readline())
            assert line["id"] == "s1"
            assert line["problem"] == "What is 2+2?"
            assert line["correct_answer"] == "4"
            assert line["response"] == "4"
            assert line["extracted_answer"] == "4"
            assert line["confidence"] == 0.99
            assert line["sources"] == ["src1"]
            assert isinstance(line["processing_time"], float)
            assert line["search_config"]["search_tool"] == "duckduckgo"


class TestRunWithDatasetClassMetricsCalculation:
    """Test metrics calculation integration."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_calculate_metrics_called_with_evaluation_file(
        self, mock_registry, mock_calc, mock_report
    ):
        """calculate_metrics is called with the evaluation file path."""
        helpers = _standard_patches()
        mock_registry.create_dataset.return_value = helpers["dataset"]
        mock_calc.return_value = helpers["metrics_return"]
        mock_report.return_value = helpers["report_return"]
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY, return_value=helpers["quick_summary_return"]
                ),
                patch(_EXTRACT_ANSWER, return_value=helpers["extract_return"]),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={}, num_examples=1, output_dir=tmpdir
                )
        mock_calc.assert_called_once()
        eval_path = mock_calc.call_args[0][0]
        assert "evaluation.jsonl" in eval_path


class TestRunWithDatasetClassReportGeneration:
    """Test report generation integration."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_generate_report_called_with_correct_params(
        self, mock_registry, mock_calc, mock_report
    ):
        """generate_report receives metrics, dataset name, and config_info."""
        helpers = _standard_patches()
        mock_registry.create_dataset.return_value = helpers["dataset"]
        mock_calc.return_value = helpers["metrics_return"]
        mock_report.return_value = "/tmp/report.md"
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(
                    _QUICK_SUMMARY, return_value=helpers["quick_summary_return"]
                ),
                patch(_EXTRACT_ANSWER, return_value=helpers["extract_return"]),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={
                        "iterations": 5,
                        "questions_per_iteration": 2,
                        "search_tool": "brave",
                    },
                    num_examples=1,
                    output_dir=tmpdir,
                )
        mock_report.assert_called_once()
        kwargs = mock_report.call_args[1]
        assert kwargs["metrics"] == helpers["metrics_return"]
        assert kwargs["dataset_name"] == "SimpleQA"
        config_info = kwargs["config_info"]
        assert config_info["Dataset"] == "SimpleQA"
        assert config_info["Examples"] == 1
        assert config_info["Iterations"] == 5
        assert config_info["Questions per iteration"] == 2
        assert config_info["Search tool"] == "brave"


class TestRunWithDatasetClassSearchParams:
    """Test search parameter extraction from system_config."""

    @patch(_GEN_REPORT)
    @patch(_CALC_METRICS)
    @patch(_DATASET_REGISTRY)
    def test_search_params_defaults(
        self, mock_registry, mock_calc, mock_report
    ):
        """When system_config has no search keys, defaults are used."""
        helpers = _standard_patches()
        mock_registry.create_dataset.return_value = helpers["dataset"]
        mock_calc.return_value = helpers["metrics_return"]
        mock_report.return_value = helpers["report_return"]
        mock_quick = MagicMock(return_value=helpers["quick_summary_return"])
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            with (
                patch(_QUICK_SUMMARY, mock_quick),
                patch(_EXTRACT_ANSWER, return_value=helpers["extract_return"]),
                patch(_GRADE_RESULTS),
            ):
                evaluator._run_with_dataset_class(
                    system_config={}, num_examples=1, output_dir=tmpdir
                )
        mock_quick.assert_called_once()
        call_kwargs = mock_quick.call_args[1]
        assert call_kwargs["iterations"] == 3
        assert call_kwargs["questions_per_iteration"] == 3
        assert call_kwargs["search_tool"] == "searxng"


class TestEvaluateDatasetExceptionReturnsErrorDict:
    """Test that evaluate returns error dict when _run_with_dataset_class raises."""

    @patch.object(SimpleQAEvaluator, "_run_with_dataset_class")
    def test_evaluate_returns_error_dict_on_exception(self, mock_run):
        """When _run_with_dataset_class raises, evaluate catches and returns error dict."""
        mock_run.side_effect = Exception("Dataset load explosion")
        evaluator = _make_evaluator()
        with tempfile.TemporaryDirectory() as tmpdir:
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
                use_direct_dataset=True,
            )
        assert result["benchmark_type"] == "simpleqa"
        assert result["quality_score"] == 0.0
        assert result["accuracy"] == 0.0
        assert "Dataset load explosion" in result["error"]