learningcircuit--local-deep-research/tests/benchmarks/test_graders_deep_coverage.py

"""
Tests for uncovered code paths in graders.py.

Targets:
- extract_answer_from_response: browsecomp mode, missing matches
- grade_single_result: various LLM interfaces (chat_messages, content attr, callable)
- grade_results / _grade_results_inner: browsecomp extraction, progress callbacks, error handling
- human_evaluation: non-interactive mode
"""

import json
from unittest.mock import Mock, patch


class TestExtractAnswerFromResponse:
    """Tests for extract_answer_from_response."""

    def test_simpleqa_returns_full_response(self):
        """SimpleQA mode returns full response as extracted answer."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "The answer is 42 [1] with citations [2]", "simpleqa"
        )

        assert "42" in result["extracted_answer"]
        assert result["confidence"] == "100"
        # Citations should be stripped
        assert "[1]" not in result["extracted_answer"]
        assert "[2]" not in result["extracted_answer"]

    def test_browsecomp_extracts_answer_and_confidence(self):
        """BrowseComp mode extracts answer and confidence from structured response."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        response = "Exact Answer: The Great Wall of China\nConfidence: 85%"
        result = extract_answer_from_response(response, "browsecomp")

        assert result["extracted_answer"] == "The Great Wall of China"
        assert result["confidence"] == "85"

    def test_browsecomp_missing_answer_field(self):
        """BrowseComp mode returns 'None' when answer field is missing."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "Some unstructured response", "browsecomp"
        )

        assert result["extracted_answer"] == "None"

    def test_browsecomp_missing_confidence(self):
        """BrowseComp mode defaults to 100 when confidence is missing."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "Exact Answer: Something\nNo confidence here", "browsecomp"
        )

        assert result["confidence"] == "100"

    def test_citations_stripped_from_response(self):
        """Citation markers like [1], [23] are removed."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "The answer [1] is 42 [23] exactly.", "simpleqa"
        )

        assert "[1]" not in result["extracted_answer"]
        assert "[23]" not in result["extracted_answer"]
        assert "42" in result["extracted_answer"]


class TestGradeSingleResult:
    """Tests for grade_single_result."""

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_single_result_with_content_attr(
        self, mock_close, mock_get_llm
    ):
        """grade_single_result handles LLM response with .content attribute."""
        from local_deep_research.benchmarks.graders import grade_single_result

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = (
            "Extracted Answer: 42\nReasoning: Matches exactly\nCorrect: yes"
        )
        mock_llm.invoke.return_value = mock_response
        mock_llm.chat_messages = None  # Has invoke but not chat_messages
        # Remove chat_messages to trigger the else branch
        del mock_llm.chat_messages
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {
                "problem": "What is 6 * 7?",
                "correct_answer": "42",
                "response": "The answer is 42",
            },
            dataset_type="simpleqa",
        )

        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "42"

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_single_result_browsecomp(self, mock_close, mock_get_llm):
        """grade_single_result handles browsecomp extraction format."""
        from local_deep_research.benchmarks.graders import grade_single_result

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = (
            "extracted_final_answer: The Great Wall\n"
            "reasoning: The response correctly identifies the structure\n\n"
            "correct: yes\n"
            "confidence: 90"
        )
        mock_llm.invoke.return_value = mock_response
        del mock_llm.chat_messages
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {
                "problem": "What is the longest wall?",
                "correct_answer": "The Great Wall",
                "response": "The Great Wall of China",
            },
            dataset_type="browsecomp",
        )

        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "The Great Wall"
        assert result["graded_confidence"] == "90"

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_single_result_callable_fallback(
        self, mock_close, mock_get_llm
    ):
        """grade_single_result falls back to calling LLM as callable."""
        from local_deep_research.benchmarks.graders import grade_single_result

        mock_llm = Mock()
        # Remove invoke to trigger callable fallback
        del mock_llm.invoke
        mock_llm.return_value = (
            "Extracted Answer: 42\nReasoning: ok\nCorrect: yes"
        )
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {
                "problem": "What?",
                "correct_answer": "42",
                "response": "42",
            },
        )

        assert result["is_correct"] is True

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_single_result_exception(self, mock_close, mock_get_llm):
        """grade_single_result handles exceptions gracefully."""
        from local_deep_research.benchmarks.graders import grade_single_result

        mock_llm = Mock()
        mock_llm.invoke.side_effect = Exception("LLM failed")
        del mock_llm.chat_messages
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "A", "response": "R"},
        )

        assert result["is_correct"] is False
        assert "grading_error" in result
        assert "LLM failed" in result["grading_error"]

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_single_result_no_correct_match(
        self, mock_close, mock_get_llm
    ):
        """grade_single_result defaults to incorrect when no 'Correct:' match."""
        from local_deep_research.benchmarks.graders import grade_single_result

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = "Some response without the expected format"
        mock_llm.invoke.return_value = mock_response
        del mock_llm.chat_messages
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "A", "response": "R"},
        )

        assert result["is_correct"] is False
        assert result["extracted_by_grader"] == "None"


class TestHumanEvaluation:
    """Tests for human_evaluation non-interactive mode."""

    def test_non_interactive_mode(self, tmp_path):
        """human_evaluation in non-interactive mode marks all as incorrect."""
        from local_deep_research.benchmarks.graders import human_evaluation

        # Create input file
        input_file = tmp_path / "results.jsonl"
        results = [
            {
                "problem": "Q1",
                "correct_answer": "A1",
                "response": "R1",
                "extracted_answer": "E1",
            },
            {
                "problem": "Q2",
                "correct_answer": "A2",
                "response": "R2",
                "extracted_answer": "E2",
            },
        ]
        input_file.write_text("\n".join(json.dumps(r) for r in results))

        output_file = tmp_path / "graded.jsonl"

        graded = human_evaluation(
            str(input_file), str(output_file), interactive=False
        )

        assert len(graded) == 2
        assert all(r["is_correct"] is False for r in graded)
        assert all(r["human_evaluation"] is True for r in graded)
        assert all(
            r["reasoning"] == "Non-interactive evaluation" for r in graded
        )

        # Check output file was written
        assert output_file.exists()
        lines = output_file.read_text().strip().split("\n")
        assert len(lines) == 2

    def test_non_interactive_empty_results(self, tmp_path):
        """human_evaluation handles empty results file."""
        from local_deep_research.benchmarks.graders import human_evaluation

        input_file = tmp_path / "empty.jsonl"
        input_file.write_text("")

        output_file = tmp_path / "graded.jsonl"

        graded = human_evaluation(
            str(input_file), str(output_file), interactive=False
        )

        assert len(graded) == 0


class TestGradeResultsInner:
    """Tests for _grade_results_inner."""

    def test_grade_results_with_progress_callback(self, tmp_path):
        """_grade_results_inner calls progress callback correctly."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        # Create input file
        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps(
                {
                    "problem": "Q1",
                    "correct_answer": "42",
                    "response": "42",
                }
            )
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = (
            "Extracted Answer: 42\nReasoning: ok\nCorrect: yes"
        )
        mock_llm.invoke.return_value = mock_response
        del mock_llm.chat_messages

        callback_calls = []

        def progress_callback(idx, total, data):
            callback_calls.append((idx, total, data))

        result = _grade_results_inner(
            mock_llm,
            str(input_file),
            str(output_file),
            "simpleqa",
            progress_callback,
        )

        assert len(result) == 1
        assert result[0]["is_correct"] is True
        # Should have at least 2 callbacks: grading + graded
        assert len(callback_calls) >= 2

    def test_grade_results_error_handling(self, tmp_path):
        """_grade_results_inner handles grading errors per result."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps({"problem": "Q", "correct_answer": "A", "response": "R"})
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        mock_llm.invoke.side_effect = Exception("LLM error")
        del mock_llm.chat_messages

        result = _grade_results_inner(
            mock_llm, str(input_file), str(output_file), "simpleqa", None
        )

        assert len(result) == 1
        assert "grading_error" in result[0]

    def test_grade_results_browsecomp_extraction(self, tmp_path):
        """_grade_results_inner uses browsecomp extraction format."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps(
                {
                    "problem": "What is X?",
                    "correct_answer": "Y",
                    "response": "Y is correct",
                }
            )
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = (
            "extracted_final_answer: Y\n"
            "reasoning: Exact match\n\n"
            "correct: yes\n"
            "confidence: 95"
        )
        mock_llm.invoke.return_value = mock_response
        del mock_llm.chat_messages

        result = _grade_results_inner(
            mock_llm, str(input_file), str(output_file), "browsecomp", None
        )

        assert len(result) == 1
        assert result[0]["is_correct"] is True
        assert result[0]["graded_confidence"] == "95"

    def test_grade_results_removes_existing_output(self, tmp_path):
        """_grade_results_inner removes existing output file before writing."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps({"problem": "Q", "correct_answer": "A", "response": "R"})
        )
        output_file = tmp_path / "graded.jsonl"
        output_file.write_text("old data\n")

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = (
            "Extracted Answer: A\nReasoning: ok\nCorrect: yes"
        )
        mock_llm.invoke.return_value = mock_response
        del mock_llm.chat_messages

        _grade_results_inner(
            mock_llm, str(input_file), str(output_file), "simpleqa", None
        )

        # Output should not contain old data
        content = output_file.read_text()
        assert "old data" not in content