learningcircuit--local-deep-research/tests/benchmarks/test_graders_extra_coverage.py

"""Extra coverage tests for benchmarks/graders.py targeting uncovered branches.

Targets:
- LLM invoke __call__ fallback (line 203)
- BrowseComp extraction edge cases (missing fields, multiline, malformed)
- SimpleQA "no" judgment
- Batch _grade_results_inner per-item exception
- grade_single_result chat_messages path
- extract_answer_from_response: SimpleQA path, BrowseComp with/without fields
- grade_single_result: exception path (grading error)
- _grade_results_inner: browsecomp dataset, progress callback, with existing output file
- get_evaluation_llm: custom config override, openai_endpoint API key from snapshot
"""

import json
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch


MODULE = "local_deep_research.benchmarks.graders"


def _make_llm_with_invoke(response_text):
    """Create a mock LLM with invoke() returning an object with .content."""
    llm = MagicMock()
    result = MagicMock()
    result.content = response_text
    llm.invoke.return_value = result
    # Ensure callable interface also exists
    llm.return_value = response_text
    return llm


def _make_llm_callable_only(response_text):
    """Create a mock LLM without invoke attribute — uses __call__ fallback."""
    llm = MagicMock(spec=[])  # No attributes
    llm.return_value = response_text
    return llm


def _make_llm_with_chat_messages(response_text):
    """Create a mock LLM with both invoke and chat_messages."""
    llm = MagicMock()
    llm.chat_messages = True
    result = MagicMock()
    result.content = response_text
    llm.invoke.return_value = result
    return llm


# ---------------------------------------------------------------------------
# LLM invoke __call__ fallback
# ---------------------------------------------------------------------------


class TestLLMCallFallback:
    def test_callable_only_llm(self):
        """LLM without invoke uses __call__ fallback in grade_single_result."""
        from local_deep_research.benchmarks.graders import grade_single_result

        llm = _make_llm_callable_only(
            "Extracted Answer: Paris\nReasoning: Capital of France\nCorrect: yes"
        )

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "Capital of France?",
                    "correct_answer": "Paris",
                    "response": "Paris is the capital.",
                },
                dataset_type="simpleqa",
            )

        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "Paris"


# ---------------------------------------------------------------------------
# BrowseComp extraction edge cases
# ---------------------------------------------------------------------------


class TestBrowseCompExtraction:
    def test_missing_reasoning_field(self):
        """Response without reasoning → still extracts answer and correct."""
        from local_deep_research.benchmarks.graders import grade_single_result

        response = "extracted_final_answer: The Moon\ncorrect: yes"
        llm = _make_llm_with_invoke(response)

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "What orbits Earth?",
                    "correct_answer": "The Moon",
                    "response": "The Moon orbits Earth",
                },
                dataset_type="browsecomp",
            )

        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "The Moon"

    def test_multiline_reasoning(self):
        """Response with multiline reasoning (re.DOTALL) → extracts correctly."""
        from local_deep_research.benchmarks.graders import grade_single_result

        response = (
            "extracted_final_answer: 42\n"
            "reasoning: The answer is 42\nbecause of the ultimate question\n\n"
            "correct: yes\nconfidence: 95"
        )
        llm = _make_llm_with_invoke(response)

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "Answer to life?",
                    "correct_answer": "42",
                    "response": "42",
                },
                dataset_type="browsecomp",
            )

        assert result["is_correct"] is True
        assert result["graded_confidence"] == "95"

    def test_completely_malformed_response(self):
        """Malformed response → returns defaults (None/False)."""
        from local_deep_research.benchmarks.graders import grade_single_result

        llm = _make_llm_with_invoke("This is totally garbled output xyz")

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "Test?",
                    "correct_answer": "A",
                    "response": "B",
                },
                dataset_type="browsecomp",
            )

        assert result["is_correct"] is False
        assert result["extracted_by_grader"] == "None"


# ---------------------------------------------------------------------------
# SimpleQA "no" judgment
# ---------------------------------------------------------------------------


class TestSimpleQANoJudgment:
    def test_correct_no(self):
        """SimpleQA with 'Correct: no' → is_correct=False."""
        from local_deep_research.benchmarks.graders import grade_single_result

        response = (
            "Extracted Answer: Wrong\nReasoning: Completely wrong\nCorrect: no"
        )
        llm = _make_llm_with_invoke(response)

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "What is 2+2?",
                    "correct_answer": "4",
                    "response": "5",
                },
                dataset_type="simpleqa",
            )

        assert result["is_correct"] is False
        assert result["extracted_by_grader"] == "Wrong"


# ---------------------------------------------------------------------------
# Batch _grade_results_inner per-item exception
# ---------------------------------------------------------------------------


class TestBatchGradePerItemException:
    def test_one_item_throws_others_still_graded(self):
        """One result throwing during grading → error recorded, others graded."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        # Create results file with 2 entries
        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(
                json.dumps(
                    {
                        "problem": "Q1",
                        "correct_answer": "A1",
                        "response": "R1",
                    }
                )
                + "\n"
            )
            f.write(
                json.dumps(
                    {
                        "problem": "Q2",
                        "correct_answer": "A2",
                        "response": "R2",
                    }
                )
                + "\n"
            )
            results_file = f.name

        output_file = results_file + ".graded"

        # LLM that fails on first call, succeeds on second
        llm = MagicMock()
        call_count = 0

        def invoke_side_effect(prompt):
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                raise RuntimeError("LLM failed")
            result = MagicMock()
            result.content = (
                "Extracted Answer: A2\nReasoning: Correct\nCorrect: yes"
            )
            return result

        llm.invoke = invoke_side_effect

        try:
            results = _grade_results_inner(
                llm, results_file, output_file, "simpleqa", None
            )

            assert len(results) == 2
            assert "grading_error" in results[0]
            assert results[1].get("is_correct") is True
        finally:
            Path(results_file).unlink(missing_ok=True)
            Path(output_file).unlink(missing_ok=True)


# ---------------------------------------------------------------------------
# grade_single_result chat_messages path
# ---------------------------------------------------------------------------


class TestChatMessagesPath:
    def test_chat_messages_format(self):
        """LLM with chat_messages attribute → uses HumanMessage format."""
        from local_deep_research.benchmarks.graders import grade_single_result

        response_text = (
            "Extracted Answer: Paris\nReasoning: Capital\nCorrect: yes"
        )
        llm = _make_llm_with_chat_messages(response_text)

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "Capital of France?",
                    "correct_answer": "Paris",
                    "response": "Paris",
                },
                dataset_type="simpleqa",
            )

        assert result["is_correct"] is True
        # Verify HumanMessage was used (invoke called with list)
        call_args = llm.invoke.call_args[0][0]
        assert isinstance(call_args, list)


# ---------------------------------------------------------------------------
# extract_answer_from_response
# ---------------------------------------------------------------------------


class TestExtractAnswerFromResponse:
    def test_simpleqa_strips_citations(self):
        """SimpleQA: citations [1] [2] removed, whole response returned."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "Paris is the capital [1] of France [2].", "simpleqa"
        )
        assert result["extracted_answer"] == "Paris is the capital  of France ."
        assert result["confidence"] == "100"

    def test_browsecomp_extracts_answer_and_confidence(self):
        """BrowseComp: extracts Exact Answer and Confidence."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "Exact Answer: The Moon\nConfidence: 95%", "browsecomp"
        )
        assert result["extracted_answer"] == "The Moon"
        assert result["confidence"] == "95"

    def test_browsecomp_no_match(self):
        """BrowseComp: no matching fields → defaults."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response("Random text here", "browsecomp")
        assert result["extracted_answer"] == "None"
        assert result["confidence"] == "100"

    def test_browsecomp_no_confidence(self):
        """BrowseComp: answer found but no confidence → default 100."""
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        result = extract_answer_from_response(
            "Exact Answer: Jupiter", "browsecomp"
        )
        assert result["extracted_answer"] == "Jupiter"
        assert result["confidence"] == "100"


# ---------------------------------------------------------------------------
# grade_single_result — exception path
# ---------------------------------------------------------------------------


class TestGradeSingleResultException:
    def test_llm_exception_returns_error(self):
        """LLM raises exception → returns grading_error dict."""
        from local_deep_research.benchmarks.graders import grade_single_result

        llm = MagicMock()
        llm.invoke.side_effect = RuntimeError("API down")

        with patch(f"{MODULE}.get_evaluation_llm", return_value=llm):
            result = grade_single_result(
                {
                    "problem": "Q?",
                    "correct_answer": "A",
                    "response": "R",
                },
                dataset_type="simpleqa",
            )

        assert "grading_error" in result
        assert result["is_correct"] is False
        assert result["graded_confidence"] == "0"


# ---------------------------------------------------------------------------
# _grade_results_inner — browsecomp + progress callback + existing output
# ---------------------------------------------------------------------------


class TestGradeResultsInnerBrowsecomp:
    def test_browsecomp_batch(self):
        """Batch grading with browsecomp dataset type."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(
                json.dumps(
                    {
                        "problem": "Q1",
                        "correct_answer": "A1",
                        "response": "R1",
                    }
                )
                + "\n"
            )
            results_file = f.name

        output_file = results_file + ".graded"

        llm = MagicMock()
        result_obj = MagicMock()
        result_obj.content = (
            "extracted_final_answer: A1\n"
            "reasoning: Looks correct\n\n"
            "correct: yes\nconfidence: 90"
        )
        llm.invoke.return_value = result_obj

        try:
            results = _grade_results_inner(
                llm, results_file, output_file, "browsecomp", None
            )

            assert len(results) == 1
            assert results[0]["is_correct"] is True
            assert results[0]["graded_confidence"] == "90"
        finally:
            Path(results_file).unlink(missing_ok=True)
            Path(output_file).unlink(missing_ok=True)

    def test_with_progress_callback(self):
        """Progress callback called for each result."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(
                json.dumps(
                    {
                        "problem": "Q1",
                        "correct_answer": "A1",
                        "response": "R1",
                    }
                )
                + "\n"
            )
            results_file = f.name

        output_file = results_file + ".graded"

        llm = MagicMock()
        result_obj = MagicMock()
        result_obj.content = (
            "Extracted Answer: A1\nReasoning: Good\nCorrect: yes"
        )
        llm.invoke.return_value = result_obj

        callback_calls = []

        def progress_cb(idx, total, meta):
            callback_calls.append((idx, total, meta["status"]))

        try:
            _grade_results_inner(
                llm, results_file, output_file, "simpleqa", progress_cb
            )

            # Should be called twice per item: "grading" and "graded"
            assert len(callback_calls) == 2
            assert callback_calls[0][2] == "grading"
            assert callback_calls[1][2] == "graded"
        finally:
            Path(results_file).unlink(missing_ok=True)
            Path(output_file).unlink(missing_ok=True)

    def test_existing_output_file_removed(self):
        """Existing output file gets removed before grading starts."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".jsonl", delete=False
        ) as f:
            f.write(
                json.dumps(
                    {
                        "problem": "Q",
                        "correct_answer": "A",
                        "response": "R",
                    }
                )
                + "\n"
            )
            results_file = f.name

        output_file = results_file + ".graded"
        # Create pre-existing output file
        Path(output_file).write_text("old data\n")

        llm = MagicMock()
        result_obj = MagicMock()
        result_obj.content = (
            "Extracted Answer: A\nReasoning: Correct\nCorrect: yes"
        )
        llm.invoke.return_value = result_obj

        try:
            _grade_results_inner(
                llm, results_file, output_file, "simpleqa", None
            )
            # Output file should only contain the new result, not "old data"
            content = Path(output_file).read_text()
            assert "old data" not in content
        finally:
            Path(results_file).unlink(missing_ok=True)
            Path(output_file).unlink(missing_ok=True)


# ---------------------------------------------------------------------------
# get_evaluation_llm
# ---------------------------------------------------------------------------


class TestGetEvaluationLLM:
    def test_custom_config_override(self):
        """Custom config merges with defaults."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        with patch(f"{MODULE}.get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            get_evaluation_llm(custom_config={"model_name": "custom-model"})

        call_kwargs = mock_get_llm.call_args.kwargs
        assert call_kwargs["model_name"] == "custom-model"

    def test_api_key_from_settings_snapshot(self):
        """API key extracted from settings snapshot dict format."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {"llm.openai_endpoint.api_key": {"value": "sk-test-key-123"}}

        with patch(f"{MODULE}.get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            get_evaluation_llm(settings_snapshot=snapshot)

        # Should not raise or warn about missing API key
        mock_get_llm.assert_called_once()

    def test_api_key_from_settings_snapshot_plain_string(self):
        """API key as plain string (not dict) from snapshot."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {"llm.openai_endpoint.api_key": "sk-plain-key"}

        with patch(f"{MODULE}.get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            get_evaluation_llm(settings_snapshot=snapshot)

        mock_get_llm.assert_called_once()

    def test_no_snapshot_no_api_key(self):
        """No snapshot, no API key → warns but proceeds."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        with patch(f"{MODULE}.get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            get_evaluation_llm()

        mock_get_llm.assert_called_once()