learningcircuit--local-deep-research/tests/benchmarks/test_graders_full_coverage.py

"""
Full coverage tests for benchmarks/graders.py.

Targets uncovered paths:
- get_evaluation_llm: API key from settings_snapshot, missing API key warning
- grade_single_result: chat_messages branch
- _grade_results_inner: callable fallback, error with progress_callback
- grade_results: cleanup with safe_close
"""

import json
from unittest.mock import Mock, patch

import pytest


class TestGetEvaluationLlm:
    """Cover get_evaluation_llm API key handling branches."""

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_api_key_from_settings_snapshot_dict(self, mock_get_llm):
        """Extracts API key from settings_snapshot when value is a dict."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {
            "llm.openai_endpoint.api_key": {
                "value": "sk-test-key",
                "type": "string",
            }
        }

        get_evaluation_llm(settings_snapshot=snapshot)

        mock_get_llm.assert_called_once()

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_api_key_from_settings_snapshot_string(self, mock_get_llm):
        """Extracts API key from settings_snapshot when value is a string."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {"llm.openai_endpoint.api_key": "sk-direct-key"}

        get_evaluation_llm(settings_snapshot=snapshot)
        mock_get_llm.assert_called_once()

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_no_settings_snapshot_warns(self, mock_get_llm):
        """Warns when no settings_snapshot provided for openai_endpoint."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        get_evaluation_llm()
        mock_get_llm.assert_called_once()

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_missing_api_key_warns(self, mock_get_llm):
        """Warns when settings_snapshot has no API key."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {"llm.openai_endpoint.api_key": None}
        get_evaluation_llm(settings_snapshot=snapshot)
        mock_get_llm.assert_called_once()

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_custom_config_overrides_defaults(self, mock_get_llm):
        """Custom config overrides default evaluation config."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        custom = {"provider": "anthropic", "model_name": "claude-3"}
        get_evaluation_llm(custom_config=custom)

        call_kwargs = mock_get_llm.call_args
        assert call_kwargs[1]["provider"] == "anthropic"
        assert call_kwargs[1]["model_name"] == "claude-3"

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_unsupported_params_filtered(self, mock_get_llm):
        """Parameters not in ldr_supported_params are filtered out."""
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        custom = {"max_tokens": 1000, "model_name": "test"}
        get_evaluation_llm(custom_config=custom)

        call_kwargs = mock_get_llm.call_args[1]
        assert "max_tokens" not in call_kwargs


class TestGradeSingleResultChatMessages:
    """Cover grade_single_result chat_messages branch."""

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_chat_messages_branch(self, mock_close, mock_get_llm):
        """Uses HumanMessage when LLM has chat_messages attribute."""
        from local_deep_research.benchmarks.graders import grade_single_result

        mock_llm = Mock()
        mock_llm.chat_messages = True  # Has chat_messages attribute
        mock_response = Mock()
        mock_response.content = (
            "Extracted Answer: 42\nReasoning: ok\nCorrect: yes"
        )
        mock_llm.invoke.return_value = mock_response
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {
                "problem": "Q",
                "correct_answer": "42",
                "response": "42",
            },
            dataset_type="simpleqa",
        )

        assert result["is_correct"] is True
        # Verify invoke was called with a list (HumanMessage)
        call_args = mock_llm.invoke.call_args[0][0]
        assert isinstance(call_args, list)


class TestGradeResultsInnerCallableFallback:
    """Cover _grade_results_inner callable fallback path."""

    def test_callable_fallback_in_batch(self, tmp_path):
        """Falls back to calling LLM as callable when no invoke method."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps(
                {
                    "problem": "Q",
                    "correct_answer": "A",
                    "response": "R",
                }
            )
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        del mock_llm.invoke  # Remove invoke to trigger callable
        mock_llm.return_value = (
            "Extracted Answer: A\nReasoning: matches\nCorrect: yes"
        )

        result = _grade_results_inner(
            mock_llm, str(input_file), str(output_file), "simpleqa", None
        )

        assert len(result) == 1
        assert result[0]["is_correct"] is True

    def test_chat_messages_branch_in_batch(self, tmp_path):
        """Batch grading uses HumanMessage when LLM has chat_messages."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps({"problem": "Q", "correct_answer": "A", "response": "R"})
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        mock_llm.chat_messages = True
        mock_response = Mock()
        mock_response.content = (
            "Extracted Answer: A\nReasoning: ok\nCorrect: yes"
        )
        mock_llm.invoke.return_value = mock_response

        result = _grade_results_inner(
            mock_llm, str(input_file), str(output_file), "simpleqa", None
        )

        assert len(result) == 1
        assert result[0]["is_correct"] is True


class TestGradeResultsInnerErrorWithCallback:
    """Cover error handling with progress_callback in _grade_results_inner."""

    def test_error_callback_called_on_exception(self, tmp_path):
        """Progress callback receives error status when grading fails."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps({"problem": "Q", "correct_answer": "A", "response": "R"})
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        mock_llm.invoke.side_effect = Exception("LLM crashed")
        del mock_llm.chat_messages

        callback_calls = []

        def callback(idx, total, data):
            callback_calls.append(data)

        result = _grade_results_inner(
            mock_llm,
            str(input_file),
            str(output_file),
            "simpleqa",
            callback,
        )

        assert len(result) == 1
        assert "grading_error" in result[0]

        # Should have grading + error callbacks
        error_calls = [c for c in callback_calls if c["status"] == "error"]
        assert len(error_calls) == 1
        assert "LLM crashed" in error_calls[0]["error"]

        # Output file should contain error result
        assert output_file.exists()


class TestGradeResults:
    """Cover grade_results wrapper with safe_close."""

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_safe_close_called_after_grading(
        self, mock_close, mock_get_llm, tmp_path
    ):
        """safe_close is called on the LLM after grading completes."""
        from local_deep_research.benchmarks.graders import grade_results

        input_file = tmp_path / "results.jsonl"
        input_file.write_text(
            json.dumps({"problem": "Q", "correct_answer": "A", "response": "R"})
        )
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()
        mock_response = Mock()
        mock_response.content = (
            "Extracted Answer: A\nReasoning: ok\nCorrect: yes"
        )
        mock_llm.invoke.return_value = mock_response
        del mock_llm.chat_messages
        mock_get_llm.return_value = mock_llm

        grade_results(str(input_file), str(output_file))

        mock_close.assert_called_once_with(mock_llm, "grader LLM")

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_safe_close_called_even_on_error(
        self, mock_close, mock_get_llm, tmp_path
    ):
        """safe_close is called even when grading raises."""
        from local_deep_research.benchmarks.graders import grade_results

        mock_llm = Mock()
        mock_get_llm.return_value = mock_llm

        # Non-existent input file will cause an error
        with pytest.raises(Exception):
            grade_results(
                str(tmp_path / "nonexistent.jsonl"),
                str(tmp_path / "output.jsonl"),
            )

        mock_close.assert_called_once_with(mock_llm, "grader LLM")


class TestGradeResultsEmptyFile:
    """Cover accuracy calculation with empty results."""

    def test_empty_results_zero_accuracy(self, tmp_path):
        """Empty results file produces zero accuracy without division error."""
        from local_deep_research.benchmarks.graders import _grade_results_inner

        input_file = tmp_path / "results.jsonl"
        input_file.write_text("")
        output_file = tmp_path / "graded.jsonl"

        mock_llm = Mock()

        result = _grade_results_inner(
            mock_llm, str(input_file), str(output_file), "simpleqa", None
        )

        assert result == []