learningcircuit--local-deep-research/tests/benchmarks/test_graders_coverage.py

"""
Coverage tests for benchmarks/graders.py.

Targets the 37 missing lines:
- get_evaluation_llm() with/without custom_config, with/without settings_snapshot
- get_evaluation_llm() openai_endpoint branch with api_key from snapshot
- extract_answer_from_response() for browsecomp and simpleqa
- grade_single_result() simpleqa success path (invoke without chat_messages)
- grade_single_result() browsecomp success path
- grade_single_result() grading error path
- grade_results() basic flow with progress_callback
- human_evaluation() non-interactive mode
"""

import json
from unittest.mock import Mock, patch

MODULE = "local_deep_research.benchmarks.graders"


# ---------------------------------------------------------------------------
# get_evaluation_llm
# ---------------------------------------------------------------------------


class TestGetEvaluationLlm:
    @patch(f"{MODULE}.get_llm")
    def test_default_config_uses_claude_sonnet(self, mock_get_llm):
        mock_get_llm.return_value = Mock()
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        get_evaluation_llm()
        call_kwargs = mock_get_llm.call_args[1]
        assert call_kwargs["model_name"] == "anthropic/claude-3.7-sonnet"
        assert call_kwargs["provider"] == "openai_endpoint"
        assert call_kwargs["temperature"] == 0

    @patch(f"{MODULE}.get_llm")
    def test_custom_config_overrides_default(self, mock_get_llm):
        mock_get_llm.return_value = Mock()
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        get_evaluation_llm(
            custom_config={"model_name": "gpt-4", "provider": "openai"}
        )
        call_kwargs = mock_get_llm.call_args[1]
        assert call_kwargs["model_name"] == "gpt-4"
        assert call_kwargs["provider"] == "openai"

    @patch(f"{MODULE}.get_llm")
    def test_settings_snapshot_dict_api_key_suppresses_warning(
        self, mock_get_llm
    ):
        mock_get_llm.return_value = Mock()
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {"llm.openai_endpoint.api_key": {"value": "sk-test-key"}}
        with patch(f"{MODULE}.logger") as mock_logger:
            get_evaluation_llm(settings_snapshot=snapshot)
        # With a valid api_key from snapshot, no "no API key found" warning
        no_key_warnings = [
            c
            for c in mock_logger.warning.call_args_list
            if "no api key" in str(c).lower()
        ]
        assert len(no_key_warnings) == 0

    @patch(f"{MODULE}.get_llm")
    def test_settings_snapshot_string_api_key_suppresses_warning(
        self, mock_get_llm
    ):
        mock_get_llm.return_value = Mock()
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        snapshot = {"llm.openai_endpoint.api_key": "sk-direct-key"}
        with patch(f"{MODULE}.logger") as mock_logger:
            get_evaluation_llm(settings_snapshot=snapshot)
        no_key_warnings = [
            c
            for c in mock_logger.warning.call_args_list
            if "no api key" in str(c).lower()
        ]
        assert len(no_key_warnings) == 0

    @patch(f"{MODULE}.get_llm")
    def test_no_settings_snapshot_logs_warning(self, mock_get_llm):
        mock_get_llm.return_value = Mock()
        from local_deep_research.benchmarks.graders import get_evaluation_llm

        with patch(f"{MODULE}.logger") as mock_logger:
            get_evaluation_llm()
        # Warning about missing settings snapshot (provider is openai_endpoint by default)
        warning_calls = [str(c) for c in mock_logger.warning.call_args_list]
        assert any(
            "settings snapshot" in w.lower() or "api key" in w.lower()
            for w in warning_calls
        )


# ---------------------------------------------------------------------------
# extract_answer_from_response
# ---------------------------------------------------------------------------


class TestExtractAnswerFromResponse:
    def test_browsecomp_extracts_exact_answer(self):
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        response = "Exact Answer: Paris\nConfidence: 95%"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "Paris"
        assert result["confidence"] == "95"

    def test_browsecomp_no_match_returns_none(self):
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        response = "I don't know"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "None"

    def test_simpleqa_returns_full_response(self):
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        response = "The answer is 42."
        result = extract_answer_from_response(response, dataset_type="simpleqa")
        assert result["extracted_answer"] == "The answer is 42."
        assert result["confidence"] == "100"

    def test_citations_removed_from_response(self):
        from local_deep_research.benchmarks.graders import (
            extract_answer_from_response,
        )

        response = "Paris[1] is the capital[2]."
        result = extract_answer_from_response(response, dataset_type="simpleqa")
        assert "[1]" not in result["extracted_answer"]
        assert "[2]" not in result["extracted_answer"]


# ---------------------------------------------------------------------------
# grade_single_result – simpleqa success
# ---------------------------------------------------------------------------


class TestGradeSingleResultSimpleqa:
    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_simpleqa_correct_answer(self, mock_safe_close, mock_get_llm):
        mock_llm = Mock(spec=[])  # no chat_messages attr, has invoke
        mock_llm.invoke = Mock(
            return_value="Extracted Answer: Paris\nReasoning: Correct.\nCorrect: yes"
        )
        mock_get_llm.return_value = mock_llm
        from local_deep_research.benchmarks.graders import grade_single_result

        result = grade_single_result(
            result_data={
                "problem": "Capital of France?",
                "correct_answer": "Paris",
                "response": "Paris is the capital of France.",
            },
            dataset_type="simpleqa",
        )
        assert result["is_correct"] is True

    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_simpleqa_incorrect_answer(self, mock_safe_close, mock_get_llm):
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(
            return_value="Extracted Answer: London\nReasoning: Wrong city.\nCorrect: no"
        )
        mock_get_llm.return_value = mock_llm
        from local_deep_research.benchmarks.graders import grade_single_result

        result = grade_single_result(
            result_data={
                "problem": "Capital of France?",
                "correct_answer": "Paris",
                "response": "London is the capital.",
            },
            dataset_type="simpleqa",
        )
        assert result["is_correct"] is False

    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_simpleqa_llm_with_content_attribute(
        self, mock_safe_close, mock_get_llm
    ):
        response_obj = Mock()
        response_obj.content = (
            "Extracted Answer: Tokyo\nReasoning: Correct.\nCorrect: yes"
        )
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(return_value=response_obj)
        mock_get_llm.return_value = mock_llm
        from local_deep_research.benchmarks.graders import grade_single_result

        result = grade_single_result(
            result_data={
                "problem": "Capital of Japan?",
                "correct_answer": "Tokyo",
                "response": "Tokyo.",
            },
            dataset_type="simpleqa",
        )
        assert result["is_correct"] is True


# ---------------------------------------------------------------------------
# grade_single_result – browsecomp success
# ---------------------------------------------------------------------------


class TestGradeSingleResultBrowsecomp:
    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_browsecomp_correct_answer(self, mock_safe_close, mock_get_llm):
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(
            return_value=(
                "extracted_final_answer: Eiffel Tower\n"
                "reasoning: The answer matches.\n"
                "correct: yes\n"
                "confidence: 90"
            )
        )
        mock_get_llm.return_value = mock_llm
        from local_deep_research.benchmarks.graders import grade_single_result

        result = grade_single_result(
            result_data={
                "problem": "Famous Paris landmark?",
                "correct_answer": "Eiffel Tower",
                "response": "The Eiffel Tower is in Paris.",
            },
            dataset_type="browsecomp",
        )
        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "Eiffel Tower"

    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_browsecomp_incorrect_returns_false(
        self, mock_safe_close, mock_get_llm
    ):
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(
            return_value=(
                "extracted_final_answer: None\n"
                "reasoning: No match.\n"
                "correct: no\n"
                "confidence: 50"
            )
        )
        mock_get_llm.return_value = mock_llm
        from local_deep_research.benchmarks.graders import grade_single_result

        result = grade_single_result(
            result_data={
                "problem": "Q?",
                "correct_answer": "A",
                "response": "wrong",
            },
            dataset_type="browsecomp",
        )
        assert result["is_correct"] is False


# ---------------------------------------------------------------------------
# grade_single_result – error path
# ---------------------------------------------------------------------------


class TestGradeSingleResultError:
    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_llm_error_returns_error_dict(self, mock_safe_close, mock_get_llm):
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(side_effect=RuntimeError("LLM crashed"))
        mock_get_llm.return_value = mock_llm
        from local_deep_research.benchmarks.graders import grade_single_result

        result = grade_single_result(
            result_data={
                "problem": "Q?",
                "correct_answer": "A",
                "response": "R",
            }
        )
        assert result["is_correct"] is False
        assert "grading_error" in result
        assert "LLM crashed" in result["grading_error"]


# ---------------------------------------------------------------------------
# grade_results – basic flow
# ---------------------------------------------------------------------------


class TestGradeResults:
    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_results_processes_all_lines(
        self, mock_safe_close, mock_get_llm, tmp_path
    ):
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(
            return_value="Extracted Answer: A\nReasoning: ok.\nCorrect: yes"
        )
        mock_get_llm.return_value = mock_llm

        results_file = tmp_path / "results.jsonl"
        output_file = tmp_path / "graded.jsonl"
        data = [
            {"problem": "Q1", "correct_answer": "A1", "response": "A1"},
            {"problem": "Q2", "correct_answer": "A2", "response": "A2"},
        ]
        results_file.write_text("\n".join(json.dumps(d) for d in data))

        from local_deep_research.benchmarks.graders import grade_results

        graded = grade_results(
            results_file=str(results_file),
            output_file=str(output_file),
            dataset_type="simpleqa",
        )
        assert len(graded) == 2

    @patch(f"{MODULE}.get_evaluation_llm")
    @patch("local_deep_research.utilities.resource_utils.safe_close")
    def test_grade_results_with_progress_callback(
        self, mock_safe_close, mock_get_llm, tmp_path
    ):
        mock_llm = Mock(spec=[])
        mock_llm.invoke = Mock(
            return_value="Extracted Answer: A\nReasoning: ok.\nCorrect: no"
        )
        mock_get_llm.return_value = mock_llm

        results_file = tmp_path / "results.jsonl"
        output_file = tmp_path / "graded.jsonl"
        data = [{"problem": "Q1", "correct_answer": "A1", "response": "wrong"}]
        results_file.write_text(json.dumps(data[0]))

        callback = Mock()
        from local_deep_research.benchmarks.graders import grade_results

        grade_results(
            results_file=str(results_file),
            output_file=str(output_file),
            dataset_type="simpleqa",
            progress_callback=callback,
        )
        assert callback.call_count >= 1


# ---------------------------------------------------------------------------
# human_evaluation – non-interactive mode
# ---------------------------------------------------------------------------


class TestHumanEvaluation:
    def test_non_interactive_returns_false_for_each(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        output_file = tmp_path / "human_graded.jsonl"
        data = [
            {"problem": "Q1", "correct_answer": "A1", "response": "R1"},
            {"problem": "Q2", "correct_answer": "A2", "response": "R2"},
        ]
        results_file.write_text("\n".join(json.dumps(d) for d in data))

        from local_deep_research.benchmarks.graders import human_evaluation

        graded = human_evaluation(
            results_file=str(results_file),
            output_file=str(output_file),
            interactive=False,
        )
        assert len(graded) == 2
        # Non-interactive mode always marks as incorrect
        for item in graded:
            assert item["is_correct"] is False
            assert item["human_evaluation"] is True