learningcircuit--local-deep-research/tests/benchmarks/test_graders_high_value.py

"""
High-value pure logic tests for graders.py.

Tests extract_answer_from_response(), DEFAULT_EVALUATION_CONFIG,
regex extraction patterns, and grade_single_result response parsing.
"""

import re
from unittest.mock import MagicMock, patch


from local_deep_research.benchmarks.graders import (
    DEFAULT_EVALUATION_CONFIG,
    extract_answer_from_response,
    get_evaluation_llm,
    grade_single_result,
)


class TestDefaultEvaluationConfig:
    """Tests for DEFAULT_EVALUATION_CONFIG dict values."""

    def test_model_name_is_set(self):
        assert "model_name" in DEFAULT_EVALUATION_CONFIG
        assert (
            DEFAULT_EVALUATION_CONFIG["model_name"]
            == "anthropic/claude-3.7-sonnet"
        )

    def test_provider_is_openai_endpoint(self):
        assert DEFAULT_EVALUATION_CONFIG["provider"] == "openai_endpoint"

    def test_openai_endpoint_url(self):
        assert (
            DEFAULT_EVALUATION_CONFIG["openai_endpoint_url"]
            == "https://openrouter.ai/api/v1"
        )

    def test_temperature_is_zero(self):
        assert DEFAULT_EVALUATION_CONFIG["temperature"] == 0

    def test_no_max_tokens_key(self):
        """max_tokens was intentionally removed per code comments."""
        assert "max_tokens" not in DEFAULT_EVALUATION_CONFIG

    def test_config_has_exactly_four_keys(self):
        assert len(DEFAULT_EVALUATION_CONFIG) == 4


class TestExtractAnswerSimpleQA:
    """Tests for extract_answer_from_response with simpleqa dataset type."""

    def test_basic_response_returned_as_is(self):
        result = extract_answer_from_response("Paris is the capital of France.")
        assert result["extracted_answer"] == "Paris is the capital of France."
        assert result["confidence"] == "100"

    def test_empty_response(self):
        result = extract_answer_from_response("")
        assert result["extracted_answer"] == ""
        assert result["confidence"] == "100"

    def test_citations_are_stripped(self):
        result = extract_answer_from_response(
            "The answer is 42[1] according to sources[23]."
        )
        assert "[1]" not in result["extracted_answer"]
        assert "[23]" not in result["extracted_answer"]
        assert "42" in result["extracted_answer"]

    def test_multiple_citations_stripped(self):
        result = extract_answer_from_response("[1][2][3] Some text [45]")
        assert result["extracted_answer"] == " Some text "

    def test_default_dataset_type_is_simpleqa(self):
        result = extract_answer_from_response("Hello world")
        assert result["extracted_answer"] == "Hello world"

    def test_simpleqa_explicit_type(self):
        result = extract_answer_from_response(
            "Hello world", dataset_type="simpleqa"
        )
        assert result["extracted_answer"] == "Hello world"

    def test_simpleqa_case_insensitive_type(self):
        result = extract_answer_from_response("Hello", dataset_type="SimpleQA")
        assert result["extracted_answer"] == "Hello"

    def test_multiline_response(self):
        text = "Line one\nLine two\nLine three"
        result = extract_answer_from_response(text, dataset_type="simpleqa")
        assert "Line one" in result["extracted_answer"]
        assert "Line three" in result["extracted_answer"]


class TestExtractAnswerBrowseComp:
    """Tests for extract_answer_from_response with browsecomp dataset type."""

    def test_exact_answer_extracted(self):
        response = "Exact Answer: John Smith\nConfidence: 85%"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "John Smith"
        assert result["confidence"] == "85"

    def test_exact_answer_at_end_of_string(self):
        response = "Exact Answer: The Eiffel Tower"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "The Eiffel Tower"

    def test_no_exact_answer_returns_none_string(self):
        response = "I don't know the answer."
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "None"

    def test_no_confidence_defaults_to_100(self):
        response = "Exact Answer: Something"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["confidence"] == "100"

    def test_confidence_extraction(self):
        response = "Exact Answer: X\nConfidence: 42%"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["confidence"] == "42"

    def test_browsecomp_case_insensitive_type(self):
        response = "Exact Answer: Yes\nConfidence: 90%"
        result = extract_answer_from_response(
            response, dataset_type="BrowseComp"
        )
        assert result["extracted_answer"] == "Yes"

    def test_citations_stripped_before_extraction(self):
        response = "Exact Answer: Paris[1]\nConfidence: 75%"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "Paris"

    def test_empty_response_browsecomp(self):
        result = extract_answer_from_response("", dataset_type="browsecomp")
        assert result["extracted_answer"] == "None"
        assert result["confidence"] == "100"

    def test_exact_answer_with_extra_whitespace(self):
        response = "Exact Answer:   spaced out   \nConfidence: 50%"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "spaced out"

    def test_exact_answer_followed_by_more_text(self):
        response = "Exact Answer: Alpha\nSome other line\nConfidence: 60%"
        result = extract_answer_from_response(
            response, dataset_type="browsecomp"
        )
        assert result["extracted_answer"] == "Alpha"
        assert result["confidence"] == "60"


class TestGradeSingleResultBrowseComp:
    """Tests for grade_single_result browsecomp regex parsing of grader LLM response."""

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_browsecomp_correct_yes(self, mock_get_llm):
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = MagicMock(
            content="extracted_final_answer: Paris\nreasoning: Matches reference\ncorrect: yes\nconfidence: 95"
        )
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {
                "problem": "Capital of France?",
                "correct_answer": "Paris",
                "response": "Paris",
            },
            dataset_type="browsecomp",
        )
        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "Paris"
        assert result["graded_confidence"] == "95"

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_browsecomp_correct_no(self, mock_get_llm):
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = MagicMock(
            content="extracted_final_answer: London\nreasoning: Wrong city\ncorrect: no\nconfidence: 10"
        )
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "Paris", "response": "London"},
            dataset_type="browsecomp",
        )
        assert result["is_correct"] is False

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_browsecomp_no_correct_field_defaults_false(self, mock_get_llm):
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = MagicMock(
            content="extracted_final_answer: Something\nreasoning: Unclear"
        )
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "A", "response": "R"},
            dataset_type="browsecomp",
        )
        assert result["is_correct"] is False
        assert result["graded_confidence"] == "100"


class TestGradeSingleResultSimpleQA:
    """Tests for grade_single_result simpleqa regex parsing of grader LLM response."""

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_simpleqa_correct_yes(self, mock_get_llm):
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = MagicMock(
            content="Extracted Answer: 42\nReasoning: Matches exactly\nCorrect: Yes"
        )
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "42", "response": "42"},
            dataset_type="simpleqa",
        )
        assert result["is_correct"] is True
        assert result["extracted_by_grader"] == "42"
        assert result["graded_confidence"] == "100"

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_simpleqa_correct_no(self, mock_get_llm):
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = MagicMock(
            content="Extracted Answer: 43\nReasoning: Off by one\nCorrect: No"
        )
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "42", "response": "43"},
            dataset_type="simpleqa",
        )
        assert result["is_correct"] is False

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_simpleqa_no_extracted_answer_defaults_none(self, mock_get_llm):
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = MagicMock(
            content="Reasoning: Cannot determine\nCorrect: no"
        )
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "A", "response": "R"},
            dataset_type="simpleqa",
        )
        assert result["extracted_by_grader"] == "None"

    @patch("local_deep_research.benchmarks.graders.get_evaluation_llm")
    def test_grade_single_result_invoke_exception_returns_error(
        self, mock_get_llm
    ):
        """When the LLM invoke raises, the try/except inside grade_single_result catches it."""
        mock_llm = MagicMock()
        mock_llm.invoke.side_effect = RuntimeError("LLM call failed")
        mock_llm.chat_messages = True
        mock_get_llm.return_value = mock_llm

        result = grade_single_result(
            {"problem": "Q", "correct_answer": "A", "response": "R"},
        )
        assert result["is_correct"] is False
        assert "grading_error" in result
        assert "LLM call failed" in result["grading_error"]
        assert result["graded_confidence"] == "0"


class TestCitationStrippingRegex:
    """Directly test the citation-stripping regex used in extract_answer_from_response."""

    def test_single_digit_citation(self):
        assert re.sub(r"\[\d+\]", "", "text[5]") == "text"

    def test_multi_digit_citation(self):
        assert re.sub(r"\[\d+\]", "", "text[123]more") == "textmore"

    def test_no_citation(self):
        assert (
            re.sub(r"\[\d+\]", "", "no citations here") == "no citations here"
        )

    def test_non_numeric_brackets_preserved(self):
        assert re.sub(r"\[\d+\]", "", "text[abc]") == "text[abc]"


class TestGetEvaluationLlm:
    """Tests for get_evaluation_llm configuration merging logic."""

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_default_config_used_when_no_custom(self, mock_get_llm):
        mock_get_llm.return_value = MagicMock()
        get_evaluation_llm()
        mock_get_llm.assert_called_once()
        call_kwargs = mock_get_llm.call_args[1]
        assert call_kwargs["model_name"] == "anthropic/claude-3.7-sonnet"
        assert call_kwargs["temperature"] == 0

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_custom_config_overrides_defaults(self, mock_get_llm):
        mock_get_llm.return_value = MagicMock()
        get_evaluation_llm(
            custom_config={"model_name": "gpt-4", "temperature": 0.5}
        )
        call_kwargs = mock_get_llm.call_args[1]
        assert call_kwargs["model_name"] == "gpt-4"
        assert call_kwargs["temperature"] == 0.5

    @patch("local_deep_research.benchmarks.graders.get_llm")
    def test_unsupported_params_filtered_out(self, mock_get_llm):
        mock_get_llm.return_value = MagicMock()
        get_evaluation_llm(custom_config={"max_tokens": 1000, "top_p": 0.9})
        call_kwargs = mock_get_llm.call_args[1]
        assert "max_tokens" not in call_kwargs
        assert "top_p" not in call_kwargs