learningcircuit--local-deep-research/tests/benchmarks/test_templates.py

"""
Tests for benchmark evaluation templates.

This module tests the prompt templates used for evaluating model outputs
against reference answers.
"""

from local_deep_research.benchmarks.templates import (
    BROWSECOMP_GRADER_TEMPLATE,
    BROWSECOMP_QUERY_TEMPLATE,
    SIMPLEQA_GRADER_TEMPLATE,
)


class TestSimpleQAGraderTemplate:
    """Tests for the SimpleQA grader template."""

    def test_contains_question_placeholder(self):
        """Template must contain the question placeholder."""
        assert "{question}" in SIMPLEQA_GRADER_TEMPLATE

    def test_contains_correct_answer_placeholder(self):
        """Template must contain the correct_answer placeholder."""
        assert "{correct_answer}" in SIMPLEQA_GRADER_TEMPLATE

    def test_contains_response_placeholder(self):
        """Template must contain the response placeholder."""
        assert "{response}" in SIMPLEQA_GRADER_TEMPLATE

    def test_can_format_with_all_placeholders(self):
        """Template can be formatted with all required placeholders."""
        formatted = SIMPLEQA_GRADER_TEMPLATE.format(
            question="What is the capital of France?",
            correct_answer="Paris",
            response="The capital of France is Paris.",
        )

        assert "What is the capital of France?" in formatted
        assert "Paris" in formatted
        assert "The capital of France is Paris." in formatted
        # Ensure no unformatted placeholders remain
        assert "{question}" not in formatted
        assert "{correct_answer}" not in formatted
        assert "{response}" not in formatted

    def test_contains_grading_instructions(self):
        """Template should contain instructions for grading."""
        assert "Correct:" in SIMPLEQA_GRADER_TEMPLATE
        assert "yes/no" in SIMPLEQA_GRADER_TEMPLATE


class TestBrowseCompGraderTemplate:
    """Tests for the BrowseComp grader template."""

    def test_contains_question_placeholder(self):
        """Template must contain the question placeholder."""
        assert "{question}" in BROWSECOMP_GRADER_TEMPLATE

    def test_contains_correct_answer_placeholder(self):
        """Template must contain the correct_answer placeholder."""
        assert "{correct_answer}" in BROWSECOMP_GRADER_TEMPLATE

    def test_contains_response_placeholder(self):
        """Template must contain the response placeholder."""
        assert "{response}" in BROWSECOMP_GRADER_TEMPLATE

    def test_can_format_with_all_placeholders(self):
        """Template can be formatted with all required placeholders."""
        formatted = BROWSECOMP_GRADER_TEMPLATE.format(
            question="Who wrote Romeo and Juliet?",
            correct_answer="William Shakespeare",
            response="William Shakespeare wrote Romeo and Juliet.",
        )

        assert "Who wrote Romeo and Juliet?" in formatted
        assert "William Shakespeare" in formatted
        # Ensure no unformatted placeholders remain
        assert "{question}" not in formatted
        assert "{correct_answer}" not in formatted
        assert "{response}" not in formatted

    def test_contains_grading_fields(self):
        """Template should contain expected grading fields."""
        assert "extracted_final_answer:" in BROWSECOMP_GRADER_TEMPLATE
        assert "reasoning:" in BROWSECOMP_GRADER_TEMPLATE
        assert "correct:" in BROWSECOMP_GRADER_TEMPLATE
        assert "confidence:" in BROWSECOMP_GRADER_TEMPLATE


class TestBrowseCompQueryTemplate:
    """Tests for the BrowseComp query template."""

    def test_contains_question_placeholder(self):
        """Template must contain the question placeholder."""
        assert "{question}" in BROWSECOMP_QUERY_TEMPLATE

    def test_can_format_with_question(self):
        """Template can be formatted with a question."""
        formatted = BROWSECOMP_QUERY_TEMPLATE.format(
            question="What year was the Eiffel Tower built?"
        )

        assert "What year was the Eiffel Tower built?" in formatted
        assert "{question}" not in formatted

    def test_contains_expected_response_format(self):
        """Template should specify the expected response format."""
        assert "Explanation:" in BROWSECOMP_QUERY_TEMPLATE
        assert "Exact Answer:" in BROWSECOMP_QUERY_TEMPLATE
        assert "Confidence:" in BROWSECOMP_QUERY_TEMPLATE

    def test_uses_escaped_braces_for_format_instructions(self):
        """Template uses escaped braces for format instructions that should remain literal."""
        # The template uses {{ and }} for literal braces in format instructions
        # After formatting, these should become { and }
        formatted = BROWSECOMP_QUERY_TEMPLATE.format(question="Test question")
        assert "{your explanation" in formatted
        assert "{your succinct" in formatted
        assert "{your confidence" in formatted