learningcircuit--local-deep-research/tests/benchmarks/evaluators/test_base.py

"""
Tests for BaseBenchmarkEvaluator class.

Tests the abstract base class functionality and interface contract.
"""

import tempfile
from pathlib import Path
from typing import Any, Dict

import pytest

from local_deep_research.benchmarks.evaluators.base import (
    BaseBenchmarkEvaluator,
)


class ConcreteBenchmarkEvaluator(BaseBenchmarkEvaluator):
    """Concrete implementation for testing the abstract base class."""

    def __init__(self, name: str = "test_benchmark"):
        super().__init__(name)
        self.evaluate_called = False

    def evaluate(
        self,
        system_config: Dict[str, Any],
        num_examples: int,
        output_dir: str,
    ) -> Dict[str, Any]:
        """Concrete implementation of evaluate."""
        self.evaluate_called = True
        return {
            "benchmark_type": self.name,
            "quality_score": 0.5,
            "num_examples": num_examples,
        }


class TestBaseBenchmarkEvaluatorInit:
    """Test initialization of BaseBenchmarkEvaluator."""

    def test_init_with_name(self):
        """Test initialization with a benchmark name."""
        evaluator = ConcreteBenchmarkEvaluator("my_benchmark")
        assert evaluator.name == "my_benchmark"

    def test_init_with_default_name(self):
        """Test initialization with default name."""
        evaluator = ConcreteBenchmarkEvaluator()
        assert evaluator.name == "test_benchmark"

    def test_init_with_empty_name(self):
        """Test initialization with empty name."""
        evaluator = ConcreteBenchmarkEvaluator("")
        assert evaluator.name == ""


class TestGetName:
    """Test get_name method."""

    def test_get_name_returns_name(self):
        """Test that get_name returns the benchmark name."""
        evaluator = ConcreteBenchmarkEvaluator("simpleqa")
        assert evaluator.get_name() == "simpleqa"

    def test_get_name_matches_attribute(self):
        """Test that get_name returns same value as name attribute."""
        evaluator = ConcreteBenchmarkEvaluator("browsecomp")
        assert evaluator.get_name() == evaluator.name


class TestCreateSubdirectory:
    """Test _create_subdirectory method."""

    def test_create_subdirectory_creates_directory(self):
        """Test that subdirectory is created."""
        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = ConcreteBenchmarkEvaluator("test_bench")
            result = evaluator._create_subdirectory(tmpdir)

            expected_path = Path(tmpdir) / "test_bench"
            assert Path(result).exists()
            assert Path(result) == expected_path

    def test_create_subdirectory_returns_string_path(self):
        """Test that subdirectory method returns string path."""
        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = ConcreteBenchmarkEvaluator("my_test")
            result = evaluator._create_subdirectory(tmpdir)

            assert isinstance(result, str)

    def test_create_subdirectory_with_nested_parent(self):
        """Test creating subdirectory in nested parent."""
        with tempfile.TemporaryDirectory() as tmpdir:
            nested_dir = str(Path(tmpdir) / "level1" / "level2")
            evaluator = ConcreteBenchmarkEvaluator("nested_bench")

            result = evaluator._create_subdirectory(nested_dir)

            assert Path(result).exists()
            assert Path(result).name == "nested_bench"

    def test_create_subdirectory_idempotent(self):
        """Test that calling _create_subdirectory multiple times is safe."""
        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = ConcreteBenchmarkEvaluator("repeat_bench")

            result1 = evaluator._create_subdirectory(tmpdir)
            result2 = evaluator._create_subdirectory(tmpdir)

            assert result1 == result2
            assert Path(result1).exists()


class TestEvaluateAbstract:
    """Test evaluate method interface."""

    def test_evaluate_is_callable(self):
        """Test that evaluate can be called."""
        evaluator = ConcreteBenchmarkEvaluator("test")
        evaluator.evaluate(
            system_config={"key": "value"},
            num_examples=10,
            output_dir="/tmp",
        )
        assert evaluator.evaluate_called

    def test_evaluate_returns_dict(self):
        """Test that evaluate returns a dictionary."""
        evaluator = ConcreteBenchmarkEvaluator("test")
        result = evaluator.evaluate(
            system_config={},
            num_examples=5,
            output_dir="/tmp",
        )
        assert isinstance(result, dict)

    def test_evaluate_includes_benchmark_type(self):
        """Test that evaluate result includes benchmark_type."""
        evaluator = ConcreteBenchmarkEvaluator("my_bench")
        result = evaluator.evaluate(
            system_config={},
            num_examples=5,
            output_dir="/tmp",
        )
        assert result["benchmark_type"] == "my_bench"

    def test_evaluate_includes_quality_score(self):
        """Test that evaluate result includes quality_score."""
        evaluator = ConcreteBenchmarkEvaluator("test")
        result = evaluator.evaluate(
            system_config={},
            num_examples=5,
            output_dir="/tmp",
        )
        assert "quality_score" in result
        assert 0 <= result["quality_score"] <= 1


class TestAbstractMethodEnforcement:
    """Test that abstract method is properly enforced."""

    def test_cannot_instantiate_base_class(self):
        """Test that BaseBenchmarkEvaluator cannot be instantiated directly."""
        with pytest.raises(TypeError):
            BaseBenchmarkEvaluator("test")