chore: import upstream snapshot with attribution

tests/benchmarks/evaluators/test_base.py

@@ -0,0 +1,168 @@
+"""
+Tests for BaseBenchmarkEvaluator class.
+
+Tests the abstract base class functionality and interface contract.
+"""
+
+import tempfile
+from pathlib import Path
+from typing import Any, Dict
+
+import pytest
+
+from local_deep_research.benchmarks.evaluators.base import (
+    BaseBenchmarkEvaluator,
+)
+
+
+class ConcreteBenchmarkEvaluator(BaseBenchmarkEvaluator):
+    """Concrete implementation for testing the abstract base class."""
+
+    def __init__(self, name: str = "test_benchmark"):
+        super().__init__(name)
+        self.evaluate_called = False
+
+    def evaluate(
+        self,
+        system_config: Dict[str, Any],
+        num_examples: int,
+        output_dir: str,
+    ) -> Dict[str, Any]:
+        """Concrete implementation of evaluate."""
+        self.evaluate_called = True
+        return {
+            "benchmark_type": self.name,
+            "quality_score": 0.5,
+            "num_examples": num_examples,
+        }
+
+
+class TestBaseBenchmarkEvaluatorInit:
+    """Test initialization of BaseBenchmarkEvaluator."""
+
+    def test_init_with_name(self):
+        """Test initialization with a benchmark name."""
+        evaluator = ConcreteBenchmarkEvaluator("my_benchmark")
+        assert evaluator.name == "my_benchmark"
+
+    def test_init_with_default_name(self):
+        """Test initialization with default name."""
+        evaluator = ConcreteBenchmarkEvaluator()
+        assert evaluator.name == "test_benchmark"
+
+    def test_init_with_empty_name(self):
+        """Test initialization with empty name."""
+        evaluator = ConcreteBenchmarkEvaluator("")
+        assert evaluator.name == ""
+
+
+class TestGetName:
+    """Test get_name method."""
+
+    def test_get_name_returns_name(self):
+        """Test that get_name returns the benchmark name."""
+        evaluator = ConcreteBenchmarkEvaluator("simpleqa")
+        assert evaluator.get_name() == "simpleqa"
+
+    def test_get_name_matches_attribute(self):
+        """Test that get_name returns same value as name attribute."""
+        evaluator = ConcreteBenchmarkEvaluator("browsecomp")
+        assert evaluator.get_name() == evaluator.name
+
+
+class TestCreateSubdirectory:
+    """Test _create_subdirectory method."""
+
+    def test_create_subdirectory_creates_directory(self):
+        """Test that subdirectory is created."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            evaluator = ConcreteBenchmarkEvaluator("test_bench")
+            result = evaluator._create_subdirectory(tmpdir)
+
+            expected_path = Path(tmpdir) / "test_bench"
+            assert Path(result).exists()
+            assert Path(result) == expected_path
+
+    def test_create_subdirectory_returns_string_path(self):
+        """Test that subdirectory method returns string path."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            evaluator = ConcreteBenchmarkEvaluator("my_test")
+            result = evaluator._create_subdirectory(tmpdir)
+
+            assert isinstance(result, str)
+
+    def test_create_subdirectory_with_nested_parent(self):
+        """Test creating subdirectory in nested parent."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            nested_dir = str(Path(tmpdir) / "level1" / "level2")
+            evaluator = ConcreteBenchmarkEvaluator("nested_bench")
+
+            result = evaluator._create_subdirectory(nested_dir)
+
+            assert Path(result).exists()
+            assert Path(result).name == "nested_bench"
+
+    def test_create_subdirectory_idempotent(self):
+        """Test that calling _create_subdirectory multiple times is safe."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            evaluator = ConcreteBenchmarkEvaluator("repeat_bench")
+
+            result1 = evaluator._create_subdirectory(tmpdir)
+            result2 = evaluator._create_subdirectory(tmpdir)
+
+            assert result1 == result2
+            assert Path(result1).exists()
+
+
+class TestEvaluateAbstract:
+    """Test evaluate method interface."""
+
+    def test_evaluate_is_callable(self):
+        """Test that evaluate can be called."""
+        evaluator = ConcreteBenchmarkEvaluator("test")
+        evaluator.evaluate(
+            system_config={"key": "value"},
+            num_examples=10,
+            output_dir="/tmp",
+        )
+        assert evaluator.evaluate_called
+
+    def test_evaluate_returns_dict(self):
+        """Test that evaluate returns a dictionary."""
+        evaluator = ConcreteBenchmarkEvaluator("test")
+        result = evaluator.evaluate(
+            system_config={},
+            num_examples=5,
+            output_dir="/tmp",
+        )
+        assert isinstance(result, dict)
+
+    def test_evaluate_includes_benchmark_type(self):
+        """Test that evaluate result includes benchmark_type."""
+        evaluator = ConcreteBenchmarkEvaluator("my_bench")
+        result = evaluator.evaluate(
+            system_config={},
+            num_examples=5,
+            output_dir="/tmp",
+        )
+        assert result["benchmark_type"] == "my_bench"
+
+    def test_evaluate_includes_quality_score(self):
+        """Test that evaluate result includes quality_score."""
+        evaluator = ConcreteBenchmarkEvaluator("test")
+        result = evaluator.evaluate(
+            system_config={},
+            num_examples=5,
+            output_dir="/tmp",
+        )
+        assert "quality_score" in result
+        assert 0 <= result["quality_score"] <= 1
+
+
+class TestAbstractMethodEnforcement:
+    """Test that abstract method is properly enforced."""
+
+    def test_cannot_instantiate_base_class(self):
+        """Test that BaseBenchmarkEvaluator cannot be instantiated directly."""
+        with pytest.raises(TypeError):
+            BaseBenchmarkEvaluator("test")