learningcircuit--local-deep-research/tests/benchmarks/test_datasets_base_high_value.py

"""High-value tests for benchmarks/datasets/base.py.

Covers BenchmarkDataset ABC enforcement, DatasetRegistry CRUD,
load() with different file formats, sampling, and get_example bounds.
"""

import json
import pytest

from local_deep_research.benchmarks.datasets.base import (
    BenchmarkDataset,
    DatasetRegistry,
)


class ConcreteBenchmarkDataset(BenchmarkDataset):
    """Minimal concrete implementation for testing."""

    @classmethod
    def get_dataset_info(cls):
        return {
            "id": "test-dataset",
            "name": "Test",
            "description": "For testing",
        }

    @classmethod
    def get_default_dataset_path(cls):
        return "/tmp/test_dataset.json"

    def process_example(self, example):
        return example


class TestBenchmarkDatasetABC:
    """Test abstract base class enforcement."""

    def test_cannot_instantiate_directly(self):
        """BenchmarkDataset cannot be instantiated."""
        with pytest.raises(TypeError):
            BenchmarkDataset()

    def test_incomplete_subclass_raises(self):
        """Subclass missing abstract methods raises TypeError."""

        class Incomplete(BenchmarkDataset):
            pass

        with pytest.raises(TypeError):
            Incomplete()

    def test_complete_subclass_instantiable(self):
        """Complete subclass can be instantiated."""
        ds = ConcreteBenchmarkDataset()
        assert isinstance(ds, BenchmarkDataset)


class TestBenchmarkDatasetInit:
    """Test BenchmarkDataset initialization."""

    def test_default_path_from_class_method(self):
        """Uses get_default_dataset_path when no path provided."""
        ds = ConcreteBenchmarkDataset()
        assert ds.dataset_path == "/tmp/test_dataset.json"

    def test_custom_path_overrides_default(self):
        """Custom path overrides the default."""
        ds = ConcreteBenchmarkDataset(dataset_path="/custom/path.json")
        assert ds.dataset_path == "/custom/path.json"

    def test_not_loaded_initially(self):
        """Dataset is not loaded on init."""
        ds = ConcreteBenchmarkDataset()
        assert ds._is_loaded is False
        assert ds.examples == []

    def test_default_seed(self):
        """Default seed is 42."""
        ds = ConcreteBenchmarkDataset()
        assert ds.seed == 42

    def test_custom_num_examples(self):
        """num_examples is stored correctly."""
        ds = ConcreteBenchmarkDataset(num_examples=10)
        assert ds.num_examples == 10


class TestBenchmarkDatasetLoad:
    """Test load() method."""

    def test_load_json_file(self, tmp_path):
        """Loads examples from a JSON file."""
        data = [
            {"problem": "Q1", "answer": "A1"},
            {"problem": "Q2", "answer": "A2"},
        ]
        f = tmp_path / "data.json"
        f.write_text(json.dumps(data))

        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        examples = ds.load()
        assert len(examples) == 2
        assert ds._is_loaded is True

    def test_load_jsonl_file(self, tmp_path):
        """Loads examples from a JSONL file."""
        lines = ['{"problem": "Q1", "answer": "A1"}\n', '{"problem": "Q2"}\n']
        f = tmp_path / "data.jsonl"
        f.write_text("".join(lines))

        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        examples = ds.load()
        assert len(examples) == 2

    def test_load_csv_file(self, tmp_path):
        """Loads examples from a CSV file."""
        f = tmp_path / "data.csv"
        f.write_text("problem,answer\nQ1,A1\nQ2,A2\n")

        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        examples = ds.load()
        assert len(examples) == 2

    def test_unsupported_format_raises(self, tmp_path):
        """Unsupported file format raises ValueError."""
        f = tmp_path / "data.xml"
        f.write_text("<data/>")

        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        with pytest.raises(ValueError, match="Unsupported file format"):
            ds.load()

    def test_load_returns_cached_on_second_call(self, tmp_path):
        """Second call to load() returns cached results."""
        f = tmp_path / "data.json"
        f.write_text(json.dumps([{"q": "1"}]))

        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        first = ds.load()
        second = ds.load()
        assert first is second

    def test_sampling_with_num_examples(self, tmp_path):
        """load() samples when num_examples < total examples."""
        data = [{"q": str(i)} for i in range(100)]
        f = tmp_path / "data.json"
        f.write_text(json.dumps(data))

        ds = ConcreteBenchmarkDataset(dataset_path=str(f), num_examples=10)
        examples = ds.load()
        assert len(examples) == 10

    def test_sampling_deterministic_with_seed(self, tmp_path):
        """Same seed produces same sample."""
        data = [{"q": str(i)} for i in range(100)]
        f = tmp_path / "data.json"
        f.write_text(json.dumps(data))

        ds1 = ConcreteBenchmarkDataset(
            dataset_path=str(f), num_examples=5, seed=42
        )
        ds2 = ConcreteBenchmarkDataset(
            dataset_path=str(f), num_examples=5, seed=42
        )
        assert ds1.load() == ds2.load()

    def test_process_example_error_counted(self, tmp_path):
        """Errors during process_example are counted, not raised."""
        data = [{"q": "1"}, {"q": "2"}, {"q": "3"}]
        f = tmp_path / "data.json"
        f.write_text(json.dumps(data))

        class FailingDataset(ConcreteBenchmarkDataset):
            def process_example(self, example):
                if example["q"] == "2":
                    raise RuntimeError("bad example")
                return example

        ds = FailingDataset(dataset_path=str(f))
        examples = ds.load()
        assert len(examples) == 2  # 3 total - 1 failed


class TestBenchmarkDatasetGetters:
    """Test get_example, get_question, get_answer."""

    def test_get_example_valid_index(self, tmp_path):
        """get_example returns the correct example."""
        f = tmp_path / "data.json"
        f.write_text(json.dumps([{"q": "A"}, {"q": "B"}]))
        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        assert ds.get_example(1)["q"] == "B"

    def test_get_example_out_of_range(self, tmp_path):
        """get_example raises IndexError for invalid index."""
        f = tmp_path / "data.json"
        f.write_text(json.dumps([{"q": "A"}]))
        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        with pytest.raises(IndexError):
            ds.get_example(5)

    def test_get_example_negative_index_raises(self, tmp_path):
        """Negative index raises IndexError."""
        f = tmp_path / "data.json"
        f.write_text(json.dumps([{"q": "A"}]))
        ds = ConcreteBenchmarkDataset(dataset_path=str(f))
        with pytest.raises(IndexError):
            ds.get_example(-1)

    def test_get_question_returns_problem_field(self):
        """get_question extracts 'problem' field."""
        ds = ConcreteBenchmarkDataset()
        assert ds.get_question({"problem": "What is X?"}) == "What is X?"

    def test_get_question_missing_field_returns_empty(self):
        """Missing 'problem' field returns empty string."""
        ds = ConcreteBenchmarkDataset()
        assert ds.get_question({}) == ""

    def test_get_answer_prefers_correct_answer(self):
        """get_answer prefers 'correct_answer' over 'answer'."""
        ds = ConcreteBenchmarkDataset()
        example = {"correct_answer": "CA", "answer": "A"}
        assert ds.get_answer(example) == "CA"

    def test_get_answer_falls_back_to_answer(self):
        """get_answer falls back to 'answer' field."""
        ds = ConcreteBenchmarkDataset()
        assert ds.get_answer({"answer": "A"}) == "A"


class TestDatasetRegistry:
    """Test DatasetRegistry registration and retrieval."""

    def setup_method(self):
        """Clear registry before each test."""
        DatasetRegistry._registry = {}

    def test_register_stores_class(self):
        """register() stores the class by its ID."""
        DatasetRegistry.register(ConcreteBenchmarkDataset)
        assert "test-dataset" in DatasetRegistry._registry

    def test_register_no_id_raises(self):
        """register() raises when dataset has no ID."""

        class NoIdDataset(ConcreteBenchmarkDataset):
            @classmethod
            def get_dataset_info(cls):
                return {"name": "No ID"}

        with pytest.raises(ValueError, match="must have an ID"):
            DatasetRegistry.register(NoIdDataset)

    def test_get_dataset_class_unknown_raises(self):
        """get_dataset_class raises for unknown ID."""
        with pytest.raises(ValueError, match="Unknown dataset"):
            DatasetRegistry.get_dataset_class("nonexistent")

    def test_create_dataset_returns_instance(self):
        """create_dataset returns a BenchmarkDataset instance."""
        DatasetRegistry.register(ConcreteBenchmarkDataset)
        ds = DatasetRegistry.create_dataset("test-dataset")
        assert isinstance(ds, BenchmarkDataset)

    def test_get_available_datasets(self):
        """get_available_datasets returns info for all registered datasets."""
        DatasetRegistry.register(ConcreteBenchmarkDataset)
        available = DatasetRegistry.get_available_datasets()
        assert len(available) == 1
        assert available[0]["id"] == "test-dataset"