""" High-value tests for benchmark schema definitions. Covers: - BenchmarkStatus enum values and completeness - DatasetType enum values and completeness - Table definition structures (columns, indexes, constraints, foreign keys) - create_benchmark_tables_simple: actual table creation with in-memory SQLite """ import pytest from sqlalchemy import create_engine, inspect from local_deep_research.web.database.benchmark_schema import ( BenchmarkStatus, DatasetType, benchmark_configs_table, benchmark_progress_table, benchmark_results_table, benchmark_runs_table, create_benchmark_tables_simple, ) # --------------------------------------------------------------------------- # Enum definitions # --------------------------------------------------------------------------- class TestBenchmarkStatus: """Tests for BenchmarkStatus enum.""" def test_all_expected_values(self): assert BenchmarkStatus.PENDING.value == "pending" assert BenchmarkStatus.IN_PROGRESS.value == "in_progress" assert BenchmarkStatus.COMPLETED.value == "completed" assert BenchmarkStatus.FAILED.value == "failed" assert BenchmarkStatus.CANCELLED.value == "cancelled" assert BenchmarkStatus.PAUSED.value == "paused" def test_member_count(self): assert len(BenchmarkStatus) == 6 def test_values_are_unique(self): values = [s.value for s in BenchmarkStatus] assert len(values) == len(set(values)) def test_lookup_by_name(self): assert BenchmarkStatus["PENDING"] is BenchmarkStatus.PENDING def test_lookup_invalid_raises(self): with pytest.raises(KeyError): BenchmarkStatus["NONEXISTENT"] class TestDatasetType: """Tests for DatasetType enum.""" def test_all_expected_values(self): assert DatasetType.SIMPLEQA.value == "simpleqa" assert DatasetType.BROWSECOMP.value == "browsecomp" assert DatasetType.CUSTOM.value == "custom" def test_member_count(self): assert len(DatasetType) == 3 def test_values_are_unique(self): values = [d.value for d in DatasetType] assert len(values) == len(set(values)) # --------------------------------------------------------------------------- # Table definition structures # --------------------------------------------------------------------------- class TestTableDefinitions: """Tests for table definition dict structures.""" def test_benchmark_runs_table_name(self): assert benchmark_runs_table["table_name"] == "benchmark_runs" def test_benchmark_runs_has_columns(self): assert len(benchmark_runs_table["columns"]) > 0 def test_benchmark_runs_has_indexes(self): assert len(benchmark_runs_table["indexes"]) >= 2 def test_benchmark_results_table_name(self): assert benchmark_results_table["table_name"] == "benchmark_results" def test_benchmark_results_has_foreign_key(self): """benchmark_results has a FK to benchmark_runs.id.""" columns = benchmark_results_table["columns"] fk_columns = [ c for c in columns if c.foreign_keys or (hasattr(c, "name") and "run_id" in str(c)) ] # At least one column should reference benchmark_runs assert len(fk_columns) >= 1 def test_benchmark_results_has_unique_constraint(self): constraints = benchmark_results_table.get("constraints", []) assert len(constraints) >= 1 def test_benchmark_configs_table_name(self): assert benchmark_configs_table["table_name"] == "benchmark_configs" def test_benchmark_progress_table_name(self): assert benchmark_progress_table["table_name"] == "benchmark_progress" def test_all_tables_have_id_column(self): for table_def in [ benchmark_runs_table, benchmark_results_table, benchmark_configs_table, benchmark_progress_table, ]: column_names = [c.name for c in table_def["columns"]] assert "id" in column_names, ( f"{table_def['table_name']} missing id column" ) # --------------------------------------------------------------------------- # create_benchmark_tables_simple with in-memory SQLite # --------------------------------------------------------------------------- class TestCreateBenchmarkTablesSimple: """Tests for actual table creation using SQLite in-memory. Uses a module-scoped engine because the Column objects in the schema module are module-level singletons that get bound to a Table on first use and cannot be re-bound to a different MetaData instance. """ @pytest.fixture(scope="class") def db_engine(self): """Create an in-memory SQLite engine shared across tests in this class.""" engine = create_engine("sqlite:///:memory:") create_benchmark_tables_simple(engine) yield engine engine.dispose() def test_creates_all_four_tables(self, db_engine): inspector = inspect(db_engine) tables = inspector.get_table_names() assert "benchmark_runs" in tables assert "benchmark_results" in tables assert "benchmark_configs" in tables assert "benchmark_progress" in tables def test_benchmark_runs_columns(self, db_engine): inspector = inspect(db_engine) columns = {c["name"] for c in inspector.get_columns("benchmark_runs")} expected = { "id", "run_name", "config_hash", "query_hash_list", "search_config", "evaluation_config", "datasets_config", "status", "created_at", "updated_at", "start_time", "end_time", "total_examples", "completed_examples", "failed_examples", "overall_accuracy", "processing_rate", "error_message", } assert expected.issubset(columns) def test_benchmark_results_columns(self, db_engine): inspector = inspect(db_engine) columns = { c["name"] for c in inspector.get_columns("benchmark_results") } expected = { "id", "benchmark_run_id", "example_id", "query_hash", "dataset_type", "question", "correct_answer", "response", "is_correct", "processing_time", "confidence", } assert expected.issubset(columns) def test_benchmark_configs_columns(self, db_engine): inspector = inspect(db_engine) columns = { c["name"] for c in inspector.get_columns("benchmark_configs") } expected = { "id", "name", "description", "config_hash", "search_config", "evaluation_config", "datasets_config", "is_default", "is_public", "usage_count", } assert expected.issubset(columns) def test_benchmark_progress_columns(self, db_engine): inspector = inspect(db_engine) columns = { c["name"] for c in inspector.get_columns("benchmark_progress") } expected = { "id", "benchmark_run_id", "timestamp", "completed_examples", "total_examples", "overall_accuracy", } assert expected.issubset(columns) def test_benchmark_results_foreign_key_exists(self, db_engine): inspector = inspect(db_engine) fks = inspector.get_foreign_keys("benchmark_results") fk_tables = [fk["referred_table"] for fk in fks] assert "benchmark_runs" in fk_tables def test_benchmark_progress_foreign_key_exists(self, db_engine): inspector = inspect(db_engine) fks = inspector.get_foreign_keys("benchmark_progress") fk_tables = [fk["referred_table"] for fk in fks] assert "benchmark_runs" in fk_tables