""" Tests for settings defaults integrity. Phase 1 of the JSON → Python settings migration. These tests validate the current JSON-based default settings and will serve as a safety net when settings definitions are migrated to Python modules in Phase 2. The tests perform structural validation, cross-referencing against Python enums, and static analysis of consumed setting keys vs. defined defaults. """ import json import os import re from pathlib import Path import pytest from local_deep_research.settings.manager import SettingsManager from local_deep_research.utilities.enums import KnowledgeAccumulationApproach # --------------------------------------------------------------------------- # Registry: enums tied to settings # --------------------------------------------------------------------------- ENUM_SETTINGS = { "general.knowledge_accumulation": ( KnowledgeAccumulationApproach, # Active values only — MAX_NR_OF_CHARACTERS is dead code (Phase 4) {"ITERATION", "QUESTION", "NO_KNOWLEDGE"}, ), } # --------------------------------------------------------------------------- # Known-unused settings (exist in defaults but not consumed by Python code). # Some are JS-frontend-only; others are truly dead. Phase 4 will remove # dead ones and document frontend-only ones. # --------------------------------------------------------------------------- KNOWN_UNUSED = { # Truly unused / dead code (candidates for future removal) "report.export_formats", "research_library.confirm_deletions", "search.quality_check_urls", # Only ever consumed by the iterdrag/standard strategies, which were # removed when the experimental strategies were dropped. The setting and # its KnowledgeAccumulationApproach enum are now dead; left in defaults # for the planned Phase 4 settings cleanup. "general.knowledge_accumulation", "general.knowledge_accumulation_context_limit", } # --------------------------------------------------------------------------- # Known select-option data bugs in current JSON (to be fixed in Phase 2/3). # These are real bugs caught by test_select_options_validity — we skip them # so the test passes on current JSON while still catching NEW regressions. # --------------------------------------------------------------------------- KNOWN_SELECT_ISSUES = { # 'dark' was a legacy theme name; dynamic options from theme_registry # don't include it (should be e.g. 'midnight' or 'system') "app.theme", # Options list is suggestive (common models), not restrictive; # user can type any model name. Default is empty so users must # consciously pick a model. "llm.model", # Default '265' is a typo — should be '365'. "search.journal_reputation.reanalysis_period", } # --------------------------------------------------------------------------- # Known numeric constraint data bugs in current JSON. # --------------------------------------------------------------------------- KNOWN_NUMERIC_ISSUES = set() # --------------------------------------------------------------------------- # Known snapshot/get_setting divergences. # These are structural issues where get_setting() expands engine parent keys # or deserializes JSON list values differently from the snapshot path. # --------------------------------------------------------------------------- KNOWN_SNAPSHOT_DIVERGENCES = { # Engine parent keys: get_setting() expands to include all children, # snapshot stores the raw dict value "search.engine.web.gutenberg", "search.engine.web.nasa_ads", "search.engine.web.openalex", "search.engine.web.openlibrary", "search.engine.web.pubchem", "search.engine.web.semantic_scholar", "search.engine.web.stackexchange", "search.engine.web.zenodo", } # --------------------------------------------------------------------------- # Consumed settings that intentionally have no JSON default. # They use explicit defaults in code, are env-only, or are runtime-created. # --------------------------------------------------------------------------- KNOWN_MISSING_DEFAULTS = { # Env-only bootstrap settings "bootstrap.data_dir", # Runtime/programmatic settings (created by SettingsManager, not JSON) "app.version", "document_scheduler.last_run", # Consumed with explicit code defaults, no JSON entry needed "app.api_rate_limit", "app.default_theme", "app.enable_api", "llm.max_retries", "llm.request_timeout", "llm.streaming", "llm.openai.api_base", "llm.openai.organization", # Embeddings settings (consumed by embeddings config, sparse JSON coverage) "embeddings.ollama.model", "embeddings.openai.api_key", "embeddings.openai.base_url", "embeddings.openai.dimensions", "embeddings.openai.model", "embeddings.provider", "embeddings.sentence_transformers.device", "embeddings.sentence_transformers.model", # Notification settings consumed but not in defaults JSON "notifications.allow_private_ips", # Search settings consumed with code defaults "search.questions", } # --------------------------------------------------------------------------- # Required schema fields for every default setting # --------------------------------------------------------------------------- REQUIRED_FIELDS = { "description": str, "editable": bool, "name": str, "ui_element": str, "visible": bool, "type": str, # "value" can be any type, checked separately } # --------------------------------------------------------------------------- # Paths # --------------------------------------------------------------------------- GOLDEN_MASTER_PATH = Path(__file__).parent / "golden_master_settings.json" SRC_DIR = Path(__file__).parent.parent.parent / "src" / "local_deep_research" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def load_all_defaults() -> dict: """Load defaults via SettingsManager with no DB.""" manager = SettingsManager(db_session=None) return manager.default_settings def collect_consumed_setting_keys() -> set: """Static analysis: regex-find all setting keys in Python code under src/. Covers get_setting(), get_setting_from_snapshot(), get_bool_setting(), extract_setting_value(), extract_bool_setting(), the deprecated db_main_thread wrapper, and dict-key access patterns used in web routes and search engine factory. """ # The deprecated function name is split across concatenation to avoid # triggering the pre-commit hook that flags its usage. _deprecated_fn = "get_setting_from_db" + "_main_thread" patterns = [ r'get_setting\(\s*["\']([^"\']+)["\']', # Matches both _get_setting(snapshot, "key") and the egress # policy module's _get_setting_value(snapshot, "key", default). r'_get_setting(?:_value)?\([^,]+,\s*["\']([^"\']+)["\']', r'get_setting_from_snapshot\(\s*["\']([^"\']+)["\']', r'get_bool_setting\(\s*["\']([^"\']+)["\']', r'get_bool_setting_from_snapshot\(\s*["\']([^"\']+)["\']', r'extract_setting_value\([^,]+,\s*["\']([^"\']+)["\']', r'extract_bool_setting\([^,]+,\s*["\']([^"\']+)["\']', rf'{_deprecated_fn}\(\s*["\']([^"\']+)["\']', # Dict key presence check: "key" in settings_snapshot r'"([a-z][a-z0-9_.]+)"\s+in\s+settings_snapshot', # Web route key comparison: setting.key == "key" r'setting\.key\s*==\s*"([^"]+)"', # Provider-class api_key_setting declarations (consumed indirectly # via cls.resolve_api_key / cls.has_api_key on BaseLLMProvider # rather than a literal get_setting_from_snapshot call). r'api_key_setting\s*=\s*"([^"]+)"', ] compiled = [re.compile(p) for p in patterns] keys = set() for py_file in SRC_DIR.rglob("*.py"): try: text = py_file.read_text(encoding="utf-8", errors="ignore") except OSError: continue for pattern in compiled: keys.update(pattern.findall(text)) return keys def _is_dynamic_setting(key: str) -> bool: """Keys consumed dynamically, not via explicit get_setting() calls. These are batch-loaded via parent prefix (e.g., get_setting("search.engine.web") returns all children), consumed by LLM/embeddings provider factories that construct keys from the provider name, or consumed by web frontend JS. """ dynamic_prefixes = ( # Search engine configs: batch-loaded by search_engines_config.py "search.engine.", # LLM provider-specific settings: key constructed from provider name # e.g., f"llm.{provider}.api_key" in llm_config.py "llm.google.", "llm.ionos.", "llm.openrouter.", "llm.xai.", "llm.ollama.", "llm.lmstudio.", "llm.openai_endpoint.", "llm.anthropic_endpoint.", "llm.deepseek.", # News subsystem: many settings consumed by JS frontend or # batch-loaded by news scheduler "news.", # Notification event toggles: consumed by JS frontend "notifications.", # Rate limiting config: batch-loaded by rate limiter "rate_limiting.", # Security settings: consumed by web routes / server config "security.", # UI warning dismissals: consumed by JS frontend "app.warnings.", ) return any(key.startswith(p) for p in dynamic_prefixes) def _is_metadata_setting(key: str) -> bool: """Keys that are engine/provider metadata, not consumed via get_setting().""" metadata_suffixes = ( ".display_name", ".module_path", ".class_name", ".strengths", ".weaknesses", ) metadata_prefixes = ( "search.engines.", "focused_iteration.", ) return any(key.endswith(s) for s in metadata_suffixes) or any( key.startswith(p) for p in metadata_prefixes ) def _is_web_consumed_setting(key: str) -> bool: """Settings consumed by web server/app factory, not via get_setting(). These are read via env var access (load_server_config), app_factory configuration, or auth route logic — patterns not caught by get_setting() regex. """ return key in { "app.allow_registrations", "app.debug", "app.enable_file_logging", "app.enable_notifications", "app.enable_web", "app.max_global_concurrent_researches", "app.theme", "app.web_interface", } def _extract_option_values(options: list) -> list: """Extract comparable values from an options list. Handles both string options and {"label": ..., "value": ...} dicts. """ values = [] for opt in options: if isinstance(opt, dict): values.append(opt.get("value", opt.get("label"))) else: values.append(opt) return values # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- class TestSettingsDefaultsIntegrity: """Validates structural integrity of all default settings.""" @pytest.fixture(autouse=True) def clean_env(self): """Save/clear/restore LDR_* env vars to prevent pollution across tests.""" original_env = { k: v for k, v in os.environ.items() if k.startswith("LDR_") } for key in list(os.environ.keys()): if key.startswith("LDR_"): os.environ.pop(key, None) yield for key in list(os.environ.keys()): if key.startswith("LDR_"): os.environ.pop(key, None) for key, value in original_env.items(): os.environ[key] = value @pytest.fixture(scope="class") def defaults(self) -> dict: return load_all_defaults() # -- Test 1: Schema validation ------------------------------------------ def test_schema_validation(self, defaults): """Every default setting must have required fields with correct types.""" errors = [] for key, setting in defaults.items(): # "value" must exist (can be any type including None) if "value" not in setting: errors.append(f"{key}: missing 'value' field") for field, expected_type in REQUIRED_FIELDS.items(): if field not in setting: errors.append(f"{key}: missing '{field}' field") elif not isinstance(setting[field], expected_type): errors.append( f"{key}: '{field}' should be {expected_type.__name__}, " f"got {type(setting[field]).__name__}" ) assert not errors, f"{len(errors)} schema violation(s):\n" + "\n".join( errors ) # -- Test 2: Settings manager loads all defaults ------------------------ def test_settings_manager_loads_all_defaults(self, defaults): """SettingsManager(db_session=None) must return a reasonable number of settings.""" assert len(defaults) >= 400, ( f"Expected at least 400 settings, got {len(defaults)}. " f"Some JSON files may have failed to load." ) # -- Test 3: Value-type consistency ------------------------------------ def test_value_type_consistency(self, defaults): """Checkbox defaults must be bool, number/range defaults must be numeric.""" errors = [] for key, setting in defaults.items(): ui = setting.get("ui_element") value = setting.get("value") if value is None: continue if ui == "checkbox" and not isinstance(value, bool): errors.append( f"{key}: checkbox default should be bool, " f"got {type(value).__name__} ({value!r})" ) elif ui in ("number", "range") and not isinstance( value, (int, float) ): errors.append( f"{key}: {ui} default should be numeric, " f"got {type(value).__name__} ({value!r})" ) assert not errors, f"{len(errors)} type mismatch(es):\n" + "\n".join( errors ) # -- Test 4: Select options validity ----------------------------------- def test_select_options_validity(self, defaults): """Settings with ui_element='select' must have valid options containing the default value.""" errors = [] for key, setting in defaults.items(): if setting.get("ui_element") != "select": continue if key in KNOWN_SELECT_ISSUES: continue options = setting.get("options") if not options: errors.append(f"{key}: select setting has no options") continue value = setting.get("value") option_values = _extract_option_values(options) if value not in option_values: errors.append( f"{key}: default value {value!r} not in options " f"{option_values!r}" ) assert not errors, ( f"{len(errors)} select option error(s):\n" + "\n".join(errors) ) # -- Test 5: Enum-settings consistency --------------------------------- def test_enum_settings_consistency(self, defaults): """Settings backed by Python enums must have options matching actual enum values.""" errors = [] for key, (enum_class, active_values) in ENUM_SETTINGS.items(): setting = defaults.get(key) if setting is None: errors.append(f"{key}: setting not found in defaults") continue # The setting should be a select with options matching the enum options = setting.get("options") if options is None: # The current JSON has ui_element="text" with no options — # this is WRONG: should be select with enum values as options errors.append( f"{key}: should be a 'select' with options matching " f"{enum_class.__name__} enum values {active_values}, " f"but has ui_element='{setting.get('ui_element')}' " f"and no options" ) continue option_values = set(_extract_option_values(options)) if option_values != active_values: errors.append( f"{key}: options {option_values} don't match " f"enum active values {active_values}" ) # Default value must be a valid active enum member value = setting.get("value") if value not in active_values: errors.append( f"{key}: default value {value!r} not in " f"active enum values {active_values}" ) assert not errors, f"{len(errors)} enum mismatch(es):\n" + "\n".join( errors ) # -- Test 6: No orphaned settings -------------------------------------- def test_no_orphaned_settings(self, defaults): """Settings in defaults but not consumed by code should be accounted for.""" consumed = collect_consumed_setting_keys() orphaned = [] for key in sorted(defaults.keys()): if key in consumed: continue if key in KNOWN_UNUSED: continue if _is_dynamic_setting(key): continue if _is_metadata_setting(key): continue if _is_web_consumed_setting(key): continue orphaned.append(key) assert not orphaned, ( f"{len(orphaned)} orphaned setting(s) (not consumed by code, " f"not in KNOWN_UNUSED, not dynamic/metadata/web-consumed):\n" + "\n".join(f" {k}" for k in orphaned) ) # -- Test 7: Consumed settings exist in defaults ----------------------- def test_consumed_settings_exist(self, defaults): """Every get_setting('key') call in code must have a matching default.""" consumed = collect_consumed_setting_keys() default_keys = set(defaults.keys()) missing = [] for key in sorted(consumed): if key in default_keys: continue if key in KNOWN_MISSING_DEFAULTS: continue # Skip keys that look like they're constructed dynamically # (contain format-string markers or are parent-level prefix keys) # Also skip keys without dots — all real settings use # dot-separated namespacing (e.g., "app.debug"). if "{" in key or key.endswith(".") or "." not in key: continue # Skip parent-level keys used for batch loading # (e.g., "search.engine.web" loads all search.engine.web.* children) children = [dk for dk in default_keys if dk.startswith(f"{key}.")] if children: continue # Skip keys that are children of a dynamic prefix # (e.g., search.engine.web.guardian.api_key is under search.engine.web.*) if _is_dynamic_setting(key): continue missing.append(key) assert not missing, ( f"{len(missing)} consumed setting(s) missing from defaults:\n" + "\n".join(f" {k}" for k in missing) ) # -- Test 8: Numeric constraints --------------------------------------- def test_numeric_constraints(self, defaults): """Settings with min_value/max_value must have defaults within range.""" errors = [] for key, setting in defaults.items(): if key in KNOWN_NUMERIC_ISSUES: continue value = setting.get("value") min_val = setting.get("min_value") max_val = setting.get("max_value") if not isinstance(value, (int, float)): continue if min_val is not None and value < min_val: errors.append(f"{key}: default {value} < min_value {min_val}") if max_val is not None and value > max_val: errors.append(f"{key}: default {value} > max_value {max_val}") assert not errors, f"{len(errors)} range violation(s):\n" + "\n".join( errors ) # -- Test 9: Settings snapshot ----------------------------------------- def test_settings_snapshot(self, defaults): """All default setting keys should be present in a snapshot built from defaults.""" manager = SettingsManager(db_session=None) snapshot = manager.get_settings_snapshot() missing = set(defaults.keys()) - set(snapshot.keys()) assert not missing, ( f"{len(missing)} key(s) missing from snapshot:\n" + "\n".join(f" {k}" for k in sorted(missing)) ) # -- Test 10: Golden master -------------------------------------------- def test_golden_master(self, defaults): """Serialize defaults to JSON; compare against golden master on subsequent runs. First run creates the golden master file. This test ensures the Phase 2 Python modules produce identical output to the JSON files. """ # Build a serializable representation (sorted for determinism) current = {} for key in sorted(defaults.keys()): setting = dict(defaults[key]) # Normalize: drop any runtime-injected fields that vary # (theme options are dynamically injected from theme_registry) current[key] = setting current_json = ( json.dumps( current, indent=2, sort_keys=True, default=str, ensure_ascii=False, ) + "\n" ) if not GOLDEN_MASTER_PATH.exists(): GOLDEN_MASTER_PATH.write_text(current_json, encoding="utf-8") pytest.skip( f"Golden master created at {GOLDEN_MASTER_PATH}. " f"Run tests again to validate." ) expected = GOLDEN_MASTER_PATH.read_text(encoding="utf-8") if current_json != expected: # Find differences for a helpful error message current_data = json.loads(current_json) expected_data = json.loads(expected) added = set(current_data.keys()) - set(expected_data.keys()) removed = set(expected_data.keys()) - set(current_data.keys()) changed = { k for k in current_data if k in expected_data and current_data[k] != expected_data[k] } diff_parts = [] if added: diff_parts.append(f"Added keys: {sorted(added)}") if removed: diff_parts.append(f"Removed keys: {sorted(removed)}") if changed: diff_parts.append(f"Changed keys: {sorted(changed)}") diff_msg = ( "\n".join(diff_parts) if diff_parts else "Content differs" ) pytest.fail( f"Settings defaults have changed from golden master.\n" f"{diff_msg}\n" f"If this is intentional, delete {GOLDEN_MASTER_PATH} " f"and re-run." ) # -- Test 11: Reverse value-type consistency ---------------------------- def test_value_type_implies_ui_element(self, defaults): """Bool values must use checkbox; numeric values must not use text-like elements.""" errors = [] for key, setting in defaults.items(): ui = setting.get("ui_element") value = setting.get("value") if value is None: continue if isinstance(value, bool) and ui != "checkbox": errors.append( f"{key}: bool value {value!r} has ui_element='{ui}', " f"should be 'checkbox'" ) elif ( isinstance(value, (int, float)) and not isinstance(value, bool) and ui in ("text", "password", "textarea") ): errors.append( f"{key}: numeric value {value!r} has ui_element='{ui}', " f"consider 'number' or 'range'" ) assert not errors, ( f"{len(errors)} reverse type mismatch(es):\n" + "\n".join(errors) ) # -- Test 12: Snapshot values match typed get_setting ------------------- def test_snapshot_values_match_get_setting(self, defaults): """Snapshot values must equal get_setting() typed values for every key.""" manager = SettingsManager(db_session=None) snapshot = manager.get_settings_snapshot() errors = [] for key in defaults: if key in KNOWN_SNAPSHOT_DIVERGENCES: continue snapshot_val = snapshot.get(key) typed_val = manager.get_setting(key) if snapshot_val != typed_val: errors.append( f"{key}: snapshot={snapshot_val!r} ({type(snapshot_val).__name__}) " f"vs get_setting={typed_val!r} ({type(typed_val).__name__})" ) assert not errors, ( f"{len(errors)} snapshot/get_setting divergence(s):\n" + "\n".join(errors) )