learningcircuit--local-deep-research/tests/settings/test_settings_defaults_integrity.py

"""
Tests for settings defaults integrity.

Phase 1 of the JSON → Python settings migration. These tests validate the
current JSON-based default settings and will serve as a safety net when
settings definitions are migrated to Python modules in Phase 2.

The tests perform structural validation, cross-referencing against Python
enums, and static analysis of consumed setting keys vs. defined defaults.
"""

import json
import os
import re
from pathlib import Path

import pytest

from local_deep_research.settings.manager import SettingsManager
from local_deep_research.utilities.enums import KnowledgeAccumulationApproach


# ---------------------------------------------------------------------------
# Registry: enums tied to settings
# ---------------------------------------------------------------------------

ENUM_SETTINGS = {
    "general.knowledge_accumulation": (
        KnowledgeAccumulationApproach,
        # Active values only — MAX_NR_OF_CHARACTERS is dead code (Phase 4)
        {"ITERATION", "QUESTION", "NO_KNOWLEDGE"},
    ),
}

# ---------------------------------------------------------------------------
# Known-unused settings (exist in defaults but not consumed by Python code).
# Some are JS-frontend-only; others are truly dead. Phase 4 will remove
# dead ones and document frontend-only ones.
# ---------------------------------------------------------------------------

KNOWN_UNUSED = {
    # Truly unused / dead code (candidates for future removal)
    "report.export_formats",
    "research_library.confirm_deletions",
    "search.quality_check_urls",
    # Only ever consumed by the iterdrag/standard strategies, which were
    # removed when the experimental strategies were dropped. The setting and
    # its KnowledgeAccumulationApproach enum are now dead; left in defaults
    # for the planned Phase 4 settings cleanup.
    "general.knowledge_accumulation",
    "general.knowledge_accumulation_context_limit",
}

# ---------------------------------------------------------------------------
# Known select-option data bugs in current JSON (to be fixed in Phase 2/3).
# These are real bugs caught by test_select_options_validity — we skip them
# so the test passes on current JSON while still catching NEW regressions.
# ---------------------------------------------------------------------------

KNOWN_SELECT_ISSUES = {
    # 'dark' was a legacy theme name; dynamic options from theme_registry
    # don't include it (should be e.g. 'midnight' or 'system')
    "app.theme",
    # Options list is suggestive (common models), not restrictive;
    # user can type any model name. Default is empty so users must
    # consciously pick a model.
    "llm.model",
    # Default '265' is a typo — should be '365'.
    "search.journal_reputation.reanalysis_period",
}

# ---------------------------------------------------------------------------
# Known numeric constraint data bugs in current JSON.
# ---------------------------------------------------------------------------

KNOWN_NUMERIC_ISSUES = set()

# ---------------------------------------------------------------------------
# Known snapshot/get_setting divergences.
# These are structural issues where get_setting() expands engine parent keys
# or deserializes JSON list values differently from the snapshot path.
# ---------------------------------------------------------------------------

KNOWN_SNAPSHOT_DIVERGENCES = {
    # Engine parent keys: get_setting() expands to include all children,
    # snapshot stores the raw dict value
    "search.engine.web.gutenberg",
    "search.engine.web.nasa_ads",
    "search.engine.web.openalex",
    "search.engine.web.openlibrary",
    "search.engine.web.pubchem",
    "search.engine.web.semantic_scholar",
    "search.engine.web.stackexchange",
    "search.engine.web.zenodo",
}

# ---------------------------------------------------------------------------
# Consumed settings that intentionally have no JSON default.
# They use explicit defaults in code, are env-only, or are runtime-created.
# ---------------------------------------------------------------------------

KNOWN_MISSING_DEFAULTS = {
    # Env-only bootstrap settings
    "bootstrap.data_dir",
    # Runtime/programmatic settings (created by SettingsManager, not JSON)
    "app.version",
    "document_scheduler.last_run",
    # Consumed with explicit code defaults, no JSON entry needed
    "app.api_rate_limit",
    "app.default_theme",
    "app.enable_api",
    "llm.max_retries",
    "llm.request_timeout",
    "llm.streaming",
    "llm.openai.api_base",
    "llm.openai.organization",
    # Embeddings settings (consumed by embeddings config, sparse JSON coverage)
    "embeddings.ollama.model",
    "embeddings.openai.api_key",
    "embeddings.openai.base_url",
    "embeddings.openai.dimensions",
    "embeddings.openai.model",
    "embeddings.provider",
    "embeddings.sentence_transformers.device",
    "embeddings.sentence_transformers.model",
    # Notification settings consumed but not in defaults JSON
    "notifications.allow_private_ips",
    # Search settings consumed with code defaults
    "search.questions",
}

# ---------------------------------------------------------------------------
# Required schema fields for every default setting
# ---------------------------------------------------------------------------

REQUIRED_FIELDS = {
    "description": str,
    "editable": bool,
    "name": str,
    "ui_element": str,
    "visible": bool,
    "type": str,
    # "value" can be any type, checked separately
}

# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------

GOLDEN_MASTER_PATH = Path(__file__).parent / "golden_master_settings.json"
SRC_DIR = Path(__file__).parent.parent.parent / "src" / "local_deep_research"


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def load_all_defaults() -> dict:
    """Load defaults via SettingsManager with no DB."""
    manager = SettingsManager(db_session=None)
    return manager.default_settings


def collect_consumed_setting_keys() -> set:
    """Static analysis: regex-find all setting keys in Python code under src/.

    Covers get_setting(), get_setting_from_snapshot(), get_bool_setting(),
    extract_setting_value(), extract_bool_setting(),
    the deprecated db_main_thread wrapper, and dict-key access patterns
    used in web routes and search engine factory.
    """
    # The deprecated function name is split across concatenation to avoid
    # triggering the pre-commit hook that flags its usage.
    _deprecated_fn = "get_setting_from_db" + "_main_thread"
    patterns = [
        r'get_setting\(\s*["\']([^"\']+)["\']',
        # Matches both _get_setting(snapshot, "key") and the egress
        # policy module's _get_setting_value(snapshot, "key", default).
        r'_get_setting(?:_value)?\([^,]+,\s*["\']([^"\']+)["\']',
        r'get_setting_from_snapshot\(\s*["\']([^"\']+)["\']',
        r'get_bool_setting\(\s*["\']([^"\']+)["\']',
        r'get_bool_setting_from_snapshot\(\s*["\']([^"\']+)["\']',
        r'extract_setting_value\([^,]+,\s*["\']([^"\']+)["\']',
        r'extract_bool_setting\([^,]+,\s*["\']([^"\']+)["\']',
        rf'{_deprecated_fn}\(\s*["\']([^"\']+)["\']',
        # Dict key presence check: "key" in settings_snapshot
        r'"([a-z][a-z0-9_.]+)"\s+in\s+settings_snapshot',
        # Web route key comparison: setting.key == "key"
        r'setting\.key\s*==\s*"([^"]+)"',
        # Provider-class api_key_setting declarations (consumed indirectly
        # via cls.resolve_api_key / cls.has_api_key on BaseLLMProvider
        # rather than a literal get_setting_from_snapshot call).
        r'api_key_setting\s*=\s*"([^"]+)"',
    ]
    compiled = [re.compile(p) for p in patterns]
    keys = set()

    for py_file in SRC_DIR.rglob("*.py"):
        try:
            text = py_file.read_text(encoding="utf-8", errors="ignore")
        except OSError:
            continue
        for pattern in compiled:
            keys.update(pattern.findall(text))

    return keys


def _is_dynamic_setting(key: str) -> bool:
    """Keys consumed dynamically, not via explicit get_setting() calls.

    These are batch-loaded via parent prefix (e.g., get_setting("search.engine.web")
    returns all children), consumed by LLM/embeddings provider factories that
    construct keys from the provider name, or consumed by web frontend JS.
    """
    dynamic_prefixes = (
        # Search engine configs: batch-loaded by search_engines_config.py
        "search.engine.",
        # LLM provider-specific settings: key constructed from provider name
        # e.g., f"llm.{provider}.api_key" in llm_config.py
        "llm.google.",
        "llm.ionos.",
        "llm.openrouter.",
        "llm.xai.",
        "llm.ollama.",
        "llm.lmstudio.",
        "llm.openai_endpoint.",
        "llm.anthropic_endpoint.",
        "llm.deepseek.",
        # News subsystem: many settings consumed by JS frontend or
        # batch-loaded by news scheduler
        "news.",
        # Notification event toggles: consumed by JS frontend
        "notifications.",
        # Rate limiting config: batch-loaded by rate limiter
        "rate_limiting.",
        # Security settings: consumed by web routes / server config
        "security.",
        # UI warning dismissals: consumed by JS frontend
        "app.warnings.",
    )
    return any(key.startswith(p) for p in dynamic_prefixes)


def _is_metadata_setting(key: str) -> bool:
    """Keys that are engine/provider metadata, not consumed via get_setting()."""
    metadata_suffixes = (
        ".display_name",
        ".module_path",
        ".class_name",
        ".strengths",
        ".weaknesses",
    )
    metadata_prefixes = (
        "search.engines.",
        "focused_iteration.",
    )
    return any(key.endswith(s) for s in metadata_suffixes) or any(
        key.startswith(p) for p in metadata_prefixes
    )


def _is_web_consumed_setting(key: str) -> bool:
    """Settings consumed by web server/app factory, not via get_setting().

    These are read via env var access (load_server_config), app_factory
    configuration, or auth route logic — patterns not caught by get_setting()
    regex.
    """
    return key in {
        "app.allow_registrations",
        "app.debug",
        "app.enable_file_logging",
        "app.enable_notifications",
        "app.enable_web",
        "app.max_global_concurrent_researches",
        "app.theme",
        "app.web_interface",
    }


def _extract_option_values(options: list) -> list:
    """Extract comparable values from an options list.

    Handles both string options and {"label": ..., "value": ...} dicts.
    """
    values = []
    for opt in options:
        if isinstance(opt, dict):
            values.append(opt.get("value", opt.get("label")))
        else:
            values.append(opt)
    return values


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------


class TestSettingsDefaultsIntegrity:
    """Validates structural integrity of all default settings."""

    @pytest.fixture(autouse=True)
    def clean_env(self):
        """Save/clear/restore LDR_* env vars to prevent pollution across tests."""
        original_env = {
            k: v for k, v in os.environ.items() if k.startswith("LDR_")
        }
        for key in list(os.environ.keys()):
            if key.startswith("LDR_"):
                os.environ.pop(key, None)
        yield
        for key in list(os.environ.keys()):
            if key.startswith("LDR_"):
                os.environ.pop(key, None)
        for key, value in original_env.items():
            os.environ[key] = value

    @pytest.fixture(scope="class")
    def defaults(self) -> dict:
        return load_all_defaults()

    # -- Test 1: Schema validation ------------------------------------------

    def test_schema_validation(self, defaults):
        """Every default setting must have required fields with correct types."""
        errors = []
        for key, setting in defaults.items():
            # "value" must exist (can be any type including None)
            if "value" not in setting:
                errors.append(f"{key}: missing 'value' field")

            for field, expected_type in REQUIRED_FIELDS.items():
                if field not in setting:
                    errors.append(f"{key}: missing '{field}' field")
                elif not isinstance(setting[field], expected_type):
                    errors.append(
                        f"{key}: '{field}' should be {expected_type.__name__}, "
                        f"got {type(setting[field]).__name__}"
                    )

        assert not errors, f"{len(errors)} schema violation(s):\n" + "\n".join(
            errors
        )

    # -- Test 2: Settings manager loads all defaults ------------------------

    def test_settings_manager_loads_all_defaults(self, defaults):
        """SettingsManager(db_session=None) must return a reasonable number of settings."""
        assert len(defaults) >= 400, (
            f"Expected at least 400 settings, got {len(defaults)}. "
            f"Some JSON files may have failed to load."
        )

    # -- Test 3: Value-type consistency ------------------------------------

    def test_value_type_consistency(self, defaults):
        """Checkbox defaults must be bool, number/range defaults must be numeric."""
        errors = []
        for key, setting in defaults.items():
            ui = setting.get("ui_element")
            value = setting.get("value")

            if value is None:
                continue

            if ui == "checkbox" and not isinstance(value, bool):
                errors.append(
                    f"{key}: checkbox default should be bool, "
                    f"got {type(value).__name__} ({value!r})"
                )
            elif ui in ("number", "range") and not isinstance(
                value, (int, float)
            ):
                errors.append(
                    f"{key}: {ui} default should be numeric, "
                    f"got {type(value).__name__} ({value!r})"
                )

        assert not errors, f"{len(errors)} type mismatch(es):\n" + "\n".join(
            errors
        )

    # -- Test 4: Select options validity -----------------------------------

    def test_select_options_validity(self, defaults):
        """Settings with ui_element='select' must have valid options containing the default value."""
        errors = []
        for key, setting in defaults.items():
            if setting.get("ui_element") != "select":
                continue
            if key in KNOWN_SELECT_ISSUES:
                continue

            options = setting.get("options")
            if not options:
                errors.append(f"{key}: select setting has no options")
                continue

            value = setting.get("value")
            option_values = _extract_option_values(options)

            if value not in option_values:
                errors.append(
                    f"{key}: default value {value!r} not in options "
                    f"{option_values!r}"
                )

        assert not errors, (
            f"{len(errors)} select option error(s):\n" + "\n".join(errors)
        )

    # -- Test 5: Enum-settings consistency ---------------------------------

    def test_enum_settings_consistency(self, defaults):
        """Settings backed by Python enums must have options matching actual enum values."""
        errors = []
        for key, (enum_class, active_values) in ENUM_SETTINGS.items():
            setting = defaults.get(key)
            if setting is None:
                errors.append(f"{key}: setting not found in defaults")
                continue

            # The setting should be a select with options matching the enum
            options = setting.get("options")

            if options is None:
                # The current JSON has ui_element="text" with no options —
                # this is WRONG: should be select with enum values as options
                errors.append(
                    f"{key}: should be a 'select' with options matching "
                    f"{enum_class.__name__} enum values {active_values}, "
                    f"but has ui_element='{setting.get('ui_element')}' "
                    f"and no options"
                )
                continue

            option_values = set(_extract_option_values(options))
            if option_values != active_values:
                errors.append(
                    f"{key}: options {option_values} don't match "
                    f"enum active values {active_values}"
                )

            # Default value must be a valid active enum member
            value = setting.get("value")
            if value not in active_values:
                errors.append(
                    f"{key}: default value {value!r} not in "
                    f"active enum values {active_values}"
                )

        assert not errors, f"{len(errors)} enum mismatch(es):\n" + "\n".join(
            errors
        )

    # -- Test 6: No orphaned settings --------------------------------------

    def test_no_orphaned_settings(self, defaults):
        """Settings in defaults but not consumed by code should be accounted for."""
        consumed = collect_consumed_setting_keys()
        orphaned = []

        for key in sorted(defaults.keys()):
            if key in consumed:
                continue
            if key in KNOWN_UNUSED:
                continue
            if _is_dynamic_setting(key):
                continue
            if _is_metadata_setting(key):
                continue
            if _is_web_consumed_setting(key):
                continue
            orphaned.append(key)

        assert not orphaned, (
            f"{len(orphaned)} orphaned setting(s) (not consumed by code, "
            f"not in KNOWN_UNUSED, not dynamic/metadata/web-consumed):\n"
            + "\n".join(f"  {k}" for k in orphaned)
        )

    # -- Test 7: Consumed settings exist in defaults -----------------------

    def test_consumed_settings_exist(self, defaults):
        """Every get_setting('key') call in code must have a matching default."""
        consumed = collect_consumed_setting_keys()
        default_keys = set(defaults.keys())
        missing = []

        for key in sorted(consumed):
            if key in default_keys:
                continue
            if key in KNOWN_MISSING_DEFAULTS:
                continue
            # Skip keys that look like they're constructed dynamically
            # (contain format-string markers or are parent-level prefix keys)
            # Also skip keys without dots — all real settings use
            # dot-separated namespacing (e.g., "app.debug").
            if "{" in key or key.endswith(".") or "." not in key:
                continue
            # Skip parent-level keys used for batch loading
            # (e.g., "search.engine.web" loads all search.engine.web.* children)
            children = [dk for dk in default_keys if dk.startswith(f"{key}.")]
            if children:
                continue
            # Skip keys that are children of a dynamic prefix
            # (e.g., search.engine.web.guardian.api_key is under search.engine.web.*)
            if _is_dynamic_setting(key):
                continue
            missing.append(key)

        assert not missing, (
            f"{len(missing)} consumed setting(s) missing from defaults:\n"
            + "\n".join(f"  {k}" for k in missing)
        )

    # -- Test 8: Numeric constraints ---------------------------------------

    def test_numeric_constraints(self, defaults):
        """Settings with min_value/max_value must have defaults within range."""
        errors = []
        for key, setting in defaults.items():
            if key in KNOWN_NUMERIC_ISSUES:
                continue

            value = setting.get("value")
            min_val = setting.get("min_value")
            max_val = setting.get("max_value")

            if not isinstance(value, (int, float)):
                continue

            if min_val is not None and value < min_val:
                errors.append(f"{key}: default {value} < min_value {min_val}")
            if max_val is not None and value > max_val:
                errors.append(f"{key}: default {value} > max_value {max_val}")

        assert not errors, f"{len(errors)} range violation(s):\n" + "\n".join(
            errors
        )

    # -- Test 9: Settings snapshot -----------------------------------------

    def test_settings_snapshot(self, defaults):
        """All default setting keys should be present in a snapshot built from defaults."""
        manager = SettingsManager(db_session=None)
        snapshot = manager.get_settings_snapshot()

        missing = set(defaults.keys()) - set(snapshot.keys())
        assert not missing, (
            f"{len(missing)} key(s) missing from snapshot:\n"
            + "\n".join(f"  {k}" for k in sorted(missing))
        )

    # -- Test 10: Golden master --------------------------------------------

    def test_golden_master(self, defaults):
        """Serialize defaults to JSON; compare against golden master on subsequent runs.

        First run creates the golden master file. This test ensures the
        Phase 2 Python modules produce identical output to the JSON files.
        """
        # Build a serializable representation (sorted for determinism)
        current = {}
        for key in sorted(defaults.keys()):
            setting = dict(defaults[key])
            # Normalize: drop any runtime-injected fields that vary
            # (theme options are dynamically injected from theme_registry)
            current[key] = setting

        current_json = (
            json.dumps(
                current,
                indent=2,
                sort_keys=True,
                default=str,
                ensure_ascii=False,
            )
            + "\n"
        )

        if not GOLDEN_MASTER_PATH.exists():
            GOLDEN_MASTER_PATH.write_text(current_json, encoding="utf-8")
            pytest.skip(
                f"Golden master created at {GOLDEN_MASTER_PATH}. "
                f"Run tests again to validate."
            )

        expected = GOLDEN_MASTER_PATH.read_text(encoding="utf-8")

        if current_json != expected:
            # Find differences for a helpful error message
            current_data = json.loads(current_json)
            expected_data = json.loads(expected)

            added = set(current_data.keys()) - set(expected_data.keys())
            removed = set(expected_data.keys()) - set(current_data.keys())
            changed = {
                k
                for k in current_data
                if k in expected_data and current_data[k] != expected_data[k]
            }

            diff_parts = []
            if added:
                diff_parts.append(f"Added keys: {sorted(added)}")
            if removed:
                diff_parts.append(f"Removed keys: {sorted(removed)}")
            if changed:
                diff_parts.append(f"Changed keys: {sorted(changed)}")

            diff_msg = (
                "\n".join(diff_parts) if diff_parts else "Content differs"
            )

            pytest.fail(
                f"Settings defaults have changed from golden master.\n"
                f"{diff_msg}\n"
                f"If this is intentional, delete {GOLDEN_MASTER_PATH} "
                f"and re-run."
            )

    # -- Test 11: Reverse value-type consistency ----------------------------

    def test_value_type_implies_ui_element(self, defaults):
        """Bool values must use checkbox; numeric values must not use text-like elements."""
        errors = []
        for key, setting in defaults.items():
            ui = setting.get("ui_element")
            value = setting.get("value")
            if value is None:
                continue
            if isinstance(value, bool) and ui != "checkbox":
                errors.append(
                    f"{key}: bool value {value!r} has ui_element='{ui}', "
                    f"should be 'checkbox'"
                )
            elif (
                isinstance(value, (int, float))
                and not isinstance(value, bool)
                and ui in ("text", "password", "textarea")
            ):
                errors.append(
                    f"{key}: numeric value {value!r} has ui_element='{ui}', "
                    f"consider 'number' or 'range'"
                )
        assert not errors, (
            f"{len(errors)} reverse type mismatch(es):\n" + "\n".join(errors)
        )

    # -- Test 12: Snapshot values match typed get_setting -------------------

    def test_snapshot_values_match_get_setting(self, defaults):
        """Snapshot values must equal get_setting() typed values for every key."""
        manager = SettingsManager(db_session=None)
        snapshot = manager.get_settings_snapshot()
        errors = []
        for key in defaults:
            if key in KNOWN_SNAPSHOT_DIVERGENCES:
                continue
            snapshot_val = snapshot.get(key)
            typed_val = manager.get_setting(key)
            if snapshot_val != typed_val:
                errors.append(
                    f"{key}: snapshot={snapshot_val!r} ({type(snapshot_val).__name__}) "
                    f"vs get_setting={typed_val!r} ({type(typed_val).__name__})"
                )
        assert not errors, (
            f"{len(errors)} snapshot/get_setting divergence(s):\n"
            + "\n".join(errors)
        )