chore: import upstream snapshot with attribution

scripts/ci/dump_url_map.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""
+Dump the Flask url_map to a file as one absolute URL per line.
+
+Used by the Nuclei DAST workflow to seed a URL list so the scanner
+probes the actual application surface (authenticated routes, API
+endpoints, blueprints) instead of just the index page.
+
+Parameterized routes (`/research/<string:research_id>/status`) are
+emitted with a converter-appropriate placeholder so Nuclei still
+exercises the path. The substituted URL will usually 404 (the resource
+doesn't exist for the test user), but that is fine — Nuclei probes path
+traversal, parameter injection, and SQLi templates against the URL
+pattern, not against a specific resource.
+
+Skips:
+  - The Flask `static` endpoint (asset serving, no app logic).
+  - Routes that don't accept GET — Nuclei templates almost exclusively
+    issue GET probes, so POST-only endpoints just generate 405s.
+
+Usage:
+    python scripts/ci/dump_url_map.py http://127.0.0.1:5000 > urls.txt
+"""
+
+import re
+import sys
+
+
+# Map Flask URL converters to a placeholder that satisfies the converter
+# so Flask routes the request to the handler instead of 404-ing at the
+# converter stage. Anything not listed falls back to a plain string.
+_PLACEHOLDERS = {
+    "int": "1",
+    "float": "1",
+    "uuid": "00000000-0000-0000-0000-000000000000",
+}
+_DEFAULT_PLACEHOLDER = "nuclei"
+
+_PARAM_RE = re.compile(
+    r"<(?:(?P<conv>[a-zA-Z_][a-zA-Z0-9_]*)(?:\([^)]*\))?:)?(?P<name>[a-zA-Z_][a-zA-Z0-9_]*)>"
+)
+
+
+def _substitute(match: "re.Match[str]") -> str:
+    return _PLACEHOLDERS.get(match.group("conv") or "", _DEFAULT_PLACEHOLDER)
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} BASE_URL", file=sys.stderr)
+        return 2
+
+    base_url = sys.argv[1].rstrip("/")
+
+    from local_deep_research.web.app_factory import create_app
+
+    app, _ = create_app()
+
+    seen: set[str] = set()
+    for rule in app.url_map.iter_rules():
+        if rule.endpoint == "static":
+            continue
+        if "GET" not in (rule.methods or set()):
+            continue
+        path = _PARAM_RE.sub(_substitute, rule.rule)
+        url = f"{base_url}{path}"
+        if url in seen:
+            continue
+        seen.add(url)
+        print(url)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/init_test_database.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+Initialize test database with pre-created test user for CI.
+
+This script creates the test_admin user BEFORE tests run, avoiding the slow
+registration process (2+ min) which:
+1. Creates encrypted SQLCipher database
+2. Derives encryption keys from password
+3. Creates 58 database tables
+4. Imports 500+ settings from JSON files
+
+Usage:
+    python scripts/ci/init_test_database.py
+
+Environment variables:
+    LDR_DATA_DIR: Directory for database files (required for path consistency)
+    TEST_ENV: Should be "true" for test environment
+
+Note: Test user credentials must match CI_TEST_USER in tests/ui_tests/auth_helper.js
+"""
+
+import os
+from pathlib import Path
+
+
+def main():
+    """Create test database and test_admin user."""
+    # Test user credentials - must match CI_TEST_USER in auth_helper.js
+    TEST_USERNAME = "test_admin"
+    TEST_PASSWORD = "testpass123"  # pragma: allowlist secret
+
+    # Use LDR_DATA_DIR if set, otherwise default
+    data_dir = Path(
+        os.environ.get(
+            "LDR_DATA_DIR",
+            Path.home() / ".local" / "share" / "local-deep-research",
+        )
+    )
+    print(f"Using data directory: {data_dir}")
+    data_dir.mkdir(parents=True, exist_ok=True)
+    (data_dir / "encrypted_databases").mkdir(parents=True, exist_ok=True)
+
+    # Import after setting up paths
+    from local_deep_research.database.auth_db import (
+        get_auth_db_session,
+        init_auth_database,
+    )
+    from local_deep_research.database.encrypted_db import db_manager
+    from local_deep_research.database.models.auth import User
+
+    # Initialize auth database
+    init_auth_database()
+
+    # Create test user in auth database (no password stored)
+    session = get_auth_db_session()
+    user = User(username=TEST_USERNAME)
+    session.add(user)
+    session.commit()
+    session.close()
+
+    # Create user's encrypted database with password
+    db_manager.create_user_database(TEST_USERNAME, TEST_PASSWORD)
+
+    print("✅ Database initialized successfully")
+    print(f"✅ Test user '{TEST_USERNAME}' created")
+    print(f"   Encrypted databases in: {data_dir / 'encrypted_databases'}")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/seed_link_analytics.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""Seed link-analytics test data for the `link-analytics` UI shard.
+
+The /metrics/links page derives its domain cards entirely from
+``ResearchResource`` rows (see ``get_link_analytics`` in
+``web/routes/metrics_routes.py``). A freshly-initialized CI database
+(``scripts/ci/init_test_database.py``) has none, so the page renders the
+"No domain data available" placeholder and ``test_link_analytics_full.js``
+times out waiting for ``#domain-list .ldr-domain-item-expanded``.
+
+This script inserts a small, deterministic fixture (2 researches, 7 resources
+across 4 domains) for the ``test_admin`` user so the page renders real domain
+cards with frequency/diversity badges and a populated "Recent Researches
+(N total)" header.
+
+It is wired into ``.github/workflows/docker-tests.yml`` **only** for the
+``link-analytics`` shard. Other shards (and the responsive / empty-state
+screenshot suites that assert the placeholder on /metrics/links) keep seeing
+an empty database.
+
+Usage:
+    python scripts/ci/seed_link_analytics.py
+
+Environment variables (must match init_test_database.py / the server):
+    LDR_DATA_DIR: Directory for database files.
+    LDR_TEST_MODE / LDR_DB_CONFIG_KDF_ITERATIONS: SQLCipher key-derivation
+        parameters — must match the values used to create the DB or the
+        encryption key will not match.
+"""
+
+import os
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+# Credentials must match CI_TEST_USER in tests/ui_tests/auth_helper.js and the
+# user created by scripts/ci/init_test_database.py.
+TEST_USERNAME = "test_admin"
+TEST_PASSWORD = "testpass123"  # pragma: allowlist secret
+
+# A stable namespace UUID so re-running on a fresh DB always produces the same
+# ids (keeps the fixture idempotent and easy to reason about).
+_SEED_NS = uuid.UUID("00000000-0000-0000-0000-00000000a11a")
+
+
+def main():
+    data_dir = Path(
+        os.environ.get(
+            "LDR_DATA_DIR",
+            Path.home() / ".local" / "share" / "local-deep-research",
+        )
+    )
+    print(f"Using data directory: {data_dir}")
+
+    # Imported after the env is set so path/SQLCipher config is picked up.
+    from local_deep_research.database.encrypted_db import db_manager
+    from local_deep_research.database.models.research import (
+        ResearchHistory,
+        ResearchResource,
+    )
+
+    engine = db_manager.open_user_database(TEST_USERNAME, TEST_PASSWORD)
+    if engine is None:
+        raise RuntimeError(
+            f"Could not open encrypted database for user '{TEST_USERNAME}'. "
+            "Was init_test_database.py run first with matching "
+            "LDR_TEST_MODE / LDR_DB_CONFIG_KDF_ITERATIONS?"
+        )
+
+    research_a = str(uuid.uuid5(_SEED_NS, "research-a"))
+    research_b = str(uuid.uuid5(_SEED_NS, "research-b"))
+
+    # created_at lands a day ago so it falls inside the default 30d window the
+    # /metrics/links page queries with. Stored as ISO strings to match the
+    # rest of the codebase (created_at columns are Text/String, not DateTime).
+    created = (datetime.now(UTC) - timedelta(days=1)).isoformat()
+
+    researches = [
+        ResearchHistory(
+            id=research_a,
+            query="Seed: deep learning survey for link analytics",
+            mode="quick_summary",
+            status="completed",
+            created_at=created,
+            completed_at=created,
+        ),
+        ResearchHistory(
+            id=research_b,
+            query="Seed: transformer architectures for link analytics",
+            mode="quick_summary",
+            status="completed",
+            created_at=created,
+            completed_at=created,
+        ),
+    ]
+
+    # (research_id, url, title, source_type). Spread across 4 domains with
+    # uneven counts so frequency_rank / usage_percentage vary, and across both
+    # researches so research_diversity is >1 for the busiest domains.
+    resources = [
+        (
+            research_a,
+            "https://en.wikipedia.org/wiki/Deep_learning",
+            "Deep learning - Wikipedia",
+            "web",
+        ),
+        (
+            research_a,
+            "https://en.wikipedia.org/wiki/Neural_network",
+            "Neural network - Wikipedia",
+            "web",
+        ),
+        (
+            research_a,
+            "https://arxiv.org/abs/1706.03762",
+            "Attention Is All You Need",
+            "academic",
+        ),
+        (
+            research_a,
+            "https://www.nature.com/articles/nature14539",
+            "Deep learning - Nature",
+            "academic",
+        ),
+        (
+            research_b,
+            "https://en.wikipedia.org/wiki/Transformer_(deep_learning_architecture)",
+            "Transformer - Wikipedia",
+            "web",
+        ),
+        (research_b, "https://arxiv.org/abs/1810.04805", "BERT", "academic"),
+        (
+            research_b,
+            "https://github.com/huggingface/transformers",
+            "huggingface/transformers",
+            "code",
+        ),
+    ]
+
+    with Session(engine) as session:
+        # Idempotent: a fresh CI DB is created per attempt, but guard anyway so
+        # a manual re-run does not duplicate the fixture.
+        already = (
+            session.query(ResearchHistory)
+            .filter(ResearchHistory.id == research_a)
+            .first()
+        )
+        if already:
+            print("Link analytics fixture already present — nothing to do.")
+            return
+
+        session.add_all(researches)
+        for research_id, url, title, source_type in resources:
+            session.add(
+                ResearchResource(
+                    research_id=research_id,
+                    url=url,
+                    title=title,
+                    source_type=source_type,
+                    created_at=created,
+                )
+            )
+        session.commit()
+
+    domains = {url.split("/")[2] for _, url, _, _ in resources}
+    print(
+        f"✅ Seeded {len(researches)} researches and {len(resources)} resources "
+        f"across {len(domains)} domains for '{TEST_USERNAME}'"
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/dev/analyze_passthroughs.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""
+Passthrough analysis: catalogue every paper that fell through every
+local tier of the journal filter and ended up at the passthrough score
+(== threshold). Helps decide whether the LLM tier is still worth keeping.
+
+Reuses the same arxiv/openalex/pubmed/s2 dataset as
+test_journal_filter_arxiv.py but routes results through __score_journal
+directly so we can also see *which* tier actually scored each paper.
+"""
+
+import argparse
+import re
+import sys
+from collections import Counter, defaultdict
+from unittest.mock import Mock
+
+from local_deep_research.advanced_search_system.filters import (
+    journal_reputation_filter as _jrf,
+)
+
+# Stub SearXNG so the filter can be instantiated standalone
+_jrf.create_search_engine = lambda *a, **kw: None
+
+from local_deep_research.utilities.thread_context import search_context  # noqa: E402
+from local_deep_research.web_search_engines.engines.search_engine_arxiv import (  # noqa: E402
+    ArXivSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_openalex import (  # noqa: E402
+    OpenAlexSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_pubmed import (  # noqa: E402
+    PubMedSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_semantic_scholar import (  # noqa: E402
+    SemanticScholarSearchEngine,
+)
+
+DOMAIN_QUERIES = {
+    "fusion": "tokamak plasma confinement",
+    "llm": "large language model alignment",
+    "graph_nn": "graph neural networks",
+    "astro": "exoplanet atmosphere spectroscopy",
+    "biomed": "CRISPR gene editing therapy",
+    "condmat": "high temperature superconductor cuprate",
+    "climate": "climate model ocean heat content",
+    "quantum": "quantum error correction surface code",
+    "math": "riemann hypothesis zeta function",
+    "robotics": "reinforcement learning robotic manipulation",
+}
+
+ENGINES = {
+    "arxiv": ArXivSearchEngine,
+    "openalex": OpenAlexSearchEngine,
+    "pubmed": PubMedSearchEngine,
+    "s2": SemanticScholarSearchEngine,
+}
+
+
+def categorize(journal_ref: str, cleaned: str) -> str:
+    """Bucket a passthrough journal_ref into a category."""
+    j = journal_ref.strip()
+    c = cleaned.strip()
+    # Citation-like strings (author initials, "et al.", quotes around title)
+    if re.search(r"^[A-Z]\.\s*[A-Z]", j):
+        return "citation_author"
+    if '"' in j or "“" in j:
+        return "citation_quoted"
+    if re.search(r"\bet al\b", j, re.I):
+        return "citation_et_al"
+    # Cleaning debris (bare year/page leftover)
+    if re.search(r"\b(19|20)\d{2}\b", c):
+        return "cleaning_debris"
+    if re.search(r"\d", c[-10:]):
+        return "cleaning_trailing_num"
+    # Conference (the regex tier already handles these via score=5)
+    if re.search(
+        r"(proc(eedings|\.)?|conference|symp|workshop|colloq)", c, re.I
+    ):
+        return "conference_uncaught"
+    # Truncated long names
+    if len(j) > 50 and j.endswith(("…", "...")):
+        return "truncated"
+    # Looks like a real journal name we just don't have
+    return "real_journal_unknown"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("-n", type=int, default=15)
+    ap.add_argument("-t", "--threshold", type=int, default=4)
+    args = ap.parse_args()
+
+    snapshot = {
+        "search.journal_reputation.threshold": args.threshold,
+        "search.journal_reputation.exclude_non_published": False,
+        "search.journal_reputation.max_context": 3000,
+        "search.journal_reputation.reanalysis_period": 365,
+        "search.engine.web.arxiv.journal_reputation.enabled": True,
+        "search.engine.web.openalex.journal_reputation.enabled": True,
+        "search.engine.web.pubmed.journal_reputation.enabled": True,
+        "search.engine.web.semantic_scholar.journal_reputation.enabled": True,
+    }
+
+    passthroughs: list[tuple[str, str, str, str, str]] = []
+    # (engine, domain, raw_journal_ref, cleaned, category)
+    totals: Counter = Counter()
+
+    for engine_name, cls in ENGINES.items():
+        for domain, q in DOMAIN_QUERIES.items():
+            print(f"  • {engine_name:<9} {domain:<10} {q!r} ...", flush=True)
+            kwargs = {
+                "max_results": args.n,
+                "llm": Mock(),
+                "settings_snapshot": snapshot,
+            }
+            if engine_name in ("pubmed", "s2"):
+                kwargs["optimize_queries"] = False
+            try:
+                with search_context({"username": None, "user_password": None}):
+                    engine = cls(**kwargs)
+                    out = engine.run(q)
+            except Exception as e:
+                print(f"      ! {e}")
+                continue
+
+            for r in out:
+                jref = r.get("journal_ref")
+                if not jref:
+                    totals["no_journal_ref"] += 1
+                    continue
+                qual = r.get("journal_quality")
+                if qual == args.threshold:
+                    # Passthrough — local tiers couldn't score
+                    cleaned = jref  # Filter caches private; approximate
+                    cat = categorize(jref, cleaned)
+                    passthroughs.append(
+                        (engine_name, domain, jref, cleaned, cat)
+                    )
+                    totals[f"passthrough_{cat}"] += 1
+                elif qual is not None:
+                    totals[f"scored_{qual}"] += 1
+
+    print()
+    print("=" * 90)
+    print("OVERALL")
+    print("=" * 90)
+    for k in sorted(totals):
+        print(f"  {k:<35} {totals[k]:>5}")
+
+    print()
+    print("=" * 90)
+    print(f"PASSTHROUGH BY CATEGORY ({len(passthroughs)} total)")
+    print("=" * 90)
+    by_cat: dict[str, list] = defaultdict(list)
+    for row in passthroughs:
+        by_cat[row[4]].append(row)
+    for cat in sorted(by_cat, key=lambda c: -len(by_cat[c])):
+        rows = by_cat[cat]
+        print(f"\n[{cat}]  ({len(rows)} entries)")
+        for engine_name, domain, jref, cleaned, _ in rows[:8]:
+            print(f"  {engine_name:<9} {domain:<10} {jref[:65]}")
+        if len(rows) > 8:
+            print(f"  ... and {len(rows) - 8} more")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev/debug_pytest.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""Debug script to run pytest with CI settings and see what's failing.
+
+Do not remove — developer utility for reproducing CI failures locally.
+"""
+
+import os
+import subprocess
+import sys
+
+# Set CI environment to mimic CI behavior
+os.environ["CI"] = "true"
+os.environ["USE_FALLBACK_LLM"] = "true"
+
+cmd = [
+    sys.executable,
+    "run_tests.py",
+    "-m",
+    "not slow",
+    "--ignore=tests/searxng/",
+    "--ignore=tests/unit/test_config.py",
+    "--ignore=tests/api_tests/",
+    "--ignore=tests/health_check/test_endpoints_health.py",
+    "-v",
+    "--tb=short",
+    "--timeout=30",
+    "-x",  # Stop on first failure to debug
+]
+
+print(f"Running: {' '.join(cmd)}")
+result = subprocess.run(cmd)
+sys.exit(result.returncode)

scripts/dev/regenerate_golden_master.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Regenerate the golden master settings snapshot.
+
+Usage:
+    python scripts/dev/regenerate_golden_master.py
+
+NOTE: This script is called by the pre-commit hook
+.pre-commit-hooks/check-golden-master-settings.py — do not delete.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Resolve paths
+# ---------------------------------------------------------------------------
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+SRC_DIR = PROJECT_ROOT / "src"
+GOLDEN_MASTER_PATH = (
+    PROJECT_ROOT / "tests" / "settings" / "golden_master_settings.json"
+)
+
+# Ensure the package is importable
+if str(SRC_DIR) not in sys.path:
+    sys.path.insert(0, str(SRC_DIR))
+
+
+def main() -> int:
+    from local_deep_research.settings.manager import SettingsManager
+
+    manager = SettingsManager(db_session=None)
+    defaults = manager.default_settings
+
+    current = {key: dict(defaults[key]) for key in sorted(defaults.keys())}
+    output = (
+        json.dumps(
+            current, indent=2, sort_keys=True, default=str, ensure_ascii=False
+        )
+        + "\n"
+    )
+
+    GOLDEN_MASTER_PATH.write_text(output, encoding="utf-8")
+
+    print(f"Wrote {len(current)} settings to {GOLDEN_MASTER_PATH}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev/restart_server.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+# Restart the LDR dev server.
+#
+# Usage:
+#   scripts/dev/restart_server.sh [PORT] [--debug] [--tmp]
+#
+#   PORT      Optional port (default: 5000). Passed via LDR_WEB_PORT.
+#   --debug   DEBUG logging (LDR_APP_DEBUG=true, LDR_LOG_SETTINGS=summary).
+#             WARNING: debug logs may contain sensitive data (queries,
+#             answers, API responses) — local dev / feature testing only,
+#             never production or real user data.
+#   --tmp     Disposable instance: point LDR_DATA_DIR at a throwaway dir
+#             (default /tmp/ldr-test) so encrypted user DBs, the auth DB,
+#             library, logs, research_outputs etc. land OUTSIDE your real
+#             data dir (~/.local/share/local-deep-research). Override the
+#             location by exporting LDR_DATA_DIR before running. Reuses one
+#             dir (doesn't accumulate throwaway databases). To start from a
+#             clean slate, wipe LDR_DATA_DIR yourself first (e.g.
+#             `rm -rf "$LDR_DATA_DIR"`) — this script never deletes data.
+#
+# Only the instance listening on the target PORT is stopped, so multiple
+# instances on different ports (e.g. several agents each on their own port)
+# coexist — restarting one does not kill the others.
+#
+# Referenced in:
+#   - tests/api_tests_with_login/README.md
+#   - examples/api_usage/http/README.md
+#   - examples/api_usage/http/advanced/simple_http_example.py
+# Do not delete without updating those references.
+
+set -u
+
+usage() {
+    cat <<'EOF'
+Restart the LDR dev server.
+
+Usage:
+  scripts/dev/restart_server.sh [PORT] [--debug] [--tmp]
+
+  PORT      Optional port (default: 5000).
+  --debug   DEBUG logging (may log sensitive data; local dev only).
+  --tmp     Disposable instance: LDR_DATA_DIR -> /tmp/ldr-test (override by
+            exporting LDR_DATA_DIR). Keeps test data out of your real dir.
+            This script never deletes data; wipe LDR_DATA_DIR yourself to
+            start from a clean slate.
+  --test    Alias for --tmp.
+
+Only the instance on the target PORT is stopped; instances on other ports
+keep running.
+EOF
+}
+
+PORT=5000
+debug=0
+use_tmp=0
+
+for arg in "$@"; do
+    case "$arg" in
+        --debug)    debug=1 ;;
+        --tmp|--test) use_tmp=1 ;;
+        -h|--help)  usage; exit 0 ;;
+        ''|*[!0-9]*) echo "Unknown argument: $arg" >&2; usage >&2; exit 2 ;;
+        *)          PORT="$arg" ;;
+    esac
+done
+
+echo "Stopping existing LDR server on port ${PORT} (if any)..."
+# Port-scoped stop: kill only the process LISTENing on PORT, leaving
+# instances on other ports running. The port isn't in the process argv
+# (it comes from LDR_WEB_PORT), so a broad pkill can't distinguish them.
+if command -v fuser >/dev/null 2>&1; then
+    fuser -k "${PORT}/tcp" 2>/dev/null || echo "No existing server on port ${PORT}"
+elif command -v lsof >/dev/null 2>&1; then
+    pids=$(lsof -ti "tcp:${PORT}" -sTCP:LISTEN 2>/dev/null)
+    if [ -n "${pids}" ]; then
+        # $pids may be several PIDs; word-splitting is intended here.
+        # shellcheck disable=SC2086
+        kill ${pids} 2>/dev/null
+    else
+        echo "No existing server on port ${PORT}"
+    fi
+else
+    echo "Warning: neither fuser nor lsof found; cannot port-scope the stop." >&2
+    echo "Skipping stop to avoid killing instances on other ports." >&2
+fi
+
+sleep 1
+
+# Change to the project root.
+cd "$(dirname "$0")/../.." || exit 1
+
+# --- environment for the launched server -------------------------------------
+env_args=(LDR_WEB_PORT="${PORT}")
+
+if [ "$use_tmp" -eq 1 ]; then
+    : "${LDR_DATA_DIR:=/tmp/ldr-test}"
+    mkdir -p "$LDR_DATA_DIR"
+    echo "Using disposable LDR_DATA_DIR=$LDR_DATA_DIR"
+    env_args+=(LDR_DATA_DIR="${LDR_DATA_DIR}")
+fi
+
+if [ "$debug" -eq 1 ]; then
+    env_args+=(LDR_APP_DEBUG=true LDR_LOG_SETTINGS=summary)
+fi
+
+LOG_FILE="/tmp/ldr_server_${PORT}.log"
+
+if [ "$debug" -eq 1 ]; then
+    echo "Starting LDR server on port ${PORT} (DEBUG)..."
+else
+    echo "Starting LDR server on port ${PORT}..."
+fi
+# Start server in background and detach from terminal. Backgrounding nohup
+# directly (rather than wrapping in an extra `( ... & ) &` subshell) means
+# $! captures the real server PID, not a transient subshell that has already
+# exited. nohup ignores SIGHUP, so the server survives the launcher exiting
+# and is reparented to init.
+nohup env "${env_args[@]}" pdm run python -m local_deep_research.web.app > "${LOG_FILE}" 2>&1 &
+SERVER_PID=$!
+
+sleep 2
+
+echo "Server started. PID: ${SERVER_PID}"
+echo "Logs: ${LOG_FILE}"
+echo "URL: http://127.0.0.1:${PORT}"
+if [ "$debug" -eq 1 ]; then
+    echo ""
+    echo "WARNING: DEBUG mode — logs may contain sensitive data; local dev only."
+fi
+echo ""
+echo "To view logs: tail -f ${LOG_FILE}"
+echo "To stop server: fuser -k ${PORT}/tcp"
+
+exit 0

scripts/dev/run_tests.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Run tests for Local Deep Research.
+
+This script runs pytest with appropriate configuration for the project.
+
+Referenced in CONTRIBUTING.md — do not delete without updating that file.
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main():
+    """Run the test suite with appropriate settings."""
+    # Get the project root directory
+    project_root = Path(__file__).parent.absolute()
+
+    # Ensure PYTHONPATH includes the project root for proper imports
+    os.environ["PYTHONPATH"] = str(project_root)
+
+    # Configure pytest arguments
+    pytest_args = [
+        sys.executable,  # Use Python interpreter
+        "-m",  # Run module
+        "pytest",  # Call pytest
+        "--verbose",  # Verbose output
+        "--color=yes",  # Force colored output
+        "--cov=src",  # Measure coverage for src directory
+        "--cov-report=term",  # Report coverage in terminal
+        "--cov-report=html:coverage_html",  # Also generate HTML report
+        "--cov-config=.coveragerc",  # Use the coverage configuration file
+    ]
+
+    # Add any command line arguments passed to this script
+    pytest_args.extend(sys.argv[1:])
+
+    # Print the command being run
+    print(f"Running: {' '.join(pytest_args)}")
+
+    # Run pytest and capture the return code
+    result = subprocess.run(pytest_args)
+
+    # Return the pytest exit code
+    return result.returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev/stop_server.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Script to stop the LDR server.
+# Do not remove — companion to restart_server.sh; handles graceful
+# and forced shutdown of the dev server.
+
+echo "Stopping LDR server..."
+
+# Kill the main server process
+if pkill -f "python -m local_deep_research.web.app" 2>/dev/null; then
+    echo "✓ Server stopped successfully"
+
+    # Wait a moment for the process to fully terminate
+    sleep 1
+
+    # Check if any processes are still running
+    if pgrep -f "python -m local_deep_research.web.app" > /dev/null; then
+        echo "Warning: Some server processes may still be running"
+        echo "Attempting force stop..."
+        pkill -9 -f "python -m local_deep_research.web.app" 2>/dev/null
+        sleep 1
+    fi
+else
+    echo "No running LDR server found"
+fi
+
+# Also stop any orphaned Flask dev servers
+pkill -f "flask run" 2>/dev/null
+
+# Check final status
+if pgrep -f "python -m local_deep_research.web.app" > /dev/null; then
+    echo "Error: Server processes still running:"
+    pgrep -af "python -m local_deep_research.web.app"
+    exit 1
+else
+    echo "All server processes stopped"
+    exit 0
+fi

scripts/dev/test_journal_filter_arxiv.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""
+Probe the journal reputation filter against live arXiv results.
+
+Runs the *full* ArXivSearchEngine pipeline (including the journal filter
+as a preview filter) and prints which papers passed/were dropped, along
+with the journal_quality score the filter attached.
+
+Usage:
+    python scripts/dev/test_journal_filter_arxiv.py "graph neural networks" -n 25
+"""
+
+import argparse
+import sys
+from unittest.mock import Mock
+
+from local_deep_research.utilities.thread_context import search_context
+from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
+    ArXivSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_openalex import (
+    OpenAlexSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_pubmed import (
+    PubMedSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_semantic_scholar import (
+    SemanticScholarSearchEngine,
+)
+
+ENGINES = {
+    "arxiv": ArXivSearchEngine,
+    "openalex": OpenAlexSearchEngine,
+    "pubmed": PubMedSearchEngine,
+    "s2": SemanticScholarSearchEngine,
+}
+
+
+DOMAIN_QUERIES = {
+    "fusion": "tokamak plasma confinement",
+    "llm": "large language model alignment",
+    "graph_nn": "graph neural networks",
+    "astro": "exoplanet atmosphere spectroscopy",
+    "biomed": "CRISPR gene editing therapy",
+    "condmat": "high temperature superconductor cuprate",
+    "climate": "climate model ocean heat content",
+    "quantum": "quantum error correction surface code",
+    "math": "riemann hypothesis zeta function",
+    "robotics": "reinforcement learning robotic manipulation",
+}
+
+
+def run_one(
+    query: str,
+    n: int,
+    threshold: int,
+    snapshot: dict,
+    engine_name: str = "arxiv",
+) -> dict:
+    cls = ENGINES[engine_name]
+    kwargs = {"max_results": n, "llm": Mock(), "settings_snapshot": snapshot}
+    # PubMed/S2 use the LLM for query optimization — Mock would corrupt
+    # the optimized query string. Disable optimization for the probe so
+    # we send the raw query verbatim.
+    if engine_name in ("pubmed", "s2"):
+        kwargs["optimize_queries"] = False
+    with search_context({"username": None, "user_password": None}):
+        try:
+            engine = cls(**kwargs)
+            out = engine.run(query)
+        except Exception as e:
+            return {
+                "query": query,
+                "n_total": 0,
+                "n_no_jref": 0,
+                "n_scored": 0,
+                "avg_q": 0,
+                "high": 0,
+                "passthrough": 0,
+                "results": [],
+                "error": str(e)[:80],
+            }
+
+    scored = [r for r in out if "journal_quality" in r]
+    no_jref = [r for r in out if not r.get("journal_ref")]
+    qualities = [r["journal_quality"] for r in scored]
+    return {
+        "query": query,
+        "n_total": len(out),
+        "n_no_jref": len(no_jref),
+        "n_scored": len(scored),
+        "avg_q": sum(qualities) / len(qualities) if qualities else 0,
+        "high": sum(1 for q in qualities if q >= 7),
+        "passthrough": sum(1 for q in qualities if q == threshold),
+        "results": out,
+    }
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument(
+        "query", nargs="?", help="arXiv search query (omit to run dataset)"
+    )
+    ap.add_argument("-n", type=int, default=20, help="number of papers")
+    ap.add_argument("-t", "--threshold", type=int, default=4)
+    ap.add_argument(
+        "--dataset",
+        action="store_true",
+        help="run the multi-domain dataset of queries",
+    )
+    ap.add_argument(
+        "--engines",
+        default="arxiv",
+        help="comma-separated list: arxiv,openalex,pubmed,s2 (or 'all')",
+    )
+    args = ap.parse_args()
+    if args.engines == "all":
+        engine_names = list(ENGINES)
+    else:
+        engine_names = [e.strip() for e in args.engines.split(",")]
+
+    # Minimal settings snapshot — defaults are returned for missing keys.
+    # Threshold and arxiv journal-filter enable flag are wired explicitly.
+    snapshot = {
+        "search.journal_reputation.threshold": args.threshold,
+        "search.journal_reputation.exclude_non_published": False,
+        "search.journal_reputation.max_context": 3000,
+        "search.journal_reputation.reanalysis_period": 365,
+        "search.engine.web.arxiv.journal_reputation.enabled": True,
+        "search.engine.web.openalex.journal_reputation.enabled": True,
+        "search.engine.web.pubmed.journal_reputation.enabled": True,
+        "search.engine.web.semantic_scholar.journal_reputation.enabled": True,
+    }
+
+    if args.dataset:
+        print(
+            f"Running dataset of {len(DOMAIN_QUERIES)} domain queries × "
+            f"{len(engine_names)} engines (n={args.n}, threshold={args.threshold})\n"
+        )
+        all_rows = []
+        for engine_name in engine_names:
+            for domain, q in DOMAIN_QUERIES.items():
+                print(
+                    f"  • {engine_name:<9} {domain:<10} {q!r} ...", flush=True
+                )
+                s = run_one(q, args.n, args.threshold, snapshot, engine_name)
+                all_rows.append((engine_name, domain, s))
+
+        print(
+            f"\n{'engine':<9} {'domain':<10} {'tot':>4} {'noJ':>4} "
+            f"{'scor':>4} {'avg':>5} {'≥7':>3} {'pass':>4}  err"
+        )
+        print("-" * 80)
+        for engine_name, domain, s in all_rows:
+            err = s.get("error", "")
+            print(
+                f"{engine_name:<9} {domain:<10} "
+                f"{s['n_total']:>4} {s['n_no_jref']:>4} {s['n_scored']:>4} "
+                f"{s['avg_q']:>5.1f} {s['high']:>3} {s['passthrough']:>4}  {err}"
+            )
+        return 0
+
+    if not args.query:
+        ap.error("query required (or use --dataset)")
+
+    with search_context({"username": None, "user_password": None}):
+        engine = ArXivSearchEngine(
+            max_results=args.n, llm=Mock(), settings_snapshot=snapshot
+        )
+        print(f"Running ArXivSearchEngine for: {args.query!r}\n")
+        out = engine.run(args.query)
+
+    print(f"{'#':<3} {'qual':<5} {'journal_ref':<45} title")
+    print("-" * 120)
+    for i, r in enumerate(out, 1):
+        q = r.get("journal_quality", "—")
+        jref = (r.get("journal_ref") or "—")[:43]
+        title = (r.get("title") or "")[:60]
+        print(f"{i:<3} {str(q):<5} {jref:<45} {title}")
+
+    print(
+        f"\nReturned {len(out)} results from pipeline "
+        f"(threshold={args.threshold})."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/generate_config_docs.py

@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+"""
+Generate CONFIGURATION.md from default settings JSON files and env_definitions.
+
+Usage:
+    python scripts/generate_config_docs.py                    # Write to docs/CONFIGURATION.md
+    python scripts/generate_config_docs.py --output /tmp/out  # Write to custom location
+    python scripts/generate_config_docs.py --check            # Exit 1 if docs are stale
+"""
+
+import argparse
+import ast
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def get_project_root() -> Path:
+    """Return the project root directory."""
+    return Path(__file__).resolve().parent.parent
+
+
+def get_env_var_name(key: str) -> str:
+    """Convert setting key to environment variable name."""
+    return f"LDR_{key.replace('.', '_').upper()}"
+
+
+def format_value(value: Any) -> str:
+    """Format default value for markdown."""
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return str(value).lower()
+    if isinstance(value, (dict, list)):
+        return f"`{json.dumps(value)}`"
+    return str(value)
+
+
+def _discover_env_definition_files(env_defs_dir: Path) -> List[Path]:
+    """Auto-discover env_definitions modules, excluding __init__.py and env_settings.py."""
+    if not env_defs_dir.is_dir():
+        return []
+    return sorted(
+        p
+        for p in env_defs_dir.glob("*.py")
+        if p.name not in ("__init__.py", "env_settings.py")
+    )
+
+
+def _category_from_filename(filename: str) -> str:
+    """Derive a human-readable category name from a filename.
+
+    Example: 'db_config.py' -> 'Db Config'
+    """
+    stem = filename.removesuffix(".py")
+    return stem.replace("_", " ").title()
+
+
+def _extract_setting_from_call(node: ast.Call) -> Optional[Dict[str, Any]]:
+    """Extract a setting dict from a *Setting() AST call node."""
+    if not (
+        isinstance(node.func, ast.Name) and node.func.id.endswith("Setting")
+    ):
+        return None
+
+    keywords = {k.arg: k.value for k in node.keywords if k.arg}
+
+    if "key" not in keywords:
+        return None
+
+    key_node = keywords["key"]
+    if not isinstance(key_node, ast.Constant):
+        return None
+    key = key_node.value  # gitleaks:allow
+
+    # Description — may be a simple string or a parenthesised concatenation
+    description = ""
+    if "description" in keywords:
+        desc_node = keywords["description"]
+        if isinstance(desc_node, ast.Constant):
+            description = desc_node.value
+        else:
+            description = ast.unparse(desc_node)
+
+    # Default
+    default_val = "None"
+    if "default" in keywords:
+        default_val = ast.unparse(keywords["default"])
+
+    # Env var (auto-generated unless explicitly overridden)
+    if "env_var" in keywords and isinstance(keywords["env_var"], ast.Constant):
+        env_var = keywords["env_var"].value
+    else:
+        env_var = get_env_var_name(key)
+
+    # Type from the class name (e.g. BooleanSetting -> Boolean)
+    setting_type = node.func.id.replace("Setting", "")
+
+    # Required
+    required = False
+    if "required" in keywords and isinstance(
+        keywords["required"], ast.Constant
+    ):
+        required = bool(keywords["required"].value)
+
+    # Min/max value
+    min_value = None
+    if "min_value" in keywords and isinstance(
+        keywords["min_value"], ast.Constant
+    ):
+        min_value = keywords["min_value"].value
+
+    max_value = None
+    if "max_value" in keywords and isinstance(
+        keywords["max_value"], ast.Constant
+    ):
+        max_value = keywords["max_value"].value
+
+    # Allowed values (ast.Set of ast.Constant)
+    allowed_values = None
+    if "allowed_values" in keywords and isinstance(
+        keywords["allowed_values"], ast.Set
+    ):
+        allowed_values = sorted(
+            elt.value
+            for elt in keywords["allowed_values"].elts
+            if isinstance(elt, ast.Constant)
+        )
+
+    # Deprecated env var
+    deprecated_env_var = None
+    if "deprecated_env_var" in keywords and isinstance(
+        keywords["deprecated_env_var"], ast.Constant
+    ):
+        deprecated_env_var = keywords["deprecated_env_var"].value
+
+    return {
+        "key": key,
+        "env_var": env_var,
+        "description": description,
+        "default": default_val,
+        "type": setting_type,
+        "required": required,
+        "min_value": min_value,
+        "max_value": max_value,
+        "allowed_values": allowed_values,
+        "deprecated_env_var": deprecated_env_var,
+    }
+
+
+def get_env_only_settings(
+    root_dir: Optional[Path] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Extract env-only settings from env_definitions/ by auto-discovering modules.
+
+    These are settings required before database initialization.
+    """
+    root_dir = root_dir or get_project_root()
+    env_defs_dir = (
+        root_dir
+        / "src"
+        / "local_deep_research"
+        / "settings"
+        / "env_definitions"
+    )
+
+    env_only: List[Dict[str, Any]] = []
+
+    for filepath in _discover_env_definition_files(env_defs_dir):
+        category = _category_from_filename(filepath.name)
+
+        try:
+            content = filepath.read_text(encoding="utf-8")
+            tree = ast.parse(content)
+        except Exception as e:
+            print(f"Warning: Could not parse {filepath}: {e}")
+            continue
+
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Call):
+                continue
+
+            setting = _extract_setting_from_call(node)
+            if setting is None:
+                continue
+
+            setting["category"] = category
+            env_only.append(setting)
+
+    return env_only
+
+
+def _format_constraints(setting: Dict[str, Any]) -> str:
+    """Build a human-readable constraints string."""
+    parts = []
+    if (
+        setting.get("min_value") is not None
+        and setting.get("max_value") is not None
+    ):
+        parts.append(f"{setting['min_value']}..{setting['max_value']}")
+    elif setting.get("min_value") is not None:
+        parts.append(f">={setting['min_value']}")
+    elif setting.get("max_value") is not None:
+        parts.append(f"<={setting['max_value']}")
+
+    if setting.get("allowed_values"):
+        parts.append(", ".join(setting["allowed_values"]))
+
+    return " | ".join(parts) if parts else ""
+
+
+def generate_docs_content(root_dir: Optional[Path] = None) -> str:
+    """Generate the full CONFIGURATION.md content as a string."""
+    root_dir = root_dir or get_project_root()
+    defaults_dir = root_dir / "src" / "local_deep_research" / "defaults"
+
+    settings: Dict[str, Any] = {}
+
+    # Recursively find all JSON files
+    for json_file in sorted(defaults_dir.rglob("*.json")):
+        try:
+            with open(json_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                settings.update(data)
+        except Exception as e:
+            print(f"Warning: Could not load {json_file}: {e}")
+
+    sorted_keys = sorted(settings.keys())
+
+    # Get env-only settings
+    env_only_settings = get_env_only_settings(root_dir)
+
+    # Build markdown
+    content = [
+        "# Configuration Reference",
+        "",
+        "This document is automatically generated from the application's default settings.",
+        "All settings can be configured via the Web UI (Settings page), or overridden via Environment Variables.",
+        "",
+        "## Environment Variables",
+        "",
+        "To override a setting using an environment variable, convert the key to uppercase, replace dots with underscores, and prefix with `LDR_`.",
+        "For example, `app.debug` becomes `LDR_APP_DEBUG`.",
+        "",
+        "Configuration Priority: Web UI Config > Environment Variables > Default Values",
+        "> Environmental Variables are used to override default values, easing installation, while allowing for adjustments to configuration via Web UI.",
+        "",
+        "### System Locking",
+        "There is a special environment variable `LDR_LOCKED_SETTINGS` that allows administrators to strictly enforce specific settings.",
+        "",
+        "*   **Variable**: `LDR_LOCKED_SETTINGS`",
+        "*   **Format**: Comma-separated list of setting keys (e.g., `llm.model,app.port`)",
+        "*   **Behavior**:",
+        "    1.  Any setting listed here **MUST** have a corresponding value defined in the environment variables (e.g., `LDR_LLM_MODEL`). If not, the application will fail to start.",
+        "    2.  The setting becomes **read-only** in the Web UI.",
+        "    3.  The **Environment Variable** value takes absolute precedence, ignoring any value in the database.",
+        "",
+        "**Priority for Locked Settings**: Environment Variable > Database (Ignored) > Default (Ignored)",
+        "",
+        "",
+    ]
+
+    # Env-only section with expanded columns
+    if env_only_settings:
+        content.extend(
+            [
+                "## Pre-Database (Env-Only) Settings",
+                "",
+                "These settings are **required before database initialization** and can only be set via environment variables.",
+                "They are not available in the Web UI because they are needed to start the application.",
+                "",
+                "| Environment Variable | Type | Default | Required | Constraints | Description | Category | Deprecated Alias |",
+                "|----------------------|------|---------|----------|-------------|-------------|----------|------------------|",
+            ]
+        )
+
+        for setting in sorted(env_only_settings, key=lambda x: x["env_var"]):
+            env_var = setting["env_var"]
+            stype = setting["type"]
+            default = setting["default"]
+            required = "Yes" if setting.get("required") else "No"
+            constraints = _format_constraints(setting).replace("|", "\\|")
+            desc = setting["description"].replace("|", "\\|").replace("\n", " ")
+            category = setting["category"]
+            deprecated = setting.get("deprecated_env_var") or ""
+
+            row = (
+                f"| `{env_var}` | {stype} | `{default}` | {required} "
+                f"| {constraints} | {desc} | {category} | {deprecated} |"
+            )
+            content.append(row)
+
+        content.extend(["", ""])
+
+    # Main settings list
+    content.extend(
+        [
+            "## Settings List",
+            "",
+            "| Key | Environment Variable | Default Value | Description | Type |",
+            "|-----|----------------------|---------------|-------------|------|",
+        ]
+    )
+
+    for key in sorted_keys:
+        setting = settings[key]
+        env_var = setting.get("env_var") or get_env_var_name(key)
+        default_val = format_value(setting.get("value"))
+        description = (
+            setting.get("description", "")
+            .replace("\n", " ")
+            .replace("|", "\\|")
+        )
+        setting_type = setting.get("type", "UNKNOWN")
+
+        row = f"| `{key}` | `{env_var}` | `{default_val}` | {description} | {setting_type} |"
+        content.append(row)
+
+    content.append("")
+    content.append("*Generated by scripts/generate_config_docs.py*")
+
+    return "\n".join(content) + "\n"
+
+
+def generate_docs(
+    output_path: Optional[Path] = None,
+    check: bool = False,
+) -> int:
+    """Generate (or check) CONFIGURATION.md.
+
+    Returns 0 on success, 1 if check finds stale docs.
+    """
+    root_dir = get_project_root()
+    output_file = output_path or (root_dir / "docs" / "CONFIGURATION.md")
+
+    new_content = generate_docs_content(root_dir)
+
+    if check:
+        if not output_file.exists():
+            print(
+                f"FAIL: {output_file} does not exist. "
+                "Run 'python scripts/generate_config_docs.py' to generate it."
+            )
+            return 1
+        existing = output_file.read_text(encoding="utf-8")
+        if existing == new_content:
+            print("OK: Configuration docs are up to date.")
+            return 0
+        print(
+            f"FAIL: {output_file} is out of date. "
+            "Run 'python scripts/generate_config_docs.py' to regenerate it."
+        )
+        return 1
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    output_file.write_text(new_content, encoding="utf-8")
+    print(f"Wrote {output_file}")
+    return 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate CONFIGURATION.md from defaults"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        help="Output file path (default: docs/CONFIGURATION.md)",
+    )
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Check if docs are up to date (exit 1 if stale)",
+    )
+    args = parser.parse_args()
+    sys.exit(generate_docs(output_path=args.output, check=args.check))

scripts/ldr-research.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""
+LDR Research Script
+
+Reads a research query from stdin and uses Local Deep Research to find
+relevant documentation, sources, and context. Returns JSON with the
+research output.
+
+Usage:
+    # Pipe a query from stdin
+    echo "What is RAG?" | python scripts/ldr-research.py
+
+    # Or pass a file
+    python scripts/ldr-research.py < query.txt
+
+    # With CLI arguments (easier local testing)
+    python scripts/ldr-research.py --provider openrouter --model gpt-4o < query.txt
+
+Output: JSON with research results, sources, and findings.
+
+Environment variables (can be overridden by CLI args):
+    OPENROUTER_API_KEY - API key for OpenRouter
+    SERPER_API_KEY - API key for Serper.dev search
+    LDR_PROVIDER - LLM provider (default: openrouter)
+    LDR_SEARCH_TOOL - Search tool (default: serper)
+    LDR_RESEARCH_MODEL - Model name (default: google/gemini-2.0-flash-001 for openrouter)
+    LDR_STRATEGY - Search strategy (default: langgraph-agent)
+
+Note: This uses the programmatic API and does NOT require a running LDR server.
+"""
+
+import argparse
+import faulthandler
+import json
+import os
+import sys
+
+# Dump a Python traceback to stderr on SIGABRT/SIGSEGV/SIGFPE/SIGBUS/SIGILL.
+faulthandler.enable()
+
+
+def make_serializable(obj):
+    """Convert objects to JSON-serializable format."""
+    if obj is None:
+        return None
+    if isinstance(obj, (str, int, float, bool)):
+        return obj
+    if isinstance(obj, dict):
+        return {k: make_serializable(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [make_serializable(item) for item in obj]
+    # Handle LangChain Document objects
+    if hasattr(obj, "page_content") and hasattr(obj, "metadata"):
+        return {
+            "content": obj.page_content,
+            "metadata": make_serializable(obj.metadata),
+        }
+    # Handle other objects with __dict__
+    if hasattr(obj, "__dict__"):
+        return make_serializable(obj.__dict__)
+    # Fallback to string representation
+    return str(obj)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Run LDR research on a query read from stdin"
+    )
+    parser.add_argument(
+        "--provider",
+        default=os.environ.get("LDR_PROVIDER", "openrouter"),
+        help="LLM provider (default: openrouter)",
+    )
+    parser.add_argument(
+        "--search-tool",
+        default=os.environ.get("LDR_SEARCH_TOOL", "serper"),
+        help="Search tool (default: serper)",
+    )
+    parser.add_argument(
+        "--model",
+        default=os.environ.get("LDR_RESEARCH_MODEL"),
+        help="Model name (default: provider's default)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=None,
+        help=(
+            "Number of research iterations. If unset, the strategy uses "
+            "its own default (e.g. langgraph-agent reads "
+            "langgraph_agent.max_iterations from settings)."
+        ),
+    )
+    parser.add_argument(
+        "--strategy",
+        default=os.environ.get("LDR_STRATEGY", "langgraph-agent"),
+        help="Search strategy name (default: langgraph-agent)",
+    )
+    return parser.parse_args()
+
+
+def main():
+    # Flush in finally: SIGABRT during interpreter shutdown won't drain the stdout buffer.
+    try:
+        args = parse_args()
+
+        # Read query from stdin
+        query = sys.stdin.read().strip()
+        if not query:
+            print(json.dumps({"error": "No query provided on stdin"}))
+            sys.exit(1)
+
+        # Default model for OpenRouter if not specified
+        model_name = args.model
+        if not model_name and args.provider == "openrouter":
+            model_name = "google/gemini-2.0-flash-001"
+
+        # Check required API keys
+        if args.provider == "openrouter" and not os.environ.get(
+            "OPENROUTER_API_KEY"
+        ):
+            print(json.dumps({"error": "OPENROUTER_API_KEY not set"}))
+            sys.exit(1)
+
+        if args.search_tool == "serper" and not os.environ.get(
+            "SERPER_API_KEY"
+        ):
+            print(json.dumps({"error": "SERPER_API_KEY not set"}))
+            sys.exit(1)
+
+        try:
+            from local_deep_research.api import quick_summary
+            from local_deep_research.api.settings_utils import (
+                create_settings_snapshot,
+            )
+
+            # Build settings overrides
+            overrides = {
+                "search.tool": args.search_tool,
+                "llm.provider": args.provider,
+            }
+            if model_name:
+                overrides["llm.model"] = model_name
+
+            # Add API keys from environment
+            if os.environ.get("OPENROUTER_API_KEY"):
+                overrides["llm.openrouter.api_key"] = os.environ[
+                    "OPENROUTER_API_KEY"
+                ]
+            if os.environ.get("SERPER_API_KEY"):
+                overrides["search.engine.web.serper.api_key"] = os.environ[
+                    "SERPER_API_KEY"
+                ]
+
+            settings = create_settings_snapshot(overrides=overrides)
+
+            # Build kwargs
+            kwargs = {
+                "query": query,
+                "provider": args.provider,
+                "search_tool": args.search_tool,
+                "settings_snapshot": settings,
+                "programmatic_mode": True,
+                "search_strategy": args.strategy,
+            }
+            if model_name:
+                kwargs["model_name"] = model_name
+            if args.iterations is not None:
+                kwargs["iterations"] = args.iterations
+
+            result = quick_summary(**kwargs)
+
+            # Use formatted_findings if available (already properly formatted with sources)
+            # Fall back to summary if not
+            research_output = result.get("formatted_findings") or result.get(
+                "summary", str(result)
+            )
+
+            # Build output - make sure everything is JSON serializable
+            output = {
+                "research": research_output,
+                "sources": make_serializable(result.get("sources", [])),
+                "findings": make_serializable(result.get("findings", [])),
+                "iterations": result.get("iterations"),
+            }
+
+            print(json.dumps(output))
+
+        except Exception as e:
+            print(json.dumps({"error": str(e)}))
+            sys.exit(1)
+    finally:
+        try:
+            sys.stdout.flush()
+        except Exception:  # noqa: silent-exception
+            pass
+
+
+if __name__ == "__main__":
+    main()

scripts/ldr_entrypoint.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -e
+
+# This entrypoint handles volume permissions for the LDR container.
+# Docker volumes are created with root ownership, but we need them
+# accessible to the ldruser (UID 1000) that runs the application.
+
+echo "Setting up /data directory permissions..."
+
+# Create required subdirectories under /data if they don't exist
+mkdir -p /data/logs
+mkdir -p /data/cache
+mkdir -p /data/cache/rag_indices
+mkdir -p /data/research_outputs
+mkdir -p /data/encrypted_databases
+
+# Set permissions to 700 (owner-only access for security)
+chmod 700 /data/logs
+chmod 700 /data/cache
+chmod 700 /data/cache/rag_indices
+chmod 700 /data/research_outputs
+chmod 700 /data/encrypted_databases
+
+# Fix ownership of /data and all subdirectories
+# This is safe because we're still root at this point (before USER directive takes effect)
+chown -R ldruser:ldruser /data
+
+# Create matplotlib cache directory for ldruser
+echo "Setting up matplotlib cache directory..."
+mkdir -p /home/ldruser/.config/matplotlib
+chown -R ldruser:ldruser /home/ldruser/.config
+chmod -R 700 /home/ldruser/.config
+
+echo "Starting LDR application as ldruser..."
+
+# Switch to ldruser and execute the command.
+# setpriv needs CAP_SETUID and CAP_SETGID to call setuid()/setgid() syscalls.
+# These are granted via cap_add in docker-compose.yml, but in restricted
+# environments (e.g. Proxmox LXC) the outer container may block them.
+if ! setpriv --reuid=ldruser --regid=ldruser --init-groups -- true 2>/dev/null; then
+    echo ""
+    echo "ERROR: Failed to switch to non-root user 'ldruser'."
+    echo ""
+    echo "  setpriv requires CAP_SETUID and CAP_SETGID Linux capabilities."
+    echo "  This typically happens in LXC containers (e.g. Proxmox) that"
+    echo "  restrict these capabilities."
+    echo ""
+    echo "  To fix, ensure your LXC container allows SETUID/SETGID:"
+    echo "    - Proxmox: check 'Features' -> 'Nesting' is enabled"
+    echo "    - Or add to LXC config: lxc.cap.keep = setuid setgid"
+    echo ""
+    echo "  See: https://github.com/LearningCircuit/local-deep-research/issues"
+    echo ""
+    exit 1
+fi
+export HOME=/home/ldruser
+exec setpriv --reuid=ldruser --regid=ldruser --init-groups -- "$@"

scripts/mcp_smoke_test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# MCP Server Smoke Test
+# Verifies that the MCP server module loads correctly and basic tools work
+
+set -e
+
+echo "=== MCP Server Smoke Test ==="
+echo ""
+
+# Test 1: Verify MCP module loads
+echo "1. Testing MCP module loading..."
+python -c "
+from local_deep_research.mcp.server import mcp, list_strategies, get_configuration
+print('   MCP server module loaded successfully')
+print(f'   Server name: {mcp.name}')
+
+# Test discovery tools
+result = list_strategies()
+assert result['status'] == 'success', f'list_strategies failed: {result}'
+print(f'   list_strategies: {len(result[\"strategies\"])} strategies available')
+
+config = get_configuration()
+assert config['status'] == 'success', f'get_configuration failed: {config}'
+print('   get_configuration: works correctly')
+"
+echo "   PASSED"
+echo ""
+
+# Test 2: Verify MCP server startup
+echo "2. Testing MCP server startup..."
+timeout 5 python -m local_deep_research.mcp 2>&1 || {
+    exit_code=$?
+    if [ $exit_code -eq 124 ]; then
+        echo "   Server started correctly (timed out waiting for STDIO input as expected)"
+        echo "   PASSED"
+    else
+        echo "   FAILED: MCP server failed to start with exit code $exit_code"
+        exit 1
+    fi
+}
+
+echo ""
+echo "=== All MCP smoke tests passed ==="

scripts/ollama_entrypoint.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+# Function to display usage information
+usage() {
+  echo "Usage: $0 <model_name>"
+  exit 1
+}
+
+# Check if a model name is provided as an argument
+if [ "$#" -ne 1 ]; then
+  usage
+fi
+
+MODEL_NAME=$1
+
+# Validate model name to prevent command injection
+if ! echo "$MODEL_NAME" | grep -qE '^[a-zA-Z0-9._:/-]+$'; then
+    echo "ERROR: Invalid model name: $MODEL_NAME"
+    exit 1
+fi
+
+# Start the main Ollama application
+ollama serve &
+
+# Wait for the Ollama application to be ready (optional, if necessary)
+while ! ollama ls; do
+  echo "Waiting for Ollama service to be ready..."
+  sleep 10
+done
+echo "Ollama service is ready."
+
+# Pull the model using ollama pull
+echo "Pulling the $MODEL_NAME with ollama pull..."
+# Check if the model was pulled successfully
+if ollama pull "$MODEL_NAME"; then
+  echo "Model pulled successfully."
+else
+  echo "Failed to pull model."
+  exit 1
+fi
+
+# Run ollama forever.
+sleep infinity

scripts/pre_commit/check_datetime_timezone.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Pre-commit hook: ensure DateTime columns in models and migrations use UtcDateTime.
+
+Scans files under ``src/local_deep_research/database/models/`` and
+``src/local_deep_research/database/migrations/versions/`` for
+``Column(...)`` / ``sa.Column(...)`` calls whose type argument is a bare
+``DateTime`` (with or without ``timezone=True``). Flags them and hints at
+the ``UtcDateTime`` replacement from ``sqlalchemy_utc``.
+
+Limitations (accepted gaps, not caught by this hook):
+- Raw SQL inside ``op.execute("... DATETIME ...")`` — the hook cannot
+  parse SQL strings.
+- Type-alias indirection: ``dt = sa.DateTime(); sa.Column("x", dt)``.
+- Fully-qualified imports without the ``sa`` alias
+  (e.g. ``import sqlalchemy; sqlalchemy.Column(...)``).
+- ``sa.TIMESTAMP`` columns.
+- Walrus expressions: ``Column((dt := DateTime()))`` wraps the call in
+  ``ast.NamedExpr``, which the helper does not traverse.
+- Import-order variations beyond the two hardcoded substring forms.
+"""
+
+import ast
+import re
+import sys
+from pathlib import Path
+from typing import List, Tuple
+
+
+def _callable_name(func_node):
+    """Return the callable's short name regardless of ``X`` or ``sa.X`` form."""
+    if isinstance(func_node, ast.Name):
+        return func_node.id
+    if isinstance(func_node, ast.Attribute):
+        return func_node.attr
+    return None
+
+
+def _resolve_type_arg(arg):
+    """Return list of ('call', Call) or ('name', str) entries for all
+    type-like nodes in arg's subtree. Returns [] when arg is not a type
+    reference.
+
+    For ast.IfExp, BOTH branches are included — returning only the
+    first-resolved branch would silently pass a violation that lives
+    in the other branch.
+    """
+    if isinstance(arg, ast.Call):
+        return [("call", arg)]
+    if isinstance(arg, ast.Name) and arg.id in {"UtcDateTime", "DateTime"}:
+        return [("name", arg.id)]
+    if isinstance(arg, ast.IfExp):
+        return _resolve_type_arg(arg.body) + _resolve_type_arg(arg.orelse)
+    return []
+
+
+def check_datetime_columns(file_path: Path) -> List[Tuple[int, str, str]]:
+    """Check a Python file for DateTime columns that should use UtcDateTime.
+
+    Returns a list of (line_number, line_content, error_message) tuples for violations.
+    """
+    violations = []
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            content = f.read()
+            lines = content.split("\n")
+    except Exception as e:
+        print(f"Error reading {file_path}: {e}", file=sys.stderr)
+        return violations
+
+    has_utc_datetime_import = (
+        "from sqlalchemy_utc import UtcDateTime" in content
+        or "from sqlalchemy_utc import utcnow, UtcDateTime" in content
+    )
+
+    try:
+        tree = ast.parse(content)
+    except SyntaxError:
+        return violations
+
+    fix_hint = (
+        "Use UtcDateTime() instead of DateTime() — "
+        "import: from sqlalchemy_utc import UtcDateTime"
+    )
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        if _callable_name(node.func) != "Column":
+            continue
+
+        type_entries = []
+        for arg in node.args:
+            type_entries = _resolve_type_arg(arg)
+            if type_entries:
+                break
+
+        for kind, payload in type_entries:
+            if kind == "call":
+                inner_name = _callable_name(payload.func)
+                if inner_name == "DateTime":
+                    line_num = node.lineno
+                    if 0 <= line_num - 1 < len(lines):
+                        violations.append(
+                            (line_num, lines[line_num - 1].strip(), fix_hint)
+                        )
+                elif inner_name == "UtcDateTime":
+                    if not has_utc_datetime_import:
+                        line_num = node.lineno
+                        if 0 <= line_num - 1 < len(lines):
+                            violations.append(
+                                (
+                                    line_num,
+                                    lines[line_num - 1].strip(),
+                                    "Missing import: from sqlalchemy_utc import UtcDateTime",
+                                )
+                            )
+            elif kind == "name":
+                if payload == "DateTime":
+                    line_num = node.lineno
+                    if 0 <= line_num - 1 < len(lines):
+                        violations.append(
+                            (line_num, lines[line_num - 1].strip(), fix_hint)
+                        )
+                elif payload == "UtcDateTime" and not has_utc_datetime_import:
+                    line_num = node.lineno
+                    if 0 <= line_num - 1 < len(lines):
+                        violations.append(
+                            (
+                                line_num,
+                                lines[line_num - 1].strip(),
+                                "Missing import: from sqlalchemy_utc import UtcDateTime",
+                            )
+                        )
+
+    for i, line in enumerate(lines, 1):
+        if "func.now()" in line and "Column" in line:
+            violations.append(
+                (
+                    i,
+                    line.strip(),
+                    "Use utcnow() instead of func.now() for timezone-aware defaults",
+                )
+            )
+        if re.search(
+            r"default\s*=\s*(lambda:\s*)?datetime\.(utcnow|now)", line
+        ):
+            violations.append(
+                (
+                    i,
+                    line.strip(),
+                    "Use utcnow() from sqlalchemy_utc instead of datetime functions for defaults",
+                )
+            )
+
+    return violations
+
+
+def main():
+    """Main entry point for the pre-commit hook."""
+    files_to_check = sys.argv[1:]
+
+    if not files_to_check:
+        print("No files to check")
+        return 0
+
+    all_violations = []
+
+    for file_path_str in files_to_check:
+        file_path = Path(file_path_str)
+
+        path_str = str(file_path)
+        in_scope = file_path.suffix == ".py" and (
+            "src/local_deep_research/database/models/" in path_str
+            or "src/local_deep_research/database/migrations/versions/"
+            in path_str
+        )
+        if not in_scope:
+            continue
+
+        violations = check_datetime_columns(file_path)
+        if violations:
+            all_violations.append((file_path, violations))
+
+    if all_violations:
+        print("\nDateTime column issues found:\n")
+        for file_path, violations in all_violations:
+            print(f"  {file_path}:")
+            for line_num, line_content, error_msg in violations:
+                print(f"    Line {line_num}: {error_msg}")
+                print(f"      > {line_content}")
+        print(
+            "\n  Fix: use UtcDateTime from sqlalchemy_utc for all datetime columns"
+        )
+        print(
+            "  (applies to both database/models/ and database/migrations/versions/)"
+        )
+        print("  Example: ")
+        print("    from sqlalchemy_utc import UtcDateTime, utcnow")
+        print("    Column(UtcDateTime, default=utcnow(), ...)\n")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())