chore: import upstream snapshot with attribution

scripts/dev/analyze_passthroughs.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""
+Passthrough analysis: catalogue every paper that fell through every
+local tier of the journal filter and ended up at the passthrough score
+(== threshold). Helps decide whether the LLM tier is still worth keeping.
+
+Reuses the same arxiv/openalex/pubmed/s2 dataset as
+test_journal_filter_arxiv.py but routes results through __score_journal
+directly so we can also see *which* tier actually scored each paper.
+"""
+
+import argparse
+import re
+import sys
+from collections import Counter, defaultdict
+from unittest.mock import Mock
+
+from local_deep_research.advanced_search_system.filters import (
+    journal_reputation_filter as _jrf,
+)
+
+# Stub SearXNG so the filter can be instantiated standalone
+_jrf.create_search_engine = lambda *a, **kw: None
+
+from local_deep_research.utilities.thread_context import search_context  # noqa: E402
+from local_deep_research.web_search_engines.engines.search_engine_arxiv import (  # noqa: E402
+    ArXivSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_openalex import (  # noqa: E402
+    OpenAlexSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_pubmed import (  # noqa: E402
+    PubMedSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_semantic_scholar import (  # noqa: E402
+    SemanticScholarSearchEngine,
+)
+
+DOMAIN_QUERIES = {
+    "fusion": "tokamak plasma confinement",
+    "llm": "large language model alignment",
+    "graph_nn": "graph neural networks",
+    "astro": "exoplanet atmosphere spectroscopy",
+    "biomed": "CRISPR gene editing therapy",
+    "condmat": "high temperature superconductor cuprate",
+    "climate": "climate model ocean heat content",
+    "quantum": "quantum error correction surface code",
+    "math": "riemann hypothesis zeta function",
+    "robotics": "reinforcement learning robotic manipulation",
+}
+
+ENGINES = {
+    "arxiv": ArXivSearchEngine,
+    "openalex": OpenAlexSearchEngine,
+    "pubmed": PubMedSearchEngine,
+    "s2": SemanticScholarSearchEngine,
+}
+
+
+def categorize(journal_ref: str, cleaned: str) -> str:
+    """Bucket a passthrough journal_ref into a category."""
+    j = journal_ref.strip()
+    c = cleaned.strip()
+    # Citation-like strings (author initials, "et al.", quotes around title)
+    if re.search(r"^[A-Z]\.\s*[A-Z]", j):
+        return "citation_author"
+    if '"' in j or "“" in j:
+        return "citation_quoted"
+    if re.search(r"\bet al\b", j, re.I):
+        return "citation_et_al"
+    # Cleaning debris (bare year/page leftover)
+    if re.search(r"\b(19|20)\d{2}\b", c):
+        return "cleaning_debris"
+    if re.search(r"\d", c[-10:]):
+        return "cleaning_trailing_num"
+    # Conference (the regex tier already handles these via score=5)
+    if re.search(
+        r"(proc(eedings|\.)?|conference|symp|workshop|colloq)", c, re.I
+    ):
+        return "conference_uncaught"
+    # Truncated long names
+    if len(j) > 50 and j.endswith(("…", "...")):
+        return "truncated"
+    # Looks like a real journal name we just don't have
+    return "real_journal_unknown"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("-n", type=int, default=15)
+    ap.add_argument("-t", "--threshold", type=int, default=4)
+    args = ap.parse_args()
+
+    snapshot = {
+        "search.journal_reputation.threshold": args.threshold,
+        "search.journal_reputation.exclude_non_published": False,
+        "search.journal_reputation.max_context": 3000,
+        "search.journal_reputation.reanalysis_period": 365,
+        "search.engine.web.arxiv.journal_reputation.enabled": True,
+        "search.engine.web.openalex.journal_reputation.enabled": True,
+        "search.engine.web.pubmed.journal_reputation.enabled": True,
+        "search.engine.web.semantic_scholar.journal_reputation.enabled": True,
+    }
+
+    passthroughs: list[tuple[str, str, str, str, str]] = []
+    # (engine, domain, raw_journal_ref, cleaned, category)
+    totals: Counter = Counter()
+
+    for engine_name, cls in ENGINES.items():
+        for domain, q in DOMAIN_QUERIES.items():
+            print(f"  • {engine_name:<9} {domain:<10} {q!r} ...", flush=True)
+            kwargs = {
+                "max_results": args.n,
+                "llm": Mock(),
+                "settings_snapshot": snapshot,
+            }
+            if engine_name in ("pubmed", "s2"):
+                kwargs["optimize_queries"] = False
+            try:
+                with search_context({"username": None, "user_password": None}):
+                    engine = cls(**kwargs)
+                    out = engine.run(q)
+            except Exception as e:
+                print(f"      ! {e}")
+                continue
+
+            for r in out:
+                jref = r.get("journal_ref")
+                if not jref:
+                    totals["no_journal_ref"] += 1
+                    continue
+                qual = r.get("journal_quality")
+                if qual == args.threshold:
+                    # Passthrough — local tiers couldn't score
+                    cleaned = jref  # Filter caches private; approximate
+                    cat = categorize(jref, cleaned)
+                    passthroughs.append(
+                        (engine_name, domain, jref, cleaned, cat)
+                    )
+                    totals[f"passthrough_{cat}"] += 1
+                elif qual is not None:
+                    totals[f"scored_{qual}"] += 1
+
+    print()
+    print("=" * 90)
+    print("OVERALL")
+    print("=" * 90)
+    for k in sorted(totals):
+        print(f"  {k:<35} {totals[k]:>5}")
+
+    print()
+    print("=" * 90)
+    print(f"PASSTHROUGH BY CATEGORY ({len(passthroughs)} total)")
+    print("=" * 90)
+    by_cat: dict[str, list] = defaultdict(list)
+    for row in passthroughs:
+        by_cat[row[4]].append(row)
+    for cat in sorted(by_cat, key=lambda c: -len(by_cat[c])):
+        rows = by_cat[cat]
+        print(f"\n[{cat}]  ({len(rows)} entries)")
+        for engine_name, domain, jref, cleaned, _ in rows[:8]:
+            print(f"  {engine_name:<9} {domain:<10} {jref[:65]}")
+        if len(rows) > 8:
+            print(f"  ... and {len(rows) - 8} more")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev/debug_pytest.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""Debug script to run pytest with CI settings and see what's failing.
+
+Do not remove — developer utility for reproducing CI failures locally.
+"""
+
+import os
+import subprocess
+import sys
+
+# Set CI environment to mimic CI behavior
+os.environ["CI"] = "true"
+os.environ["USE_FALLBACK_LLM"] = "true"
+
+cmd = [
+    sys.executable,
+    "run_tests.py",
+    "-m",
+    "not slow",
+    "--ignore=tests/searxng/",
+    "--ignore=tests/unit/test_config.py",
+    "--ignore=tests/api_tests/",
+    "--ignore=tests/health_check/test_endpoints_health.py",
+    "-v",
+    "--tb=short",
+    "--timeout=30",
+    "-x",  # Stop on first failure to debug
+]
+
+print(f"Running: {' '.join(cmd)}")
+result = subprocess.run(cmd)
+sys.exit(result.returncode)

scripts/dev/regenerate_golden_master.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Regenerate the golden master settings snapshot.
+
+Usage:
+    python scripts/dev/regenerate_golden_master.py
+
+NOTE: This script is called by the pre-commit hook
+.pre-commit-hooks/check-golden-master-settings.py — do not delete.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Resolve paths
+# ---------------------------------------------------------------------------
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+SRC_DIR = PROJECT_ROOT / "src"
+GOLDEN_MASTER_PATH = (
+    PROJECT_ROOT / "tests" / "settings" / "golden_master_settings.json"
+)
+
+# Ensure the package is importable
+if str(SRC_DIR) not in sys.path:
+    sys.path.insert(0, str(SRC_DIR))
+
+
+def main() -> int:
+    from local_deep_research.settings.manager import SettingsManager
+
+    manager = SettingsManager(db_session=None)
+    defaults = manager.default_settings
+
+    current = {key: dict(defaults[key]) for key in sorted(defaults.keys())}
+    output = (
+        json.dumps(
+            current, indent=2, sort_keys=True, default=str, ensure_ascii=False
+        )
+        + "\n"
+    )
+
+    GOLDEN_MASTER_PATH.write_text(output, encoding="utf-8")
+
+    print(f"Wrote {len(current)} settings to {GOLDEN_MASTER_PATH}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev/restart_server.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+# Restart the LDR dev server.
+#
+# Usage:
+#   scripts/dev/restart_server.sh [PORT] [--debug] [--tmp]
+#
+#   PORT      Optional port (default: 5000). Passed via LDR_WEB_PORT.
+#   --debug   DEBUG logging (LDR_APP_DEBUG=true, LDR_LOG_SETTINGS=summary).
+#             WARNING: debug logs may contain sensitive data (queries,
+#             answers, API responses) — local dev / feature testing only,
+#             never production or real user data.
+#   --tmp     Disposable instance: point LDR_DATA_DIR at a throwaway dir
+#             (default /tmp/ldr-test) so encrypted user DBs, the auth DB,
+#             library, logs, research_outputs etc. land OUTSIDE your real
+#             data dir (~/.local/share/local-deep-research). Override the
+#             location by exporting LDR_DATA_DIR before running. Reuses one
+#             dir (doesn't accumulate throwaway databases). To start from a
+#             clean slate, wipe LDR_DATA_DIR yourself first (e.g.
+#             `rm -rf "$LDR_DATA_DIR"`) — this script never deletes data.
+#
+# Only the instance listening on the target PORT is stopped, so multiple
+# instances on different ports (e.g. several agents each on their own port)
+# coexist — restarting one does not kill the others.
+#
+# Referenced in:
+#   - tests/api_tests_with_login/README.md
+#   - examples/api_usage/http/README.md
+#   - examples/api_usage/http/advanced/simple_http_example.py
+# Do not delete without updating those references.
+
+set -u
+
+usage() {
+    cat <<'EOF'
+Restart the LDR dev server.
+
+Usage:
+  scripts/dev/restart_server.sh [PORT] [--debug] [--tmp]
+
+  PORT      Optional port (default: 5000).
+  --debug   DEBUG logging (may log sensitive data; local dev only).
+  --tmp     Disposable instance: LDR_DATA_DIR -> /tmp/ldr-test (override by
+            exporting LDR_DATA_DIR). Keeps test data out of your real dir.
+            This script never deletes data; wipe LDR_DATA_DIR yourself to
+            start from a clean slate.
+  --test    Alias for --tmp.
+
+Only the instance on the target PORT is stopped; instances on other ports
+keep running.
+EOF
+}
+
+PORT=5000
+debug=0
+use_tmp=0
+
+for arg in "$@"; do
+    case "$arg" in
+        --debug)    debug=1 ;;
+        --tmp|--test) use_tmp=1 ;;
+        -h|--help)  usage; exit 0 ;;
+        ''|*[!0-9]*) echo "Unknown argument: $arg" >&2; usage >&2; exit 2 ;;
+        *)          PORT="$arg" ;;
+    esac
+done
+
+echo "Stopping existing LDR server on port ${PORT} (if any)..."
+# Port-scoped stop: kill only the process LISTENing on PORT, leaving
+# instances on other ports running. The port isn't in the process argv
+# (it comes from LDR_WEB_PORT), so a broad pkill can't distinguish them.
+if command -v fuser >/dev/null 2>&1; then
+    fuser -k "${PORT}/tcp" 2>/dev/null || echo "No existing server on port ${PORT}"
+elif command -v lsof >/dev/null 2>&1; then
+    pids=$(lsof -ti "tcp:${PORT}" -sTCP:LISTEN 2>/dev/null)
+    if [ -n "${pids}" ]; then
+        # $pids may be several PIDs; word-splitting is intended here.
+        # shellcheck disable=SC2086
+        kill ${pids} 2>/dev/null
+    else
+        echo "No existing server on port ${PORT}"
+    fi
+else
+    echo "Warning: neither fuser nor lsof found; cannot port-scope the stop." >&2
+    echo "Skipping stop to avoid killing instances on other ports." >&2
+fi
+
+sleep 1
+
+# Change to the project root.
+cd "$(dirname "$0")/../.." || exit 1
+
+# --- environment for the launched server -------------------------------------
+env_args=(LDR_WEB_PORT="${PORT}")
+
+if [ "$use_tmp" -eq 1 ]; then
+    : "${LDR_DATA_DIR:=/tmp/ldr-test}"
+    mkdir -p "$LDR_DATA_DIR"
+    echo "Using disposable LDR_DATA_DIR=$LDR_DATA_DIR"
+    env_args+=(LDR_DATA_DIR="${LDR_DATA_DIR}")
+fi
+
+if [ "$debug" -eq 1 ]; then
+    env_args+=(LDR_APP_DEBUG=true LDR_LOG_SETTINGS=summary)
+fi
+
+LOG_FILE="/tmp/ldr_server_${PORT}.log"
+
+if [ "$debug" -eq 1 ]; then
+    echo "Starting LDR server on port ${PORT} (DEBUG)..."
+else
+    echo "Starting LDR server on port ${PORT}..."
+fi
+# Start server in background and detach from terminal. Backgrounding nohup
+# directly (rather than wrapping in an extra `( ... & ) &` subshell) means
+# $! captures the real server PID, not a transient subshell that has already
+# exited. nohup ignores SIGHUP, so the server survives the launcher exiting
+# and is reparented to init.
+nohup env "${env_args[@]}" pdm run python -m local_deep_research.web.app > "${LOG_FILE}" 2>&1 &
+SERVER_PID=$!
+
+sleep 2
+
+echo "Server started. PID: ${SERVER_PID}"
+echo "Logs: ${LOG_FILE}"
+echo "URL: http://127.0.0.1:${PORT}"
+if [ "$debug" -eq 1 ]; then
+    echo ""
+    echo "WARNING: DEBUG mode — logs may contain sensitive data; local dev only."
+fi
+echo ""
+echo "To view logs: tail -f ${LOG_FILE}"
+echo "To stop server: fuser -k ${PORT}/tcp"
+
+exit 0

scripts/dev/run_tests.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Run tests for Local Deep Research.
+
+This script runs pytest with appropriate configuration for the project.
+
+Referenced in CONTRIBUTING.md — do not delete without updating that file.
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main():
+    """Run the test suite with appropriate settings."""
+    # Get the project root directory
+    project_root = Path(__file__).parent.absolute()
+
+    # Ensure PYTHONPATH includes the project root for proper imports
+    os.environ["PYTHONPATH"] = str(project_root)
+
+    # Configure pytest arguments
+    pytest_args = [
+        sys.executable,  # Use Python interpreter
+        "-m",  # Run module
+        "pytest",  # Call pytest
+        "--verbose",  # Verbose output
+        "--color=yes",  # Force colored output
+        "--cov=src",  # Measure coverage for src directory
+        "--cov-report=term",  # Report coverage in terminal
+        "--cov-report=html:coverage_html",  # Also generate HTML report
+        "--cov-config=.coveragerc",  # Use the coverage configuration file
+    ]
+
+    # Add any command line arguments passed to this script
+    pytest_args.extend(sys.argv[1:])
+
+    # Print the command being run
+    print(f"Running: {' '.join(pytest_args)}")
+
+    # Run pytest and capture the return code
+    result = subprocess.run(pytest_args)
+
+    # Return the pytest exit code
+    return result.returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev/stop_server.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Script to stop the LDR server.
+# Do not remove — companion to restart_server.sh; handles graceful
+# and forced shutdown of the dev server.
+
+echo "Stopping LDR server..."
+
+# Kill the main server process
+if pkill -f "python -m local_deep_research.web.app" 2>/dev/null; then
+    echo "✓ Server stopped successfully"
+
+    # Wait a moment for the process to fully terminate
+    sleep 1
+
+    # Check if any processes are still running
+    if pgrep -f "python -m local_deep_research.web.app" > /dev/null; then
+        echo "Warning: Some server processes may still be running"
+        echo "Attempting force stop..."
+        pkill -9 -f "python -m local_deep_research.web.app" 2>/dev/null
+        sleep 1
+    fi
+else
+    echo "No running LDR server found"
+fi
+
+# Also stop any orphaned Flask dev servers
+pkill -f "flask run" 2>/dev/null
+
+# Check final status
+if pgrep -f "python -m local_deep_research.web.app" > /dev/null; then
+    echo "Error: Server processes still running:"
+    pgrep -af "python -m local_deep_research.web.app"
+    exit 1
+else
+    echo "All server processes stopped"
+    exit 0
+fi

scripts/dev/test_journal_filter_arxiv.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""
+Probe the journal reputation filter against live arXiv results.
+
+Runs the *full* ArXivSearchEngine pipeline (including the journal filter
+as a preview filter) and prints which papers passed/were dropped, along
+with the journal_quality score the filter attached.
+
+Usage:
+    python scripts/dev/test_journal_filter_arxiv.py "graph neural networks" -n 25
+"""
+
+import argparse
+import sys
+from unittest.mock import Mock
+
+from local_deep_research.utilities.thread_context import search_context
+from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
+    ArXivSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_openalex import (
+    OpenAlexSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_pubmed import (
+    PubMedSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_semantic_scholar import (
+    SemanticScholarSearchEngine,
+)
+
+ENGINES = {
+    "arxiv": ArXivSearchEngine,
+    "openalex": OpenAlexSearchEngine,
+    "pubmed": PubMedSearchEngine,
+    "s2": SemanticScholarSearchEngine,
+}
+
+
+DOMAIN_QUERIES = {
+    "fusion": "tokamak plasma confinement",
+    "llm": "large language model alignment",
+    "graph_nn": "graph neural networks",
+    "astro": "exoplanet atmosphere spectroscopy",
+    "biomed": "CRISPR gene editing therapy",
+    "condmat": "high temperature superconductor cuprate",
+    "climate": "climate model ocean heat content",
+    "quantum": "quantum error correction surface code",
+    "math": "riemann hypothesis zeta function",
+    "robotics": "reinforcement learning robotic manipulation",
+}
+
+
+def run_one(
+    query: str,
+    n: int,
+    threshold: int,
+    snapshot: dict,
+    engine_name: str = "arxiv",
+) -> dict:
+    cls = ENGINES[engine_name]
+    kwargs = {"max_results": n, "llm": Mock(), "settings_snapshot": snapshot}
+    # PubMed/S2 use the LLM for query optimization — Mock would corrupt
+    # the optimized query string. Disable optimization for the probe so
+    # we send the raw query verbatim.
+    if engine_name in ("pubmed", "s2"):
+        kwargs["optimize_queries"] = False
+    with search_context({"username": None, "user_password": None}):
+        try:
+            engine = cls(**kwargs)
+            out = engine.run(query)
+        except Exception as e:
+            return {
+                "query": query,
+                "n_total": 0,
+                "n_no_jref": 0,
+                "n_scored": 0,
+                "avg_q": 0,
+                "high": 0,
+                "passthrough": 0,
+                "results": [],
+                "error": str(e)[:80],
+            }
+
+    scored = [r for r in out if "journal_quality" in r]
+    no_jref = [r for r in out if not r.get("journal_ref")]
+    qualities = [r["journal_quality"] for r in scored]
+    return {
+        "query": query,
+        "n_total": len(out),
+        "n_no_jref": len(no_jref),
+        "n_scored": len(scored),
+        "avg_q": sum(qualities) / len(qualities) if qualities else 0,
+        "high": sum(1 for q in qualities if q >= 7),
+        "passthrough": sum(1 for q in qualities if q == threshold),
+        "results": out,
+    }
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument(
+        "query", nargs="?", help="arXiv search query (omit to run dataset)"
+    )
+    ap.add_argument("-n", type=int, default=20, help="number of papers")
+    ap.add_argument("-t", "--threshold", type=int, default=4)
+    ap.add_argument(
+        "--dataset",
+        action="store_true",
+        help="run the multi-domain dataset of queries",
+    )
+    ap.add_argument(
+        "--engines",
+        default="arxiv",
+        help="comma-separated list: arxiv,openalex,pubmed,s2 (or 'all')",
+    )
+    args = ap.parse_args()
+    if args.engines == "all":
+        engine_names = list(ENGINES)
+    else:
+        engine_names = [e.strip() for e in args.engines.split(",")]
+
+    # Minimal settings snapshot — defaults are returned for missing keys.
+    # Threshold and arxiv journal-filter enable flag are wired explicitly.
+    snapshot = {
+        "search.journal_reputation.threshold": args.threshold,
+        "search.journal_reputation.exclude_non_published": False,
+        "search.journal_reputation.max_context": 3000,
+        "search.journal_reputation.reanalysis_period": 365,
+        "search.engine.web.arxiv.journal_reputation.enabled": True,
+        "search.engine.web.openalex.journal_reputation.enabled": True,
+        "search.engine.web.pubmed.journal_reputation.enabled": True,
+        "search.engine.web.semantic_scholar.journal_reputation.enabled": True,
+    }
+
+    if args.dataset:
+        print(
+            f"Running dataset of {len(DOMAIN_QUERIES)} domain queries × "
+            f"{len(engine_names)} engines (n={args.n}, threshold={args.threshold})\n"
+        )
+        all_rows = []
+        for engine_name in engine_names:
+            for domain, q in DOMAIN_QUERIES.items():
+                print(
+                    f"  • {engine_name:<9} {domain:<10} {q!r} ...", flush=True
+                )
+                s = run_one(q, args.n, args.threshold, snapshot, engine_name)
+                all_rows.append((engine_name, domain, s))
+
+        print(
+            f"\n{'engine':<9} {'domain':<10} {'tot':>4} {'noJ':>4} "
+            f"{'scor':>4} {'avg':>5} {'≥7':>3} {'pass':>4}  err"
+        )
+        print("-" * 80)
+        for engine_name, domain, s in all_rows:
+            err = s.get("error", "")
+            print(
+                f"{engine_name:<9} {domain:<10} "
+                f"{s['n_total']:>4} {s['n_no_jref']:>4} {s['n_scored']:>4} "
+                f"{s['avg_q']:>5.1f} {s['high']:>3} {s['passthrough']:>4}  {err}"
+            )
+        return 0
+
+    if not args.query:
+        ap.error("query required (or use --dataset)")
+
+    with search_context({"username": None, "user_password": None}):
+        engine = ArXivSearchEngine(
+            max_results=args.n, llm=Mock(), settings_snapshot=snapshot
+        )
+        print(f"Running ArXivSearchEngine for: {args.query!r}\n")
+        out = engine.run(args.query)
+
+    print(f"{'#':<3} {'qual':<5} {'journal_ref':<45} title")
+    print("-" * 120)
+    for i, r in enumerate(out, 1):
+        q = r.get("journal_quality", "—")
+        jref = (r.get("journal_ref") or "—")[:43]
+        title = (r.get("title") or "")[:60]
+        print(f"{i:<3} {str(q):<5} {jref:<45} {title}")
+
+    print(
+        f"\nReturned {len(out)} results from pipeline "
+        f"(threshold={args.threshold})."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())