chore: import upstream snapshot with attribution

scripts/ci/dump_url_map.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""
+Dump the Flask url_map to a file as one absolute URL per line.
+
+Used by the Nuclei DAST workflow to seed a URL list so the scanner
+probes the actual application surface (authenticated routes, API
+endpoints, blueprints) instead of just the index page.
+
+Parameterized routes (`/research/<string:research_id>/status`) are
+emitted with a converter-appropriate placeholder so Nuclei still
+exercises the path. The substituted URL will usually 404 (the resource
+doesn't exist for the test user), but that is fine — Nuclei probes path
+traversal, parameter injection, and SQLi templates against the URL
+pattern, not against a specific resource.
+
+Skips:
+  - The Flask `static` endpoint (asset serving, no app logic).
+  - Routes that don't accept GET — Nuclei templates almost exclusively
+    issue GET probes, so POST-only endpoints just generate 405s.
+
+Usage:
+    python scripts/ci/dump_url_map.py http://127.0.0.1:5000 > urls.txt
+"""
+
+import re
+import sys
+
+
+# Map Flask URL converters to a placeholder that satisfies the converter
+# so Flask routes the request to the handler instead of 404-ing at the
+# converter stage. Anything not listed falls back to a plain string.
+_PLACEHOLDERS = {
+    "int": "1",
+    "float": "1",
+    "uuid": "00000000-0000-0000-0000-000000000000",
+}
+_DEFAULT_PLACEHOLDER = "nuclei"
+
+_PARAM_RE = re.compile(
+    r"<(?:(?P<conv>[a-zA-Z_][a-zA-Z0-9_]*)(?:\([^)]*\))?:)?(?P<name>[a-zA-Z_][a-zA-Z0-9_]*)>"
+)
+
+
+def _substitute(match: "re.Match[str]") -> str:
+    return _PLACEHOLDERS.get(match.group("conv") or "", _DEFAULT_PLACEHOLDER)
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} BASE_URL", file=sys.stderr)
+        return 2
+
+    base_url = sys.argv[1].rstrip("/")
+
+    from local_deep_research.web.app_factory import create_app
+
+    app, _ = create_app()
+
+    seen: set[str] = set()
+    for rule in app.url_map.iter_rules():
+        if rule.endpoint == "static":
+            continue
+        if "GET" not in (rule.methods or set()):
+            continue
+        path = _PARAM_RE.sub(_substitute, rule.rule)
+        url = f"{base_url}{path}"
+        if url in seen:
+            continue
+        seen.add(url)
+        print(url)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/init_test_database.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+Initialize test database with pre-created test user for CI.
+
+This script creates the test_admin user BEFORE tests run, avoiding the slow
+registration process (2+ min) which:
+1. Creates encrypted SQLCipher database
+2. Derives encryption keys from password
+3. Creates 58 database tables
+4. Imports 500+ settings from JSON files
+
+Usage:
+    python scripts/ci/init_test_database.py
+
+Environment variables:
+    LDR_DATA_DIR: Directory for database files (required for path consistency)
+    TEST_ENV: Should be "true" for test environment
+
+Note: Test user credentials must match CI_TEST_USER in tests/ui_tests/auth_helper.js
+"""
+
+import os
+from pathlib import Path
+
+
+def main():
+    """Create test database and test_admin user."""
+    # Test user credentials - must match CI_TEST_USER in auth_helper.js
+    TEST_USERNAME = "test_admin"
+    TEST_PASSWORD = "testpass123"  # pragma: allowlist secret
+
+    # Use LDR_DATA_DIR if set, otherwise default
+    data_dir = Path(
+        os.environ.get(
+            "LDR_DATA_DIR",
+            Path.home() / ".local" / "share" / "local-deep-research",
+        )
+    )
+    print(f"Using data directory: {data_dir}")
+    data_dir.mkdir(parents=True, exist_ok=True)
+    (data_dir / "encrypted_databases").mkdir(parents=True, exist_ok=True)
+
+    # Import after setting up paths
+    from local_deep_research.database.auth_db import (
+        get_auth_db_session,
+        init_auth_database,
+    )
+    from local_deep_research.database.encrypted_db import db_manager
+    from local_deep_research.database.models.auth import User
+
+    # Initialize auth database
+    init_auth_database()
+
+    # Create test user in auth database (no password stored)
+    session = get_auth_db_session()
+    user = User(username=TEST_USERNAME)
+    session.add(user)
+    session.commit()
+    session.close()
+
+    # Create user's encrypted database with password
+    db_manager.create_user_database(TEST_USERNAME, TEST_PASSWORD)
+
+    print("✅ Database initialized successfully")
+    print(f"✅ Test user '{TEST_USERNAME}' created")
+    print(f"   Encrypted databases in: {data_dir / 'encrypted_databases'}")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/seed_link_analytics.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""Seed link-analytics test data for the `link-analytics` UI shard.
+
+The /metrics/links page derives its domain cards entirely from
+``ResearchResource`` rows (see ``get_link_analytics`` in
+``web/routes/metrics_routes.py``). A freshly-initialized CI database
+(``scripts/ci/init_test_database.py``) has none, so the page renders the
+"No domain data available" placeholder and ``test_link_analytics_full.js``
+times out waiting for ``#domain-list .ldr-domain-item-expanded``.
+
+This script inserts a small, deterministic fixture (2 researches, 7 resources
+across 4 domains) for the ``test_admin`` user so the page renders real domain
+cards with frequency/diversity badges and a populated "Recent Researches
+(N total)" header.
+
+It is wired into ``.github/workflows/docker-tests.yml`` **only** for the
+``link-analytics`` shard. Other shards (and the responsive / empty-state
+screenshot suites that assert the placeholder on /metrics/links) keep seeing
+an empty database.
+
+Usage:
+    python scripts/ci/seed_link_analytics.py
+
+Environment variables (must match init_test_database.py / the server):
+    LDR_DATA_DIR: Directory for database files.
+    LDR_TEST_MODE / LDR_DB_CONFIG_KDF_ITERATIONS: SQLCipher key-derivation
+        parameters — must match the values used to create the DB or the
+        encryption key will not match.
+"""
+
+import os
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+# Credentials must match CI_TEST_USER in tests/ui_tests/auth_helper.js and the
+# user created by scripts/ci/init_test_database.py.
+TEST_USERNAME = "test_admin"
+TEST_PASSWORD = "testpass123"  # pragma: allowlist secret
+
+# A stable namespace UUID so re-running on a fresh DB always produces the same
+# ids (keeps the fixture idempotent and easy to reason about).
+_SEED_NS = uuid.UUID("00000000-0000-0000-0000-00000000a11a")
+
+
+def main():
+    data_dir = Path(
+        os.environ.get(
+            "LDR_DATA_DIR",
+            Path.home() / ".local" / "share" / "local-deep-research",
+        )
+    )
+    print(f"Using data directory: {data_dir}")
+
+    # Imported after the env is set so path/SQLCipher config is picked up.
+    from local_deep_research.database.encrypted_db import db_manager
+    from local_deep_research.database.models.research import (
+        ResearchHistory,
+        ResearchResource,
+    )
+
+    engine = db_manager.open_user_database(TEST_USERNAME, TEST_PASSWORD)
+    if engine is None:
+        raise RuntimeError(
+            f"Could not open encrypted database for user '{TEST_USERNAME}'. "
+            "Was init_test_database.py run first with matching "
+            "LDR_TEST_MODE / LDR_DB_CONFIG_KDF_ITERATIONS?"
+        )
+
+    research_a = str(uuid.uuid5(_SEED_NS, "research-a"))
+    research_b = str(uuid.uuid5(_SEED_NS, "research-b"))
+
+    # created_at lands a day ago so it falls inside the default 30d window the
+    # /metrics/links page queries with. Stored as ISO strings to match the
+    # rest of the codebase (created_at columns are Text/String, not DateTime).
+    created = (datetime.now(UTC) - timedelta(days=1)).isoformat()
+
+    researches = [
+        ResearchHistory(
+            id=research_a,
+            query="Seed: deep learning survey for link analytics",
+            mode="quick_summary",
+            status="completed",
+            created_at=created,
+            completed_at=created,
+        ),
+        ResearchHistory(
+            id=research_b,
+            query="Seed: transformer architectures for link analytics",
+            mode="quick_summary",
+            status="completed",
+            created_at=created,
+            completed_at=created,
+        ),
+    ]
+
+    # (research_id, url, title, source_type). Spread across 4 domains with
+    # uneven counts so frequency_rank / usage_percentage vary, and across both
+    # researches so research_diversity is >1 for the busiest domains.
+    resources = [
+        (
+            research_a,
+            "https://en.wikipedia.org/wiki/Deep_learning",
+            "Deep learning - Wikipedia",
+            "web",
+        ),
+        (
+            research_a,
+            "https://en.wikipedia.org/wiki/Neural_network",
+            "Neural network - Wikipedia",
+            "web",
+        ),
+        (
+            research_a,
+            "https://arxiv.org/abs/1706.03762",
+            "Attention Is All You Need",
+            "academic",
+        ),
+        (
+            research_a,
+            "https://www.nature.com/articles/nature14539",
+            "Deep learning - Nature",
+            "academic",
+        ),
+        (
+            research_b,
+            "https://en.wikipedia.org/wiki/Transformer_(deep_learning_architecture)",
+            "Transformer - Wikipedia",
+            "web",
+        ),
+        (research_b, "https://arxiv.org/abs/1810.04805", "BERT", "academic"),
+        (
+            research_b,
+            "https://github.com/huggingface/transformers",
+            "huggingface/transformers",
+            "code",
+        ),
+    ]
+
+    with Session(engine) as session:
+        # Idempotent: a fresh CI DB is created per attempt, but guard anyway so
+        # a manual re-run does not duplicate the fixture.
+        already = (
+            session.query(ResearchHistory)
+            .filter(ResearchHistory.id == research_a)
+            .first()
+        )
+        if already:
+            print("Link analytics fixture already present — nothing to do.")
+            return
+
+        session.add_all(researches)
+        for research_id, url, title, source_type in resources:
+            session.add(
+                ResearchResource(
+                    research_id=research_id,
+                    url=url,
+                    title=title,
+                    source_type=source_type,
+                    created_at=created,
+                )
+            )
+        session.commit()
+
+    domains = {url.split("/")[2] for _, url, _, _ in resources}
+    print(
+        f"✅ Seeded {len(researches)} researches and {len(resources)} resources "
+        f"across {len(domains)} domains for '{TEST_USERNAME}'"
+    )
+
+
+if __name__ == "__main__":
+    main()