learningcircuit--local-deep-research/scripts/ci/seed_link_analytics.py

#!/usr/bin/env python3
"""Seed link-analytics test data for the `link-analytics` UI shard.

The /metrics/links page derives its domain cards entirely from
``ResearchResource`` rows (see ``get_link_analytics`` in
``web/routes/metrics_routes.py``). A freshly-initialized CI database
(``scripts/ci/init_test_database.py``) has none, so the page renders the
"No domain data available" placeholder and ``test_link_analytics_full.js``
times out waiting for ``#domain-list .ldr-domain-item-expanded``.

This script inserts a small, deterministic fixture (2 researches, 7 resources
across 4 domains) for the ``test_admin`` user so the page renders real domain
cards with frequency/diversity badges and a populated "Recent Researches
(N total)" header.

It is wired into ``.github/workflows/docker-tests.yml`` **only** for the
``link-analytics`` shard. Other shards (and the responsive / empty-state
screenshot suites that assert the placeholder on /metrics/links) keep seeing
an empty database.

Usage:
    python scripts/ci/seed_link_analytics.py

Environment variables (must match init_test_database.py / the server):
    LDR_DATA_DIR: Directory for database files.
    LDR_TEST_MODE / LDR_DB_CONFIG_KDF_ITERATIONS: SQLCipher key-derivation
        parameters — must match the values used to create the DB or the
        encryption key will not match.
"""

import os
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path

from sqlalchemy.orm import Session

# Credentials must match CI_TEST_USER in tests/ui_tests/auth_helper.js and the
# user created by scripts/ci/init_test_database.py.
TEST_USERNAME = "test_admin"
TEST_PASSWORD = "testpass123"  # pragma: allowlist secret

# A stable namespace UUID so re-running on a fresh DB always produces the same
# ids (keeps the fixture idempotent and easy to reason about).
_SEED_NS = uuid.UUID("00000000-0000-0000-0000-00000000a11a")


def main():
    data_dir = Path(
        os.environ.get(
            "LDR_DATA_DIR",
            Path.home() / ".local" / "share" / "local-deep-research",
        )
    )
    print(f"Using data directory: {data_dir}")

    # Imported after the env is set so path/SQLCipher config is picked up.
    from local_deep_research.database.encrypted_db import db_manager
    from local_deep_research.database.models.research import (
        ResearchHistory,
        ResearchResource,
    )

    engine = db_manager.open_user_database(TEST_USERNAME, TEST_PASSWORD)
    if engine is None:
        raise RuntimeError(
            f"Could not open encrypted database for user '{TEST_USERNAME}'. "
            "Was init_test_database.py run first with matching "
            "LDR_TEST_MODE / LDR_DB_CONFIG_KDF_ITERATIONS?"
        )

    research_a = str(uuid.uuid5(_SEED_NS, "research-a"))
    research_b = str(uuid.uuid5(_SEED_NS, "research-b"))

    # created_at lands a day ago so it falls inside the default 30d window the
    # /metrics/links page queries with. Stored as ISO strings to match the
    # rest of the codebase (created_at columns are Text/String, not DateTime).
    created = (datetime.now(UTC) - timedelta(days=1)).isoformat()

    researches = [
        ResearchHistory(
            id=research_a,
            query="Seed: deep learning survey for link analytics",
            mode="quick_summary",
            status="completed",
            created_at=created,
            completed_at=created,
        ),
        ResearchHistory(
            id=research_b,
            query="Seed: transformer architectures for link analytics",
            mode="quick_summary",
            status="completed",
            created_at=created,
            completed_at=created,
        ),
    ]

    # (research_id, url, title, source_type). Spread across 4 domains with
    # uneven counts so frequency_rank / usage_percentage vary, and across both
    # researches so research_diversity is >1 for the busiest domains.
    resources = [
        (
            research_a,
            "https://en.wikipedia.org/wiki/Deep_learning",
            "Deep learning - Wikipedia",
            "web",
        ),
        (
            research_a,
            "https://en.wikipedia.org/wiki/Neural_network",
            "Neural network - Wikipedia",
            "web",
        ),
        (
            research_a,
            "https://arxiv.org/abs/1706.03762",
            "Attention Is All You Need",
            "academic",
        ),
        (
            research_a,
            "https://www.nature.com/articles/nature14539",
            "Deep learning - Nature",
            "academic",
        ),
        (
            research_b,
            "https://en.wikipedia.org/wiki/Transformer_(deep_learning_architecture)",
            "Transformer - Wikipedia",
            "web",
        ),
        (research_b, "https://arxiv.org/abs/1810.04805", "BERT", "academic"),
        (
            research_b,
            "https://github.com/huggingface/transformers",
            "huggingface/transformers",
            "code",
        ),
    ]

    with Session(engine) as session:
        # Idempotent: a fresh CI DB is created per attempt, but guard anyway so
        # a manual re-run does not duplicate the fixture.
        already = (
            session.query(ResearchHistory)
            .filter(ResearchHistory.id == research_a)
            .first()
        )
        if already:
            print("Link analytics fixture already present — nothing to do.")
            return

        session.add_all(researches)
        for research_id, url, title, source_type in resources:
            session.add(
                ResearchResource(
                    research_id=research_id,
                    url=url,
                    title=title,
                    source_type=source_type,
                    created_at=created,
                )
            )
        session.commit()

    domains = {url.split("/")[2] for _, url, _, _ in resources}
    print(
        f"✅ Seeded {len(researches)} researches and {len(resources)} resources "
        f"across {len(domains)} domains for '{TEST_USERNAME}'"
    )


if __name__ == "__main__":
    main()