chore: import upstream snapshot with attribution

surfsense_backend/scripts/check_migration_flow.py

@@ -0,0 +1,142 @@
+"""Self-check for the alembic fast-forward/adoption flow in alembic/env.py.
+
+Verifies ``alembic upgrade head`` succeeds on the three DB states it must
+handle without replaying pre-workspace-rename history against a
+workspace-shape schema:
+
+  1. fresh   -- empty database (fast-forward: create_all + stamp head)
+  2. bootstrap -- create_all-created schema + zero_publication, no alembic
+     history (adoption: stamp head)
+  3. midcrash -- bootstrap schema whose alembic_version is stuck at a
+     pre-rename revision from a failed replay (adoption: stamp head)
+
+Run:  python scripts/check_migration_flow.py
+"""
+
+import asyncio
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+BACKEND_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(BACKEND_DIR))
+
+ADMIN_URL = os.getenv(
+    "ADMIN_DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/postgres"
+)
+SCRATCH_DB = "surfsense_check_migration_flow"
+SCRATCH_URL = ADMIN_URL.rsplit("/", 1)[0] + f"/{SCRATCH_DB}"
+SCRATCH_URL_ASYNC = SCRATCH_URL.replace("postgresql://", "postgresql+asyncpg://")
+
+
+async def recreate_scratch_db() -> None:
+    import asyncpg
+
+    admin = await asyncpg.connect(ADMIN_URL)
+    await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
+    await admin.execute(f'CREATE DATABASE "{SCRATCH_DB}"')
+    await admin.close()
+
+
+async def drop_scratch_db() -> None:
+    import asyncpg
+
+    admin = await asyncpg.connect(ADMIN_URL)
+    await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
+    await admin.close()
+
+
+def run_alembic_upgrade() -> None:
+    env = dict(os.environ, DATABASE_URL=SCRATCH_URL_ASYNC)
+    subprocess.run(
+        [sys.executable, "-m", "alembic", "upgrade", "head"],
+        cwd=BACKEND_DIR,
+        env=env,
+        check=True,
+    )
+
+
+async def bootstrap_schema() -> None:
+    """Mimic app startup bootstrap: create_all + ensure_publication, no stamp."""
+    from sqlalchemy import text
+    from sqlalchemy.ext.asyncio import create_async_engine
+
+    from app.db import Base
+    from app.zero_publication import ensure_publication
+
+    engine = create_async_engine(SCRATCH_URL_ASYNC)
+    async with engine.begin() as conn:
+        await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+        await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
+        await conn.run_sync(Base.metadata.create_all)
+        await conn.run_sync(ensure_publication)
+    await engine.dispose()
+
+
+async def set_version(version: str | None) -> None:
+    import asyncpg
+
+    conn = await asyncpg.connect(SCRATCH_URL)
+    if version is None:
+        await conn.execute("DROP TABLE IF EXISTS alembic_version")
+    else:
+        await conn.execute(
+            "CREATE TABLE IF NOT EXISTS alembic_version ("
+            "version_num VARCHAR(32) NOT NULL PRIMARY KEY)"
+        )
+        await conn.execute("DELETE FROM alembic_version")
+        await conn.execute(
+            "INSERT INTO alembic_version (version_num) VALUES ($1)", version
+        )
+    await conn.close()
+
+
+async def assert_at_head() -> None:
+    import asyncpg
+    from alembic.script import ScriptDirectory
+
+    head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head()
+    conn = await asyncpg.connect(SCRATCH_URL)
+    version = await conn.fetchval("SELECT version_num FROM alembic_version")
+    workspaces = await conn.fetchval("SELECT to_regclass('workspaces')")
+    publication = await conn.fetchval(
+        "SELECT 1 FROM pg_publication WHERE pubname = 'zero_publication'"
+    )
+    await conn.close()
+    assert version == head, f"expected version {head}, got {version}"
+    assert workspaces, "workspaces table missing"
+    assert publication, "zero_publication missing"
+
+
+async def main() -> None:
+    try:
+        # 1. Fresh empty DB -> fast-forward.
+        await recreate_scratch_db()
+        run_alembic_upgrade()
+        await assert_at_head()
+        print("OK: fresh DB fast-forwards to head")
+
+        # 2. Bootstrap-created schema, no alembic history -> adoption.
+        await recreate_scratch_db()
+        await bootstrap_schema()
+        await set_version(None)
+        run_alembic_upgrade()
+        await assert_at_head()
+        print("OK: bootstrap-created schema adopted (stamped head)")
+
+        # 3. Bootstrap schema stuck at a pre-rename revision -> adoption.
+        await set_version("4")
+        run_alembic_upgrade()
+        await assert_at_head()
+        print("OK: pre-rename stuck revision adopted (stamped head)")
+
+        # Re-run must be a clean no-op.
+        run_alembic_upgrade()
+        await assert_at_head()
+        print("OK: repeat upgrade is a no-op")
+    finally:
+        await drop_scratch_db()
+
+
+asyncio.run(main())

surfsense_backend/scripts/check_zero_publication_bootstrap.py

@@ -0,0 +1,48 @@
+"""Self-check for ensure_publication on a create_all-bootstrapped scratch DB."""
+
+import asyncio
+
+import asyncpg
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+
+SCRATCH_DB = "surfsense_zero_pub_check"
+ADMIN_DSN = "postgresql://postgres:postgres@localhost:5432/postgres"
+SCRATCH_URL = f"postgresql+asyncpg://postgres:postgres@localhost:5432/{SCRATCH_DB}"
+
+
+async def main() -> None:
+    admin = await asyncpg.connect(ADMIN_DSN)
+    await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
+    await admin.execute(f'CREATE DATABASE "{SCRATCH_DB}"')
+    await admin.close()
+
+    from app.db import Base
+    from app.zero_publication import ensure_publication, verify_publication
+
+    engine = create_async_engine(SCRATCH_URL)
+    try:
+        async with engine.begin() as conn:
+            await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+            await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
+            await conn.run_sync(Base.metadata.create_all)
+            await conn.run_sync(ensure_publication)
+            mismatches = await conn.run_sync(verify_publication)
+            assert not mismatches, f"shape wrong after ensure: {mismatches}"
+
+            # Second call must be a no-op that leaves a verified shape.
+            await conn.run_sync(ensure_publication)
+            mismatches = await conn.run_sync(verify_publication)
+            assert not mismatches, f"shape wrong after re-ensure: {mismatches}"
+    finally:
+        await engine.dispose()
+        admin = await asyncpg.connect(ADMIN_DSN)
+        await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
+        await admin.close()
+
+    print(
+        "OK: ensure_publication creates and verifies on a create_all DB, idempotently."
+    )
+
+
+asyncio.run(main())

surfsense_backend/scripts/create_sandbox_snapshot.py

@@ -0,0 +1,105 @@
+"""Create the Daytona snapshot used by SurfSense code-execution sandboxes.
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/create_sandbox_snapshot.py
+
+Prerequisites:
+    - DAYTONA_API_KEY set in surfsense_backend/.env (or exported in shell)
+    - DAYTONA_API_URL=https://app.daytona.io/api
+    - DAYTONA_TARGET=us  (or eu)
+
+After this script succeeds, add to surfsense_backend/.env:
+    DAYTONA_SNAPSHOT_ID=surfsense-sandbox
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+_here = Path(__file__).parent
+for candidate in [
+    _here / "../surfsense_backend/.env",
+    _here / ".env",
+    _here / "../.env",
+]:
+    if candidate.exists():
+        load_dotenv(candidate)
+        break
+
+from daytona import CreateSnapshotParams, Daytona, Image  # noqa: E402
+
+SNAPSHOT_NAME = "surfsense-sandbox"
+
+PACKAGES = [
+    "pandas",
+    "numpy",
+    "matplotlib",
+    "scipy",
+    "scikit-learn",
+]
+
+
+def build_image() -> Image:
+    """Build the sandbox image with data-science packages and a /documents symlink."""
+    return (
+        Image.debian_slim("3.12")
+        .pip_install(*PACKAGES)
+        # Symlink /documents → /home/daytona/documents so the LLM can use
+        # the same /documents/ path it sees in the virtual filesystem.
+        .run_commands(
+            "mkdir -p /home/daytona/documents",
+            "ln -sfn /home/daytona/documents /documents",
+        )
+    )
+
+
+def main() -> None:
+    api_key = os.environ.get("DAYTONA_API_KEY")
+    if not api_key:
+        print("ERROR: DAYTONA_API_KEY is not set.", file=sys.stderr)
+        print(
+            "Add it to surfsense_backend/.env or export it in your shell.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    daytona = Daytona()
+
+    try:
+        existing = daytona.snapshot.get(SNAPSHOT_NAME)
+        print(f"Deleting existing snapshot '{SNAPSHOT_NAME}' …")
+        daytona.snapshot.delete(existing)
+        print(f"Deleted '{SNAPSHOT_NAME}'. Waiting for removal to propagate …")
+        for _attempt in range(30):
+            time.sleep(2)
+            try:
+                daytona.snapshot.get(SNAPSHOT_NAME)
+            except Exception:
+                print(f"Confirmed '{SNAPSHOT_NAME}' is gone.\n")
+                break
+        else:
+            print(
+                f"WARNING: '{SNAPSHOT_NAME}' may still exist after 60s. Proceeding anyway.\n"
+            )
+    except Exception:
+        pass
+
+    print(f"Building snapshot '{SNAPSHOT_NAME}' …")
+    print(f"Packages: {', '.join(PACKAGES)}\n")
+
+    daytona.snapshot.create(
+        CreateSnapshotParams(name=SNAPSHOT_NAME, image=build_image()),
+        on_logs=lambda chunk: print(chunk, end="", flush=True),
+    )
+
+    print(f"\n\nSnapshot '{SNAPSHOT_NAME}' is ready.")
+    print("\nAdd this to surfsense_backend/.env:")
+    print(f"    DAYTONA_SNAPSHOT_ID={SNAPSHOT_NAME}")
+
+
+if __name__ == "__main__":
+    main()

surfsense_backend/scripts/docker/entrypoint.e2e.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# =============================================================================
+# E2E entrypoint for the multi-stage Dockerfile's `e2e` target.
+#
+# Dispatches on SERVICE_ROLE to the test-only entrypoints under tests/e2e/.
+# Those scripts apply sys.modules hijacks and LLM/embedding patches BEFORE
+# importing production app code (see tests/e2e/run_backend.py for rationale).
+#
+# Production never sees this file: tests/ is excluded from the production
+# stage, and the production stage uses scripts/docker/entrypoint.sh.
+# =============================================================================
+set -euo pipefail
+
+SERVICE_ROLE="${SERVICE_ROLE:-api}"
+echo "[e2e-entrypoint] starting role=${SERVICE_ROLE}"
+
+wait_for_db() {
+    # Block until the database is reachable. We don't loop forever — Compose
+    # depends_on/healthchecks already gate on db readiness, this is just
+    # belt-and-suspenders so a slow first connection doesn't race migrations.
+    for i in {1..60}; do
+        echo "[e2e-entrypoint] db check attempt ${i}/60"
+        if python -c "from app.db import engine; import asyncio; asyncio.run(engine.dispose())"; then
+            echo "[e2e-entrypoint] db reachable after ${i} attempts"
+            return 0
+        fi
+        sleep 1
+    done
+    echo "[e2e-entrypoint] ERROR: db not reachable after 60s" >&2
+    return 1
+}
+
+case "${SERVICE_ROLE}" in
+    api)
+        wait_for_db
+        echo "[e2e-entrypoint] running alembic upgrade head"
+        alembic upgrade head
+        # `exec` so SIGTERM from `docker stop` reaches Python directly,
+        # without a shell wrapper interposing.
+        exec python tests/e2e/run_backend.py
+        ;;
+    worker)
+        # Worker doesn't run migrations — the api role does that exactly once.
+        # We still wait for db so Celery's broker connection check doesn't
+        # race against an unready Postgres on cold start.
+        wait_for_db
+        exec python tests/e2e/run_celery.py
+        ;;
+    *)
+        echo "[e2e-entrypoint] ERROR: unknown SERVICE_ROLE='${SERVICE_ROLE}' (expected: api | worker)" >&2
+        exit 1
+        ;;
+esac

surfsense_backend/scripts/docker/entrypoint.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+set -e
+
+# ─────────────────────────────────────────────────────────────
+# SERVICE_ROLE controls which process(es) this container runs.
+#
+#   migrate – Run `alembic upgrade head`, verify zero_publication,
+#             then exit 0. Used by the dedicated `migrations` service
+#             in docker-compose.yml so downstream services can gate
+#             on `condition: service_completed_successfully`.
+#   api     – FastAPI backend only (does NOT run migrations)
+#   worker  – Celery worker only
+#   beat    – Celery beat scheduler only
+#   all     – migrations + api + worker + beat in one container
+#             (legacy / dev default; fails fast on migration error)
+#
+# Set SERVICE_ROLE as an environment variable in Coolify for
+# each service deployment.
+# ─────────────────────────────────────────────────────────────
+SERVICE_ROLE="${SERVICE_ROLE:-all}"
+echo "Starting SurfSense with SERVICE_ROLE=${SERVICE_ROLE}"
+
+# ── Autoscale defaults (override via env) ────────────────────
+#   CELERY_MAX_WORKERS  – max concurrent worker processes
+#   CELERY_MIN_WORKERS  – min workers kept warm
+#   CELERY_QUEUES       – comma-separated queues to consume
+#                         (empty = all queues for backward compat)
+CELERY_MAX_WORKERS="${CELERY_MAX_WORKERS:-10}"
+CELERY_MIN_WORKERS="${CELERY_MIN_WORKERS:-2}"
+CELERY_MAX_TASKS_PER_CHILD="${CELERY_MAX_TASKS_PER_CHILD:-50}"
+CELERY_QUEUES="${CELERY_QUEUES:-}"
+
+# ── Graceful shutdown ────────────────────────────────────────
+PIDS=()
+
+cleanup() {
+    echo "Shutting down services..."
+    for pid in "${PIDS[@]}"; do
+        kill -TERM "$pid" 2>/dev/null || true
+    done
+    for pid in "${PIDS[@]}"; do
+        wait "$pid" 2>/dev/null || true
+    done
+    exit 0
+}
+
+trap cleanup SIGTERM SIGINT
+
+# ── Database migrations (only for migrate / all) ─────────────
+# Fail-fast contract:
+#   - alembic upgrade head must succeed within ${MIGRATION_TIMEOUT:-900}s
+#   - zero_publication must match the canonical app.zero_publication shape
+# Either failure exits non-zero so the dedicated `migrations` compose
+# service exits non-zero, halting the rest of the stack instead of
+# silently producing a drifted Zero publication.
+run_migrations() {
+    echo "Running database migrations..."
+    for i in {1..30}; do
+        if python -c "from app.db import engine; import asyncio; asyncio.run(engine.dispose())" 2>/dev/null; then
+            echo "Database is ready."
+            break
+        fi
+        echo "Waiting for database... ($i/30)"
+        sleep 1
+    done
+
+    local timeout_secs="${MIGRATION_TIMEOUT:-900}"
+    echo "Running alembic upgrade head (timeout=${timeout_secs}s)..."
+    if ! timeout "${timeout_secs}" alembic upgrade head; then
+        echo "ERROR: alembic upgrade head failed (or exceeded ${timeout_secs}s timeout)." >&2
+        echo "Refusing to start. Inspect the error above and re-run." >&2
+        exit 1
+    fi
+    echo "Migrations completed successfully."
+
+    echo "Verifying zero_publication matches the canonical shape..."
+    if ! python -m app.zero_publication --verify; then
+        echo "ERROR: zero_publication does not match the canonical shape." >&2
+        echo "Inspect alembic state with:" >&2
+        echo "  docker compose exec db psql -U \"\$DB_USER\" -d \"\$DB_NAME\" -c 'SELECT * FROM alembic_version;'" >&2
+        exit 1
+    fi
+}
+
+# ── Service starters ─────────────────────────────────────────
+start_api() {
+    echo "Starting FastAPI Backend..."
+    python main.py &
+    PIDS+=($!)
+    echo "  FastAPI PID=${PIDS[-1]}"
+}
+
+start_worker() {
+    QUEUE_ARGS=""
+    if [ -n "${CELERY_QUEUES}" ]; then
+        QUEUE_ARGS="--queues=${CELERY_QUEUES}"
+    else
+        # When no queues specified, consume from the default, connectors, and
+        # gateway maintenance queues. Without --queues, Celery only consumes
+        # from the default queue, leaving connector/gateway maintenance tasks stuck.
+        DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}"
+        QUEUE_ARGS="--queues=${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway"
+    fi
+
+    echo "Starting Celery Worker (autoscale=${CELERY_MAX_WORKERS},${CELERY_MIN_WORKERS}, max-tasks-per-child=${CELERY_MAX_TASKS_PER_CHILD}, queues=${CELERY_QUEUES:-all})..."
+    celery -A app.celery_app worker \
+        --loglevel=info \
+        --autoscale="${CELERY_MAX_WORKERS},${CELERY_MIN_WORKERS}" \
+        --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD}" \
+        --prefetch-multiplier=1 \
+        -Ofair \
+        ${QUEUE_ARGS} &
+    PIDS+=($!)
+    echo "  Celery Worker PID=${PIDS[-1]}"
+}
+
+start_beat() {
+    echo "Starting Celery Beat..."
+    celery -A app.celery_app beat --loglevel=info &
+    PIDS+=($!)
+    echo "  Celery Beat PID=${PIDS[-1]}"
+}
+
+# ── Main: run based on role ──────────────────────────────────
+case "${SERVICE_ROLE}" in
+    migrate)
+        run_migrations
+        echo "Migrations complete; exiting cleanly."
+        exit 0
+        ;;
+    api)
+        start_api
+        ;;
+    worker)
+        start_worker
+        ;;
+    beat)
+        start_beat
+        ;;
+    all)
+        run_migrations
+        start_api
+        sleep 5
+        start_worker
+        sleep 3
+        start_beat
+        ;;
+    *)
+        echo "ERROR: Unknown SERVICE_ROLE '${SERVICE_ROLE}'. Use: migrate, api, worker, beat, or all"
+        exit 1
+        ;;
+esac
+
+echo "All requested services started. PIDs: ${PIDS[*]}"
+
+# Wait for any process to exit
+wait -n
+
+# If we get here, one process exited unexpectedly
+exit $?

surfsense_backend/scripts/e2e_google_maps_deep.py

@@ -0,0 +1,824 @@
+"""Deep functional verification for the Google Maps scraper (live network).
+
+Complements the fast smoke e2e (e2e_google_maps_scraper.py) with breadth:
+diverse places (countries, scripts, categories), URL-kind coverage (name-only
+URLs, CID), and review semantics (sort order, date cutoff, pagination
+uniqueness, personal-data stripping, localization).
+
+Verification style: ground-truth invariants instead of screenshots — known
+coordinates/websites/address keywords for world-famous places, and internal
+consistency rules (newest sort is monotonically non-increasing, cutoff dates
+hold, review IDs are unique across pages, etc.).
+
+Run from the backend directory:
+    .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_deep.py
+"""
+
+import asyncio
+import itertools
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.platforms.google_maps import (  # noqa: E402
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    scrape_places,
+    scrape_reviews,
+)
+
+_CHECKS: list[tuple[str, bool, str]] = []
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    _CHECKS.append((label, ok, detail))
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _near(actual: float | None, expected: float, tol: float = 0.05) -> bool:
+    return actual is not None and abs(actual - expected) <= tol
+
+
+def _iso(s: str | None) -> datetime | None:
+    if not s:
+        return None
+    try:
+        return datetime.fromisoformat(s.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+# Ground truth: world-famous places whose facts don't drift. Name-only URLs
+# exercise the HTML -> fid resolution path for every one of them.
+_PLACES = [
+    {
+        "label": "Eiffel Tower (FR landmark)",
+        "url": "https://www.google.com/maps/place/Eiffel+Tower/",
+        "title_contains": "eiffel",
+        "lat": 48.8584,
+        "lng": 2.2945,
+        "address_contains": ["paris", "75007", "france"],
+        "website_contains": "toureiffel",
+    },
+    {
+        "label": "Tokyo Tower (JP, non-Latin locale)",
+        "url": "https://www.google.com/maps/place/Tokyo+Tower/",
+        "title_contains": "tokyo tower",
+        "lat": 35.6586,
+        "lng": 139.7454,
+        "address_contains": ["tokyo", "japan", "minato"],
+    },
+    {
+        "label": "Sydney Opera House (AU)",
+        "url": "https://www.google.com/maps/place/Sydney+Opera+House/",
+        "title_contains": "opera house",
+        "lat": -33.8568,
+        "lng": 151.2153,
+        "address_contains": ["sydney", "nsw", "australia"],
+    },
+    {
+        "label": "The Plaza Hotel (US hotel category)",
+        "url": "https://www.google.com/maps/place/The+Plaza+Hotel+New+York/",
+        "title_contains": "plaza",
+        "lat": 40.7646,
+        "lng": -73.9744,
+        "address_contains": ["new york", "ny"],
+        "category_contains": "hotel",
+    },
+]
+
+_KIMS_CID_URL = "https://maps.google.com/?cid=7838756667406262025"  # Kim's Island
+_KIMS_PLACE_ID = "ChIJJQz5EZzKw4kRCZ95UajbyGw"
+_LOUVRE_URL = "https://www.google.com/maps/place/Louvre+Museum/"
+
+
+async def scrape_one(url: str, **kwargs) -> dict | None:
+    items = await scrape_places(
+        GoogleMapsScrapeInput(startUrls=[{"url": url}], **kwargs)
+    )
+    return items[0] if items else None
+
+
+async def step_diverse_places() -> None:
+    _hr("A — diverse places via name-only URLs (HTML -> fid path)")
+    for spec in _PLACES:
+        it = await scrape_one(spec["url"])
+        if it is None:
+            _check(spec["label"], False, "no item returned")
+            continue
+        title = (it.get("title") or "").lower()
+        loc = it.get("location") or {}
+        addr = (it.get("address") or "").lower()
+        problems = []
+        if spec["title_contains"] not in title:
+            problems.append(f"title={it.get('title')!r}")
+        if not _near(loc.get("lat"), spec["lat"]) or not _near(
+            loc.get("lng"), spec["lng"]
+        ):
+            problems.append(f"loc={loc}")
+        if not any(k in addr for k in spec["address_contains"]):
+            problems.append(f"address={it.get('address')!r}")
+        if not it.get("placeId"):
+            problems.append("no placeId")
+        if not it.get("categories"):
+            problems.append("no categories")
+        if spec.get("website_contains") and spec["website_contains"] not in (
+            it.get("website") or ""
+        ):
+            problems.append(f"website={it.get('website')!r}")
+        if (
+            spec.get("category_contains")
+            and spec["category_contains"]
+            not in (
+                (it.get("categoryName") or "") + " ".join(it.get("categories") or [])
+            ).lower()
+        ):
+            problems.append(f"categories={it.get('categories')}")
+        _check(
+            spec["label"],
+            not problems,
+            "; ".join(problems)
+            or f"{it.get('title')!r} @ ({loc.get('lat'):.4f},{loc.get('lng'):.4f}) "
+            f"cat={it.get('categoryName')!r} score={it.get('totalScore')}",
+        )
+
+
+async def step_cid_url() -> None:
+    _hr("B — CID URL dispatch")
+    it = await scrape_one(_KIMS_CID_URL)
+    ok = it is not None and it.get("title") == "Kim's Island"
+    _check(
+        "?cid=... resolves to the right place",
+        ok,
+        f"title={it.get('title')!r}" if it else "no item",
+    )
+    if it:
+        _check(
+            "cid place has full detail (phone+hours)",
+            bool(it.get("phone")) and bool(it.get("openingHours")),
+            f"phone={it.get('phone')!r}, hours={len(it.get('openingHours') or [])} days",
+        )
+
+
+async def step_review_sorts() -> None:
+    _hr("C — review sort semantics (Kim's Island)")
+    newest = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10)
+    )
+    dates = [_iso(r.get("publishedAtDate")) for r in newest]
+    dated = [d for d in dates if d]
+    _check(
+        "newest: publishedAtDate non-increasing",
+        len(dated) >= 5 and all(a >= b for a, b in itertools.pairwise(dated)),
+        f"{len(newest)} reviews, first={newest[0].get('publishAt') if newest else None}",
+    )
+
+    lowest = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="lowestRanking"
+        )
+    )
+    highest = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="highestRanking"
+        )
+    )
+    lo = [r["stars"] for r in lowest if r.get("stars")]
+    hi = [r["stars"] for r in highest if r.get("stars")]
+    lo_avg = sum(lo) / len(lo) if lo else 0
+    hi_avg = sum(hi) / len(hi) if hi else 0
+    _check(
+        "lowestRanking avg < highestRanking avg",
+        bool(lo and hi) and lo_avg < hi_avg,
+        f"lowest avg={lo_avg:.2f} (first={lo[:3]}), highest avg={hi_avg:.2f} (first={hi[:3]})",
+    )
+    _check(
+        "highestRanking page is all 5 stars",
+        bool(hi) and all(s == 5 for s in hi),
+        f"stars={hi}",
+    )
+
+
+async def step_start_date_cutoff() -> None:
+    _hr("D — reviewsStartDate cutoff")
+    baseline = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10)
+    )
+    dated = [(_iso(r.get("publishedAtDate")), r) for r in baseline]
+    dated = [(d, r) for d, r in dated if d]
+    if len(dated) < 6:
+        _check("cutoff test has enough dated reviews", False, f"only {len(dated)}")
+        return
+    # Cut between the 4th and 5th newest review -> expect exactly 4 back.
+    cutoff_dt = dated[4][0]
+    cutoff = (
+        dated[3][0].strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        if dated[3][0] == cutoff_dt
+        else cutoff_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+    )
+    got = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=100, reviewsStartDate=cutoff
+        )
+    )
+    got_dates = [_iso(r.get("publishedAtDate")) for r in got]
+    cutoff_parsed = _iso(cutoff)
+    _check(
+        "all returned reviews >= cutoff",
+        bool(got) and all(d is None or d >= cutoff_parsed for d in got_dates),
+        f"cutoff={cutoff}, returned={len(got)} (expected ~4)",
+    )
+    _check(
+        "cutoff actually limits the result",
+        0 < len(got) < len(baseline) + 1 and len(got) <= 6,
+        f"{len(got)} vs baseline {len(baseline)}",
+    )
+
+
+async def step_personal_data() -> None:
+    _hr("E — personalData=false stripping")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=3, personalData=False
+        )
+    )
+    if not items:
+        _check("reviews returned", False)
+        return
+    leaked = [
+        k
+        for k in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl")
+        if any(r.get(k) for r in items)
+    ]
+    _check(
+        "reviewer identity fields are absent",
+        not leaked,
+        f"leaked={leaked}" if leaked else f"{len(items)} reviews, ids+stars kept",
+    )
+    _check(
+        "non-personal fields survive",
+        all(r.get("reviewId") and r.get("stars") for r in items),
+    )
+
+
+async def step_localization() -> None:
+    _hr("F — localization (language=fr)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            startUrls=[{"url": "https://www.google.com/maps/place/Eiffel+Tower/"}],
+            maxReviews=5,
+            language="fr",
+        )
+    )
+    if not items:
+        _check("french reviews returned", False)
+        return
+    rel = [r.get("publishAt") or "" for r in items]
+    french = [s for s in rel if "il y a" in s or "mois" in s or "semaine" in s]
+    _check(
+        "relative dates come back in French",
+        len(french) >= 3,
+        f"publishAt={rel}",
+    )
+    _check(
+        "items stamped language=fr",
+        all(r.get("language") == "fr" for r in items),
+    )
+
+
+async def step_big_place_pagination() -> None:
+    _hr("G — big place, 30 reviews across >=3 pages (Louvre)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(startUrls=[{"url": _LOUVRE_URL}], maxReviews=30)
+    )
+    ids = [r.get("reviewId") for r in items]
+    _check(
+        "30 reviews with unique IDs",
+        len(items) == 30 and len(set(ids)) == 30,
+        f"{len(items)} reviews, {len(set(ids))} unique",
+    )
+    ok_fields = all(
+        r.get("name") and r.get("stars") is not None and r.get("publishedAtDate")
+        for r in items
+    )
+    _check("every review has author/stars/date", ok_fields)
+    _check(
+        "place header stamped on all (Louvre)",
+        all("louvre" in (r.get("title") or "").lower() for r in items),
+        f"title={items[0].get('title')!r}" if items else "",
+    )
+
+
+async def step_search_discovery() -> None:
+    _hr("I — search discovery: paging, rank, dedupe (pizza in New York)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["pizza"],
+            locationQuery="New York, NY",
+            maxCrawledPlacesPerSearch=25,
+        )
+    )
+    fids = [i.get("fid") for i in items]
+    _check(
+        "25 places across >1 page, all unique fids",
+        len(items) == 25 and len(set(fids)) == 25,
+        f"{len(items)} items, {len(set(fids))} unique",
+    )
+    _check(
+        "ranks are 1..25 in order",
+        [i.get("rank") for i in items] == list(range(1, 26)),
+    )
+    ny = sum(
+        1
+        for i in items
+        if i.get("location")
+        and 40.4 < (i["location"]["lat"] or 0) < 41.1
+        and -74.3 < (i["location"]["lng"] or 0) < -73.6
+    )
+    _check(
+        "locationQuery scopes results to NYC",
+        ny >= 23,
+        f"{ny}/25 within NYC bounds",
+    )
+    with_core = sum(
+        1 for i in items if i.get("title") and i.get("placeId") and i.get("address")
+    )
+    _check("all items have title/placeId/address", with_core == 25, f"{with_core}/25")
+
+
+async def step_search_filters() -> None:
+    _hr("J — search filters (stars, website, matching)")
+    starred = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["restaurant"],
+            locationQuery="Chicago, IL",
+            maxCrawledPlacesPerSearch=10,
+            placeMinimumStars="fourAndHalf",
+        )
+    )
+    scores = [i.get("totalScore") for i in starred]
+    _check(
+        "placeMinimumStars=fourAndHalf holds",
+        bool(scores) and all(s is not None and s >= 4.5 for s in scores),
+        f"scores={scores}",
+    )
+
+    no_web = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["restaurant"],
+            locationQuery="Chicago, IL",
+            maxCrawledPlacesPerSearch=5,
+            website="withoutWebsite",
+        )
+    )
+    _check(
+        "website=withoutWebsite holds",
+        bool(no_web) and all(not i.get("website") for i in no_web),
+        f"{len(no_web)} items, websites={[i.get('website') for i in no_web]}",
+    )
+
+    matching = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["pizza"],
+            locationQuery="Boston, MA",
+            maxCrawledPlacesPerSearch=5,
+            searchMatching="only_includes",
+        )
+    )
+    titles = [i.get("title") for i in matching]
+    _check(
+        "searchMatching=only_includes keeps 'pizza' in titles",
+        bool(titles) and all("pizza" in (t or "").lower() for t in titles),
+        f"titles={titles}",
+    )
+
+
+async def step_search_closed() -> None:
+    _hr("K — closed-place detection + skipClosedPlaces (Dean & DeLuca)")
+    q = "Dean DeLuca 560 Broadway New York"
+    found = await scrape_places(
+        GoogleMapsScrapeInput(searchStringsArray=[q], maxCrawledPlacesPerSearch=3)
+    )
+    dean = next((i for i in found if "DeLuca" in (i.get("title") or "")), None)
+    _check(
+        "permanently closed place flagged",
+        dean is not None and dean.get("permanentlyClosed") is True,
+        f"title={dean.get('title') if dean else None}, closed={dean.get('permanentlyClosed') if dean else None}",
+    )
+    skipped = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=[q],
+            maxCrawledPlacesPerSearch=3,
+            skipClosedPlaces=True,
+        )
+    )
+    _check(
+        "skipClosedPlaces filters it out",
+        not any("DeLuca" in (i.get("title") or "") for i in skipped),
+        f"{len(skipped)} items after skip",
+    )
+
+
+async def step_search_url_and_geo() -> None:
+    _hr("L — /maps/search/ URL routing + customGeolocation")
+    via_url = await scrape_places(
+        GoogleMapsScrapeInput(
+            startUrls=[
+                {"url": "https://www.google.com/maps/search/ramen+in+Osaka+Japan/"}
+            ],
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    osaka = sum(
+        1
+        for i in via_url
+        if i.get("location") and 34.4 < (i["location"]["lat"] or 0) < 35.0
+    )
+    _check(
+        "/maps/search/ startUrl yields Osaka ramen places",
+        len(via_url) == 5 and osaka >= 4,
+        f"{len(via_url)} items, {osaka} in Osaka",
+    )
+
+    geo = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["museum"],
+            customGeolocation={
+                "type": "Point",
+                "coordinates": [2.3522, 48.8566],  # [lng, lat] Paris
+                "radiusKm": 10,
+            },
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    paris = sum(
+        1
+        for i in geo
+        if i.get("location")
+        and 48.5 < (i["location"]["lat"] or 0) < 49.2
+        and 1.9 < (i["location"]["lng"] or 0) < 2.8
+    )
+    _check(
+        "customGeolocation Point scopes to Paris",
+        len(geo) >= 3 and paris >= 3,
+        f"{len(geo)} items, {paris} in Paris: {[i.get('title') for i in geo]}",
+    )
+
+
+async def step_search_pagination_stress() -> None:
+    _hr("M — search pagination stress (60 results / 3+ pages, exhaustion)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["restaurant"],
+            locationQuery="Manhattan, New York",
+            maxCrawledPlacesPerSearch=60,
+        )
+    )
+    fids = [i.get("fid") for i in items]
+    _check(
+        "60 places, all fids unique (offset paging + dedupe)",
+        len(items) == 60 and len(set(fids)) == 60,
+        f"{len(items)} items, {len(set(fids))} unique",
+    )
+    _check(
+        "ranks strictly sequential 1..60",
+        [i.get("rank") for i in items] == list(range(1, 61)),
+    )
+    manhattan = sum(
+        1
+        for i in items
+        if i.get("location") and 40.68 < (i["location"]["lat"] or 0) < 40.9
+    )
+    _check(
+        "results stay in Manhattan across pages",
+        manhattan >= 55,
+        f"{manhattan}/60 in bounds",
+    )
+
+    # A hyper-specific query has few results: paging must terminate on its
+    # own (no infinite loop, no error) well before the requested cap.
+    sparse = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["Kim's Island Staten Island"],
+            maxCrawledPlacesPerSearch=100,
+        )
+    )
+    _check(
+        "sparse query exhausts naturally below the cap",
+        0 < len(sparse) < 25,
+        f"{len(sparse)} results",
+    )
+
+
+async def step_detail_extras() -> None:
+    _hr("N — detail-page extras (kgmid/cid/additionalInfo/links)")
+    # pin the exact place by fid — a name search returns a different Joe's
+    # branch run to run, which makes magnitude assertions flaky
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            startUrls=[
+                {
+                    "url": "https://www.google.com/maps/place/Joe's+Pizza+Broadway/"
+                    "data=!4m2!3m1!1s0x89c259ab3c1ef289:0x3b67a41175949f55"
+                }
+            ],
+            maxImages=5,
+        )
+    )
+    if not items:
+        _check("place returned", False)
+        return
+    it = items[0]
+    dist = it.get("reviewsDistribution") or {}
+    _check(
+        "reviewsCount + distribution (NID-session fields)",
+        (it.get("reviewsCount") or 0) > 20_000
+        and (it.get("reviewsCount") == sum(dist.values())),
+        f"count={it.get('reviewsCount')}, dist={dist}",
+    )
+    hist = it.get("popularTimesHistogram") or {}
+    _check(
+        "popular times histogram covers the week",
+        set(hist) == {"Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"}
+        and all(
+            {"hour", "occupancyPercent"} <= set(slot)
+            for d in hist.values()
+            for slot in d
+        ),
+        f"days={sorted(hist)}",
+    )
+    _check(
+        "image gallery fields (count/categories/urls capped at maxImages)",
+        (it.get("imagesCount") or 0) > 1000
+        and "All" in (it.get("imageCategories") or [])
+        and len(it.get("imageUrls") or []) == 5,
+        f"count={it.get('imagesCount')}, cats={len(it.get('imageCategories') or [])}, "
+        f"urls={len(it.get('imageUrls') or [])}",
+    )
+    tags = it.get("reviewsTags") or []
+    _check(
+        "reviewsTags with counts",
+        len(tags) >= 5 and all(t.get("title") and t.get("count") for t in tags),
+        f"{tags[:2]}",
+    )
+    _check(
+        "additionalInfo has full section set (not just Accessibility)",
+        len(it.get("additionalInfo") or {}) >= 8,
+        f"sections={list((it.get('additionalInfo') or {}).keys())}",
+    )
+
+    # scrapePlaceDetailPage on the SEARCH flow: search darrays lack the
+    # session-gated fields, so each hit must get enriched via a detail RPC.
+    enriched = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["ramen"],
+            locationQuery="Chicago, IL",
+            maxCrawledPlacesPerSearch=2,
+            scrapePlaceDetailPage=True,
+        )
+    )
+    _check(
+        "search flow + scrapePlaceDetailPage enriches every hit",
+        len(enriched) == 2
+        and all(
+            (e.get("reviewsCount") or 0) > 0 and e.get("reviewsDistribution")
+            for e in enriched
+        ),
+        f"counts={[e.get('reviewsCount') for e in enriched]}",
+    )
+    fid = it.get("fid") or ""
+    cid_ok = bool(fid) and it.get("cid") == str(int(fid.split(":")[1], 16))
+    _check(
+        "kgmid + cid derived from fid",
+        bool(it.get("kgmid", "").startswith("/g/")) and cid_ok,
+        f"kgmid={it.get('kgmid')}, cid={it.get('cid')}",
+    )
+    info = it.get("additionalInfo") or {}
+    _check(
+        "additionalInfo has sections with boolean options",
+        bool(info)
+        and all(
+            isinstance(v, list) and all(isinstance(e, dict) for e in v)
+            for v in info.values()
+        ),
+        f"sections={list(info.keys())}",
+    )
+
+
+async def step_hotel_fields() -> None:
+    _hr("O — hotel fields (The Plaza: stars, dates, similar, ads)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            startUrls=[
+                {
+                    "url": "https://www.google.com/maps/place/The+Plaza/"
+                    "data=!4m2!3m1!1s0x89c258f07d5da561:0x61f6aa300ba8339d"
+                }
+            ],
+        )
+    )
+    if not items:
+        _check("hotel returned", False)
+        return
+    it = items[0]
+    _check(
+        "hotelStars + check-in/out dates",
+        it.get("hotelStars") == "5 stars"
+        and (it.get("checkInDate") or "") < (it.get("checkOutDate") or ""),
+        f"stars={it.get('hotelStars')}, {it.get('checkInDate')}..{it.get('checkOutDate')}",
+    )
+    similar = it.get("similarHotelsNearby") or []
+    _check(
+        "similarHotelsNearby with fid/score",
+        len(similar) >= 3 and all(h.get("title") and h.get("fid") for h in similar),
+        f"{len(similar)} hotels, first={similar[0].get('title') if similar else None}",
+    )
+    ads = it.get("hotelAds") or []
+    _check(
+        "hotelAds with booking links",
+        bool(ads) and all(a.get("url", "").startswith("https://") for a in ads),
+        f"{len(ads)} ads",
+    )
+
+
+async def step_all_places_scan() -> None:
+    _hr("P — allPlacesNoSearchAction area scan (Times Square 400m)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            allPlacesNoSearchAction="all_places_no_search_mouse",
+            customGeolocation={
+                "type": "Point",
+                "coordinates": [-73.9855, 40.758],
+                "radiusKm": 0.4,
+            },
+            maxCrawledPlacesPerSearch=25,
+        )
+    )
+    fids = [i.get("fid") for i in items]
+    _check(
+        "25 unique places without any search term",
+        len(items) == 25 and len(set(fids)) == 25,
+        f"{len(items)} items",
+    )
+    cats = {i.get("categoryName") for i in items if i.get("categoryName")}
+    _check(
+        "multiple categories represented (sweep, not one query)",
+        len(cats) >= 5,
+        f"{len(cats)} categories",
+    )
+    in_view = sum(
+        1 for i in items if i.get("location") and 40.74 < i["location"]["lat"] < 40.78
+    )
+    _check("scan respects the viewport", in_view >= 23, f"{in_view}/25 in bounds")
+
+
+async def step_short_url() -> None:
+    _hr("Q — short link (maps.app.goo.gl Firebase redirect -> place)")
+    # A real shared short link -> "Aux Merveilleux de Fred" (NYC). These are
+    # Firebase Dynamic Links (JS interstitial), so this exercises the browser-
+    # render redirect path in resolve_fid. fid is the ground-truth invariant.
+    it = await scrape_one("https://maps.app.goo.gl/8YUvDPbQPrasqC528")
+    _check(
+        "maps.app.goo.gl short link resolves to the right place",
+        it is not None
+        and it.get("fid") == "0x89c259957da502cd:0xed3eb58a4ca08a95"
+        and "merveilleux" in (it.get("title") or "").lower(),
+        f"title={it.get('title') if it else None!r}, fid={it.get('fid') if it else None}",
+    )
+
+
+async def step_filter_variants() -> None:
+    _hr("R — search filter variants (only_exact, categoryFilterWords)")
+    # only_exact: the parsed title must equal the query exactly (Seattle
+    # Starbucks are titled "Starbucks Coffee Company", not "Starbucks").
+    exact = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["Starbucks Coffee Company"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=8,
+            searchMatching="only_exact",
+        )
+    )
+    titles = [i.get("title") for i in exact]
+    _check(
+        "searchMatching=only_exact keeps only exact-title matches",
+        bool(titles)
+        and all((t or "").lower() == "starbucks coffee company" for t in titles),
+        f"{len(titles)} items, titles={titles[:3]}",
+    )
+    # categoryFilterWords: drop places whose categories don't include a word.
+    coffee = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["food"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=6,
+            categoryFilterWords=["coffee"],
+        )
+    )
+    _check(
+        "categoryFilterWords keeps only matching categories",
+        bool(coffee)
+        and all(
+            any("coffee" in c.lower() for c in (i.get("categories") or []))
+            for i in coffee
+        ),
+        f"{len(coffee)} items, cats={[i.get('categories') for i in coffee][:2]}",
+    )
+
+
+async def step_reviews_origin() -> None:
+    _hr("S — reviewsOrigin=google filter")
+    # The public BOQ feed only carries Google-origin reviews (partner reviews
+    # aren't exposed anonymously), so the invariant is: nothing non-Google
+    # leaks through when origin is pinned to google.
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsOrigin="google"
+        )
+    )
+    origins = {(r.get("reviewOrigin") or "Google") for r in items}
+    _check(
+        "reviewsOrigin=google -> only Google-origin reviews",
+        bool(items) and origins <= {"Google"},
+        f"{len(items)} reviews, origins={origins}",
+    )
+
+
+async def step_inline_consistency() -> None:
+    _hr("H — inline reviews[] match the standalone reviews endpoint")
+    place = await scrape_one(
+        f"https://www.google.com/maps/place/?q=place_id:{_KIMS_PLACE_ID}",
+        maxReviews=5,
+    )
+    standalone = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=5)
+    )
+    if not place or not standalone:
+        _check("both sources returned data", False)
+        return
+    inline_ids = {r.get("reviewId") for r in (place.get("reviews") or [])}
+    standalone_ids = {r.get("reviewId") for r in standalone}
+    overlap = len(inline_ids & standalone_ids)
+    _check(
+        "inline and standalone reviews overlap (same feed)",
+        overlap >= 3,  # feed ordering can shift slightly between calls
+        f"{overlap}/5 shared review IDs",
+    )
+
+
+async def main() -> int:
+    steps = [
+        step_diverse_places(),
+        step_cid_url(),
+        step_review_sorts(),
+        step_start_date_cutoff(),
+        step_personal_data(),
+        step_localization(),
+        step_big_place_pagination(),
+        step_inline_consistency(),
+        step_search_discovery(),
+        step_search_filters(),
+        step_search_closed(),
+        step_search_url_and_geo(),
+        step_search_pagination_stress(),
+        step_detail_extras(),
+        step_hotel_fields(),
+        step_all_places_scan(),
+        step_short_url(),
+        step_filter_variants(),
+        step_reviews_origin(),
+    ]
+    for coro in steps:
+        try:
+            await coro
+        except Exception as e:  # keep going; report the step as failed
+            _check(f"step crashed: {coro}", False, repr(e))
+
+    _hr("SUMMARY")
+    passed = sum(1 for _, ok, _ in _CHECKS if ok)
+    for label, ok, detail in _CHECKS:
+        if not ok:
+            print(f"  FAILED: {label} — {detail}")
+    print(f"  {passed}/{len(_CHECKS)} checks passed")
+    return 0 if passed == len(_CHECKS) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_google_maps_scraper.py

@@ -0,0 +1,241 @@
+"""Manual functional e2e for the Google Maps scraper (app/proprietary/platforms/google_maps).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_google_maps_scraper.py
+    # or: .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_scraper.py
+
+NOT a pytest test (needs live network + optional proxy creds). It:
+  Step 1 — scrapes a known place URL and prints the core fields.
+  Step 2 — scrapes the same place by bare placeId (HTML -> fid -> RPC path).
+  Step 3 — dumps the raw place darray (jd[6] of /maps/preview/place) to
+      tests/unit/platforms/google_maps/fixtures/ for the offline parser test.
+  Step 4 — scrapes reviews via the Reviews endpoint (BOQ feed), checks fields.
+  Step 5 — paginates past one page (maxReviews=15) and checks the count.
+  Step 6 — place scrape with maxReviews>0 attaches inline reviews[].
+  Step 7 — dumps a raw BOQ reviews page fixture for the offline parser test.
+  Step 8 — search discovery (searchStringsArray + locationQuery), checks items.
+  Step 9 — dumps a raw map-search response fixture for the offline parser test.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Windows consoles default to cp1252; reviews/ratings contain non-latin chars.
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.platforms.google_maps import (  # noqa: E402
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    scrape_places,
+    scrape_reviews,
+)
+from app.proprietary.platforms.google_maps.fetch import (  # noqa: E402
+    build_search_url,
+    fetch_place_darray,
+    fetch_rpc_json,
+    iter_reviews_pages,
+)
+from app.proprietary.platforms.google_maps.url_resolver import extract_fid  # noqa: E402
+
+# A well-known, stable place (the restaurant used in the Apify output example).
+_PLACE_URL = (
+    "https://www.google.com/maps/place/Kim's+Island/"
+    "@40.5107736,-74.2482624,17z/data=!4m6!3m5!1s0x89c3ca9c11f90c25:"
+    "0x6cc8dba851799f09!8m2!3d40.5107736!4d-74.2482624!16s%2Fg%2F1tmgdcj8?hl=en"
+)
+
+_FIXTURE_DIR = (
+    _BACKEND_ROOT / "tests" / "unit" / "platforms" / "google_maps" / "fixtures"
+)
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+async def step1_place() -> bool:
+    _hr("STEP 1 — scrape a known place URL")
+    items = await scrape_places(GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}]))
+    if not items:
+        return _check("place scraped", False, "no items returned")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:2500])
+    ok = bool(it.get("title")) and it.get("placeId") is not None
+    return _check(
+        "place has title + placeId",
+        ok,
+        f"{it.get('title')!r} / {it.get('totalScore')}★ / {it.get('reviewsCount')} reviews",
+    )
+
+
+async def step2_place_id() -> bool:
+    _hr("STEP 2 — scrape by bare placeId (HTML -> fid -> RPC path)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"])
+    )
+    if not items:
+        return _check("placeId scraped", False, "no items returned")
+    it = items[0]
+    return _check(
+        "placeId resolves to same place",
+        it.get("title") == "Kim's Island",
+        f"{it.get('title')!r}",
+    )
+
+
+async def step3_dump_fixture() -> bool:
+    _hr("STEP 3 — dump raw place darray fixture for offline test")
+    fid = extract_fid(_PLACE_URL)
+    if not fid:
+        return _check("extracted fid from URL", False)
+    darray = await fetch_place_darray(fid)
+    if not darray:
+        return _check("fetched place darray via RPC", False)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    (_FIXTURE_DIR / "place_darray.json").write_text(
+        json.dumps(darray), encoding="utf-8"
+    )
+    return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'place_darray.json'}")
+
+
+async def step4_reviews() -> bool:
+    _hr("STEP 4 — reviews endpoint (one page, newest first)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(startUrls=[{"url": _PLACE_URL}], maxReviews=5)
+    )
+    if not items:
+        return _check("reviews returned", False, "no items")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:1800])
+    ok = (
+        bool(it.get("name"))
+        and it.get("stars") is not None
+        and bool(it.get("reviewId"))
+        and it.get("title") == "Kim's Island"  # place header stamped on
+        and bool(it.get("publishedAtDate"))
+    )
+    return _check(
+        "review has author/stars/id/place header",
+        ok and len(items) == 5,
+        f"{len(items)} reviews, first by {it.get('name')!r} ({it.get('stars')}★)",
+    )
+
+
+async def step5_pagination() -> bool:
+    _hr("STEP 5 — pagination past one page (maxReviews=15)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], maxReviews=15)
+    )
+    dates = [i.get("publishedAtDate") for i in items[:3]]
+    return _check(
+        "got 15 reviews across >1 page",
+        len(items) == 15,
+        f"{len(items)} reviews; newest: {dates}",
+    )
+
+
+async def step6_inline_reviews() -> bool:
+    _hr("STEP 6 — place scrape with maxReviews=3 attaches inline reviews[]")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}], maxReviews=3)
+    )
+    if not items:
+        return _check("place scraped", False, "no items")
+    reviews = items[0].get("reviews") or []
+    return _check(
+        "place item carries 3 inline reviews",
+        len(reviews) == 3 and bool(reviews[0].get("name")),
+        f"{len(reviews)} reviews inline",
+    )
+
+
+async def step7_dump_reviews_fixture() -> bool:
+    _hr("STEP 7 — dump raw BOQ reviews page fixture for offline test")
+    fid = extract_fid(_PLACE_URL)
+    async for raw_page in iter_reviews_pages(fid, sort="newest", max_pages=1):
+        _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+        (_FIXTURE_DIR / "boq_reviews_page.json").write_text(
+            json.dumps(raw_page), encoding="utf-8"
+        )
+        return _check(
+            "dumped fixture",
+            len(raw_page) > 0,
+            f"{len(raw_page)} reviews -> {_FIXTURE_DIR / 'boq_reviews_page.json'}",
+        )
+    return _check("fetched a reviews page", False)
+
+
+async def step8_search() -> bool:
+    _hr("STEP 8 — search discovery (query + locationQuery)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["coffee shop"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    if not items:
+        return _check("search returned items", False, "no items")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:1500])
+    ok = (
+        len(items) == 5
+        and all(i.get("title") and i.get("placeId") and i.get("fid") for i in items)
+        and [i.get("rank") for i in items] == [1, 2, 3, 4, 5]
+        and it.get("searchString") == "coffee shop"
+    )
+    return _check(
+        "5 ranked places with title/placeId/fid",
+        ok,
+        f"first: {it.get('title')!r} ({it.get('totalScore')}★, {it.get('city')})",
+    )
+
+
+async def step9_dump_search_fixture() -> bool:
+    _hr("STEP 9 — dump raw map-search response fixture for offline test")
+    url = build_search_url("pizza new york")
+    jd = await fetch_rpc_json(url)
+    if not isinstance(jd, list):
+        return _check("fetched search response", False)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    (_FIXTURE_DIR / "search_response.json").write_text(json.dumps(jd), encoding="utf-8")
+    return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'search_response.json'}")
+
+
+async def main() -> int:
+    results = [
+        await step1_place(),
+        await step2_place_id(),
+        await step3_dump_fixture(),
+        await step4_reviews(),
+        await step5_pagination(),
+        await step6_inline_reviews(),
+        await step7_dump_reviews_fixture(),
+        await step8_search(),
+        await step9_dump_search_fixture(),
+    ]
+    _hr("SUMMARY")
+    print(f"  {sum(results)}/{len(results)} steps passed")
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_google_search.py

@@ -0,0 +1,206 @@
+"""Live end-to-end checks for the Google Search scraper (needs proxy + browser).
+
+    .venv/Scripts/python.exe scripts/e2e_google_search.py
+
+Covers: a plain query, a site: filter, text ads, product ads, the
+focusOnPaidAds retry (commercial = ads found; non-commercial = retries capped,
+organic still returned), People-Also-Ask answer expansion, sitelinks, the AI
+Overview, the mobile layout, filter=0, base64 icons, and Google AI Mode.
+
+Pass case names as args to run a subset, e.g.:
+
+    .venv/Scripts/python.exe scripts/e2e_google_search.py paa
+"""
+
+import asyncio
+import logging
+import sys
+import time
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_ROOT))
+load_dotenv(_ROOT / ".env")
+
+logging.basicConfig(level=logging.WARNING)
+logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(
+    logging.INFO
+)
+
+from app.proprietary.platforms.google_search import (  # noqa: E402
+    GoogleSearchScrapeInput,
+    scrape_serps,
+)
+from app.proprietary.platforms.google_search.fetch import close_sessions  # noqa: E402
+
+
+async def run_ai_mode(label: str, *, queries: str) -> None:
+    print(f"\n=== {label} ===")
+    t0 = time.perf_counter()
+    inp = GoogleSearchScrapeInput(
+        queries=queries,
+        countryCode="us",
+        languageCode="en",
+        aiModeSearch={"enableAiMode": True},
+    )
+    items = await scrape_serps(inp, limit=2)
+    ai_items = [i for i in items if i["aiModeResult"]]
+    assert ai_items, f"{label}: no aiModeResult item emitted"
+    res = ai_items[0]["aiModeResult"]
+    print(
+        f"  text={len(res['text'])} chars, sources={len(res['sources'])} "
+        f"({time.perf_counter() - t0:.0f}s)"
+    )
+    print(f"  {res['text'][:130]!r}")
+    for s in res["sources"][:3]:
+        print(f"    src: {(s['title'] or '')[:60]!r}")
+    assert res["text"] and len(res["text"]) > 100, f"{label}: answer too short"
+    assert res["sources"], f"{label}: no cited sources"
+    assert "udm=50" in ai_items[0]["searchQuery"]["url"]
+
+
+async def run(
+    label: str,
+    *,
+    expect_ads=False,
+    expect_products=False,
+    expect_paa_answers=False,
+    expect_sitelinks=False,
+    expect_aio=False,
+    expect_device=None,
+    expect_icons=False,
+    **kwargs,
+) -> None:
+    print(f"\n=== {label} ===")
+    t0 = time.perf_counter()
+    inp = GoogleSearchScrapeInput(countryCode="us", languageCode="en", **kwargs)
+    items = await scrape_serps(inp, limit=1)
+    assert items, f"{label}: no SERP item"
+    it = items[0]
+    paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]]
+    sitelinked = [o for o in it["organicResults"] if o["siteLinks"]]
+    print(f"  term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}")
+    print(
+        f"  organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} "
+        f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} "
+        f"suggested={len(it['suggestedResults'])} "
+        f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) "
+        f"({time.perf_counter() - t0:.0f}s)"
+    )
+    for o in sitelinked[:2]:
+        print(
+            f"    [sitelinks on #{o['position']}] "
+            + ", ".join(s["title"] for s in o["siteLinks"][:5])
+        )
+    aio = it["aiOverview"]
+    if aio:
+        print(
+            f"    [aiOverview] content={len(aio['content'])} chars, "
+            f"sources={len(aio['sources'])}"
+        )
+        print(f"        {aio['content'][:110]!r}")
+        for s in aio["sources"][:3]:
+            print(f"        src: {(s['title'] or '')[:55]!r}")
+    for a in it["paidResults"][:3]:
+        print(f"    [ad {a['adPosition']}] {a['title'][:44]!r} {(a['url'] or '')[:45]}")
+    for p in it["paidProducts"][:3]:
+        print(f"    [pla] {p['title'][:40]!r} {p['prices']} {p['displayedUrl']}")
+    for p in paa_answered[:3]:
+        print(f"    [paa] {p['question'][:48]!r}")
+        print(f"          A: {p['answer'][:90]!r}")
+        print(f"          src: {p['url'] or '-'} | {(p['title'] or '-')[:45]}")
+    assert it["organicResults"], f"{label}: no organic results"
+    if expect_ads:
+        assert it["paidResults"], f"{label}: expected text ads, got none"
+    if expect_products:
+        assert it["paidProducts"], f"{label}: expected product ads, got none"
+    if expect_paa_answers:
+        assert paa_answered, f"{label}: expected PAA answers, got none"
+    if expect_sitelinks:
+        assert sitelinked, f"{label}: expected sitelinks, got none"
+        assert it["suggestedResults"], f"{label}: expected suggestedResults"
+    if expect_aio:
+        assert aio and aio["content"], f"{label}: expected an AI Overview"
+        assert aio["sources"], f"{label}: expected AI Overview sources"
+    if expect_device:
+        assert it["searchQuery"]["device"] == expect_device, (
+            f"{label}: device={it['searchQuery']['device']}"
+        )
+    if expect_icons:
+        iconed = [
+            o
+            for o in it["organicResults"]
+            if (o["icon"] or "").startswith("data:image")
+        ]
+        print(
+            f"    [icons] {len(iconed)}/{len(it['organicResults'])} organic "
+            f"carry a base64 favicon"
+        )
+        assert iconed, f"{label}: expected base64 icons on organic results"
+
+
+_CASES = {
+    "plain": lambda: run("plain query", queries="python asyncio tutorial"),
+    "site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"),
+    "ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True),
+    "products": lambda: run(
+        "product ads", queries="buy running shoes", expect_products=True
+    ),
+    "focus": lambda: run(
+        "focusOnPaidAds (commercial)",
+        queries="car insurance quotes",
+        focusOnPaidAds=True,
+        expect_ads=True,
+    ),
+    "focus-neg": lambda: run(
+        "focusOnPaidAds (non-commercial, retries capped)",
+        queries="python asyncio tutorial",
+        focusOnPaidAds=True,
+    ),
+    "paa": lambda: run(
+        "people also ask", queries="what is seo", expect_paa_answers=True
+    ),
+    "sitelinks": lambda: run(
+        "sitelinks + suggested (brand query)", queries="amazon", expect_sitelinks=True
+    ),
+    "aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True),
+    "mobile": lambda: run(
+        "mobile layout (mobileResults)",
+        queries="best seo tools",
+        mobileResults=True,
+        expect_device="MOBILE",
+    ),
+    "unfiltered": lambda: run(
+        "includeUnfilteredResults (filter=0)",
+        queries="python asyncio tutorial",
+        includeUnfilteredResults=True,
+    ),
+    "icons": lambda: run(
+        "includeIcons (base64 favicons)",
+        queries="github",
+        includeIcons=True,
+        expect_icons=True,
+    ),
+    "aimode": lambda: run_ai_mode(
+        "Google AI Mode (udm=50)", queries="what is quantum computing"
+    ),
+}
+
+
+async def main() -> None:
+    names = sys.argv[1:] or list(_CASES)
+    try:
+        for name in names:
+            await _CASES[name]()
+    finally:
+        await close_sessions()
+    print("\nALL E2E OK")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

surfsense_backend/scripts/e2e_phase3_crawl_billing.py

@@ -0,0 +1,140 @@
+"""Manual functional e2e for Phase 3 crawler core (3a / 3b).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_phase3_crawl_billing.py
+    # or: .\\.venv\\Scripts\\python.exe scripts/e2e_phase3_crawl_billing.py
+
+What it exercises (everything REAL — live network, live proxy, live DB reads):
+
+  Stage 1 (3a + 3b) — direct fetch + proxy egress-IP proof + crawl_url ladder.
+
+Crawl billing now lives entirely in the ``web.crawl`` capability (charged
+directly on the wallet via ``charge_capability``); there is no longer a
+chat-turn "fold" surface to exercise here.
+
+This is NOT a pytest test (it needs a live stack + proxy creds + network). It
+is the manual functional counterpart to the unit suites; the undetectability /
+anti-bot scorecard is a separate deliverable (03f), after 03d/03e.
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+
+# Content-rich, generally crawl-friendly targets (real extraction expected).
+_ARTICLE_URLS = [
+    "https://en.wikipedia.org/wiki/Competitive_intelligence",
+    "https://en.wikipedia.org/wiki/Market_research",
+]
+_IP_ECHO = "https://api.ipify.org?format=json"
+
+
+def _mask(url: str | None) -> str:
+    if not url:
+        return "<none>"
+    p = urlsplit(url)
+    host = p.hostname or "?"
+    port = f":{p.port}" if p.port else ""
+    creds = "***@" if p.username else ""
+    return f"{p.scheme}://{creds}{host}{port}"
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+# ===========================================================================
+# Stage 1 — crawl core (3a) + proxy routing (3b)
+# ===========================================================================
+async def stage1_crawl_and_proxy() -> bool:
+    _hr("STAGE 1 — crawl_url ladder (3a) + proxy egress (3b)")
+    from scrapling.fetchers import AsyncFetcher
+
+    from app.proprietary.web_crawler import CrawlOutcomeStatus, WebCrawlerConnector
+    from app.utils.proxy import get_active_provider, get_proxy_url, is_pool_backed
+
+    ok = True
+    provider = get_active_provider()
+    proxy_url = get_proxy_url()
+    print(f"  active proxy provider : {provider.name}")
+    print(f"  proxy url             : {_mask(proxy_url)}")
+    print(f"  pool-backed (rotates) : {is_pool_backed()}")
+
+    # Proxy egress-IP proof: direct IP vs proxied IP should differ.
+    direct_ip = proxied_ip = None
+    try:
+        direct = await AsyncFetcher.get(_IP_ECHO, impersonate="chrome", timeout=30)
+        direct_ip = direct.json().get("ip")
+    except Exception as exc:
+        print(f"  [INFO] direct IP fetch failed: {exc}")
+    if proxy_url:
+        try:
+            via = await AsyncFetcher.get(
+                _IP_ECHO, impersonate="chrome", proxy=proxy_url, timeout=45
+            )
+            proxied_ip = via.json().get("ip")
+        except Exception as exc:
+            print(f"  [INFO] proxied IP fetch failed: {exc}")
+    print(f"  egress IP (direct)    : {direct_ip}")
+    print(f"  egress IP (via proxy) : {proxied_ip}")
+    if proxy_url:
+        ok &= _check(
+            "proxy changes egress IP",
+            bool(proxied_ip) and proxied_ip != direct_ip,
+            f"{direct_ip} -> {proxied_ip}",
+        )
+    else:
+        print("  [INFO] no proxy configured — skipping egress-IP assertion")
+
+    # crawl_url end-to-end on a content-rich page.
+    crawler = WebCrawlerConnector()
+    outcome = await crawler.crawl_url(_ARTICLE_URLS[0])
+    content = (outcome.result or {}).get("content", "") if outcome.result else ""
+    tier = (outcome.result or {}).get("crawler_type", "?") if outcome.result else "?"
+    ok &= _check(
+        "crawl_url returns SUCCESS with content",
+        outcome.status is CrawlOutcomeStatus.SUCCESS and len(content) > 200,
+        f"status={outcome.status.value} tier={tier} chars={len(content)}",
+    )
+    return ok
+
+
+async def main() -> int:
+    print("Phase 3 functional e2e (3a/3b) — live network + proxy, DB rolled back")
+    results: dict[str, bool] = {}
+    for name, coro in (("Stage 1 crawl+proxy", stage1_crawl_and_proxy),):
+        try:
+            results[name] = await coro()
+        except Exception as exc:
+            import traceback
+
+            traceback.print_exc()
+            print(f"  [ERROR] {name} raised: {exc}")
+            results[name] = False
+
+    _hr("SUMMARY")
+    for name, ok in results.items():
+        print(f"  {'PASS' if ok else 'FAIL/SKIP'} — {name}")
+    return 0 if all(results.values()) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_reddit_scraper.py

@@ -0,0 +1,211 @@
+"""Manual functional e2e for the Reddit scraper (app/proprietary/platforms/reddit).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_reddit_scraper.py
+    # or: .venv/bin/python scripts/e2e_reddit_scraper.py
+
+This is NOT a pytest test (it needs live network + a residential/custom proxy).
+It:
+
+  Step 0 — go/no-go probe (folds in the old scripts/reddit_probe.py): open a
+      proxy session, warm a ``loid`` (svc/shreddit first, old.reddit fallback),
+      then do sequential ``.json`` fetches on the SAME sticky IP and assert each
+      returns a Reddit Listing. If this fails the whole approach is invalid —
+      later steps are skipped.
+  Step 1 — scrape a discovered post URL (post + a few comments).
+  Step 2 — scrape a subreddit listing.
+  Step 3 — run a search query.
+  Step 4 — scrape a user profile.
+  Step 5 — dump trimmed raw ``.json`` fixtures into
+      tests/unit/platforms/reddit/fixtures/ for the offline parser tests.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.platforms.reddit import (  # noqa: E402
+    RedditScrapeInput,
+    scrape_reddit,
+)
+from app.proprietary.platforms.reddit.fetch import (  # noqa: E402
+    fetch_json,
+    proxy_session,
+    warm_session,
+)
+from app.proprietary.platforms.reddit.parsers import children  # noqa: E402
+
+_SUBREDDIT = "python"
+_SEARCH_TERM = "async web scraping"
+_USER = "spez"
+
+_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "reddit" / "fixtures"
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+def _listing_count(data) -> int | None:
+    """Child count if ``data`` looks like a Reddit Listing (or post+comments)."""
+    try:
+        if isinstance(data, dict) and data.get("kind") == "Listing":
+            return len(data["data"]["children"])
+        if isinstance(data, list):
+            return sum(
+                len(x["data"]["children"])
+                for x in data
+                if isinstance(x, dict) and x.get("kind") == "Listing"
+            )
+    except Exception:
+        return None
+    return None
+
+
+async def _first_post_permalink() -> str | None:
+    """Discover a live post URL from the subreddit's hot listing."""
+    listing = await fetch_json(f"r/{_SUBREDDIT}/hot", {"limit": 5})
+    kids = children(listing)
+    if not kids:
+        return None
+    permalink = (kids[0].get("data") or {}).get("permalink")
+    return f"https://www.reddit.com{permalink}" if permalink else None
+
+
+async def step0_probe() -> bool:
+    _hr("STEP 0 — go/no-go: loid warm-up + sticky .json")
+    async with proxy_session() as holder:
+        if holder.session is None:
+            return _check(
+                "proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds"
+            )
+        minted = await warm_session(holder.session)
+        holder.warmed = True  # don't let fetch_json re-warm; we just warmed it
+        _check("loid warm-up minted a session", minted)
+        oks: list[bool] = []
+        for path in (f"r/{_SUBREDDIT}/hot", "r/programming/new", f"r/{_SUBREDDIT}/hot"):
+            data = await fetch_json(path, {"limit": 5})
+            n = _listing_count(data)
+            print(f"    {path} -> listing_count={n}")
+            oks.append(n is not None and n > 0)
+            await asyncio.sleep(1.0)
+        return _check("sequential .json on sticky IP", minted and all(oks))
+
+
+async def step1_post() -> bool:
+    _hr("STEP 1 — scrape a discovered post (post + comments)")
+    url = await _first_post_permalink()
+    if not url:
+        return _check("discovered a post URL", False)
+    items = await scrape_reddit(
+        RedditScrapeInput(startUrls=[{"url": url}], maxComments=5)
+    )
+    posts = [i for i in items if i.get("dataType") == "post"]
+    comments = [i for i in items if i.get("dataType") == "comment"]
+    print(f"    posts={len(posts)} comments={len(comments)} url={url}")
+    return _check("post scraped", bool(posts) and bool(posts[0].get("id")))
+
+
+async def step2_subreddit() -> bool:
+    _hr("STEP 2 — scrape a subreddit listing")
+    items = await scrape_reddit(
+        RedditScrapeInput(
+            startUrls=[{"url": f"https://www.reddit.com/r/{_SUBREDDIT}/hot"}],
+            maxPostCount=5,
+            skipComments=True,
+        )
+    )
+    posts = [i for i in items if i.get("dataType") == "post"]
+    for it in posts[:5]:
+        print(f"    - {it.get('id')} | {it.get('title')}")
+    return _check("subreddit returned posts", len(posts) > 0, f"{len(posts)} posts")
+
+
+async def step3_search() -> bool:
+    _hr("STEP 3 — search query")
+    items = await scrape_reddit(
+        RedditScrapeInput(searches=[_SEARCH_TERM], sort="relevance", maxItems=5)
+    )
+    for it in items[:5]:
+        print(f"    - {it.get('id')} | {it.get('title')}")
+    return _check("search returned results", len(items) > 0, f"{len(items)} items")
+
+
+async def step4_user() -> bool:
+    _hr("STEP 4 — user profile")
+    items = await scrape_reddit(
+        RedditScrapeInput(
+            startUrls=[{"url": f"https://www.reddit.com/user/{_USER}"}], maxItems=5
+        )
+    )
+    print(f"    {len(items)} items for u/{_USER}")
+    return _check("user returned items", len(items) > 0, f"{len(items)} items")
+
+
+async def step5_dump_fixtures() -> bool:
+    _hr("STEP 5 — dump trimmed .json fixtures for offline tests")
+    listing = await fetch_json(f"r/{_SUBREDDIT}/hot", {"limit": 25})
+    url = await _first_post_permalink()
+    post = None
+    if url:
+        path = url.split("reddit.com/")[-1].strip("/")
+        post = await fetch_json(path, {"limit": 20})
+
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    wrote = []
+    if _listing_count(listing):
+        (_FIXTURE_DIR / "sample_listing.json").write_text(
+            json.dumps(listing), encoding="utf-8"
+        )
+        wrote.append("sample_listing.json")
+    if isinstance(post, list) and post:
+        (_FIXTURE_DIR / "sample_post.json").write_text(
+            json.dumps(post), encoding="utf-8"
+        )
+        wrote.append("sample_post.json")
+        # A single comment thing, for the comment-mapping fixture.
+        comment_kids = children(post[1]) if len(post) > 1 else []
+        first_comment = next((c for c in comment_kids if c.get("kind") == "t1"), None)
+        if first_comment:
+            (_FIXTURE_DIR / "sample_comment.json").write_text(
+                json.dumps(first_comment), encoding="utf-8"
+            )
+            wrote.append("sample_comment.json")
+    return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}")
+
+
+async def main() -> int:
+    results = [await step0_probe()]
+    if not results[-1]:
+        print("\nloid probe failed — the approach is invalid on this IP/proxy.")
+        print("Aborting remaining steps.")
+        return 1
+    results.append(await step1_post())
+    results.append(await step2_subreddit())
+    results.append(await step3_search())
+    results.append(await step4_user())
+    results.append(await step5_dump_fixtures())
+    _hr("SUMMARY")
+    print(f"  {sum(results)}/{len(results)} steps passed")
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_youtube_scraper.py

@@ -0,0 +1,205 @@
+"""Manual functional e2e for the YouTube scraper (app/proprietary/platforms/youtube).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_youtube_scraper.py
+    # or: .\\.venv\\Scripts\\python.exe scripts/e2e_youtube_scraper.py
+
+This is NOT a pytest test (it needs live network + optional proxy creds). It:
+
+  Step 0 — validates the keyless InnerTube ``search`` POST works; if it 400s the
+      scraper transparently retries with the public web key (proven here).
+  Step 1 — scrapes a known video URL (metadata + optional subtitles).
+  Step 2 — runs a search query and prints the first few results.
+  Step 3 — scrapes a small channel's latest videos.
+  Step 4 — dumps trimmed raw ytInitialData / ytInitialPlayerResponse fixtures to
+      tests/unit/platforms/youtube/fixtures/ for the offline parser test.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.platforms.youtube import (  # noqa: E402
+    YouTubeCommentsInput,
+    YouTubeScrapeInput,
+    scrape_comments,
+    scrape_youtube,
+)
+from app.proprietary.platforms.youtube.innertube import (  # noqa: E402
+    INNERTUBE_PUBLIC_API_KEY,
+    INNERTUBE_SEARCH_URL,
+    build_innertube_payload,
+    fetch_html,
+    post_innertube,
+)
+from app.proprietary.platforms.youtube.parsers import (  # noqa: E402
+    extract_yt_initial_data,
+    extract_yt_initial_player_response,
+)
+
+_VIDEO_URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
+_SEARCH_TERM = "web scraping tutorials"
+_CHANNEL_URL = "https://www.youtube.com/@YouTube"
+# A geo-tagged walking tour + a multi-owner collaboration video (long-tail fields).
+_LOCATION_VIDEO_URL = "https://www.youtube.com/watch?v=bhJU_fVHMmY"
+_COLLAB_VIDEO_URL = "https://www.youtube.com/watch?v=AI2BwwLX_7s"
+# MrBeast localizes titles/descriptions into many languages (translation flow).
+_TRANSLATED_VIDEO_URL = "https://www.youtube.com/watch?v=iYlODtkyw_I"
+
+_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "youtube" / "fixtures"
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+async def step0_validate_innertube() -> bool:
+    _hr("STEP 0 — InnerTube search POST (keyless, then public-key fallback)")
+    payload = build_innertube_payload(search_query=_SEARCH_TERM)
+    keyless = await post_innertube(INNERTUBE_SEARCH_URL, payload)
+    if keyless is not None:
+        return _check("keyless search POST", True, "keyless works")
+    keyed = await post_innertube(
+        INNERTUBE_SEARCH_URL, payload, api_key=INNERTUBE_PUBLIC_API_KEY
+    )
+    return _check("public-key search POST", keyed is not None, "keyless 400 -> key")
+
+
+async def step1_video() -> bool:
+    _hr("STEP 1 — scrape a known video")
+    inp = YouTubeScrapeInput(startUrls=[{"url": _VIDEO_URL}], downloadSubtitles=True)
+    items = await scrape_youtube(inp)
+    print(json.dumps(items[:1], indent=2)[:2000])
+    ok = bool(items) and items[0].get("id") == "dQw4w9WgXcQ"
+    return _check("video scraped with id + title", ok and bool(items[0].get("title")))
+
+
+async def step2_search() -> bool:
+    _hr("STEP 2 — search query")
+    inp = YouTubeScrapeInput(searchQueries=[_SEARCH_TERM], maxResults=5)
+    items = await scrape_youtube(inp)
+    for it in items[:5]:
+        print(f"  - {it.get('id')} | {it.get('title')}")
+    return _check("search returned results", len(items) > 0, f"{len(items)} items")
+
+
+async def step3_channel() -> bool:
+    _hr("STEP 3 — channel latest videos")
+    inp = YouTubeScrapeInput(startUrls=[{"url": _CHANNEL_URL}], maxResults=3)
+    items = await scrape_youtube(inp)
+    for it in items[:3]:
+        print(f"  - {it.get('id')} | {it.get('title')}")
+    return _check("channel returned videos", len(items) > 0, f"{len(items)} items")
+
+
+async def step4_dump_fixtures() -> bool:
+    _hr("STEP 4 — dump raw fixtures for offline test")
+    html = await fetch_html(_VIDEO_URL)
+    if not html:
+        return _check("fetched video HTML", False)
+    initial = extract_yt_initial_data(html)
+    player = extract_yt_initial_player_response(html)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    if player:
+        (_FIXTURE_DIR / "video_player_response.json").write_text(
+            json.dumps(player), encoding="utf-8"
+        )
+    if initial:
+        (_FIXTURE_DIR / "video_initial_data.json").write_text(
+            json.dumps(initial), encoding="utf-8"
+        )
+    return _check("dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}")
+
+
+async def step5_comments() -> bool:
+    _hr("STEP 5 — comments (+ replies) for a video")
+    inp = YouTubeCommentsInput(
+        startUrls=[{"url": _VIDEO_URL}], maxComments=6, sortCommentsBy="NEWEST_FIRST"
+    )
+    items = await scrape_comments(inp)
+    for it in items[:6]:
+        print(
+            f"  - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes"
+        )
+    ok = bool(items) and all(it.get("cid") and it.get("videoId") for it in items)
+    has_reply = any(it.get("type") == "reply" and it.get("replyToCid") for it in items)
+    return _check(
+        "comments scraped (cid+videoId, reply linkage)",
+        ok and has_reply,
+        f"{len(items)} items",
+    )
+
+
+async def step6_location_collaborators() -> bool:
+    _hr("STEP 6 — long-tail fields: location + collaborators")
+    loc_items = await scrape_youtube(
+        YouTubeScrapeInput(startUrls=[{"url": _LOCATION_VIDEO_URL}])
+    )
+    location = loc_items[0].get("location") if loc_items else None
+    print(f"  location: {location!r}")
+    collab_items = await scrape_youtube(
+        YouTubeScrapeInput(startUrls=[{"url": _COLLAB_VIDEO_URL}])
+    )
+    collaborators = collab_items[0].get("collaborators") if collab_items else None
+    print(f"  collaborators: {collaborators}")
+    return _check(
+        "location + collaborators populated",
+        bool(location) and bool(collaborators) and len(collaborators) >= 2,
+    )
+
+
+async def step7_translation() -> bool:
+    _hr("STEP 7 — translatedTitle/translatedText (subtitlesLanguage=es)")
+    items = await scrape_youtube(
+        YouTubeScrapeInput(
+            startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es"
+        )
+    )
+    it = items[0] if items else {}
+    print(f"  title          : {it.get('title')}")
+    print(f"  translatedTitle: {it.get('translatedTitle')}")
+    # A localized video's translated title differs from the canonical English one.
+    ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get(
+        "title"
+    )
+    return _check("translatedTitle differs from original", ok)
+
+
+async def main() -> int:
+    results = []
+    results.append(await step0_validate_innertube())
+    if not results[-1]:
+        print("\nInnerTube unreachable — aborting remaining steps.")
+        return 1
+    results.append(await step1_video())
+    results.append(await step2_search())
+    results.append(await step3_channel())
+    results.append(await step4_dump_fixtures())
+    results.append(await step5_comments())
+    results.append(await step6_location_collaborators())
+    results.append(await step7_translation())
+
+    _hr("SUMMARY")
+    print(f"  {sum(results)}/{len(results)} steps passed")
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/register_webhook.py

@@ -0,0 +1,62 @@
+"""Register the SurfSense Telegram webhook."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import sys
+
+from dotenv import load_dotenv
+from telegram import Bot
+
+from app.db import async_session_maker
+from app.gateway.accounts import get_or_create_system_telegram_account
+
+load_dotenv()
+
+WEBHOOK_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{1,256}$")
+
+
+async def main() -> int:
+    token = os.getenv("TELEGRAM_SHARED_BOT_TOKEN")
+    secret = os.getenv("TELEGRAM_WEBHOOK_SECRET")
+    base_url = os.getenv("GATEWAY_BASE_URL") or os.getenv("BACKEND_URL")
+    if not token or not secret or not base_url:
+        print(
+            "Missing TELEGRAM_SHARED_BOT_TOKEN, TELEGRAM_WEBHOOK_SECRET, or GATEWAY_BASE_URL/BACKEND_URL",
+            file=sys.stderr,
+        )
+        return 1
+    if not WEBHOOK_SECRET_RE.fullmatch(secret):
+        print(
+            "TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, '_' or '-'",
+            file=sys.stderr,
+        )
+        return 1
+
+    async with async_session_maker() as session:
+        account = await get_or_create_system_telegram_account(session)
+        account.webhook_secret = secret
+        await session.commit()
+        account_id = int(account.id)
+
+    webhook_url = (
+        f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}"
+    )
+    bot = Bot(token=token)
+    ok = await bot.set_webhook(
+        url=webhook_url,
+        secret_token=secret,
+        allowed_updates=["message", "edited_message"],
+        drop_pending_updates=True,
+    )
+    if not ok:
+        print("Telegram rejected webhook registration", file=sys.stderr)
+        return 1
+    print(f"Registered Telegram webhook: {webhook_url}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/revoke_refresh_tokens_cutover.py

@@ -0,0 +1,69 @@
+"""One-shot cutover helper to revoke every refresh token.
+
+Run with --yes during the auth-hardening cutover, alongside setting
+MIN_ISSUED_AT to the deploy epoch.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+
+from sqlalchemy import text
+
+from app.db import async_session_maker
+
+
+async def _count_active_tokens() -> int:
+    async with async_session_maker() as session:
+        result = await session.execute(
+            text(
+                """
+                SELECT count(*)
+                FROM refresh_tokens
+                WHERE revoked_at IS NULL
+                  AND expires_at > NOW()
+                """
+            )
+        )
+        return int(result.scalar_one())
+
+
+async def _revoke_all_tokens() -> int:
+    async with async_session_maker() as session:
+        result = await session.execute(
+            text(
+                """
+                UPDATE refresh_tokens
+                SET revoked_at = NOW(),
+                    expires_at = NOW()
+                WHERE revoked_at IS NULL
+                   OR expires_at > NOW()
+                """
+            )
+        )
+        await session.commit()
+        return int(result.rowcount or 0)
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--yes",
+        action="store_true",
+        help="Actually revoke tokens. Without this flag the command is a dry run.",
+    )
+    args = parser.parse_args()
+
+    active_count = await _count_active_tokens()
+    if not args.yes:
+        print(f"Dry run: {active_count} active refresh token(s) would be revoked.")
+        print("Re-run with --yes during the auth-hardening cutover to revoke them.")
+        return
+
+    updated_count = await _revoke_all_tokens()
+    print(f"Revoked {updated_count} refresh token row(s).")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

surfsense_backend/scripts/verify_chat_image_capability.py

@@ -0,0 +1,502 @@
+"""End-to-end smoke test for vision / image config wiring.
+
+Loads the live ``global_llm_config.yaml`` (no mocking, no fixtures) and
+exercises every chat / vision / image-generation config + the OpenRouter
+dynamic catalog. For each config the script:
+
+1. Reports the resolver classification (catalog-allow vs strict-block).
+2. Optionally fires a tiny live API call against the provider:
+   - Chat configs: ``litellm.acompletion`` with a 1x1 PNG and the prompt
+     ``"reply with one word: ok"``.
+   - Vision configs: same, against the dedicated vision router pool.
+   - Image-gen configs: ``litellm.aimage_generation`` with a single tiny
+     prompt and ``n=1``.
+   - OpenRouter integration: samples one chat, one vision, one image-gen
+     model from the dynamically fetched catalog.
+
+Usage::
+
+    python -m scripts.verify_chat_image_capability             # capability + connectivity
+    python -m scripts.verify_chat_image_capability --no-live   # capability resolver only
+
+The script is meant to be runnable from the repository root or from
+``surfsense_backend/`` and prints a short PASS/FAIL/SKIP summary at the
+end so it's usable as a CI smoke check too.
+
+Live-mode caveat: each successful call costs a small amount of provider
+credit (a few tokens or one tiny generated image per config). The
+default size for image generation is ``1024x1024`` because Azure
+GPT-image deployments reject smaller sizes; OpenRouter image-gen models
+generally accept the same size.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import os
+import sys
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+# Bootstrap the surfsense_backend package on sys.path so the script runs
+# from the repo root or from `surfsense_backend/` interchangeably.
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_BACKEND_ROOT = os.path.dirname(_HERE)
+if _BACKEND_ROOT not in sys.path:
+    sys.path.insert(0, _BACKEND_ROOT)
+
+import litellm  # noqa: E402
+
+from app.config import config  # noqa: E402
+from app.services.openrouter_integration_service import (  # noqa: E402
+    _OPENROUTER_DYNAMIC_MARKER,
+    OpenRouterIntegrationService,
+)
+from app.services.provider_capabilities import (  # noqa: E402
+    derive_supports_image_input,
+    is_known_text_only_chat_model,
+)
+
+logging.basicConfig(
+    level=logging.WARNING,
+    format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
+)
+# Quiet down LiteLLM's verbose router/cost logs so the script output is
+# scannable.
+logging.getLogger("LiteLLM").setLevel(logging.ERROR)
+logging.getLogger("litellm").setLevel(logging.ERROR)
+logging.getLogger("httpx").setLevel(logging.ERROR)
+
+# 1x1 transparent PNG — used as the cheapest possible vision payload.
+_TINY_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+_TINY_PNG_DATA_URL = f"data:image/png;base64,{_TINY_PNG_B64}"
+
+
+# ---------------------------------------------------------------------------
+# Result accounting
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ProbeResult:
+    label: str
+    surface: str
+    config_id: int | str
+    capability_ok: bool | None = None
+    capability_note: str = ""
+    live_ok: bool | None = None
+    live_note: str = ""
+    duration_s: float = 0.0
+
+
+@dataclass
+class Report:
+    results: list[ProbeResult] = field(default_factory=list)
+
+    def add(self, r: ProbeResult) -> None:
+        self.results.append(r)
+
+    def render(self) -> int:
+        passed = failed = skipped = 0
+        print()
+        print("=" * 92)
+        print(
+            f"{'Surface':<14}{'ID':>8}  {'Cap':>5}  {'Live':>5}  {'Time':>6}  Label  /  notes"
+        )
+        print("-" * 92)
+        for r in self.results:
+
+            def _flag(value: bool | None) -> str:
+                if value is None:
+                    return "skip"
+                return "ok" if value else "fail"
+
+            cap = _flag(r.capability_ok)
+            live = _flag(r.live_ok)
+            if r.capability_ok is False or r.live_ok is False:
+                failed += 1
+            elif r.capability_ok is None and r.live_ok is None:
+                skipped += 1
+            else:
+                passed += 1
+            print(
+                f"{r.surface:<14}{r.config_id!s:>8}  {cap:>5}  {live:>5}  "
+                f"{r.duration_s:>5.2f}s  {r.label}"
+            )
+            if r.capability_note:
+                print(f"               cap:  {r.capability_note}")
+            if r.live_note:
+                print(f"               live: {r.live_note}")
+        print("-" * 92)
+        print(
+            f"Total: {passed} ok / {failed} fail / {skipped} skip "
+            f"(of {len(self.results)} probes)"
+        )
+        print("=" * 92)
+        return failed
+
+
+# ---------------------------------------------------------------------------
+# Capability probes (no network)
+# ---------------------------------------------------------------------------
+
+
+def _probe_chat_capability(cfg: dict) -> tuple[bool, str]:
+    """For chat configs the catalog flag is *expected* True (vision-capable
+    pool). The probe reports both the resolver value and the strict
+    safety-net value to surface any drift between them."""
+    litellm_params = cfg.get("litellm_params") or {}
+    base_model = (
+        litellm_params.get("base_model") if isinstance(litellm_params, dict) else None
+    )
+    cap = derive_supports_image_input(
+        provider=cfg.get("litellm_provider"),
+        model_name=cfg.get("model_name"),
+        base_model=base_model,
+        custom_provider=cfg.get("custom_provider"),
+    )
+    block = is_known_text_only_chat_model(
+        provider=cfg.get("litellm_provider"),
+        model_name=cfg.get("model_name"),
+        base_model=base_model,
+        custom_provider=cfg.get("custom_provider"),
+    )
+    note = f"derive={cap} strict_block={block}"
+    if not cap and not block:
+        # Resolver said False but strict gate is also False — that means
+        # OR modalities published [text] explicitly. Surface it.
+        note += " (OR modality says text-only)"
+    # We accept a True derive *or* (False derive AND False block) as
+    # 'capability ok' — either way, the streaming task will flow through.
+    ok = cap or not block
+    return ok, note
+
+
+def _build_chat_model_string(cfg: dict) -> str:
+    if cfg.get("custom_provider"):
+        return f"{cfg['custom_provider']}/{cfg['model_name']}"
+    prefix = cfg.get("litellm_provider") or "openai"
+    return f"{prefix}/{cfg['model_name']}"
+
+
+# ---------------------------------------------------------------------------
+# Live probes (network calls)
+# ---------------------------------------------------------------------------
+
+
+async def _live_chat_image_call(cfg: dict) -> tuple[bool, str]:
+    """Send a 1x1 PNG + `reply with one word: ok` to the chat config."""
+    model_string = _build_chat_model_string(cfg)
+    kwargs: dict[str, Any] = {
+        "model": model_string,
+        "api_key": cfg.get("api_key"),
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "reply with one word: ok"},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": _TINY_PNG_DATA_URL},
+                    },
+                ],
+            }
+        ],
+        "max_tokens": 16,
+        "timeout": 60,
+    }
+    if cfg.get("api_base"):
+        kwargs["api_base"] = cfg["api_base"]
+    if cfg.get("litellm_params"):
+        # Strip pricing keys — they're tracking-only and confuse some
+        # provider validators (e.g. azure/openai reject unknown kwargs
+        # in strict mode).
+        merged = {
+            k: v
+            for k, v in dict(cfg["litellm_params"]).items()
+            if k
+            not in {
+                "input_cost_per_token",
+                "output_cost_per_token",
+                "input_cost_per_pixel",
+                "output_cost_per_pixel",
+            }
+        }
+        kwargs.update(merged)
+    try:
+        resp = await litellm.acompletion(**kwargs)
+    except Exception as exc:
+        return False, f"{type(exc).__name__}: {exc}"
+    text = resp.choices[0].message.content if resp.choices else ""
+    return True, f"got reply ({(text or '').strip()[:40]!r})"
+
+
+# Gemini image models occasionally return zero-length ``data`` for the
+# minimal "red dot on white" prompt (provider-side safety / empty-output
+# quirk reproducible against ``google/gemini-2.5-flash-image`` even when
+# the request itself succeeds). Use a more naturalistic prompt and
+# retry once with a different one before giving up.
+_IMAGE_GEN_PROMPTS: tuple[str, ...] = (
+    "A simple icon of a coffee cup, flat illustration",
+    "A small green leaf on a white background",
+)
+
+
+async def _live_image_gen_call(cfg: dict) -> tuple[bool, str]:
+    """Generate one tiny image to verify the deployment is reachable."""
+    if cfg.get("custom_provider"):
+        prefix = cfg["custom_provider"]
+    else:
+        prefix = cfg.get("litellm_provider") or "openai"
+    model_string = f"{prefix}/{cfg['model_name']}"
+    base_kwargs: dict[str, Any] = {
+        "model": model_string,
+        "api_key": cfg.get("api_key"),
+        "n": 1,
+        "size": "1024x1024",
+        "timeout": 120,
+    }
+    if cfg.get("api_base"):
+        base_kwargs["api_base"] = cfg["api_base"]
+    if cfg.get("api_version"):
+        base_kwargs["api_version"] = cfg["api_version"]
+    if cfg.get("litellm_params"):
+        base_kwargs.update(
+            {
+                k: v
+                for k, v in dict(cfg["litellm_params"]).items()
+                if k
+                not in {
+                    "input_cost_per_token",
+                    "output_cost_per_token",
+                    "input_cost_per_pixel",
+                    "output_cost_per_pixel",
+                }
+            }
+        )
+
+    last_note = ""
+    for attempt, prompt in enumerate(_IMAGE_GEN_PROMPTS, start=1):
+        try:
+            resp = await litellm.aimage_generation(prompt=prompt, **base_kwargs)
+        except Exception as exc:
+            last_note = f"{type(exc).__name__}: {exc}"
+            continue
+        data_count = len(getattr(resp, "data", None) or [])
+        if data_count > 0:
+            return True, (
+                f"received {data_count} image(s) on attempt {attempt} "
+                f"(prompt={prompt!r})"
+            )
+        last_note = (
+            f"call ok but received 0 images on attempt {attempt} (prompt={prompt!r})"
+        )
+    return False, last_note
+
+
+# ---------------------------------------------------------------------------
+# Probe drivers
+# ---------------------------------------------------------------------------
+
+
+def _is_or_dynamic(cfg: dict) -> bool:
+    return bool(cfg.get(_OPENROUTER_DYNAMIC_MARKER))
+
+
+async def probe_chat_configs(report: Report, *, live: bool) -> None:
+    print("\n[chat configs from global_llm_configs (YAML-static)]")
+    for cfg in config.GLOBAL_LLM_CONFIGS:
+        # Skip OR dynamic entries here — handled in the OR section so
+        # the YAML / OR split stays clear in the report.
+        if _is_or_dynamic(cfg):
+            continue
+        result = ProbeResult(
+            label=str(cfg.get("name") or cfg.get("model_name")),
+            surface="chat-yaml",
+            config_id=cfg.get("id"),
+        )
+        cap_ok, cap_note = _probe_chat_capability(cfg)
+        result.capability_ok = cap_ok
+        result.capability_note = cap_note
+        if live:
+            t0 = time.perf_counter()
+            ok, note = await _live_chat_image_call(cfg)
+            result.live_ok = ok
+            result.live_note = note
+            result.duration_s = time.perf_counter() - t0
+        report.add(result)
+
+
+async def probe_image_gen_configs(report: Report, *, live: bool) -> None:
+    print(
+        "\n[image generation configs from global_image_generation_configs (YAML-static)]"
+    )
+    for cfg in config.GLOBAL_IMAGE_GEN_CONFIGS:
+        if _is_or_dynamic(cfg):
+            continue
+        result = ProbeResult(
+            label=str(cfg.get("name") or cfg.get("model_name")),
+            surface="image-gen",
+            config_id=cfg.get("id"),
+        )
+        # Image gen configs don't have a "supports_image_input" flag;
+        # the catalog tracks output, not input. Mark capability as None
+        # (skip) for the report.
+        if live:
+            t0 = time.perf_counter()
+            ok, note = await _live_image_gen_call(cfg)
+            result.live_ok = ok
+            result.live_note = note
+            result.duration_s = time.perf_counter() - t0
+        report.add(result)
+
+
+async def probe_openrouter_catalog(report: Report, *, live: bool) -> None:
+    """Sample chat/vision-capable and image-gen models
+    from the live OpenRouter catalogue. Doesn't iterate the full pool
+    (would be hundreds of probes); just validates the integration end-
+    to-end on a representative model from each surface."""
+    print("\n[OpenRouter integration: sampled probes]")
+    settings = config.OPENROUTER_INTEGRATION_SETTINGS
+    if not settings:
+        report.add(
+            ProbeResult(
+                label="OpenRouter integration",
+                surface="openrouter",
+                config_id="settings",
+                capability_ok=None,
+                capability_note="openrouter_integration disabled in YAML — skipping",
+                live_ok=None,
+            )
+        )
+        return
+
+    service = OpenRouterIntegrationService.get_instance()
+    or_chat = [
+        c
+        for c in config.GLOBAL_LLM_CONFIGS
+        if c.get("provider") == "OPENROUTER" and c.get("supports_image_input")
+    ]
+    or_image_gen = [
+        c for c in config.GLOBAL_IMAGE_GEN_CONFIGS if c.get("provider") == "OPENROUTER"
+    ]
+
+    # Pick one representative per provider family per surface so a single
+    # broken vendor (e.g. Anthropic key revoked, Google quota exceeded)
+    # surfaces independently of the others. Each needle matches the
+    # OpenRouter ``model_name`` prefix; the first match wins.
+    def _pick_first(pool: list[dict], needle: str) -> dict | None:
+        for c in pool:
+            if (c.get("model_name") or "").lower().startswith(needle):
+                return c
+        return None
+
+    chat_picks = [
+        ("or-chat", _pick_first(or_chat, "openai/gpt-4o")),
+        ("or-chat", _pick_first(or_chat, "anthropic/claude")),
+        ("or-chat", _pick_first(or_chat, "google/gemini-2.5-flash")),
+    ]
+    image_picks = [
+        ("or-image", _pick_first(or_image_gen, "google/gemini-2.5-flash-image")),
+        # OpenRouter publishes OpenAI image models as ``openai/gpt-5-image*``
+        # / ``openai/gpt-5.4-image-2`` (no ``gpt-image`` literal). Match
+        # the actual prefix.
+        ("or-image", _pick_first(or_image_gen, "openai/gpt-5-image")),
+    ]
+
+    print(
+        f"  catalog: chat_vision={len(or_chat)} image_gen={len(or_image_gen)} "
+        f"(service initialized={service.is_initialized() if hasattr(service, 'is_initialized') else 'n/a'})"
+    )
+
+    for surface, picked in chat_picks + image_picks:
+        if not picked:
+            report.add(
+                ProbeResult(
+                    label=f"<no candidate for {surface}>",
+                    surface=surface,
+                    config_id="-",
+                    capability_ok=None,
+                    capability_note="no candidate found in OR catalog",
+                )
+            )
+            continue
+        runner = (
+            _live_image_gen_call if surface == "or-image" else _live_chat_image_call
+        )
+        result = ProbeResult(
+            label=str(picked.get("model_name")),
+            surface=surface,
+            config_id=picked.get("id"),
+        )
+        if surface != "or-image":
+            cap_ok, cap_note = _probe_chat_capability(picked)
+            result.capability_ok = cap_ok
+            result.capability_note = cap_note
+        if live:
+            t0 = time.perf_counter()
+            ok, note = await runner(picked)
+            result.live_ok = ok
+            result.live_note = note
+            result.duration_s = time.perf_counter() - t0
+        report.add(result)
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+async def main(args: argparse.Namespace) -> int:
+    print("Loaded global configs:")
+    print(f"  chat:      {len(config.GLOBAL_LLM_CONFIGS)} entries")
+    print(f"  image-gen: {len(config.GLOBAL_IMAGE_GEN_CONFIGS)} entries")
+    print(f"  OR settings present: {bool(config.OPENROUTER_INTEGRATION_SETTINGS)}")
+
+    # Initialize the OpenRouter integration so the catalog is populated
+    # (this is what main.py does at startup). It's idempotent.
+    if config.OPENROUTER_INTEGRATION_SETTINGS:
+        try:
+            from app.config import initialize_openrouter_integration
+
+            initialize_openrouter_integration()
+        except Exception as exc:
+            print(f"  WARNING: OpenRouter integration init failed: {exc}")
+
+    print(
+        f"\nMode: {'LIVE (will hit providers)' if args.live else 'DRY (capability only)'}"
+    )
+
+    report = Report()
+    if not args.skip_chat:
+        await probe_chat_configs(report, live=args.live)
+    if not args.skip_image_gen:
+        await probe_image_gen_configs(report, live=args.live)
+    if not args.skip_openrouter:
+        await probe_openrouter_catalog(report, live=args.live)
+
+    failed = report.render()
+    return 1 if failed else 0
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--no-live",
+        dest="live",
+        action="store_false",
+        help="Skip live API calls — capability resolver only.",
+    )
+    parser.set_defaults(live=True)
+    parser.add_argument("--skip-chat", action="store_true")
+    parser.add_argument("--skip-image-gen", action="store_true")
+    parser.add_argument("--skip-openrouter", action="store_true")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = _parse_args()
+    sys.exit(asyncio.run(main(args)))

surfsense_backend/scripts/whatsapp-bridge/Dockerfile

@@ -0,0 +1,15 @@
+FROM node:20-alpine
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci --silent
+
+COPY . .
+
+ENV WHATSAPP_SESSION_DIR=/data/sessions
+EXPOSE 9929
+
+HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://127.0.0.1:9929/health || exit 1
+
+CMD ["node", "bridge.js"]

surfsense_backend/scripts/whatsapp-bridge/bridge.js

@@ -0,0 +1,343 @@
+#!/usr/bin/env node
+
+import {
+  DisconnectReason,
+  fetchLatestBaileysVersion,
+  makeWASocket,
+  useMultiFileAuthState,
+} from "@whiskeysockets/baileys";
+import { Boom } from "@hapi/boom";
+import express from "express";
+import { mkdirSync, readdirSync, rmSync } from "node:fs";
+import path from "node:path";
+import pino from "pino";
+import qrcode from "qrcode-terminal";
+
+const PORT = Number(process.env.PORT || "9929");
+const SESSION_DIR = process.env.WHATSAPP_SESSION_DIR || "/data/sessions";
+const SEND_TIMEOUT_MS = Number(process.env.WHATSAPP_SEND_TIMEOUT_MS || "60000");
+const PAIRING_TIMEOUT_MS = Number(process.env.WHATSAPP_PAIRING_TIMEOUT_MS || "30000");
+const MAX_QUEUE_SIZE = Number(process.env.WHATSAPP_MAX_QUEUE_SIZE || "100");
+const WHATSAPP_MODE = process.env.WHATSAPP_MODE || "self-chat";
+const SENT_ECHO_TTL_MS = 60_000;
+
+mkdirSync(SESSION_DIR, { recursive: true });
+
+const app = express();
+app.use(express.json({ limit: "2mb" }));
+
+const logger = pino({ level: process.env.WHATSAPP_DEBUG ? "debug" : "warn" });
+const messageQueue = [];
+const sentKeys = new Map();
+const recentlySentIds = new Set();
+
+let sock = null;
+let connectionState = "disconnected";
+let latestQr = null;
+let starting = null;
+let pendingPairing = null;
+
+function resetSessionState() {
+  sock = null;
+  latestQr = null;
+  sentKeys.clear();
+  recentlySentIds.clear();
+  mkdirSync(SESSION_DIR, { recursive: true });
+  for (const entry of readdirSync(SESSION_DIR)) {
+    rmSync(path.join(SESSION_DIR, entry), { recursive: true, force: true });
+  }
+}
+
+function resolvePendingPairing(payload) {
+  if (!pendingPairing) return;
+  clearTimeout(pendingPairing.timer);
+  pendingPairing.resolve(payload);
+  pendingPairing = null;
+}
+
+function rejectPendingPairing(error) {
+  if (!pendingPairing) return;
+  clearTimeout(pendingPairing.timer);
+  pendingPairing.reject(error);
+  pendingPairing = null;
+}
+
+async function maybeRequestPairingCode(update = {}) {
+  if (!pendingPairing || pendingPairing.inFlight || !sock) return;
+
+  const canRequestPairingCode =
+    update.connection === "connecting" ||
+    Boolean(update.qr) ||
+    Boolean(latestQr);
+
+  if (!canRequestPairingCode) return;
+
+  pendingPairing.inFlight = true;
+  connectionState = "pairing";
+  try {
+    const code = await sock.requestPairingCode(pendingPairing.phoneNumber);
+    resolvePendingPairing({ status: "pairing", pairing_code: code, expires_in: 60 });
+  } catch (error) {
+    rejectPendingPairing(error);
+  }
+}
+
+function requestPairingCodeWhenReady(phoneNumber) {
+  if (connectionState === "connected") {
+    return Promise.resolve({ status: "connected", pairing_code: null, expires_in: 0 });
+  }
+
+  if (pendingPairing) {
+    return Promise.reject(new Error("A WhatsApp pairing request is already in progress"));
+  }
+
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      pendingPairing = null;
+      reject(new Error("Timed out waiting for WhatsApp to become ready for pairing"));
+    }, PAIRING_TIMEOUT_MS);
+
+    pendingPairing = {
+      phoneNumber,
+      resolve,
+      reject,
+      timer,
+      inFlight: false,
+    };
+
+    void startSocket()
+      .then(() => maybeRequestPairingCode())
+      .catch((error) => rejectPendingPairing(error));
+    void maybeRequestPairingCode();
+  });
+}
+
+function normalizeText(message) {
+  const content = message?.message || {};
+  return (
+    content.conversation ||
+    content.extendedTextMessage?.text ||
+    content.imageMessage?.caption ||
+    content.videoMessage?.caption ||
+    content.documentMessage?.caption ||
+    ""
+  );
+}
+
+function enqueueMessage(message) {
+  const remoteJid = message?.key?.remoteJid;
+  const id = message?.key?.id;
+  if (!remoteJid || !id || !message?.message) return;
+  if (messageQueue.length >= MAX_QUEUE_SIZE) messageQueue.shift();
+  messageQueue.push({
+    event: "messages.upsert",
+    key: message.key,
+    chatId: remoteJid,
+    senderId: message.key.participant || remoteJid,
+    messageId: id,
+    fromMe: Boolean(message.key.fromMe),
+    isGroup: remoteJid.endsWith("@g.us"),
+    body: normalizeText(message),
+    timestamp: Number(message.messageTimestamp || Date.now() / 1000),
+    raw: message,
+  });
+}
+
+function rememberSentMessage(sent) {
+  const sentId = sent?.key?.id;
+  if (!sentId) return;
+  sentKeys.set(sentId, sent.key);
+  recentlySentIds.add(sentId);
+  setTimeout(() => {
+    recentlySentIds.delete(sentId);
+  }, SENT_ECHO_TTL_MS).unref?.();
+}
+
+function withTimeout(promise, timeoutMs) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(
+      () => reject(new Error(`sendMessage timed out after ${timeoutMs}ms`)),
+      timeoutMs,
+    );
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+
+async function startSocket() {
+  if (starting) return starting;
+  starting = (async () => {
+    connectionState = "connecting";
+    const { state, saveCreds } = await useMultiFileAuthState(SESSION_DIR);
+    const { version } = await fetchLatestBaileysVersion();
+    sock = makeWASocket({
+      version,
+      auth: state,
+      logger,
+      printQRInTerminal: false,
+      browser: ["SurfSense", "Chrome", "120.0"],
+      syncFullHistory: false,
+      markOnlineOnConnect: false,
+      getMessage: async () => ({ conversation: "" }),
+    });
+
+    sock.ev.on("creds.update", saveCreds);
+    sock.ev.on("connection.update", (update) => {
+      const { connection, lastDisconnect, qr } = update;
+      if (qr) {
+        latestQr = qr;
+        connectionState = "qr";
+        qrcode.generate(qr, { small: true });
+        void maybeRequestPairingCode(update);
+      }
+      if (connection === "open") {
+        latestQr = null;
+        connectionState = "connected";
+        console.log("WhatsApp connected");
+        resolvePendingPairing({ status: "connected", pairing_code: null, expires_in: 0 });
+      }
+      if (connection === "close") {
+        const reason = new Boom(lastDisconnect?.error)?.output?.statusCode;
+        connectionState = "disconnected";
+        if (reason === DisconnectReason.loggedOut) {
+          console.error("WhatsApp logged out; clearing session and waiting for pairing.");
+          connectionState = "logged_out";
+          resetSessionState();
+          setTimeout(() => {
+            starting = null;
+            void startSocket();
+          }, 1000);
+          return;
+        }
+        setTimeout(() => {
+          starting = null;
+          void startSocket();
+        }, reason === 515 ? 1000 : 3000);
+      }
+      void maybeRequestPairingCode(update);
+    });
+
+    sock.ev.on("messages.upsert", ({ messages, type }) => {
+      if (type !== "notify" && type !== "append") return;
+      for (const message of messages || []) {
+        const chatId = message?.key?.remoteJid;
+        if (!chatId) continue;
+        if (chatId.endsWith("@g.us") || chatId.includes("status@broadcast")) continue;
+
+        if (message?.key?.fromMe) {
+          if (WHATSAPP_MODE !== "self-chat") continue;
+          if (recentlySentIds.has(message.key.id)) continue;
+
+          const myNumber = (sock.user?.id || "").replace(/:.*@/, "@").replace(/@.*/, "");
+          const myLid = (sock.user?.lid || "").replace(/:.*@/, "@").replace(/@.*/, "");
+          const chatNumber = chatId.replace(/@.*/, "");
+          const isSelfChat =
+            (myNumber && chatNumber === myNumber) || (myLid && chatNumber === myLid);
+          if (!isSelfChat) continue;
+        } else if (WHATSAPP_MODE === "self-chat") {
+          continue;
+        }
+
+        enqueueMessage(message);
+      }
+    });
+  })();
+  try {
+    await starting;
+  } finally {
+    starting = null;
+  }
+}
+
+app.get("/health", (_req, res) => {
+  res.json({
+    status: connectionState,
+    hasQr: Boolean(latestQr),
+    qr: latestQr,
+    queueDepth: messageQueue.length,
+    user: sock?.user || null,
+  });
+});
+
+app.get("/messages", (_req, res) => {
+  const messages = messageQueue.splice(0, messageQueue.length);
+  res.json(messages);
+});
+
+app.post("/send", async (req, res) => {
+  try {
+    if (!sock || connectionState !== "connected") {
+      return res.status(503).json({ error: "WhatsApp is not connected" });
+    }
+    const { chatId, message, replyTo } = req.body || {};
+    if (!chatId || !message) {
+      return res.status(400).json({ error: "chatId and message are required" });
+    }
+    const payload = { text: String(message) };
+    if (replyTo) {
+      payload.contextInfo = { stanzaId: String(replyTo) };
+    }
+    const sent = await withTimeout(sock.sendMessage(chatId, payload), SEND_TIMEOUT_MS);
+    rememberSentMessage(sent);
+    res.json({ messageId: sent?.key?.id || null, raw: sent });
+  } catch (error) {
+    res.status(500).json({ error: error?.message || "send failed" });
+  }
+});
+
+app.post("/edit", async (req, res) => {
+  try {
+    if (!sock || connectionState !== "connected") {
+      return res.status(503).json({ error: "WhatsApp is not connected" });
+    }
+    const { chatId, messageId, message } = req.body || {};
+    if (!chatId || !messageId || !message) {
+      return res.status(400).json({ error: "chatId, messageId and message are required" });
+    }
+    const key = sentKeys.get(String(messageId)) || {
+      remoteJid: chatId,
+      id: String(messageId),
+      fromMe: true,
+    };
+    const sent = await withTimeout(
+      sock.sendMessage(chatId, { text: String(message), edit: key }),
+      SEND_TIMEOUT_MS,
+    );
+    rememberSentMessage(sent);
+    res.json({ messageId: sent?.key?.id || messageId, raw: sent });
+  } catch (error) {
+    res.status(500).json({ error: error?.message || "edit failed" });
+  }
+});
+
+app.post("/typing", async (req, res) => {
+  try {
+    if (!sock || connectionState !== "connected") return res.status(204).end();
+    const { chatId } = req.body || {};
+    if (chatId) {
+      await sock.sendPresenceUpdate("composing", chatId);
+    }
+    res.status(204).end();
+  } catch {
+    res.status(204).end();
+  }
+});
+
+app.post("/pair", async (req, res) => {
+  try {
+    const phoneNumber = String(req.body?.phoneNumber || req.body?.phone_number || "").replace(/\D/g, "");
+    if (!phoneNumber) {
+      return res.status(400).json({ error: "phoneNumber is required for pairing code" });
+    }
+    res.json(await requestPairingCodeWhenReady(phoneNumber));
+  } catch (error) {
+    res.status(500).json({ error: error?.message || "pairing failed" });
+  }
+});
+
+app.listen(PORT, "0.0.0.0", () => {
+  console.log(
+    `SurfSense WhatsApp bridge listening on ${PORT}; session=${path.resolve(SESSION_DIR)}; mode=${WHATSAPP_MODE}`,
+  );
+  void startSocket();
+});

surfsense_backend/scripts/whatsapp-bridge/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "surfsense-whatsapp-bridge",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "node bridge.js"
+  },
+  "dependencies": {
+    "@hapi/boom": "latest",
+    "@whiskeysockets/baileys": "latest",
+    "express": "latest",
+    "pino": "latest",
+    "qrcode-terminal": "latest"
+  }
+}