chore: import upstream snapshot with attribution

2026-07-13 13:28:29 +08:00
commit fed8b2eed7
1531 changed files with 1107494 additions and 0 deletions
@@ -0,0 +1,108 @@
+"""Build a Daytona snapshot preloaded with the artifact-render libraries.
+
+The Daytona managed sandbox backend (``SANDBOX_BACKEND=daytona``) creates each
+session from a snapshot. The default snapshot is a plain Python image, so the
+``artifact`` tool's renderers — which ``import`` ``python-pptx`` / ``python-docx``
+/ ``openpyxl`` / ``reportlab`` inside the sandbox — fail with
+``render failed: ExecutionError``. This script bakes those libraries into a
+snapshot once; point ``DAYTONA_SNAPSHOT`` at its name to fix rendering on Daytona.
+
+Usage::
+
+    # Reads DAYTONA_API_KEY / DAYTONA_API_URL / DAYTONA_TARGET from .env (settings):
+    python scripts/build_daytona_snapshot.py
+    python scripts/build_daytona_snapshot.py --name docsgpt-artifacts-py312 --python 3.12
+
+Then set in .env::
+
+    DAYTONA_SNAPSHOT=docsgpt-artifacts-py312
+
+Keep the pins in sync with the backend venv (python-pptx / openpyxl / lxml /
+pillow are in application/requirements.txt; python-docx and reportlab arrive
+transitively) so the Daytona render output matches the Jupyter-backend output.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+# Render libraries imported by the artifact tool's renderers, pinned to the
+# versions installed in the backend venv as of this writing.
+RENDER_PINS = [
+    "python-pptx==1.0.2",
+    "python-docx==1.2.0",
+    "openpyxl==3.1.5",
+    "reportlab==4.5.1",
+    "lxml==6.0.2",
+    "pillow==11.3.0",
+]
+
+DEFAULT_NAME = "docsgpt-artifacts-py312"
+DEFAULT_PYTHON = "3.12"
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    """Parse CLI arguments for the snapshot name and Python series."""
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--name", default=DEFAULT_NAME, help="snapshot name (default: %(default)s)")
+    parser.add_argument("--python", default=DEFAULT_PYTHON, help="Python series, e.g. 3.12")
+    parser.add_argument(
+        "--force", action="store_true", help="rebuild even if a snapshot with that name exists"
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+    """Build (or skip) the snapshot and print the value to set as DAYTONA_SNAPSHOT."""
+    args = _parse_args(argv)
+
+    from application.core.settings import settings
+
+    if not settings.DAYTONA_API_KEY:
+        print("DAYTONA_API_KEY is not set (check .env).", file=sys.stderr)
+        return 2
+
+    from daytona import (
+        CreateSnapshotParams,
+        Daytona,
+        DaytonaConfig,
+        DaytonaConflictError,
+        Image,
+    )
+
+    cfg: dict[str, object] = {"api_key": settings.DAYTONA_API_KEY}
+    if settings.DAYTONA_API_URL:
+        cfg["api_url"] = settings.DAYTONA_API_URL
+    if settings.DAYTONA_TARGET:
+        cfg["target"] = settings.DAYTONA_TARGET
+    client = Daytona(DaytonaConfig(**cfg))
+
+    if not args.force:
+        try:
+            existing = client.snapshot.get(args.name)
+            print(f"snapshot {args.name!r} already exists (state={getattr(existing, 'state', '?')}).")
+            print(f"set DAYTONA_SNAPSHOT={args.name}")
+            return 0
+        except Exception as exc:  # noqa: BLE001 - "not found" is the happy path; build it
+            if type(exc).__name__ != "DaytonaNotFoundError" and "not found" not in str(exc).lower():
+                print(f"warning: get({args.name!r}) probe: {type(exc).__name__}: {exc}", file=sys.stderr)
+
+    image = Image.debian_slim(args.python).pip_install(RENDER_PINS)
+    print(f"building snapshot {args.name!r} (python {args.python}) with: {', '.join(RENDER_PINS)}")
+    print("--- build logs ---")
+    try:
+        snap = client.snapshot.create(
+            CreateSnapshotParams(name=args.name, image=image), on_logs=print, timeout=600
+        )
+    except DaytonaConflictError:
+        print(f"snapshot {args.name!r} already exists (conflict) — reuse it or pass --force a new name.")
+        return 0
+    print("--- created ---")
+    print(f"name={getattr(snap, 'name', args.name)} state={getattr(snap, 'state', '?')}")
+    print(f"\nNow set in .env:\n    DAYTONA_SNAPSHOT={args.name}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))
@@ -0,0 +1,174 @@
+"""Backfill ``token_usage.model_id`` for rows written before the column.
+
+New rows get ``model_id`` stamped at write time (see
+``application.llm.llm_creator`` / ``application.usage``). This script
+fills the historical NULLs by deriving the model from data we already
+trust, in priority order. A row is only ever filled by the
+highest-priority tier that matches it; tiers run in one transaction so
+each later tier sees only the rows still NULL.
+
+Tiers (both touch only ``source='agent_stream'`` rows)
+-----
+1. ``request_id`` join (high confidence). The route stamps the same
+   ``request_id`` on the token_usage row and the assistant message, so
+   ``conversation_messages.model_id`` is authoritative for the call.
+2. ``agent_id`` + nearest message (medium confidence). For primary rows
+   with no usable ``request_id`` (legacy), copy ``model_id`` from the
+   closest-in-time message of any conversation belonging to the same
+   agent, within ``--window-minutes`` (ties broken toward the later
+   message so re-runs are reproducible).
+
+Side-channel rows (``fallback`` / ``compression`` / ``title`` /
+``rag_condense`` / ``schedule``) are left NULL: they share the primary
+turn's ``request_id`` or agent but often ran a *different* model (a
+backup, a compression override), so copying the primary turn's model
+onto them would mis-attribute spend. New rows already get the correct
+per-call model stamped at write time, so this only concerns history.
+
+Rows that match neither tier are left NULL on purpose — the partial
+index ``token_usage_model_ts_idx`` excludes them, and a model we can't
+tie to the specific call (e.g. the agent's configured default) would
+poison the analytics it feeds.
+
+Both ``model_id`` columns store the canonical id (catalog name for
+built-ins, UUID for BYOM), so BYOM rows backfill to the UUID unchanged.
+
+Usage::
+
+    # Dry-run (default): runs the fills in a rolled-back transaction and
+    # reports exactly how many rows each tier would touch.
+    python scripts/db/backfill_token_usage_model_id.py
+
+    # Commit the backfill.
+    python scripts/db/backfill_token_usage_model_id.py --apply
+
+    # Widen the tier-2 match window (default 5 minutes).
+    python scripts/db/backfill_token_usage_model_id.py --window-minutes 10 --apply
+
+Exit codes:
+    0 — success (dry-run or apply)
+    1 — bad arguments
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import text  # noqa: E402
+
+from application.storage.db.engine import get_engine  # noqa: E402
+
+
+# Tier 1: same request -> same model, primary (agent_stream) rows only.
+# conversation_messages.model_id is authoritative for that turn; fallback
+# / compression rows share the request_id but ran a different model.
+_TIER1 = text(
+    """
+    UPDATE token_usage tu
+    SET model_id = cm.model_id
+    FROM conversation_messages cm
+    WHERE cm.request_id = tu.request_id
+      AND cm.model_id IS NOT NULL
+      AND tu.model_id IS NULL
+      AND tu.request_id IS NOT NULL
+      AND tu.source = 'agent_stream'
+    """
+)
+
+# Tier 2: nearest message of the same agent within the window, primary
+# (agent_stream) rows only. The EXISTS mirror skips rows with no match
+# (else the subquery would set NULL); the ORDER BY tiebreak (later message
+# wins) keeps the pick reproducible across re-runs.
+_TIER2 = text(
+    """
+    UPDATE token_usage tu
+    SET model_id = (
+        SELECT cm.model_id
+        FROM conversation_messages cm
+        JOIN conversations c ON c.id = cm.conversation_id
+        WHERE c.agent_id = tu.agent_id
+          AND cm.model_id IS NOT NULL
+          AND cm.timestamp BETWEEN tu.timestamp - make_interval(mins => :win)
+                               AND tu.timestamp + make_interval(mins => :win)
+        ORDER BY abs(extract(epoch FROM (cm.timestamp - tu.timestamp))), cm.timestamp DESC
+        LIMIT 1
+    )
+    WHERE tu.model_id IS NULL
+      AND tu.agent_id IS NOT NULL
+      AND tu.source = 'agent_stream'
+      AND EXISTS (
+        SELECT 1
+        FROM conversation_messages cm
+        JOIN conversations c ON c.id = cm.conversation_id
+        WHERE c.agent_id = tu.agent_id
+          AND cm.model_id IS NOT NULL
+          AND cm.timestamp BETWEEN tu.timestamp - make_interval(mins => :win)
+                               AND tu.timestamp + make_interval(mins => :win)
+      )
+    """
+)
+
+_COUNT_NULL = text("SELECT count(*) FROM token_usage WHERE model_id IS NULL")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Backfill token_usage.model_id from existing data.",
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Commit the backfill. Default is a rolled-back dry-run.",
+    )
+    parser.add_argument(
+        "--window-minutes",
+        type=int,
+        default=5,
+        metavar="N",
+        help="Tier-2 nearest-message match window, in minutes (default 5).",
+    )
+    args = parser.parse_args()
+
+    if args.window_minutes < 0:
+        print("--window-minutes must be >= 0", file=sys.stderr)
+        return 1
+
+    engine = get_engine()
+    with engine.connect() as conn:
+        trans = conn.begin()
+        try:
+            # A one-shot maintenance UPDATE can run well past the engine's
+            # 30s per-statement guardrail; lift it for this transaction.
+            conn.execute(text("SET LOCAL statement_timeout = 0"))
+
+            before = conn.execute(_COUNT_NULL).scalar_one()
+
+            t1 = conn.execute(_TIER1).rowcount or 0
+            t2 = conn.execute(_TIER2, {"win": args.window_minutes}).rowcount or 0
+
+            after = conn.execute(_COUNT_NULL).scalar_one()
+
+            print(f"NULL model_id rows before:         {before}")
+            print(f"  tier 1 (request_id):             {t1}")
+            print(f"  tier 2 (agent + nearest msg):    {t2}")
+            print(f"NULL model_id rows remaining:      {after}")
+
+            if args.apply:
+                trans.commit()
+                print("\nCommitted.")
+            else:
+                trans.rollback()
+                print("\nDry run — rolled back. Re-run with --apply to commit.")
+        except Exception:
+            trans.rollback()
+            raise
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,110 @@
+"""Backfill ``tool_call_attempts.user_id`` / ``agent_id`` (migration 0018).
+
+New rows are stamped at propose time by the tool executor. This script
+fills historical rows from data we already trust; tiers run in one
+transaction so each later tier sees only the rows still NULL.
+
+Tiers
+-----
+1. Parent message (high confidence). Rows with a ``message_id`` copy the
+   message's ``user_id`` and the conversation's ``agent_id``.
+
+Message-less rows (headless: scheduled / webhook runs, plus pre-0018
+parse-failure rows) are left NULL on purpose: there is no FK linking an
+attempt to its run, so any inference from a schedule-run *time window*
+would also catch webhook attempts and misattribute them to an unrelated
+tenant whose run happened to span the same instant. The analytics reader
+treats unattributable rows as invisible rather than guessing an owner,
+and new headless rows are stamped at propose time by the executor.
+
+Usage::
+
+    # Dry-run (default): runs the fills in a rolled-back transaction and
+    # reports exactly how many rows each tier would touch.
+    python scripts/db/backfill_tool_attempts_attribution.py
+
+    # Commit the backfill.
+    python scripts/db/backfill_tool_attempts_attribution.py --apply
+
+Exit codes:
+    0 — success (dry-run or apply)
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import text  # noqa: E402
+
+from application.storage.db.engine import get_engine  # noqa: E402
+
+
+# Tier 1: parent message → user, conversation → agent.
+_TIER1 = text(
+    """
+    UPDATE tool_call_attempts t
+       SET user_id = m.user_id,
+           agent_id = c.agent_id
+      FROM conversation_messages m
+      LEFT JOIN conversations c ON c.id = m.conversation_id
+     WHERE t.message_id = m.id
+       AND t.user_id IS NULL
+    """
+)
+
+_COUNT_NULL = text(
+    "SELECT count(*) FROM tool_call_attempts WHERE user_id IS NULL"
+)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Backfill tool_call_attempts.user_id/agent_id from existing data."
+        ),
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Commit the backfill. Default is a rolled-back dry-run.",
+    )
+    args = parser.parse_args()
+
+    engine = get_engine()
+    with engine.connect() as conn:
+        trans = conn.begin()
+        try:
+            # A one-shot maintenance UPDATE can run well past the engine's
+            # 30s per-statement guardrail; lift it for this transaction.
+            conn.execute(text("SET LOCAL statement_timeout = 0"))
+
+            before = conn.execute(_COUNT_NULL).scalar_one()
+
+            t1 = conn.execute(_TIER1).rowcount or 0
+
+            after = conn.execute(_COUNT_NULL).scalar_one()
+
+            print(f"NULL user_id rows before:           {before}")
+            print(f"  tier 1 (parent message):          {t1}")
+            print(f"NULL user_id rows remaining:        {after}")
+            print("  (message-less headless rows left NULL by design)")
+
+            if args.apply:
+                trans.commit()
+                print("\nCommitted.")
+            else:
+                trans.rollback()
+                print("\nDry run — rolled back. Re-run with --apply to commit.")
+        except Exception:
+            trans.rollback()
+            raise
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,55 @@
+"""One-shot bootstrap: run all Alembic migrations against POSTGRES_URI.
+
+Intended use:
+
+  * local dev, after setting ``POSTGRES_URI`` in ``.env``::
+
+        python scripts/db/init_postgres.py
+
+  * CI, as a step before running the pytest suite.
+
+  * Docker image build or container start, if the operator wants the
+    migrations applied automatically on first boot.
+
+This script is a thin wrapper around ``alembic upgrade head``. It exists
+separately so the same command is discoverable from the repo root without
+remembering the ``-c application/alembic.ini`` invocation.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from alembic import command
+from alembic.config import Config
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+ALEMBIC_INI = REPO_ROOT / "application" / "alembic.ini"
+
+
+def main() -> int:
+    """Apply every pending migration up to ``head``.
+
+    Returns:
+        ``0`` on success, ``1`` on failure. Non-zero is propagated as the
+        process exit code so CI jobs fail loudly.
+    """
+    if not ALEMBIC_INI.exists():
+        print(f"alembic.ini not found at {ALEMBIC_INI}", file=sys.stderr)
+        return 1
+
+    cfg = Config(str(ALEMBIC_INI))
+    # Make `script_location` resolve correctly when invoked from any cwd.
+    cfg.set_main_option("script_location", str(ALEMBIC_INI.parent / "alembic"))
+
+    try:
+        command.upgrade(cfg, "head")
+    except Exception as exc:  # noqa: BLE001 — surface everything to the operator
+        print(f"alembic upgrade failed: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,207 @@
+"""Rewrite stored model IDs across active config tables.
+
+Run this when a provider renames or deprecates a model ID. The catalog
+in ``application/core/models/<provider>.yaml`` is updated to the new ID,
+but existing agents and schedules still reference the old one and will
+fail on the next call. This script rewrites every active config row
+in-place inside a single transaction.
+
+Tables touched (active config — would fail against the provider):
+
+  * ``agents.default_model_id``   (Text)
+  * ``agents.models``             (JSONB array of model-id strings)
+  * ``schedules.model_id``        (Text)
+
+Tables intentionally NOT touched (history):
+
+  * ``conversation_messages.model_id`` — records which model wrote each
+    assistant turn. Rewriting it would falsify history.
+  * ``sources.model`` — stores the *embeddings* model name captured at
+    ingestion, not a chat LLM.
+  * ``user_custom_models.upstream_model_id`` — user-supplied BYOM config
+    against a non-catalog endpoint. Out of scope for catalog rewrites.
+
+Usage::
+
+    # Dry-run with the built-in Gemini preview -> GA mapping (default).
+    python scripts/db/migrate_model_ids.py
+
+    # Apply the built-in mapping.
+    python scripts/db/migrate_model_ids.py --apply
+
+    # Custom mapping (replaces the built-in; repeat --map per pair).
+    python scripts/db/migrate_model_ids.py \\
+        --map gemini-3-flash-preview=gemini-3.5-flash \\
+        --map gemini-3.1-flash-lite-preview=gemini-3.1-flash-lite \\
+        --apply
+
+Exit codes:
+    0 — success (dry-run or apply)
+    1 — bad arguments
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Dict, Iterable
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import text  # noqa: E402
+
+from application.storage.db.session import db_session  # noqa: E402
+
+
+# Built-in mapping reflects the 2026-05-25 Google preview -> GA swap.
+# Update when a new round of catalog churn happens.
+DEFAULT_MAPPING: Dict[str, str] = {
+    "gemini-3-flash-preview": "gemini-3.5-flash",
+    "gemini-3.1-flash-lite-preview": "gemini-3.1-flash-lite",
+}
+
+
+# JSONB array element rewrite. The ``@>`` containment check in the
+# WHERE clause skips rows that don't reference the old ID — without it
+# every agent would be touched on every iteration.
+_UPDATE_AGENTS_MODELS = text(
+    """
+    UPDATE agents
+    SET models = (
+        SELECT jsonb_agg(
+            CASE WHEN elem = to_jsonb(CAST(:old AS text))
+                 THEN to_jsonb(CAST(:new AS text))
+                 ELSE elem
+            END
+        )
+        FROM jsonb_array_elements(models) AS elem
+    )
+    WHERE models IS NOT NULL
+      AND models @> to_jsonb(ARRAY[CAST(:old AS text)])
+    """
+)
+
+
+def _parse_overrides(pairs: Iterable[str]) -> Dict[str, str]:
+    mapping: Dict[str, str] = {}
+    for raw in pairs:
+        if "=" not in raw:
+            raise SystemExit(f"--map expects OLD=NEW, got {raw!r}")
+        old, new = (s.strip() for s in raw.split("=", 1))
+        if not old or not new:
+            raise SystemExit(f"--map values must be non-empty, got {raw!r}")
+        mapping[old] = new
+    return mapping
+
+
+def _count_pre(conn, mapping: Dict[str, str]) -> Dict[str, int]:
+    """Count rows that match the OLD IDs across all target columns."""
+    out = {
+        "agents.default_model_id": 0,
+        "agents.models": 0,
+        "schedules.model_id": 0,
+    }
+    for old in mapping:
+        out["agents.default_model_id"] += conn.execute(
+            text("SELECT count(*) FROM agents WHERE default_model_id = :old"),
+            {"old": old},
+        ).scalar_one()
+        out["agents.models"] += conn.execute(
+            text(
+                "SELECT count(*) FROM agents "
+                "WHERE models IS NOT NULL "
+                "AND models @> to_jsonb(ARRAY[CAST(:old AS text)])"
+            ),
+            {"old": old},
+        ).scalar_one()
+        out["schedules.model_id"] += conn.execute(
+            text("SELECT count(*) FROM schedules WHERE model_id = :old"),
+            {"old": old},
+        ).scalar_one()
+    return out
+
+
+def _apply(conn, mapping: Dict[str, str]) -> Dict[str, int]:
+    """Execute the rewrites inside the caller's transaction."""
+    out = {
+        "agents.default_model_id": 0,
+        "agents.models": 0,
+        "schedules.model_id": 0,
+    }
+    for old, new in mapping.items():
+        res = conn.execute(
+            text(
+                "UPDATE agents SET default_model_id = :new "
+                "WHERE default_model_id = :old"
+            ),
+            {"new": new, "old": old},
+        )
+        out["agents.default_model_id"] += res.rowcount or 0
+
+        res = conn.execute(_UPDATE_AGENTS_MODELS, {"old": old, "new": new})
+        out["agents.models"] += res.rowcount or 0
+
+        res = conn.execute(
+            text("UPDATE schedules SET model_id = :new WHERE model_id = :old"),
+            {"new": new, "old": old},
+        )
+        out["schedules.model_id"] += res.rowcount or 0
+    return out
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Rewrite stored model IDs across active config tables.",
+    )
+    parser.add_argument(
+        "--map",
+        action="append",
+        default=[],
+        metavar="OLD=NEW",
+        help=(
+            "Replace the built-in mapping. Repeat for each pair. "
+            "If any --map is given, the built-in mapping is replaced, "
+            "not merged."
+        ),
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Execute the UPDATEs. Default is dry-run.",
+    )
+    args = parser.parse_args()
+
+    mapping = _parse_overrides(args.map) if args.map else dict(DEFAULT_MAPPING)
+
+    print("Mapping:")
+    for old, new in mapping.items():
+        print(f"  {old}  ->  {new}")
+    print()
+
+    with db_session() as conn:
+        counts = _count_pre(conn, mapping)
+        print("Rows matching old IDs (pre-update):")
+        for col, n in counts.items():
+            print(f"  {col:30s}  {n}")
+        print()
+
+        if sum(counts.values()) == 0:
+            print("Nothing to do.")
+            return 0
+
+        if not args.apply:
+            print("Dry run. Re-run with --apply to commit.")
+            return 0
+
+        updated = _apply(conn, mapping)
+        print("Rows updated:")
+        for col, n in updated.items():
+            print(f"  {col:30s}  {n}")
+
+    print("\nDone.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# One-time bootstrap for the DocsGPT e2e Postgres template DB.
+#
+# Creates two databases on the native DBngin Postgres instance:
+#   * docsgpt_e2e_template  — schema-only (alembic head), marked as a PG template
+#   * docsgpt_e2e           — the live DB the first `up.sh` run connects to
+#
+# Idempotent. Safe to re-run after schema changes to refresh the template.
+
+set -euo pipefail
+
+PG_BIN="/Users/Shared/DBngin/postgresql/16.2/bin"
+PSQL="${PG_BIN}/psql"
+PG_ISREADY="${PG_BIN}/pg_isready"
+
+PG_HOST="127.0.0.1"
+PG_PORT="5432"
+PG_SUPERUSER="postgres"
+
+TEMPLATE_DB="docsgpt_e2e_template"
+E2E_DB="docsgpt_e2e"
+OWNER_ROLE="docsgpt"
+OWNER_PASSWORD="docsgpt"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+log() {
+  printf '[bake_template] %s\n' "$*" >&2
+}
+
+if [[ ! -x "${PSQL}" ]]; then
+  log "psql not found at ${PSQL} — is DBngin Postgres 16.2 installed?"
+  exit 1
+fi
+
+log "Checking Postgres is up at ${PG_HOST}:${PG_PORT}..."
+if ! "${PG_ISREADY}" -h "${PG_HOST}" -p "${PG_PORT}" -q; then
+  log "Postgres is not accepting connections at ${PG_HOST}:${PG_PORT}."
+  log "Start DBngin's Postgres 16.2 instance and try again."
+  exit 1
+fi
+
+log "Dropping and recreating ${TEMPLATE_DB} as superuser ${PG_SUPERUSER}..."
+"${PSQL}" -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_SUPERUSER}" -d postgres \
+  -v ON_ERROR_STOP=1 -X -q <<SQL
+-- Clear the template flag first so DROP DATABASE is allowed.
+UPDATE pg_database SET datistemplate = FALSE WHERE datname = '${TEMPLATE_DB}';
+UPDATE pg_database SET datallowconn = TRUE  WHERE datname = '${TEMPLATE_DB}';
+
+-- Evict any lingering backends on the template DB.
+SELECT pg_terminate_backend(pid)
+  FROM pg_stat_activity
+ WHERE datname = '${TEMPLATE_DB}'
+   AND pid <> pg_backend_pid();
+
+DROP DATABASE IF EXISTS ${TEMPLATE_DB};
+CREATE DATABASE ${TEMPLATE_DB} OWNER ${OWNER_ROLE};
+SQL
+
+log "Applying Alembic schema to ${TEMPLATE_DB}..."
+
+if [[ -x "${REPO_ROOT}/.venv/bin/python" ]]; then
+  PYTHON="${REPO_ROOT}/.venv/bin/python"
+else
+  PYTHON="python3"
+fi
+
+(
+  cd "${REPO_ROOT}"
+  POSTGRES_URI="postgresql://${OWNER_ROLE}:${OWNER_PASSWORD}@${PG_HOST}:${PG_PORT}/${TEMPLATE_DB}" \
+    "${PYTHON}" scripts/db/init_postgres.py
+)
+
+log "Marking ${TEMPLATE_DB} as a Postgres template (datistemplate=TRUE, datallowconn=FALSE)..."
+"${PSQL}" -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_SUPERUSER}" -d postgres \
+  -v ON_ERROR_STOP=1 -X -q <<SQL
+UPDATE pg_database SET datistemplate = TRUE  WHERE datname = '${TEMPLATE_DB}';
+UPDATE pg_database SET datallowconn = FALSE WHERE datname = '${TEMPLATE_DB}';
+SQL
+
+log "Cloning ${E2E_DB} from ${TEMPLATE_DB}..."
+"${PSQL}" -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_SUPERUSER}" -d postgres \
+  -v ON_ERROR_STOP=1 -X -q <<SQL
+SELECT pg_terminate_backend(pid)
+  FROM pg_stat_activity
+ WHERE datname = '${E2E_DB}'
+   AND pid <> pg_backend_pid();
+
+DROP DATABASE IF EXISTS ${E2E_DB};
+CREATE DATABASE ${E2E_DB} TEMPLATE ${TEMPLATE_DB} OWNER ${OWNER_ROLE};
+SQL
+
+log "Done. Template ${TEMPLATE_DB} is baked; ${E2E_DB} is ready for the first e2e run."
+exit 0
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# scripts/e2e/down.sh
+#
+# Tear down the DocsGPT end-to-end test stack started by up.sh.
+# Reads pidfiles from /tmp/docsgpt-e2e/*.pid, sends SIGTERM, waits up to 3s,
+# escalates to SIGKILL if still alive, removes each pidfile.
+#
+# Constraints:
+#   - Idempotent: exits 0 even when no pidfiles exist.
+#   - NEVER uses pkill/killall (CLAUDE.md: don't risk killing native
+#     Postgres/Redis/Mongo processes).
+#   - NEVER touches shared DB/Redis services.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+PIDDIR="/tmp/docsgpt-e2e"
+
+log() {
+    echo "[down.sh] $*" >&2
+}
+
+# Stop a single service given its pidfile. Best-effort; never fatal.
+stop_one() {
+    local pidfile="$1"
+    local svc
+    svc="$(basename "$pidfile" .pid)"
+
+    if [[ ! -s "$pidfile" ]]; then
+        log "$svc: pidfile empty or missing — removing"
+        rm -f "$pidfile"
+        return 0
+    fi
+
+    local pid
+    pid="$(cat "$pidfile" 2>/dev/null || true)"
+
+    # Guard against garbage in the pidfile (non-numeric or empty).
+    if ! [[ "$pid" =~ ^[0-9]+$ ]]; then
+        log "$svc: pidfile contents not numeric ('$pid') — removing"
+        rm -f "$pidfile"
+        return 0
+    fi
+
+    if ! kill -0 "$pid" 2>/dev/null; then
+        log "$svc: pid $pid not running — removing pidfile"
+        rm -f "$pidfile"
+        return 0
+    fi
+
+    log "$svc: sending SIGTERM to pid $pid"
+    kill "$pid" 2>/dev/null || true
+
+    # Poll up to 3 seconds for graceful exit.
+    local waited=0
+    while (( waited < 3 )); do
+        if ! kill -0 "$pid" 2>/dev/null; then
+            break
+        fi
+        sleep 1
+        waited=$(( waited + 1 ))
+    done
+
+    if kill -0 "$pid" 2>/dev/null; then
+        log "$svc: pid $pid still alive after 3s — SIGKILL"
+        kill -9 "$pid" 2>/dev/null || true
+    else
+        log "$svc: pid $pid exited gracefully"
+    fi
+
+    rm -f "$pidfile"
+}
+
+if [[ ! -d "$PIDDIR" ]]; then
+    log "no pid directory at $PIDDIR — nothing to stop"
+    exit 0
+fi
+
+shopt -s nullglob
+pidfiles=( "$PIDDIR"/*.pid )
+shopt -u nullglob
+
+if (( ${#pidfiles[@]} == 0 )); then
+    log "no pidfiles in $PIDDIR — nothing to stop"
+    exit 0
+fi
+
+for pidfile in "${pidfiles[@]}"; do
+    stop_one "$pidfile"
+done
+
+log "teardown complete"
+exit 0
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# scripts/e2e/env.sh
+#
+# Environment variables for the DocsGPT end-to-end test stack.
+# This file is intentionally passive: it exports variables and nothing else.
+# It is `source`d by scripts/e2e/up.sh (and potentially by developers who want
+# to run Flask/Celery manually against the e2e stack).
+#
+# Mirrors `Appendix A — .env.e2e reference` in e2e-plan.md. If you add/remove
+# a variable here, update the plan doc as well.
+#
+# DO NOT run commands (mkdir, touch, etc.) from this file — keep it pure.
+
+# -----------------------------------------------------------------------------
+# Postgres
+# -----------------------------------------------------------------------------
+export POSTGRES_URI="postgresql://docsgpt:docsgpt@127.0.0.1:5432/docsgpt_e2e"
+
+# -----------------------------------------------------------------------------
+# Redis (dev uses DBs 0/1/2; e2e uses 11/12/13 to stay isolated)
+# -----------------------------------------------------------------------------
+export CELERY_BROKER_URL="redis://127.0.0.1:6379/11"
+export CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/12"
+export CACHE_REDIS_URL="redis://127.0.0.1:6379/13"
+
+# -----------------------------------------------------------------------------
+# Mongo — unused in the e2e stack (app fully cut over on this branch)
+# -----------------------------------------------------------------------------
+export MONGO_URI=""
+
+# -----------------------------------------------------------------------------
+# Vector + storage
+# -----------------------------------------------------------------------------
+export VECTOR_STORE="faiss"
+export EMBEDDINGS_NAME="huggingface_sentence-transformers/all-mpnet-base-v2"
+export STORAGE_TYPE="local"
+export URL_STRATEGY="backend"
+export UPLOAD_FOLDER=".e2e-tmp/inputs"
+
+# -----------------------------------------------------------------------------
+# Flask
+# -----------------------------------------------------------------------------
+export API_URL="http://127.0.0.1:7099"
+export FLASK_DEBUG_MODE="false"
+
+# -----------------------------------------------------------------------------
+# Auth (specs can override AUTH_TYPE per-launch via process env)
+# -----------------------------------------------------------------------------
+export AUTH_TYPE="${AUTH_TYPE:-session_jwt}"
+export JWT_SECRET_KEY="e2e-fixed-secret-never-use-in-prod"
+export ENCRYPTION_SECRET_KEY="e2e-fixed-encryption-key-never-use-in-prod"
+
+# OIDC mode (AUTH_TYPE=oidc) — points at the mock IdP that oidc.spec.ts
+# spawns on demand (scripts/e2e/mock_oidc_idp.py, port 7999). Discovery is
+# lazy, so Flask boots fine before the IdP is up. Every OIDC_* var is pinned
+# here because the app's load_dotenv() walks up and reads the repo .env —
+# whatever a developer keeps there must not leak into the e2e stack.
+if [[ "${AUTH_TYPE}" == "oidc" ]]; then
+    export OIDC_ISSUER="${OIDC_ISSUER:-http://127.0.0.1:7999}"
+    export OIDC_CLIENT_ID="${OIDC_CLIENT_ID:-docsgpt-e2e}"
+    export OIDC_FRONTEND_URL="${OIDC_FRONTEND_URL:-http://127.0.0.1:5179}"
+    export OIDC_CLIENT_SECRET="${OIDC_CLIENT_SECRET:-}"
+    export OIDC_SCOPES="${OIDC_SCOPES:-openid profile email}"
+    export OIDC_USER_ID_CLAIM="${OIDC_USER_ID_CLAIM:-sub}"
+    export OIDC_REDIRECT_URI="${OIDC_REDIRECT_URI:-}"
+    export OIDC_SESSION_LIFETIME_SECONDS="${OIDC_SESSION_LIFETIME_SECONDS:-28800}"
+    export OIDC_PROVIDER_NAME="${OIDC_PROVIDER_NAME:-}"
+    export OIDC_ALLOWED_GROUPS="${OIDC_ALLOWED_GROUPS:-}"
+    export OIDC_GROUPS_CLAIM="${OIDC_GROUPS_CLAIM:-groups}"
+    export SCIM_ENABLED="${SCIM_ENABLED:-false}"
+    export SCIM_TOKEN="${SCIM_TOKEN:-}"
+fi
+
+# -----------------------------------------------------------------------------
+# LLM → mock stub on 127.0.0.1:7899
+# -----------------------------------------------------------------------------
+export LLM_PROVIDER="openai"
+export LLM_NAME="gpt-4o-mini"
+export API_KEY="e2e-fake-key"
+export OPENAI_API_KEY="e2e-fake-key"
+export OPENAI_BASE_URL="http://127.0.0.1:7899/v1"
+export EMBEDDINGS_BASE_URL="http://127.0.0.1:7899/v1"
+export EMBEDDINGS_KEY="e2e-fake-key"
+
+# -----------------------------------------------------------------------------
+# Determinism — disable features that introduce non-determinism in tests
+# -----------------------------------------------------------------------------
+export ENABLE_CONVERSATION_COMPRESSION="false"
+export ENABLE_TOOL_PREFETCH="false"
+export PARSE_PDF_AS_IMAGE="false"
+export DOCLING_OCR_ENABLED="false"
@@ -0,0 +1,509 @@
+"""OpenAI-compatible stub server for the DocsGPT e2e test suite.
+
+Speaks the minimum subset of the OpenAI HTTP API that DocsGPT's ``openai``
+Python client needs:
+
+* ``POST /v1/chat/completions`` (streaming + non-streaming, tool calls via fixture)
+* ``POST /v1/embeddings`` (deterministic hash-seeded vectors)
+* ``GET /healthz`` (liveness probe for ``scripts/e2e/up.sh``)
+
+The server is **deterministic**: the same request always returns the same
+response. Requests are fingerprinted by SHA-256 of a canonical JSON encoding
+of ``(model, messages, tool_choice)``. If a fixture file matching that hash
+exists under ``mock_llm_fixtures/<hash>.json`` it wins; otherwise a generic
+"I don't know" fallback is returned and the hash + request is logged to stderr
+so a developer can promote it into a fixture later.
+
+Run standalone (does NOT import anything from ``application/``). Python 3.11+.
+Flask is the only non-stdlib dependency and is already in
+``application/requirements.txt``.
+
+Usage::
+
+    python scripts/e2e/mock_llm.py
+
+Defaults to ``127.0.0.1:7899`` to match the ``OPENAI_BASE_URL`` referenced in
+``e2e-plan.md`` Appendix A.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import random
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from flask import Flask, Response, jsonify, request, stream_with_context
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+HOST = os.environ.get("MOCK_LLM_HOST", "127.0.0.1")
+PORT = int(os.environ.get("MOCK_LLM_PORT", "7899"))
+FIXTURES_DIR = Path(__file__).parent / "mock_llm_fixtures"
+EMBEDDING_DIM = 768
+GENERIC_FALLBACK_TEXT = (
+    "I don't have enough information to answer that from the provided sources."
+)
+STREAM_CHUNK_COUNT = 5
+
+app = Flask(__name__)
+
+
+# ---------------------------------------------------------------------------
+# CORS — permissive; stub trusts its port
+# ---------------------------------------------------------------------------
+
+
+@app.after_request
+def _add_cors_headers(response: Response) -> Response:
+    response.headers["Access-Control-Allow-Origin"] = "*"
+    response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+    response.headers["Access-Control-Allow-Headers"] = "*"
+    return response
+
+
+@app.route("/v1/chat/completions", methods=["OPTIONS"])
+@app.route("/v1/embeddings", methods=["OPTIONS"])
+def _cors_preflight() -> Response:
+    return Response(status=204)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _canonical_messages(messages: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
+    """Return a minimal, stable representation of the messages array.
+
+    We keep only fields that are semantically meaningful for fingerprinting a
+    request. Extra keys from the OpenAI client (e.g. ``name``, ``tool_call_id``)
+    are preserved because they *do* change the intended response.
+    """
+
+    if not messages:
+        return []
+    out: list[dict[str, Any]] = []
+    for msg in messages:
+        if not isinstance(msg, dict):
+            continue
+        # Content may be a string OR a list of content-part dicts (vision / tool).
+        # Serialize both forms deterministically.
+        entry: dict[str, Any] = {
+            "role": msg.get("role"),
+            "content": msg.get("content"),
+        }
+        for key in ("name", "tool_call_id", "tool_calls"):
+            if key in msg:
+                entry[key] = msg[key]
+        out.append(entry)
+    return out
+
+
+def _compute_request_digest(payload: dict[str, Any]) -> str:
+    """SHA-256 fingerprint of ``(model, messages, tool_choice)``.
+
+    Kept narrow on purpose — temperature / top_p / seed / max_tokens should
+    NOT influence which canned answer we return; those are knobs the app may
+    flap on across runs.
+    """
+
+    canonical = {
+        "model": payload.get("model"),
+        "messages": _canonical_messages(payload.get("messages")),
+        "tool_choice": payload.get("tool_choice"),
+    }
+    blob = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+
+
+def _load_fixture(digest: str) -> dict[str, Any] | None:
+    """Return the parsed fixture dict for ``digest``, or ``None`` if missing/bad."""
+
+    path = FIXTURES_DIR / f"{digest}.json"
+    if not path.is_file():
+        return None
+    try:
+        with path.open("r", encoding="utf-8") as fh:
+            data = json.load(fh)
+    except (OSError, json.JSONDecodeError) as exc:
+        sys.stderr.write(f"[mock-llm] failed to load fixture {path}: {exc}\n")
+        sys.stderr.flush()
+        return None
+    return data
+
+
+def _estimate_tokens(text: str) -> int:
+    """Rough 4-chars-per-token estimate (OpenAI's own ballpark)."""
+
+    if not text:
+        return 0
+    return max(1, len(text) // 4)
+
+
+def _messages_text(messages: list[dict[str, Any]] | None) -> str:
+    """Concatenate message contents for prompt-token estimation."""
+
+    if not messages:
+        return ""
+    parts: list[str] = []
+    for msg in messages:
+        if not isinstance(msg, dict):
+            continue
+        content = msg.get("content")
+        if isinstance(content, str):
+            parts.append(content)
+        elif isinstance(content, list):
+            for part in content:
+                if isinstance(part, dict):
+                    text = part.get("text")
+                    if isinstance(text, str):
+                        parts.append(text)
+    return "\n".join(parts)
+
+
+def _split_into_chunks(text: str, count: int) -> list[str]:
+    """Split ``text`` into roughly ``count`` pieces by character length.
+
+    Guarantees at least one chunk even for the empty string (so streaming
+    clients still see a delta before ``[DONE]``).
+    """
+
+    if count <= 0:
+        return [text]
+    if not text:
+        return [""]
+    n = len(text)
+    size = max(1, (n + count - 1) // count)
+    chunks = [text[i : i + size] for i in range(0, n, size)]
+    if not chunks:
+        chunks = [""]
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# Chat completions
+# ---------------------------------------------------------------------------
+
+
+def _resolve_chat_response(
+    payload: dict[str, Any], digest: str
+) -> tuple[str, list[dict[str, Any]] | None, str, dict[str, int]]:
+    """Return ``(content, tool_calls, finish_reason, usage)`` for ``payload``.
+
+    Looks up a fixture by digest first; falls back to the generic response if
+    no fixture is present, and logs the miss so the dev can convert it.
+    """
+
+    fixture = _load_fixture(digest)
+    if fixture is None:
+        sys.stderr.write(f"[mock-llm] unknown fixture hash {digest}\n")
+        try:
+            sys.stderr.write(
+                "[mock-llm] request: "
+                + json.dumps(payload, sort_keys=True, ensure_ascii=False)
+                + "\n"
+            )
+        except (TypeError, ValueError):
+            sys.stderr.write("[mock-llm] request: <unserializable>\n")
+        sys.stderr.flush()
+        content = GENERIC_FALLBACK_TEXT
+        tool_calls: list[dict[str, Any]] | None = None
+        finish_reason = "stop"
+        prompt_tokens = _estimate_tokens(_messages_text(payload.get("messages")))
+        completion_tokens = _estimate_tokens(content)
+        usage = {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+        }
+        return content, tool_calls, finish_reason, usage
+
+    response = fixture.get("response") or {}
+    content = response.get("content") or ""
+    tool_calls = response.get("tool_calls")
+    finish_reason = response.get("finish_reason") or "stop"
+    fixture_usage = response.get("usage") or {}
+    prompt_tokens = int(
+        fixture_usage.get(
+            "prompt_tokens",
+            _estimate_tokens(_messages_text(payload.get("messages"))),
+        )
+    )
+    completion_tokens = int(
+        fixture_usage.get("completion_tokens", _estimate_tokens(content))
+    )
+    usage = {
+        "prompt_tokens": prompt_tokens,
+        "completion_tokens": completion_tokens,
+        "total_tokens": prompt_tokens + completion_tokens,
+    }
+    return content, tool_calls, finish_reason, usage
+
+
+def _chat_completion_envelope(
+    *,
+    digest: str,
+    model: str,
+    content: str,
+    tool_calls: list[dict[str, Any]] | None,
+    finish_reason: str,
+    usage: dict[str, int],
+) -> dict[str, Any]:
+    message: dict[str, Any] = {"role": "assistant", "content": content}
+    if tool_calls:
+        message["tool_calls"] = tool_calls
+    return {
+        "id": f"chatcmpl-e2e-{digest[:12]}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "message": message,
+                "finish_reason": finish_reason,
+            }
+        ],
+        "usage": usage,
+    }
+
+
+def _sse(payload: dict[str, Any]) -> str:
+    return "data: " + json.dumps(payload, ensure_ascii=False) + "\n\n"
+
+
+def _stream_chat_response(
+    *,
+    digest: str,
+    model: str,
+    content: str,
+    tool_calls: list[dict[str, Any]] | None,
+    finish_reason: str,
+    chunk_delay_ms: int = 0,
+):
+    """Generator yielding SSE frames that match the OpenAI streaming protocol.
+
+    ``chunk_delay_ms`` (controlled by ``X-Mock-LLM-Stream-Chunk-Delay-Ms``
+    header) sleeps that many milliseconds between successive SSE frames.
+    Used by durability E2E tests to simulate slow streams that survive a
+    mid-flight ``kill -9`` against the consumer.
+    """
+
+    created = int(time.time())
+    completion_id = f"chatcmpl-e2e-{digest[:12]}"
+
+    def _base_chunk(delta: dict[str, Any], final: bool = False) -> dict[str, Any]:
+        return {
+            "id": completion_id,
+            "object": "chat.completion.chunk",
+            "created": created,
+            "model": model,
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": delta,
+                    "finish_reason": finish_reason if final else None,
+                }
+            ],
+        }
+
+    def _maybe_sleep() -> None:
+        if chunk_delay_ms > 0:
+            time.sleep(chunk_delay_ms / 1000.0)
+
+    # Opening role delta — matches OpenAI's real behavior.
+    yield _sse(_base_chunk({"role": "assistant", "content": ""}))
+
+    if tool_calls:
+        # Emit tool calls in one delta; content streaming is skipped when
+        # tool_calls are present, matching what RAG code paths expect.
+        _maybe_sleep()
+        yield _sse(_base_chunk({"tool_calls": tool_calls}))
+        yield _sse(_base_chunk({}, final=True))
+    else:
+        chunks = _split_into_chunks(content, STREAM_CHUNK_COUNT)
+        last_index = len(chunks) - 1
+        for i, piece in enumerate(chunks):
+            _maybe_sleep()
+            yield _sse(_base_chunk({"content": piece}, final=(i == last_index)))
+
+    yield "data: [DONE]\n\n"
+
+
+def _read_int_header(name: str, default: int = 0, ceiling: int = 600_000) -> int:
+    """Parse an integer header with a sane upper bound (10 minutes)."""
+    raw = request.headers.get(name)
+    if not raw:
+        return default
+    try:
+        value = int(raw)
+    except (TypeError, ValueError):
+        return default
+    if value < 0:
+        return default
+    return min(value, ceiling)
+
+
+def _read_int_env(name: str, default: int = 0, ceiling: int = 600_000) -> int:
+    """Same as ``_read_int_header`` but for env vars — the durability E2E
+    script sets ``MOCK_LLM_FORCE_*_DELAY_MS`` so it can drive slow streams
+    through DocsGPT's OpenAI client without injecting per-request
+    headers."""
+    raw = os.environ.get(name)
+    if not raw:
+        return default
+    try:
+        value = int(raw)
+    except (TypeError, ValueError):
+        return default
+    if value < 0:
+        return default
+    return min(value, ceiling)
+
+
+@app.post("/v1/chat/completions")
+def chat_completions() -> Response:
+    payload = request.get_json(silent=True) or {}
+    model = payload.get("model") or "gpt-4o-mini"
+    stream = bool(payload.get("stream"))
+
+    digest = _compute_request_digest(payload)
+    content, tool_calls, finish_reason, usage = _resolve_chat_response(payload, digest)
+
+    # Durability E2E hooks: per-request OR per-process delays so tests can
+    # simulate slow providers without touching fixtures or recompiling the
+    # stub. Headers win over env so a single fixture run can opt in/out.
+    upfront_delay_ms = _read_int_header("X-Mock-LLM-Total-Delay-Ms") or _read_int_env(
+        "MOCK_LLM_FORCE_TOTAL_DELAY_MS"
+    )
+    chunk_delay_ms = _read_int_header(
+        "X-Mock-LLM-Stream-Chunk-Delay-Ms"
+    ) or _read_int_env("MOCK_LLM_FORCE_STREAM_CHUNK_DELAY_MS")
+    if upfront_delay_ms > 0:
+        time.sleep(upfront_delay_ms / 1000.0)
+
+    if stream:
+        generator = _stream_chat_response(
+            digest=digest,
+            model=model,
+            content=content,
+            tool_calls=tool_calls,
+            finish_reason=finish_reason,
+            chunk_delay_ms=chunk_delay_ms,
+        )
+        response = Response(
+            stream_with_context(generator),
+            mimetype="text/event-stream",
+        )
+        response.headers["Cache-Control"] = "no-cache"
+        response.headers["X-Accel-Buffering"] = "no"
+        # Flask strips Content-Length on streamed responses; make sure we don't
+        # accidentally set one. Nothing to do here — just documenting.
+        return response
+
+    envelope = _chat_completion_envelope(
+        digest=digest,
+        model=model,
+        content=content,
+        tool_calls=tool_calls,
+        finish_reason=finish_reason,
+        usage=usage,
+    )
+    return jsonify(envelope)
+
+
+# ---------------------------------------------------------------------------
+# Embeddings
+# ---------------------------------------------------------------------------
+
+
+def _deterministic_embedding(text: str) -> list[float]:
+    """Hash-seeded 768-dim float vector in [-1, 1).
+
+    Never all-zero: seeded RNG on a non-trivial hash of ``text`` plus a small
+    non-zero offset so degenerate vector-store checks pass even if
+    ``text`` itself is empty.
+    """
+
+    seed = int(hashlib.sha256(text.encode("utf-8")).hexdigest()[:16], 16) & 0xFFFFFFFF
+    rng = random.Random(seed)
+    vec = [rng.uniform(-1.0, 1.0) for _ in range(EMBEDDING_DIM)]
+    # Guarantee non-degeneracy: nudge the first component away from 0 if the
+    # seeded draw happens to produce a very small value.
+    if abs(vec[0]) < 1e-6:
+        vec[0] = 0.1
+    return vec
+
+
+@app.post("/v1/embeddings")
+@app.post("/v1/v1/embeddings")
+def embeddings() -> Response:
+    payload = request.get_json(silent=True) or {}
+    model = payload.get("model") or "text-embedding-3-small"
+    raw_input = payload.get("input", "")
+
+    if isinstance(raw_input, str):
+        inputs: list[str] = [raw_input]
+    elif isinstance(raw_input, list):
+        inputs = [str(item) if not isinstance(item, str) else item for item in raw_input]
+    else:
+        inputs = [str(raw_input)]
+
+    data = [
+        {
+            "object": "embedding",
+            "index": i,
+            "embedding": _deterministic_embedding(text),
+        }
+        for i, text in enumerate(inputs)
+    ]
+    total_tokens = sum(_estimate_tokens(text) for text in inputs)
+    return jsonify(
+        {
+            "object": "list",
+            "data": data,
+            "model": model,
+            "usage": {
+                "prompt_tokens": total_tokens,
+                "total_tokens": total_tokens,
+            },
+        }
+    )
+
+
+# ---------------------------------------------------------------------------
+# Health
+# ---------------------------------------------------------------------------
+
+
+@app.get("/healthz")
+def healthz() -> Response:
+    return jsonify({"ok": True})
+
+
+# ---------------------------------------------------------------------------
+# Entrypoint
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    FIXTURES_DIR.mkdir(parents=True, exist_ok=True)
+    sys.stderr.write(
+        f"[mock-llm] listening on http://{HOST}:{PORT} "
+        f"(fixtures: {FIXTURES_DIR})\n"
+    )
+    sys.stderr.flush()
+    # threaded=True so that concurrent streaming + embeddings requests from
+    # the Flask backend + Celery worker don't serialize behind each other.
+    app.run(host=HOST, port=PORT, debug=False, use_reloader=False, threaded=True)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,17 @@
+{
+  "_comment": "Tester fixture: 'please run whoami on my remote device' -> run_command tool call (whoami). Used to verify full mode auto-approves a safe non-denylisted command.",
+  "response": {
+    "content": "",
+    "tool_calls": [
+      {
+        "id": "call_remote_tester_whoami_1",
+        "type": "function",
+        "function": {
+          "name": "run_command",
+          "arguments": "{\"command\":\"whoami\",\"working_directory\":\"/tmp\",\"timeout_ms\":5000}"
+        }
+      }
+    ],
+    "finish_reason": "tool_calls"
+  }
+}
@@ -0,0 +1,167 @@
+# Mock LLM fixtures
+
+This directory holds **canned OpenAI Chat Completions responses** keyed by a
+SHA-256 fingerprint of the request. The stub server at
+`scripts/e2e/mock_llm.py` looks up `<hash>.json` here for every
+`POST /v1/chat/completions` request; if no fixture matches, the server returns
+a generic deterministic fallback and logs the missing hash to stderr so you
+can promote it into a fixture later.
+
+Embeddings do not use fixtures — they are generated on the fly from a
+hash-seeded RNG. Only chat completions are fixtured.
+
+## Filename format
+
+```
+<sha256-hex>.json
+```
+
+The hash is a lowercase hex SHA-256 digest (64 characters, no prefix, no
+extension beyond `.json`). Example:
+
+```
+3f5a7b9c...d12ef0.json
+```
+
+## How the hash is computed
+
+The fingerprint covers only the fields that should control which canned answer
+is returned:
+
+```python
+canonical = {
+    "model":       payload.get("model"),
+    "messages":    [minimal({role, content, name?, tool_call_id?, tool_calls?}) ...],
+    "tool_choice": payload.get("tool_choice"),
+}
+blob = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+digest = hashlib.sha256(blob.encode("utf-8")).hexdigest()
+```
+
+Notes:
+
+- `temperature`, `top_p`, `seed`, `max_tokens`, `stream`, and any other
+  sampling/transport knobs **do not influence the hash**. Streaming and
+  non-streaming variants of the same request resolve to the same fixture.
+- Message `content` is kept verbatim — if the app passes a list of content
+  parts (vision / multi-modal), the list is hashed as-is.
+- `tool_calls` on assistant messages and `tool_call_id` / `name` on tool
+  messages are included because they change what the model is *replying to*.
+
+The canonical source of the hashing logic is `_compute_request_digest` in
+`scripts/e2e/mock_llm.py`. If you change that function, regenerate all
+fixtures.
+
+## Computing a fixture hash from the command line
+
+The easiest path: run the e2e suite once with your new flow, grep stderr for
+`[mock-llm] unknown fixture hash <hash>` along with the request dump on the
+following line, and save the canned answer under `<hash>.json`. The up.sh log
+tail preserves both lines.
+
+If you need to compute a hash by hand from a request payload:
+
+```python
+# scripts/e2e/compute_hash.py (not committed; run ad hoc)
+import hashlib, json, sys
+
+payload = json.load(sys.stdin)
+canonical = {
+    "model": payload.get("model"),
+    "messages": [
+        {k: v for k, v in msg.items() if k in {"role", "content", "name", "tool_call_id", "tool_calls"}}
+        for msg in payload.get("messages", [])
+    ],
+    "tool_choice": payload.get("tool_choice"),
+}
+blob = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+print(hashlib.sha256(blob.encode("utf-8")).hexdigest())
+```
+
+```bash
+cat request.json | python scripts/e2e/compute_hash.py
+```
+
+## Fixture JSON schema
+
+```json
+{
+  "request_digest": "<hash>",
+  "description": "Human description of when this is used",
+  "response": {
+    "content": "The canned assistant text, may include markdown.",
+    "tool_calls": null,
+    "finish_reason": "stop",
+    "usage": {"prompt_tokens": 12, "completion_tokens": 34}
+  }
+}
+```
+
+### Field reference
+
+| Field | Type | Required | Notes |
+|---|---|---|---|
+| `request_digest` | string | no (documentation only) | Must match the filename. The loader does not re-verify this, it's here for human review. |
+| `description` | string | no | Short note on what flow this covers — makes grepping fixtures easier. |
+| `response.content` | string | yes (if no `tool_calls`) | The assistant's reply body. Plain text or markdown. Empty string is legal when `tool_calls` is set. |
+| `response.tool_calls` | array \| null | no | OpenAI tool-call shape: `[{"id": "call_x", "type": "function", "function": {"name": "...", "arguments": "{...}"}}]`. Arguments must be a JSON **string**, not an object. |
+| `response.finish_reason` | string | no (defaults to `"stop"`) | Use `"tool_calls"` when returning tool calls, `"length"` to simulate truncation. |
+| `response.usage.prompt_tokens` | number | no | Used verbatim in the non-streaming envelope. Default: estimated from request messages. |
+| `response.usage.completion_tokens` | number | no | Default: estimated from `content`. |
+
+`response.usage.total_tokens` is always recomputed as the sum — do not set it.
+
+### Streaming behavior
+
+The stub handles streaming vs non-streaming transparently for both content
+and tool-call fixtures:
+
+- **Content fixtures** are split into ~5 SSE deltas by character length. Only
+  the last delta carries `finish_reason`.
+- **Tool-call fixtures** are emitted as a single delta containing the full
+  `tool_calls` array, followed by a final empty delta carrying `finish_reason`.
+
+No fixture change is needed to toggle between streaming and non-streaming;
+the app's `stream=true` flag alone controls it.
+
+## Tool-call example
+
+```json
+{
+  "request_digest": "abc123...",
+  "description": "Agent calls the weather tool for 'weather in London?'",
+  "response": {
+    "content": "",
+    "tool_calls": [
+      {
+        "id": "call_e2e_weather_1",
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "arguments": "{\"city\":\"London\"}"
+        }
+      }
+    ],
+    "finish_reason": "tool_calls"
+  }
+}
+```
+
+## Workflow for adding a fixture
+
+1. Run the failing e2e spec. Watch `scripts/e2e/up.sh`'s log tail (or
+   `/tmp/docsgpt-e2e/mock_llm.log` depending on how orchestration pipes it).
+2. Find the `[mock-llm] unknown fixture hash <hash>` warning and the request
+   dump on the following line.
+3. Create `mock_llm_fixtures/<hash>.json` with the schema above.
+4. Re-run the spec — the warning should disappear and the spec should pass.
+5. Commit the fixture. Fixtures are checked into the repo so every developer
+   and every CI run gets the same canned answers.
+
+## Determinism guarantees
+
+- Same request → same hash → same fixture → same response, always.
+- No `time.time()`, no random seeds, no environment dependence in the hash.
+- Embeddings are hash-seeded but never all-zero (the stub nudges the first
+  component away from 0 if the seeded draw is near zero), so the vector store
+  ingest path never rejects them.
@@ -0,0 +1,17 @@
+{
+  "_comment": "Tester fixture: 'please run mkdir /tmp/rd_v3_test_dir on my remote device' -> run_command tool call (mkdir /tmp/rd_v3_test_dir). Used to verify a write command in ask mode still pauses on the approval bar.",
+  "response": {
+    "content": "",
+    "tool_calls": [
+      {
+        "id": "call_remote_tester_mkdir_1",
+        "type": "function",
+        "function": {
+          "name": "run_command",
+          "arguments": "{\"command\":\"mkdir /tmp/rd_v3_test_dir\",\"working_directory\":\"/tmp\",\"timeout_ms\":5000}"
+        }
+      }
+    ],
+    "finish_reason": "tool_calls"
+  }
+}
@@ -0,0 +1,17 @@
+{
+  "_comment": "Tester fixture: 'please run rm -rf / on my remote device' -> run_command tool call (rm -rf /). Used to verify full mode still pauses on the denylist-forced-prompt circuit-breaker.",
+  "response": {
+    "content": "",
+    "tool_calls": [
+      {
+        "id": "call_remote_tester_rm_1",
+        "type": "function",
+        "function": {
+          "name": "run_command",
+          "arguments": "{\"command\":\"rm -rf /\",\"working_directory\":\"/tmp\",\"timeout_ms\":5000}"
+        }
+      }
+    ],
+    "finish_reason": "tool_calls"
+  }
+}
@@ -0,0 +1,17 @@
+{
+  "_comment": "Tester fixture: 'please run ls -la /tmp on my remote device' -> run_command tool call (ls -la /tmp).",
+  "response": {
+    "content": "",
+    "tool_calls": [
+      {
+        "id": "call_remote_tester_1",
+        "type": "function",
+        "function": {
+          "name": "run_command",
+          "arguments": "{\"command\":\"ls -la /tmp\",\"working_directory\":\"/tmp\",\"timeout_ms\":5000}"
+        }
+      }
+    ],
+    "finish_reason": "tool_calls"
+  }
+}
@@ -0,0 +1,307 @@
+"""Mock OpenID Connect provider for the DocsGPT e2e suite and local dev.
+
+Speaks the minimum OIDC surface the backend's ``AUTH_TYPE=oidc`` flow needs:
+
+* ``GET /.well-known/openid-configuration`` (discovery)
+* ``GET /authorize`` (auto-approves and redirects back with a code)
+* ``POST /token`` (single-use code + PKCE S256 check, RS256 ``id_token``,
+  single-use rotating ``refresh_token``; also ``grant_type=refresh_token``)
+* ``GET /jwks`` (public key for ID-token verification)
+* ``GET /userinfo`` (Bearer access token from ``/token``)
+* ``GET /end-session`` (honors ``post_logout_redirect_uri``)
+* ``POST /trigger-backchannel-logout`` (test hook: signs and delivers a
+  back-channel logout token to a given URL)
+* ``GET /healthz`` (liveness probe)
+
+There is no login form: every ``/authorize`` request is approved as the user
+configured via ``MOCK_OIDC_SUB`` / ``MOCK_OIDC_EMAIL`` (overridable per
+request with ``?sub=``/``?email=`` for multi-user tests). Group membership
+comes from ``MOCK_OIDC_GROUPS`` (comma-separated).
+
+Run standalone (does NOT import anything from ``application/``). Dependencies
+(flask, python-jose, cryptography, requests) are all in
+``application/requirements.txt``.
+
+Usage::
+
+    python scripts/e2e/mock_oidc_idp.py
+
+Defaults to ``127.0.0.1:7999`` to match ``scripts/e2e/env.sh``'s
+``OIDC_ISSUER`` default for ``AUTH_TYPE=oidc`` runs.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import os
+import secrets
+import sys
+import time
+from urllib.parse import urlencode
+
+import requests
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from flask import Flask, Response, jsonify, redirect, request
+from jose import jwk
+from jose import jwt as jose_jwt
+
+HOST = os.environ.get("MOCK_OIDC_HOST", "127.0.0.1")
+PORT = int(os.environ.get("MOCK_OIDC_PORT", "7999"))
+ISSUER = f"http://{HOST}:{PORT}"
+DEFAULT_SUB = os.environ.get("MOCK_OIDC_SUB", "mock-oidc-user")
+DEFAULT_EMAIL = os.environ.get("MOCK_OIDC_EMAIL", "mock-oidc-user@example.com")
+DEFAULT_NAME = os.environ.get("MOCK_OIDC_NAME", "Mock OIDC User")
+DEFAULT_GROUPS = [
+    group.strip()
+    for group in os.environ.get("MOCK_OIDC_GROUPS", "docsgpt-users").split(",")
+    if group.strip()
+]
+DEFAULT_CLIENT_ID = os.environ.get("MOCK_OIDC_CLIENT_ID", "docsgpt-e2e")
+ID_TOKEN_TTL_SECONDS = 300
+LOGOUT_TOKEN_TTL_SECONDS = 120
+BACKCHANNEL_LOGOUT_EVENT = "http://schemas.openid.net/event/backchannel-logout"
+# Random per process: each restart generates a fresh RSA key, and a fresh
+# kid lets relying parties detect the change via their kid-miss refetch
+# path instead of failing signature checks against a stale cached JWKS.
+KID = f"mock-oidc-key-{secrets.token_hex(4)}"
+
+app = Flask(__name__)
+
+_private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+PRIVATE_PEM = _private_key.private_bytes(
+    encoding=serialization.Encoding.PEM,
+    format=serialization.PrivateFormat.PKCS8,
+    encryption_algorithm=serialization.NoEncryption(),
+).decode("ascii")
+PUBLIC_JWK = {
+    **jwk.construct(PRIVATE_PEM, algorithm="RS256").public_key().to_dict(),
+    "kid": KID,
+    "use": "sig",
+}
+
+# code -> {client_id, redirect_uri, code_challenge, nonce, sub, email, name, groups}
+_codes: dict[str, dict] = {}
+# access_token -> {client_id, sub, email, name, groups}
+_access_tokens: dict[str, dict] = {}
+# refresh_token -> {client_id, sub, email, name, groups} (single-use, rotated)
+_refresh_tokens: dict[str, dict] = {}
+_last_client_id: str | None = None
+
+
+def _log(message: str) -> None:
+    sys.stderr.write(f"[mock-oidc] {message}\n")
+    sys.stderr.flush()
+
+
+def _pkce_challenge(verifier: str) -> str:
+    digest = hashlib.sha256(verifier.encode("ascii")).digest()
+    return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")
+
+
+def _user_record(record: dict) -> dict:
+    """Identity fields carried from /authorize through tokens and userinfo."""
+    return {
+        "client_id": record["client_id"],
+        "sub": record["sub"],
+        "email": record["email"],
+        "name": record["name"],
+        "groups": list(record.get("groups") or DEFAULT_GROUPS),
+    }
+
+
+def _issue_tokens(record: dict, nonce: str | None) -> dict:
+    """Mint an id_token (+ tracked access/refresh tokens) for ``record``."""
+    now = int(time.time())
+    claims = {
+        "iss": ISSUER,
+        "aud": record["client_id"],
+        "sub": record["sub"],
+        "email": record["email"],
+        "name": record["name"],
+        "groups": list(record.get("groups") or DEFAULT_GROUPS),
+        "iat": now,
+        "exp": now + ID_TOKEN_TTL_SECONDS,
+    }
+    if nonce:
+        claims["nonce"] = nonce
+    id_token = jose_jwt.encode(claims, PRIVATE_PEM, algorithm="RS256", headers={"kid": KID})
+    access_token = secrets.token_urlsafe(24)
+    refresh_token = secrets.token_urlsafe(24)
+    _access_tokens[access_token] = _user_record(record)
+    _refresh_tokens[refresh_token] = _user_record(record)
+    return {
+        "access_token": access_token,
+        "token_type": "Bearer",
+        "expires_in": ID_TOKEN_TTL_SECONDS,
+        "id_token": id_token,
+        "refresh_token": refresh_token,
+    }
+
+
+@app.get("/.well-known/openid-configuration")
+def discovery() -> Response:
+    return jsonify(
+        {
+            "issuer": ISSUER,
+            "authorization_endpoint": f"{ISSUER}/authorize",
+            "token_endpoint": f"{ISSUER}/token",
+            "jwks_uri": f"{ISSUER}/jwks",
+            "userinfo_endpoint": f"{ISSUER}/userinfo",
+            "end_session_endpoint": f"{ISSUER}/end-session",
+            "backchannel_logout_supported": True,
+            "response_types_supported": ["code"],
+            "grant_types_supported": ["authorization_code", "refresh_token"],
+            "token_endpoint_auth_methods_supported": ["client_secret_post", "client_secret_basic"],
+            "id_token_signing_alg_values_supported": ["RS256"],
+            "code_challenge_methods_supported": ["S256"],
+            "scopes_supported": ["openid", "profile", "email"],
+        }
+    )
+
+
+@app.get("/jwks")
+def jwks() -> Response:
+    return jsonify({"keys": [PUBLIC_JWK]})
+
+
+@app.get("/authorize")
+def authorize() -> Response:
+    args = request.args
+    missing = [
+        name
+        for name in ("client_id", "redirect_uri", "code_challenge", "state")
+        if not args.get(name)
+    ]
+    if missing:
+        return Response(f"missing params: {', '.join(missing)}", status=400)
+    if args.get("response_type") != "code":
+        return Response("unsupported response_type", status=400)
+    if args.get("code_challenge_method") != "S256":
+        return Response("unsupported code_challenge_method", status=400)
+
+    code = secrets.token_urlsafe(24)
+    _codes[code] = {
+        "client_id": args["client_id"],
+        "redirect_uri": args["redirect_uri"],
+        "code_challenge": args["code_challenge"],
+        "nonce": args.get("nonce"),
+        "sub": args.get("sub") or DEFAULT_SUB,
+        "email": args.get("email") or DEFAULT_EMAIL,
+        "name": DEFAULT_NAME,
+        "groups": list(DEFAULT_GROUPS),
+    }
+    _log(f"authorize: auto-approved sub={_codes[code]['sub']}")
+    separator = "&" if "?" in args["redirect_uri"] else "?"
+    query = urlencode({"code": code, "state": args["state"]})
+    return redirect(f"{args['redirect_uri']}{separator}{query}", code=302)
+
+
+@app.post("/token")
+def token() -> Response:
+    global _last_client_id
+    form = request.form
+    grant_type = form.get("grant_type")
+
+    if grant_type == "refresh_token":
+        record = _refresh_tokens.pop(form.get("refresh_token", ""), None)  # single-use
+        if record is None:
+            _log("token: unknown or reused refresh_token")
+            return jsonify({"error": "invalid_grant"}), 400
+        _last_client_id = record["client_id"]
+        _log(f"token: refreshed tokens for sub={record['sub']}")
+        return jsonify(_issue_tokens(record, nonce=None))
+
+    if grant_type != "authorization_code":
+        return jsonify({"error": "unsupported_grant_type"}), 400
+    record = _codes.pop(form.get("code", ""), None)  # single-use
+    if record is None:
+        _log("token: unknown or replayed code")
+        return jsonify({"error": "invalid_grant"}), 400
+    if form.get("redirect_uri") != record["redirect_uri"]:
+        return jsonify({"error": "invalid_grant", "error_description": "redirect_uri mismatch"}), 400
+    if form.get("client_id") != record["client_id"]:
+        return jsonify({"error": "invalid_client"}), 400
+    verifier = form.get("code_verifier", "")
+    if not verifier or _pkce_challenge(verifier) != record["code_challenge"]:
+        _log("token: PKCE verification failed")
+        return jsonify({"error": "invalid_grant", "error_description": "PKCE failed"}), 400
+
+    _last_client_id = record["client_id"]
+    _log(f"token: issued id_token for sub={record['sub']}")
+    return jsonify(_issue_tokens(record, nonce=record["nonce"]))
+
+
+@app.get("/userinfo")
+def userinfo() -> Response:
+    header = request.headers.get("Authorization", "")
+    access_token = header[len("Bearer "):] if header.startswith("Bearer ") else ""
+    record = _access_tokens.get(access_token)
+    if record is None:
+        _log("userinfo: unknown access token")
+        return jsonify({"error": "invalid_token"}), 401
+    return jsonify(
+        {
+            "sub": record["sub"],
+            "email": record["email"],
+            "name": record["name"],
+            "groups": record["groups"],
+        }
+    )
+
+
+@app.post("/trigger-backchannel-logout")
+def trigger_backchannel_logout() -> Response:
+    """Test hook: sign a back-channel logout token and POST it to ``url``."""
+    body = request.get_json(silent=True) or {}
+    url = body.get("url")
+    sub = body.get("sub")
+    sid = body.get("sid")
+    if not url or not (sub or sid):
+        return jsonify({"error": "url and sub (or sid) required"}), 400
+
+    now = int(time.time())
+    claims = {
+        "iss": ISSUER,
+        "aud": body.get("client_id") or _last_client_id or DEFAULT_CLIENT_ID,
+        "iat": now,
+        "exp": now + LOGOUT_TOKEN_TTL_SECONDS,
+        "jti": secrets.token_urlsafe(16),
+        "events": {BACKCHANNEL_LOGOUT_EVENT: {}},
+    }
+    if sub:
+        claims["sub"] = sub
+    if sid:
+        claims["sid"] = sid
+    logout_token = jose_jwt.encode(claims, PRIVATE_PEM, algorithm="RS256", headers={"kid": KID})
+    try:
+        downstream = requests.post(url, data={"logout_token": logout_token}, timeout=10)
+    except requests.RequestException as exc:
+        _log(f"trigger-backchannel-logout: delivery to {url} failed: {exc}")
+        return jsonify({"error": "delivery_failed", "detail": str(exc)}), 502
+    _log(f"trigger-backchannel-logout: {url} responded {downstream.status_code}")
+    return jsonify({"status": downstream.status_code})
+
+
+@app.get("/end-session")
+def end_session() -> Response:
+    target = request.args.get("post_logout_redirect_uri")
+    _log(f"end-session: redirect={target or '<none>'}")
+    if target:
+        return redirect(target, code=302)
+    return Response("Signed out of mock IdP.", status=200, mimetype="text/plain")
+
+
+@app.get("/healthz")
+def healthz() -> Response:
+    return jsonify({"ok": True})
+
+
+def main() -> None:
+    _log(f"listening on {ISSUER} (sub={DEFAULT_SUB}, groups={','.join(DEFAULT_GROUPS)})")
+    app.run(host=HOST, port=PORT, debug=False, use_reloader=False, threaded=True)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Reset the DocsGPT e2e DB by cloning it from the baked template.
+#
+# Called by scripts/e2e/up.sh at the start of each e2e run.
+# Fast path: a single psql session does terminate + drop + clone.
+
+set -euo pipefail
+
+PG_BIN="/Users/Shared/DBngin/postgresql/16.2/bin"
+PSQL="${PG_BIN}/psql"
+PG_ISREADY="${PG_BIN}/pg_isready"
+
+PG_HOST="127.0.0.1"
+PG_PORT="5432"
+PG_SUPERUSER="postgres"
+
+TEMPLATE_DB="docsgpt_e2e_template"
+E2E_DB="docsgpt_e2e"
+OWNER_ROLE="docsgpt"
+
+log() {
+  printf '[reset_db] %s\n' "$*" >&2
+}
+
+if [[ ! -x "${PSQL}" ]]; then
+  log "psql not found at ${PSQL} — is DBngin Postgres 16.2 installed?"
+  exit 1
+fi
+
+if ! "${PG_ISREADY}" -h "${PG_HOST}" -p "${PG_PORT}" -q; then
+  log "Postgres is not accepting connections at ${PG_HOST}:${PG_PORT}."
+  exit 1
+fi
+
+# Verify the template exists before attempting to clone from it.
+template_exists="$(
+  "${PSQL}" -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_SUPERUSER}" -d postgres \
+    -tAX -c "SELECT 1 FROM pg_database WHERE datname = '${TEMPLATE_DB}';"
+)"
+
+if [[ "${template_exists}" != "1" ]]; then
+  log "Template DB '${TEMPLATE_DB}' does not exist."
+  log "Run scripts/e2e/bake_template.sh once before the first e2e run."
+  exit 1
+fi
+
+"${PSQL}" -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_SUPERUSER}" -d postgres \
+  -v ON_ERROR_STOP=1 -X -q <<SQL
+SELECT pg_terminate_backend(pid)
+  FROM pg_stat_activity
+ WHERE datname = '${E2E_DB}'
+   AND pid <> pg_backend_pid();
+
+DROP DATABASE IF EXISTS ${E2E_DB};
+CREATE DATABASE ${E2E_DB} TEMPLATE ${TEMPLATE_DB} OWNER ${OWNER_ROLE};
+SQL
+
+exit 0
@@ -0,0 +1,12 @@
+DO $$
+DECLARE r record;
+BEGIN
+  FOR r IN
+    SELECT tablename
+      FROM pg_tables
+     WHERE schemaname='public'
+       AND tablename <> 'alembic_version'
+  LOOP
+    EXECUTE format('TRUNCATE TABLE %I RESTART IDENTITY CASCADE', r.tablename);
+  END LOOP;
+END $$;
@@ -0,0 +1,356 @@
+#!/usr/bin/env bash
+# scripts/e2e/up.sh
+#
+# Boot the DocsGPT end-to-end test stack on this machine, natively.
+# See e2e-plan.md (Phase 0 / P0-A) for the contract.
+#
+# Happy path:
+#   1. Preflight shared services (Postgres, Redis). Fail loud if down.
+#   2. Reset state: Postgres template clone, Redis FLUSHDB 11/12/13, wipe .e2e-tmp.
+#   3. Export env.
+#   4. Start mock LLM (7899) → Flask (7099) → Celery → Vite (5179), each in
+#      background, each with its own pidfile + log + readiness probe.
+#   5. Exit 0, leaving services running. Playwright (or the user) invokes
+#      down.sh separately when done.
+#
+# On error before handoff: tear everything down, non-zero exit.
+# We explicitly DO NOT tear down on the happy-path exit — that would defeat
+# the purpose of "up".
+
+set -euo pipefail
+
+# -----------------------------------------------------------------------------
+# Paths
+# -----------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+PG_BIN="/Users/Shared/DBngin/postgresql/16.2/bin"
+DBNGIN_REDIS_BIN="/Users/Shared/DBngin/redis/7.0.0/bin"
+
+# Resolve redis-cli — PATH first, then DBngin's bundled copy.
+if command -v redis-cli >/dev/null 2>&1; then
+    REDIS_CLI="$(command -v redis-cli)"
+elif [[ -x "$DBNGIN_REDIS_BIN/redis-cli" ]]; then
+    REDIS_CLI="$DBNGIN_REDIS_BIN/redis-cli"
+else
+    REDIS_CLI=""
+fi
+PIDDIR="/tmp/docsgpt-e2e"
+E2E_TMP="$REPO_ROOT/.e2e-tmp"
+LOGDIR="$E2E_TMP/logs"
+BOOT_LOG="$LOGDIR/up.log"
+SVC_LOGDIR="$PIDDIR"   # per-service logs live with the pidfiles per the brief
+
+MOCK_LLM_PORT=7899
+FLASK_PORT=7099
+VITE_PORT=5179
+
+# -----------------------------------------------------------------------------
+# Bookkeeping — track which services we successfully started so we can tear
+# them down if something later fails.
+# -----------------------------------------------------------------------------
+HANDOFF_OK=0
+STARTED_SERVICES=()
+
+log() {
+    local msg="[up.sh] $*"
+    # Goes to stderr so stdout stays clean; also mirrored to the boot log.
+    echo "$msg" >&2
+    if [[ -n "${BOOT_LOG:-}" ]] && [[ -d "$(dirname "$BOOT_LOG")" ]]; then
+        echo "$msg" >> "$BOOT_LOG"
+    fi
+}
+
+die() {
+    log "ERROR: $*"
+    exit 1
+}
+
+# Trap: if we exit before handoff (failure or Ctrl-C), clean up. The happy
+# path sets HANDOFF_OK=1 just before `exit 0`, so the trap becomes a no-op.
+cleanup_on_failure() {
+    local rc=$?
+    if [[ "$HANDOFF_OK" -eq 1 ]]; then
+        return 0
+    fi
+    log "aborting — tearing down any services that started (rc=$rc)"
+    if [[ -x "$SCRIPT_DIR/down.sh" ]]; then
+        "$SCRIPT_DIR/down.sh" || true
+    fi
+}
+trap cleanup_on_failure EXIT INT TERM
+
+# -----------------------------------------------------------------------------
+# Helpers
+# -----------------------------------------------------------------------------
+
+# Wait for a shell predicate to succeed. Args: <label> <timeout-seconds> <cmd...>
+wait_for() {
+    local label="$1"
+    local timeout="$2"
+    shift 2
+    local elapsed=0
+    while (( elapsed < timeout )); do
+        if "$@" >/dev/null 2>&1; then
+            log "  -> $label ready after ${elapsed}s"
+            return 0
+        fi
+        sleep 1
+        elapsed=$(( elapsed + 1 ))
+    done
+    return 1
+}
+
+# Wait for a substring to appear in a log file.
+wait_for_log() {
+    local label="$1"
+    local timeout="$2"
+    local logfile="$3"
+    local needle="$4"
+    local elapsed=0
+    while (( elapsed < timeout )); do
+        if [[ -f "$logfile" ]] && grep -qF -- "$needle" "$logfile"; then
+            log "  -> $label saw '$needle' after ${elapsed}s"
+            return 0
+        fi
+        sleep 1
+        elapsed=$(( elapsed + 1 ))
+    done
+    return 1
+}
+
+# Dump last 50 lines of a log file to stderr (for boot-failure diagnostics).
+dump_tail() {
+    local label="$1"
+    local logfile="$2"
+    echo "---- last 50 lines of $label ($logfile) ----" >&2
+    if [[ -f "$logfile" ]]; then
+        tail -n 50 "$logfile" >&2 || true
+    else
+        echo "(log file does not exist)" >&2
+    fi
+    echo "---- end $label ----" >&2
+}
+
+# Boot-failure handler: dump the log, then let the trap tear everything down.
+boot_fail() {
+    local svc="$1"
+    local logfile="$2"
+    local reason="$3"
+    log "boot failure: $svc — $reason"
+    dump_tail "$svc" "$logfile"
+    exit 1
+}
+
+# -----------------------------------------------------------------------------
+# 1. Preflight
+# -----------------------------------------------------------------------------
+log "preflight: checking shared native services"
+
+if [[ ! -x "$PG_BIN/pg_isready" ]]; then
+    die "pg_isready not found at $PG_BIN/pg_isready — is DBngin Postgres 16.2 installed?"
+fi
+
+if ! "$PG_BIN/pg_isready" -h 127.0.0.1 -p 5432 -U docsgpt -d postgres >/dev/null 2>&1; then
+    die "Postgres not reachable at 127.0.0.1:5432 as user 'docsgpt'. Start DBngin Postgres 16.2. (CLAUDE.md: do not kill/start this process from scripts.)"
+fi
+log "  -> postgres OK"
+
+if [[ -z "$REDIS_CLI" ]]; then
+    die "redis-cli not found on PATH nor at $DBNGIN_REDIS_BIN/redis-cli — install redis or adjust DBNGIN_REDIS_BIN"
+fi
+
+if ! "$REDIS_CLI" -h 127.0.0.1 -p 6379 PING 2>/dev/null | grep -q '^PONG$'; then
+    die "Redis not reachable at 127.0.0.1:6379. Start the native redis-server. (CLAUDE.md: do not kill/start this process from scripts.)"
+fi
+log "  -> redis OK"
+
+# -----------------------------------------------------------------------------
+# 2. Reset state
+# -----------------------------------------------------------------------------
+log "resetting state"
+
+# Wipe & recreate .e2e-tmp first so BOOT_LOG has a home.
+rm -rf "$E2E_TMP"
+mkdir -p "$E2E_TMP/inputs" "$E2E_TMP/indexes" "$LOGDIR"
+: > "$BOOT_LOG"
+log "  -> .e2e-tmp wiped; logs at $LOGDIR"
+
+mkdir -p "$PIDDIR"
+# Leave existing per-service logs alone until we overwrite them at launch time;
+# that way a prior failure log isn't immediately erased if someone re-runs up.
+
+# Postgres reset — delegated to reset_db.sh (owned by track P0-B).
+RESET_DB_SCRIPT="$SCRIPT_DIR/reset_db.sh"
+if [[ ! -x "$RESET_DB_SCRIPT" ]]; then
+    die "reset_db.sh missing or not executable at $RESET_DB_SCRIPT — has track P0-B landed?"
+fi
+log "  -> invoking reset_db.sh"
+if ! "$RESET_DB_SCRIPT" >> "$BOOT_LOG" 2>&1; then
+    die "reset_db.sh failed — see $BOOT_LOG"
+fi
+
+# Redis reset — three dedicated DB indices.
+for db in 11 12 13; do
+    if ! "$REDIS_CLI" -h 127.0.0.1 -p 6379 -n "$db" FLUSHDB >/dev/null 2>&1; then
+        die "redis-cli FLUSHDB failed on db $db"
+    fi
+done
+log "  -> redis dbs 11/12/13 flushed"
+
+# -----------------------------------------------------------------------------
+# 3. Load env
+# -----------------------------------------------------------------------------
+log "sourcing env.sh"
+# shellcheck source=./env.sh
+source "$SCRIPT_DIR/env.sh"
+
+# -----------------------------------------------------------------------------
+# 4. Start services
+# -----------------------------------------------------------------------------
+
+# Pick Flask / python binaries from the repo venv when present.
+if [[ -x "$REPO_ROOT/.venv/bin/flask" ]]; then
+    FLASK_BIN="$REPO_ROOT/.venv/bin/flask"
+else
+    FLASK_BIN="$(command -v flask || true)"
+fi
+if [[ -z "$FLASK_BIN" ]]; then
+    die "flask binary not found (.venv/bin/flask missing and no 'flask' on PATH)"
+fi
+
+if [[ -x "$REPO_ROOT/.venv/bin/python" ]]; then
+    PY_BIN="$REPO_ROOT/.venv/bin/python"
+else
+    PY_BIN="$(command -v python3 || command -v python || true)"
+fi
+if [[ -z "$PY_BIN" ]]; then
+    die "python binary not found (.venv/bin/python missing and no 'python3' on PATH)"
+fi
+
+log "using flask=$FLASK_BIN python=$PY_BIN"
+
+# ---- 4a. Mock LLM ------------------------------------------------------------
+MOCK_LLM_LOG="$SVC_LOGDIR/mock-llm.log"
+MOCK_LLM_PID="$PIDDIR/mock-llm.pid"
+log "starting mock LLM on 127.0.0.1:$MOCK_LLM_PORT"
+(
+    cd "$REPO_ROOT"
+    # Port can be read from env by the script; we also export it for clarity.
+    MOCK_LLM_PORT="$MOCK_LLM_PORT" PYTHONUNBUFFERED=1 nohup "$PY_BIN" scripts/e2e/mock_llm.py \
+        >"$MOCK_LLM_LOG" 2>&1 &
+    echo $! > "$MOCK_LLM_PID"
+)
+STARTED_SERVICES+=("mock-llm")
+
+if ! wait_for "mock-llm /healthz" 10 \
+        curl -sf "http://127.0.0.1:${MOCK_LLM_PORT}/healthz"; then
+    boot_fail "mock-llm" "$MOCK_LLM_LOG" "healthz did not respond within 10s"
+fi
+
+# ---- 4b. Flask ---------------------------------------------------------------
+FLASK_LOG="$SVC_LOGDIR/flask.log"
+FLASK_PID="$PIDDIR/flask.pid"
+log "starting Flask on 127.0.0.1:$FLASK_PORT"
+(
+    cd "$E2E_TMP"
+    PYTHONUNBUFFERED=1 nohup "$FLASK_BIN" --app ../application/app.py run \
+        --host 127.0.0.1 --port "$FLASK_PORT" \
+        >"$FLASK_LOG" 2>&1 &
+    echo $! > "$FLASK_PID"
+)
+STARTED_SERVICES+=("flask")
+
+if ! wait_for "flask /api/config" 30 \
+        curl -sf "http://127.0.0.1:${FLASK_PORT}/api/config"; then
+    boot_fail "flask" "$FLASK_LOG" "/api/config did not respond within 30s"
+fi
+
+# ---- 4c. Celery --------------------------------------------------------------
+CELERY_LOG="$SVC_LOGDIR/celery.log"
+CELERY_PID="$PIDDIR/celery.pid"
+log "starting Celery worker (solo pool)"
+(
+    cd "$E2E_TMP"
+    PYTHONPATH="$REPO_ROOT${PYTHONPATH:+:$PYTHONPATH}" \
+    PYTHONUNBUFFERED=1 \
+    nohup "$PY_BIN" -m celery -A application.app.celery worker \
+        -l INFO --pool=solo -Q docsgpt,parsing \
+        --without-gossip --without-mingle --without-heartbeat \
+        >"$CELERY_LOG" 2>&1 &
+    echo $! > "$CELERY_PID"
+)
+STARTED_SERVICES+=("celery")
+
+# Celery's "ready" banner contains both "celery@<host>" and "ready.". Wait for
+# both in sequence so we know the worker actually finished bootstrapping.
+if ! wait_for_log "celery 'celery@'" 30 "$CELERY_LOG" "celery@"; then
+    boot_fail "celery" "$CELERY_LOG" "never emitted 'celery@' banner within 30s"
+fi
+
+# Ready check via `celery inspect ping`. We can't grep the log for 'ready'
+# because application/core/logging_config.py calls dictConfig with the default
+# disable_existing_loggers=True, which silences celery.worker's ready banner.
+# `inspect ping` queries the worker over the broker — it's the canonical
+# responsiveness check and doesn't depend on log output.
+CELERY_INSPECT_TIMEOUT=45
+elapsed=0
+ping_ok=0
+while (( elapsed < CELERY_INSPECT_TIMEOUT )); do
+    if ( cd "$E2E_TMP" && \
+         PYTHONPATH="$REPO_ROOT${PYTHONPATH:+:$PYTHONPATH}" \
+         PYTHONUNBUFFERED=1 \
+         "$PY_BIN" -m celery -A application.app.celery inspect ping \
+             --timeout 2 >/dev/null 2>&1 ); then
+        ping_ok=1
+        log "  -> celery inspect ping OK after ${elapsed}s"
+        break
+    fi
+    sleep 1
+    elapsed=$(( elapsed + 1 ))
+done
+if (( ping_ok == 0 )); then
+    boot_fail "celery" "$CELERY_LOG" "worker did not respond to 'inspect ping' within ${CELERY_INSPECT_TIMEOUT}s"
+fi
+
+# ---- 4d. Vite dev server -----------------------------------------------------
+VITE_LOG="$SVC_LOGDIR/vite.log"
+VITE_PID="$PIDDIR/vite.pid"
+log "starting Vite dev server on 127.0.0.1:$VITE_PORT"
+(
+    cd "$REPO_ROOT/frontend"
+    VITE_API_HOST="http://127.0.0.1:${FLASK_PORT}" nohup npm run dev -- \
+        --host 127.0.0.1 --port "$VITE_PORT" --strictPort \
+        >"$VITE_LOG" 2>&1 &
+    echo $! > "$VITE_PID"
+)
+STARTED_SERVICES+=("vite")
+
+# Prefer nc; fall back to lsof. Either succeeding means the port is LISTEN.
+vite_ready() {
+    if command -v nc >/dev/null 2>&1; then
+        nc -z 127.0.0.1 "$VITE_PORT" >/dev/null 2>&1 && return 0
+    fi
+    if command -v lsof >/dev/null 2>&1; then
+        [[ -n "$(lsof -nP -iTCP:"$VITE_PORT" -sTCP:LISTEN -t 2>/dev/null)" ]] && return 0
+    fi
+    return 1
+}
+
+if ! wait_for "vite LISTEN on $VITE_PORT" 20 vite_ready; then
+    boot_fail "vite" "$VITE_LOG" "port $VITE_PORT never entered LISTEN within 20s"
+fi
+
+# -----------------------------------------------------------------------------
+# 5. Handoff
+# -----------------------------------------------------------------------------
+log "all services up:"
+log "  mock-llm  pid=$(cat "$MOCK_LLM_PID") log=$MOCK_LLM_LOG"
+log "  flask     pid=$(cat "$FLASK_PID")    log=$FLASK_LOG"
+log "  celery    pid=$(cat "$CELERY_PID")   log=$CELERY_LOG"
+log "  vite      pid=$(cat "$VITE_PID")     log=$VITE_LOG"
+log "handoff complete — exiting 0, services remain running. Run scripts/e2e/down.sh to stop."
+
+HANDOFF_OK=1
+exit 0
@@ -0,0 +1,130 @@
+"""Grant, revoke, or list the ``admin`` role for DocsGPT users.
+
+Manual admin grants are the bootstrap mechanism for RBAC: the first admin is
+created here (there is no UI to grant admin until you already are one), and that
+admin can then manage others. Grants are written to ``user_roles`` with
+``source='manual'`` and take effect on the user's next request (persisted RBAC
+applies under ``AUTH_TYPE=oidc``; ``user_id`` is the OIDC ``sub``).
+
+Usage::
+
+    python scripts/grant_admin.py <user_id>            # grant admin
+    python scripts/grant_admin.py <user_id> --revoke   # revoke the manual admin grant
+    python scripts/grant_admin.py --list               # list current admins
+    python scripts/grant_admin.py <user_id> --force    # grant even if no users row exists
+
+Exit codes:
+    0 — success
+    1 — bad usage / user not found (without --force)
+    2 — database error
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+# Make the project root importable regardless of cwd.
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import dotenv  # noqa: E402
+
+dotenv.load_dotenv()
+
+from application.storage.db.repositories.auth_events import AuthEventsRepository  # noqa: E402
+from application.storage.db.repositories.user_roles import UserRolesRepository  # noqa: E402
+from application.storage.db.repositories.users import UsersRepository  # noqa: E402
+from application.storage.db.session import db_readonly, db_session  # noqa: E402
+
+logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s")
+logger = logging.getLogger("grant_admin")
+
+ACTOR = "cli"
+
+
+def _list_admins() -> int:
+    with db_readonly() as conn:
+        admins = UserRolesRepository(conn).list_admins()
+    if not admins:
+        print("No admins found.")
+        return 0
+    print(f"{'user_id':40}  {'sources':20}  granted_at")
+    for row in admins:
+        sources = ",".join(row.get("sources") or [])
+        print(f"{row['user_id']:40}  {sources:20}  {row.get('granted_at')}")
+    return 0
+
+
+def _grant(user_id: str, force: bool) -> int:
+    with db_session() as conn:
+        if not force:
+            if UsersRepository(conn).get(user_id) is None:
+                print(
+                    f"No users row for {user_id!r}. The user must have signed in at least "
+                    f"once, or pass --force to grant anyway (creates a dangling grant).",
+                    file=sys.stderr,
+                )
+                return 1
+        inserted = UserRolesRepository(conn).grant(
+            user_id, "admin", source="manual", granted_by=ACTOR
+        )
+        if inserted:
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "role_granted",
+                metadata={"role": "admin", "source": "manual", "granted_by": ACTOR},
+            )
+            print(f"Granted admin to {user_id!r}.")
+        else:
+            print(f"{user_id!r} already has a manual admin grant; nothing to do.")
+    return 0
+
+
+def _revoke(user_id: str) -> int:
+    with db_session() as conn:
+        removed = UserRolesRepository(conn).revoke(user_id, "admin", source="manual")
+        if removed:
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "role_revoked",
+                metadata={"role": "admin", "source": "manual", "revoked_by": ACTOR},
+            )
+            print(f"Revoked the manual admin grant from {user_id!r}.")
+        else:
+            print(
+                f"{user_id!r} has no manual admin grant. "
+                f"(OIDC-group grants are managed by group membership, not this script.)"
+            )
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Manage the admin role for DocsGPT users.")
+    parser.add_argument("user_id", nargs="?", help="The user's auth sub (OIDC subject id).")
+    parser.add_argument("--revoke", action="store_true", help="Revoke the manual admin grant.")
+    parser.add_argument("--list", action="store_true", help="List current admins and exit.")
+    parser.add_argument(
+        "--force", action="store_true", help="Grant even if no users row exists yet."
+    )
+    args = parser.parse_args(argv)
+
+    if args.list:
+        action = _list_admins
+    elif not args.user_id:
+        parser.error("user_id is required unless --list is given.")
+    elif args.revoke:
+        action = lambda: _revoke(args.user_id)  # noqa: E731
+    else:
+        action = lambda: _grant(args.user_id, args.force)  # noqa: E731
+
+    try:
+        return action()
+    except Exception:
+        logger.error("Database operation failed", exc_info=True)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""
+Migration script to convert conversation_id from DBRef to ObjectId in shared_conversations collection.
+"""
+
+import pymongo
+import logging
+from tqdm import tqdm
+from bson.dbref import DBRef
+from bson.objectid import ObjectId
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger()
+
+# Configuration
+MONGO_URI = "mongodb://localhost:27017/"
+DB_NAME = "docsgpt"
+
+def backup_collection(collection, backup_collection_name):
+    """Backup collection before migration."""
+    logger.info(f"Backing up collection {collection.name} to {backup_collection_name}")
+    collection.aggregate([{"$out": backup_collection_name}])
+    logger.info("Backup completed")
+
+def migrate_conversation_id_dbref_to_objectid():
+    """Migrate conversation_id from DBRef to ObjectId."""
+    client = pymongo.MongoClient(MONGO_URI)
+    db = client[DB_NAME]
+    shared_conversations_collection = db["shared_conversations"]
+
+    try:
+        # Backup collection before migration
+        backup_collection(shared_conversations_collection, "shared_conversations_backup")
+
+        # Find all documents and filter for DBRef conversation_id in Python
+        all_documents = list(shared_conversations_collection.find({}))
+        documents_with_dbref = []
+
+        for doc in all_documents:
+            conversation_id_field = doc.get("conversation_id")
+            if isinstance(conversation_id_field, DBRef):
+                documents_with_dbref.append(doc)
+
+        if not documents_with_dbref:
+            logger.info("No documents with DBRef conversation_id found. Migration not needed.")
+            return
+
+        logger.info(f"Found {len(documents_with_dbref)} documents with DBRef conversation_id")
+
+        # Process each document
+        migrated_count = 0
+        error_count = 0
+
+        for doc in tqdm(documents_with_dbref, desc="Migrating conversation_id"):
+            try:
+                conversation_id_field = doc.get("conversation_id")
+
+                # Extract the ObjectId from the DBRef
+                dbref_id = conversation_id_field.id
+
+                if dbref_id and ObjectId.is_valid(dbref_id):
+                    # Update the document to use direct ObjectId
+                    result = shared_conversations_collection.update_one(
+                        {"_id": doc["_id"]},
+                        {"$set": {"conversation_id": dbref_id}}
+                    )
+
+                    if result.modified_count > 0:
+                        migrated_count += 1
+                        logger.debug(f"Successfully migrated document {doc['_id']}")
+                    else:
+                        error_count += 1
+                        logger.warning(f"Failed to update document {doc['_id']}")
+                else:
+                    error_count += 1
+                    logger.warning(f"Invalid ObjectId in DBRef for document {doc['_id']}: {dbref_id}")
+
+            except Exception as e:
+                error_count += 1
+                logger.error(f"Error migrating document {doc['_id']}: {e}")
+
+        # Final verification
+        all_docs_after = list(shared_conversations_collection.find({}))
+        remaining_dbref = 0
+        for doc in all_docs_after:
+            if isinstance(doc.get("conversation_id"), DBRef):
+                remaining_dbref += 1
+
+        logger.info("Migration completed:")
+        logger.info(f"  - Total documents processed: {len(documents_with_dbref)}")
+        logger.info(f"  - Successfully migrated: {migrated_count}")
+        logger.info(f"  - Errors encountered: {error_count}")
+        logger.info(f"  - Remaining DBRef documents: {remaining_dbref}")
+
+        if remaining_dbref == 0:
+            logger.info("✅ Migration successful: All DBRef conversation_id fields have been converted to ObjectId")
+        else:
+            logger.warning(f"⚠️ Migration incomplete: {remaining_dbref} DBRef documents still exist")
+
+    except Exception as e:
+        logger.error(f"Migration failed: {e}")
+        raise
+    finally:
+        client.close()
+
+if __name__ == "__main__":
+    try:
+        logger.info("Starting conversation_id DBRef to ObjectId migration...")
+        migrate_conversation_id_dbref_to_objectid()
+        logger.info("Migration completed successfully!")
+    except Exception as e:
+        logger.error(f"Migration failed due to error: {e}")
+        logger.warning("Please verify database state or restore from backups if necessary.")
@@ -0,0 +1,104 @@
+import pymongo
+import os
+import shutil
+import logging
+from tqdm import tqdm
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger()
+
+# Configuration
+MONGO_URI = "mongodb://localhost:27017/"
+MONGO_ATLAS_URI = "mongodb+srv://<username>:<password>@<cluster>/<dbname>?retryWrites=true&w=majority"
+DB_NAME = "docsgpt"
+
+def backup_collection(collection, backup_collection_name):
+    logger.info(f"Backing up collection {collection.name} to {backup_collection_name}")
+    collection.aggregate([{"$out": backup_collection_name}])
+    logger.info("Backup completed")
+
+def migrate_to_v1_vectorstore_mongo():
+    client = pymongo.MongoClient(MONGO_URI)
+    db = client[DB_NAME]
+    vectors_collection = db["vectors"]
+    sources_collection = db["sources"]
+
+    # Backup collections before migration
+    backup_collection(vectors_collection, "vectors_backup")
+    backup_collection(sources_collection, "sources_backup")
+
+    vectors = list(vectors_collection.find())
+    for vector in tqdm(vectors, desc="Updating vectors"):
+        if "location" in vector:
+            del vector["location"]
+        if "retriever" not in vector:
+            vector["retriever"] = "classic"
+            vector["remote_data"] = None
+        vectors_collection.update_one({"_id": vector["_id"]}, {"$set": vector})
+
+    # Move data from vectors_collection to sources_collection
+    for vector in tqdm(vectors, desc="Moving to sources"):
+        sources_collection.insert_one(vector)
+
+    vectors_collection.drop()
+    client.close()
+    logger.info("Migration completed")
+
+def migrate_faiss_to_v1_vectorstore():
+    client = pymongo.MongoClient(MONGO_URI)
+    db = client[DB_NAME]
+    vectors_collection = db["vectors"]
+
+    vectors = list(vectors_collection.find())
+    for vector in tqdm(vectors, desc="Migrating FAISS vectors"):
+        old_path = f"./application/indexes/{vector['user']}/{vector['name']}"
+        new_path = f"./application/indexes/{vector['_id']}"
+        try:
+            os.makedirs(os.path.dirname(new_path), exist_ok=True)
+            shutil.move(old_path, new_path)
+        except OSError as e:
+            logger.error(f"Error moving {old_path} to {new_path}: {e}")
+
+    client.close()
+    logger.info("FAISS migration completed")
+
+def migrate_mongo_atlas_vector_to_v1_vectorstore():
+    client = pymongo.MongoClient(MONGO_ATLAS_URI)
+    db = client[DB_NAME]
+    vectors_collection = db["vectors"]
+    documents_collection = db["documents"]
+
+    # Backup collections before migration
+    backup_collection(vectors_collection, "vectors_backup")
+    backup_collection(documents_collection, "documents_backup")
+
+    vectors = list(vectors_collection.find())
+    for vector in tqdm(vectors, desc="Updating Mongo Atlas vectors"):
+        documents_collection.update_many(
+            {"store": vector["user"] + "/" + vector["name"]},
+            {"$set": {"source_id": str(vector["_id"])}}
+        )
+
+    client.close()
+    logger.info("Mongo Atlas migration completed")
+
+if __name__ == "__main__":
+    try:
+        logger.info("Starting FAISS migration...")
+        migrate_faiss_to_v1_vectorstore()
+        logger.info("FAISS migration completed successfully ")
+
+        logger.info("Starting local Mongo migration...")
+        migrate_to_v1_vectorstore_mongo()
+        logger.info("Local Mongo migration completed successfully ")
+
+        logger.info("Starting Mongo Atlas migration...")
+        migrate_mongo_atlas_vector_to_v1_vectorstore()
+        logger.info("Mongo Atlas migration completed successfully ")
+
+        logger.info(" All migrations completed successfully!")
+
+    except Exception as e:
+        logger.error(f" Migration failed due to error: {e}")
+        logger.warning(" Please verify database state or restore from backups if necessary.")
@@ -0,0 +1,209 @@
+"""Mock OpenAI-compatible LLM server for benchmarking.
+
+Fixed 5-second generation (100 tokens × 50 ms/token). No auth. Emits SSE
+chunks in OpenAI's chat.completions streaming format, or a single response
+when stream=false. Run on 127.0.0.1:8090 — point DocsGPT at it via
+OPENAI_BASE_URL=http://127.0.0.1:8090/v1.
+
+Flags:
+    --tool-calls   First response returns a tool call instead of text.
+                   Subsequent responses (after a tool_result) return text.
+                   Useful for triggering the tool-execution loop.
+"""
+
+import argparse
+import json
+import logging
+import time
+import uuid
+
+from flask import Flask, Response, request, jsonify
+
+TOKEN_COUNT = 100
+TOKEN_DELAY_S = 0.05  # 100 * 0.05 = 5.0 s
+TOOL_CALL_MODE = False
+
+logger = logging.getLogger("mock_llm")
+logging.basicConfig(level=logging.INFO, format="%(asctime)s mock: %(message)s")
+
+FILLER_TOKENS = [
+    "Lorem", " ipsum", " dolor", " sit", " amet", ",", " consectetur",
+    " adipiscing", " elit", ".", " Sed", " do", " eiusmod", " tempor",
+    " incididunt", " ut", " labore", " et", " dolore", " magna", " aliqua",
+    ".", " Ut", " enim", " ad", " minim", " veniam", ",", " quis", " nostrud",
+    " exercitation", " ullamco", " laboris", " nisi", " ut", " aliquip",
+    " ex", " ea", " commodo", " consequat", ".", " Duis", " aute", " irure",
+    " dolor", " in", " reprehenderit", " in", " voluptate", " velit",
+    " esse", " cillum", " dolore", " eu", " fugiat", " nulla", " pariatur",
+    ".", " Excepteur", " sint", " occaecat", " cupidatat", " non", " proident",
+    ",", " sunt", " in", " culpa", " qui", " officia", " deserunt",
+    " mollit", " anim", " id", " est", " laborum", ".", " Curabitur",
+    " pretium", " tincidunt", " lacus", ".", " Nulla", " gravida", " orci",
+    " a", " odio", ".", " Nullam", " varius", ",", " turpis", " et",
+    " commodo", " pharetra", ",", " est", " eros", " bibendum", " elit",
+    ".",
+]
+
+app = Flask(__name__)
+
+
+def _token_stream_id() -> str:
+    return f"chatcmpl-mock-{uuid.uuid4().hex[:12]}"
+
+
+def _sse_chunk(completion_id: str, model: str, delta: dict, finish_reason=None) -> str:
+    payload = {
+        "id": completion_id,
+        "object": "chat.completion.chunk",
+        "created": int(time.time()),
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "delta": delta,
+                "finish_reason": finish_reason,
+            }
+        ],
+    }
+    return f"data: {json.dumps(payload)}\n\n"
+
+
+def _gen_tool_call_stream(model: str, req_id: str):
+    """Emit two tool_calls (search) in streaming format.
+
+    Two calls ensure the handler executes the first (which can return a
+    huge result), then hits _check_context_limit before the second.
+    """
+    completion_id = _token_stream_id()
+    call_id_1 = f"call_{uuid.uuid4().hex[:12]}"
+    call_id_2 = f"call_{uuid.uuid4().hex[:12]}"
+
+    yield _sse_chunk(completion_id, model, {
+        "role": "assistant",
+        "content": None,
+        "tool_calls": [
+            {
+                "index": 0,
+                "id": call_id_1,
+                "type": "function",
+                "function": {"name": "search", "arguments": ""},
+            },
+            {
+                "index": 1,
+                "id": call_id_2,
+                "type": "function",
+                "function": {"name": "search", "arguments": ""},
+            },
+        ],
+    })
+    args_json = json.dumps({"query": "Python programming basics"})
+    for ch in args_json:
+        time.sleep(TOKEN_DELAY_S)
+        yield _sse_chunk(completion_id, model, {
+            "tool_calls": [
+                {"index": 0, "function": {"arguments": ch}},
+                {"index": 1, "function": {"arguments": ch}},
+            ],
+        })
+    yield _sse_chunk(completion_id, model, {}, finish_reason="tool_calls")
+    yield "data: [DONE]\n\n"
+    logger.info("[%s] tool_call stream done (ids=%s, %s)", req_id, call_id_1, call_id_2)
+
+
+def _has_tool_result(messages: list) -> bool:
+    return any(m.get("role") == "tool" for m in messages)
+
+
+def _gen_text_stream(model: str, req_id: str):
+    completion_id = _token_stream_id()
+    yield _sse_chunk(completion_id, model, {"role": "assistant", "content": ""})
+    for tok in FILLER_TOKENS[:TOKEN_COUNT]:
+        time.sleep(TOKEN_DELAY_S)
+        yield _sse_chunk(completion_id, model, {"content": tok})
+    yield _sse_chunk(completion_id, model, {}, finish_reason="stop")
+    yield "data: [DONE]\n\n"
+    logger.info("[%s] stream done", req_id)
+
+
+@app.post("/v1/chat/completions")
+def chat_completions():
+    body = request.get_json(force=True)
+    model = body.get("model", "mock")
+    stream = bool(body.get("stream", False))
+    messages = body.get("messages", [])
+    tools = body.get("tools")
+    req_id = uuid.uuid4().hex[:8]
+    logger.info(
+        "[%s] /chat/completions stream=%s model=%s tools=%s msgs=%d",
+        req_id, stream, model, bool(tools), len(messages),
+    )
+
+    use_tool_call = (
+        TOOL_CALL_MODE
+        and tools
+        and not _has_tool_result(messages)
+    )
+
+    if stream:
+        gen = (
+            _gen_tool_call_stream(model, req_id) if use_tool_call
+            else _gen_text_stream(model, req_id)
+        )
+        return Response(
+            gen,
+            mimetype="text/event-stream",
+            headers={
+                "Cache-Control": "no-cache, no-transform",
+                "X-Accel-Buffering": "no",
+            },
+        )
+
+    time.sleep(TOKEN_COUNT * TOKEN_DELAY_S)
+    logger.info("[%s] non-stream done", req_id)
+    text = "".join(FILLER_TOKENS[:TOKEN_COUNT])
+    completion_id = _token_stream_id()
+    return jsonify({
+        "id": completion_id,
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": text},
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {
+            "prompt_tokens": 10,
+            "completion_tokens": TOKEN_COUNT,
+            "total_tokens": 10 + TOKEN_COUNT,
+        },
+    })
+
+
+@app.get("/v1/models")
+def list_models():
+    return jsonify({
+        "object": "list",
+        "data": [{"id": "mock", "object": "model", "owned_by": "mock"}],
+    })
+
+
+@app.get("/health")
+def health():
+    return jsonify({"status": "ok"})
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--tool-calls", action="store_true",
+        help="First response returns a tool_call; subsequent responses return text.",
+    )
+    parser.add_argument("--port", type=int, default=8090)
+    args = parser.parse_args()
+    TOOL_CALL_MODE = args.tool_calls
+    if TOOL_CALL_MODE:
+        logger.info("Tool-call mode enabled")
+    app.run(host="127.0.0.1", port=args.port, debug=False, threaded=True)