chore: import upstream snapshot with attribution

scripts/db/backfill_token_usage_model_id.py

@@ -0,0 +1,174 @@
+"""Backfill ``token_usage.model_id`` for rows written before the column.
+
+New rows get ``model_id`` stamped at write time (see
+``application.llm.llm_creator`` / ``application.usage``). This script
+fills the historical NULLs by deriving the model from data we already
+trust, in priority order. A row is only ever filled by the
+highest-priority tier that matches it; tiers run in one transaction so
+each later tier sees only the rows still NULL.
+
+Tiers (both touch only ``source='agent_stream'`` rows)
+-----
+1. ``request_id`` join (high confidence). The route stamps the same
+   ``request_id`` on the token_usage row and the assistant message, so
+   ``conversation_messages.model_id`` is authoritative for the call.
+2. ``agent_id`` + nearest message (medium confidence). For primary rows
+   with no usable ``request_id`` (legacy), copy ``model_id`` from the
+   closest-in-time message of any conversation belonging to the same
+   agent, within ``--window-minutes`` (ties broken toward the later
+   message so re-runs are reproducible).
+
+Side-channel rows (``fallback`` / ``compression`` / ``title`` /
+``rag_condense`` / ``schedule``) are left NULL: they share the primary
+turn's ``request_id`` or agent but often ran a *different* model (a
+backup, a compression override), so copying the primary turn's model
+onto them would mis-attribute spend. New rows already get the correct
+per-call model stamped at write time, so this only concerns history.
+
+Rows that match neither tier are left NULL on purpose — the partial
+index ``token_usage_model_ts_idx`` excludes them, and a model we can't
+tie to the specific call (e.g. the agent's configured default) would
+poison the analytics it feeds.
+
+Both ``model_id`` columns store the canonical id (catalog name for
+built-ins, UUID for BYOM), so BYOM rows backfill to the UUID unchanged.
+
+Usage::
+
+    # Dry-run (default): runs the fills in a rolled-back transaction and
+    # reports exactly how many rows each tier would touch.
+    python scripts/db/backfill_token_usage_model_id.py
+
+    # Commit the backfill.
+    python scripts/db/backfill_token_usage_model_id.py --apply
+
+    # Widen the tier-2 match window (default 5 minutes).
+    python scripts/db/backfill_token_usage_model_id.py --window-minutes 10 --apply
+
+Exit codes:
+    0 — success (dry-run or apply)
+    1 — bad arguments
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import text  # noqa: E402
+
+from application.storage.db.engine import get_engine  # noqa: E402
+
+
+# Tier 1: same request -> same model, primary (agent_stream) rows only.
+# conversation_messages.model_id is authoritative for that turn; fallback
+# / compression rows share the request_id but ran a different model.
+_TIER1 = text(
+    """
+    UPDATE token_usage tu
+    SET model_id = cm.model_id
+    FROM conversation_messages cm
+    WHERE cm.request_id = tu.request_id
+      AND cm.model_id IS NOT NULL
+      AND tu.model_id IS NULL
+      AND tu.request_id IS NOT NULL
+      AND tu.source = 'agent_stream'
+    """
+)
+
+# Tier 2: nearest message of the same agent within the window, primary
+# (agent_stream) rows only. The EXISTS mirror skips rows with no match
+# (else the subquery would set NULL); the ORDER BY tiebreak (later message
+# wins) keeps the pick reproducible across re-runs.
+_TIER2 = text(
+    """
+    UPDATE token_usage tu
+    SET model_id = (
+        SELECT cm.model_id
+        FROM conversation_messages cm
+        JOIN conversations c ON c.id = cm.conversation_id
+        WHERE c.agent_id = tu.agent_id
+          AND cm.model_id IS NOT NULL
+          AND cm.timestamp BETWEEN tu.timestamp - make_interval(mins => :win)
+                               AND tu.timestamp + make_interval(mins => :win)
+        ORDER BY abs(extract(epoch FROM (cm.timestamp - tu.timestamp))), cm.timestamp DESC
+        LIMIT 1
+    )
+    WHERE tu.model_id IS NULL
+      AND tu.agent_id IS NOT NULL
+      AND tu.source = 'agent_stream'
+      AND EXISTS (
+        SELECT 1
+        FROM conversation_messages cm
+        JOIN conversations c ON c.id = cm.conversation_id
+        WHERE c.agent_id = tu.agent_id
+          AND cm.model_id IS NOT NULL
+          AND cm.timestamp BETWEEN tu.timestamp - make_interval(mins => :win)
+                               AND tu.timestamp + make_interval(mins => :win)
+      )
+    """
+)
+
+_COUNT_NULL = text("SELECT count(*) FROM token_usage WHERE model_id IS NULL")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Backfill token_usage.model_id from existing data.",
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Commit the backfill. Default is a rolled-back dry-run.",
+    )
+    parser.add_argument(
+        "--window-minutes",
+        type=int,
+        default=5,
+        metavar="N",
+        help="Tier-2 nearest-message match window, in minutes (default 5).",
+    )
+    args = parser.parse_args()
+
+    if args.window_minutes < 0:
+        print("--window-minutes must be >= 0", file=sys.stderr)
+        return 1
+
+    engine = get_engine()
+    with engine.connect() as conn:
+        trans = conn.begin()
+        try:
+            # A one-shot maintenance UPDATE can run well past the engine's
+            # 30s per-statement guardrail; lift it for this transaction.
+            conn.execute(text("SET LOCAL statement_timeout = 0"))
+
+            before = conn.execute(_COUNT_NULL).scalar_one()
+
+            t1 = conn.execute(_TIER1).rowcount or 0
+            t2 = conn.execute(_TIER2, {"win": args.window_minutes}).rowcount or 0
+
+            after = conn.execute(_COUNT_NULL).scalar_one()
+
+            print(f"NULL model_id rows before:         {before}")
+            print(f"  tier 1 (request_id):             {t1}")
+            print(f"  tier 2 (agent + nearest msg):    {t2}")
+            print(f"NULL model_id rows remaining:      {after}")
+
+            if args.apply:
+                trans.commit()
+                print("\nCommitted.")
+            else:
+                trans.rollback()
+                print("\nDry run — rolled back. Re-run with --apply to commit.")
+        except Exception:
+            trans.rollback()
+            raise
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/db/backfill_tool_attempts_attribution.py

@@ -0,0 +1,110 @@
+"""Backfill ``tool_call_attempts.user_id`` / ``agent_id`` (migration 0018).
+
+New rows are stamped at propose time by the tool executor. This script
+fills historical rows from data we already trust; tiers run in one
+transaction so each later tier sees only the rows still NULL.
+
+Tiers
+-----
+1. Parent message (high confidence). Rows with a ``message_id`` copy the
+   message's ``user_id`` and the conversation's ``agent_id``.
+
+Message-less rows (headless: scheduled / webhook runs, plus pre-0018
+parse-failure rows) are left NULL on purpose: there is no FK linking an
+attempt to its run, so any inference from a schedule-run *time window*
+would also catch webhook attempts and misattribute them to an unrelated
+tenant whose run happened to span the same instant. The analytics reader
+treats unattributable rows as invisible rather than guessing an owner,
+and new headless rows are stamped at propose time by the executor.
+
+Usage::
+
+    # Dry-run (default): runs the fills in a rolled-back transaction and
+    # reports exactly how many rows each tier would touch.
+    python scripts/db/backfill_tool_attempts_attribution.py
+
+    # Commit the backfill.
+    python scripts/db/backfill_tool_attempts_attribution.py --apply
+
+Exit codes:
+    0 — success (dry-run or apply)
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import text  # noqa: E402
+
+from application.storage.db.engine import get_engine  # noqa: E402
+
+
+# Tier 1: parent message → user, conversation → agent.
+_TIER1 = text(
+    """
+    UPDATE tool_call_attempts t
+       SET user_id = m.user_id,
+           agent_id = c.agent_id
+      FROM conversation_messages m
+      LEFT JOIN conversations c ON c.id = m.conversation_id
+     WHERE t.message_id = m.id
+       AND t.user_id IS NULL
+    """
+)
+
+_COUNT_NULL = text(
+    "SELECT count(*) FROM tool_call_attempts WHERE user_id IS NULL"
+)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Backfill tool_call_attempts.user_id/agent_id from existing data."
+        ),
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Commit the backfill. Default is a rolled-back dry-run.",
+    )
+    args = parser.parse_args()
+
+    engine = get_engine()
+    with engine.connect() as conn:
+        trans = conn.begin()
+        try:
+            # A one-shot maintenance UPDATE can run well past the engine's
+            # 30s per-statement guardrail; lift it for this transaction.
+            conn.execute(text("SET LOCAL statement_timeout = 0"))
+
+            before = conn.execute(_COUNT_NULL).scalar_one()
+
+            t1 = conn.execute(_TIER1).rowcount or 0
+
+            after = conn.execute(_COUNT_NULL).scalar_one()
+
+            print(f"NULL user_id rows before:           {before}")
+            print(f"  tier 1 (parent message):          {t1}")
+            print(f"NULL user_id rows remaining:        {after}")
+            print("  (message-less headless rows left NULL by design)")
+
+            if args.apply:
+                trans.commit()
+                print("\nCommitted.")
+            else:
+                trans.rollback()
+                print("\nDry run — rolled back. Re-run with --apply to commit.")
+        except Exception:
+            trans.rollback()
+            raise
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/db/init_postgres.py

@@ -0,0 +1,55 @@
+"""One-shot bootstrap: run all Alembic migrations against POSTGRES_URI.
+
+Intended use:
+
+  * local dev, after setting ``POSTGRES_URI`` in ``.env``::
+
+        python scripts/db/init_postgres.py
+
+  * CI, as a step before running the pytest suite.
+
+  * Docker image build or container start, if the operator wants the
+    migrations applied automatically on first boot.
+
+This script is a thin wrapper around ``alembic upgrade head``. It exists
+separately so the same command is discoverable from the repo root without
+remembering the ``-c application/alembic.ini`` invocation.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from alembic import command
+from alembic.config import Config
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+ALEMBIC_INI = REPO_ROOT / "application" / "alembic.ini"
+
+
+def main() -> int:
+    """Apply every pending migration up to ``head``.
+
+    Returns:
+        ``0`` on success, ``1`` on failure. Non-zero is propagated as the
+        process exit code so CI jobs fail loudly.
+    """
+    if not ALEMBIC_INI.exists():
+        print(f"alembic.ini not found at {ALEMBIC_INI}", file=sys.stderr)
+        return 1
+
+    cfg = Config(str(ALEMBIC_INI))
+    # Make `script_location` resolve correctly when invoked from any cwd.
+    cfg.set_main_option("script_location", str(ALEMBIC_INI.parent / "alembic"))
+
+    try:
+        command.upgrade(cfg, "head")
+    except Exception as exc:  # noqa: BLE001 — surface everything to the operator
+        print(f"alembic upgrade failed: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/db/migrate_model_ids.py

@@ -0,0 +1,207 @@
+"""Rewrite stored model IDs across active config tables.
+
+Run this when a provider renames or deprecates a model ID. The catalog
+in ``application/core/models/<provider>.yaml`` is updated to the new ID,
+but existing agents and schedules still reference the old one and will
+fail on the next call. This script rewrites every active config row
+in-place inside a single transaction.
+
+Tables touched (active config — would fail against the provider):
+
+  * ``agents.default_model_id``   (Text)
+  * ``agents.models``             (JSONB array of model-id strings)
+  * ``schedules.model_id``        (Text)
+
+Tables intentionally NOT touched (history):
+
+  * ``conversation_messages.model_id`` — records which model wrote each
+    assistant turn. Rewriting it would falsify history.
+  * ``sources.model`` — stores the *embeddings* model name captured at
+    ingestion, not a chat LLM.
+  * ``user_custom_models.upstream_model_id`` — user-supplied BYOM config
+    against a non-catalog endpoint. Out of scope for catalog rewrites.
+
+Usage::
+
+    # Dry-run with the built-in Gemini preview -> GA mapping (default).
+    python scripts/db/migrate_model_ids.py
+
+    # Apply the built-in mapping.
+    python scripts/db/migrate_model_ids.py --apply
+
+    # Custom mapping (replaces the built-in; repeat --map per pair).
+    python scripts/db/migrate_model_ids.py \\
+        --map gemini-3-flash-preview=gemini-3.5-flash \\
+        --map gemini-3.1-flash-lite-preview=gemini-3.1-flash-lite \\
+        --apply
+
+Exit codes:
+    0 — success (dry-run or apply)
+    1 — bad arguments
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Dict, Iterable
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import text  # noqa: E402
+
+from application.storage.db.session import db_session  # noqa: E402
+
+
+# Built-in mapping reflects the 2026-05-25 Google preview -> GA swap.
+# Update when a new round of catalog churn happens.
+DEFAULT_MAPPING: Dict[str, str] = {
+    "gemini-3-flash-preview": "gemini-3.5-flash",
+    "gemini-3.1-flash-lite-preview": "gemini-3.1-flash-lite",
+}
+
+
+# JSONB array element rewrite. The ``@>`` containment check in the
+# WHERE clause skips rows that don't reference the old ID — without it
+# every agent would be touched on every iteration.
+_UPDATE_AGENTS_MODELS = text(
+    """
+    UPDATE agents
+    SET models = (
+        SELECT jsonb_agg(
+            CASE WHEN elem = to_jsonb(CAST(:old AS text))
+                 THEN to_jsonb(CAST(:new AS text))
+                 ELSE elem
+            END
+        )
+        FROM jsonb_array_elements(models) AS elem
+    )
+    WHERE models IS NOT NULL
+      AND models @> to_jsonb(ARRAY[CAST(:old AS text)])
+    """
+)
+
+
+def _parse_overrides(pairs: Iterable[str]) -> Dict[str, str]:
+    mapping: Dict[str, str] = {}
+    for raw in pairs:
+        if "=" not in raw:
+            raise SystemExit(f"--map expects OLD=NEW, got {raw!r}")
+        old, new = (s.strip() for s in raw.split("=", 1))
+        if not old or not new:
+            raise SystemExit(f"--map values must be non-empty, got {raw!r}")
+        mapping[old] = new
+    return mapping
+
+
+def _count_pre(conn, mapping: Dict[str, str]) -> Dict[str, int]:
+    """Count rows that match the OLD IDs across all target columns."""
+    out = {
+        "agents.default_model_id": 0,
+        "agents.models": 0,
+        "schedules.model_id": 0,
+    }
+    for old in mapping:
+        out["agents.default_model_id"] += conn.execute(
+            text("SELECT count(*) FROM agents WHERE default_model_id = :old"),
+            {"old": old},
+        ).scalar_one()
+        out["agents.models"] += conn.execute(
+            text(
+                "SELECT count(*) FROM agents "
+                "WHERE models IS NOT NULL "
+                "AND models @> to_jsonb(ARRAY[CAST(:old AS text)])"
+            ),
+            {"old": old},
+        ).scalar_one()
+        out["schedules.model_id"] += conn.execute(
+            text("SELECT count(*) FROM schedules WHERE model_id = :old"),
+            {"old": old},
+        ).scalar_one()
+    return out
+
+
+def _apply(conn, mapping: Dict[str, str]) -> Dict[str, int]:
+    """Execute the rewrites inside the caller's transaction."""
+    out = {
+        "agents.default_model_id": 0,
+        "agents.models": 0,
+        "schedules.model_id": 0,
+    }
+    for old, new in mapping.items():
+        res = conn.execute(
+            text(
+                "UPDATE agents SET default_model_id = :new "
+                "WHERE default_model_id = :old"
+            ),
+            {"new": new, "old": old},
+        )
+        out["agents.default_model_id"] += res.rowcount or 0
+
+        res = conn.execute(_UPDATE_AGENTS_MODELS, {"old": old, "new": new})
+        out["agents.models"] += res.rowcount or 0
+
+        res = conn.execute(
+            text("UPDATE schedules SET model_id = :new WHERE model_id = :old"),
+            {"new": new, "old": old},
+        )
+        out["schedules.model_id"] += res.rowcount or 0
+    return out
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Rewrite stored model IDs across active config tables.",
+    )
+    parser.add_argument(
+        "--map",
+        action="append",
+        default=[],
+        metavar="OLD=NEW",
+        help=(
+            "Replace the built-in mapping. Repeat for each pair. "
+            "If any --map is given, the built-in mapping is replaced, "
+            "not merged."
+        ),
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Execute the UPDATEs. Default is dry-run.",
+    )
+    args = parser.parse_args()
+
+    mapping = _parse_overrides(args.map) if args.map else dict(DEFAULT_MAPPING)
+
+    print("Mapping:")
+    for old, new in mapping.items():
+        print(f"  {old}  ->  {new}")
+    print()
+
+    with db_session() as conn:
+        counts = _count_pre(conn, mapping)
+        print("Rows matching old IDs (pre-update):")
+        for col, n in counts.items():
+            print(f"  {col:30s}  {n}")
+        print()
+
+        if sum(counts.values()) == 0:
+            print("Nothing to do.")
+            return 0
+
+        if not args.apply:
+            print("Dry run. Re-run with --apply to commit.")
+            return 0
+
+        updated = _apply(conn, mapping)
+        print("Rows updated:")
+        for col, n in updated.items():
+            print(f"  {col:30s}  {n}")
+
+    print("\nDone.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())