chore: import upstream snapshot with attribution

application/api/__init__.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/api/admin/__init__.py

@@ -0,0 +1,3 @@
+from .routes import admin_ns
+
+__all__ = ["admin_ns"]

application/api/admin/routes.py

@@ -0,0 +1,375 @@
+"""Admin-gated management endpoints (RBAC ``admin`` role required).
+
+Every resource here is behind ``@admin_required``. The frontend route guard is
+cosmetic — this server-side decorator is the actual security boundary, so any
+new endpoint added to this namespace MUST carry it.
+
+Reads are cross-user aggregates/feeds (deliberately *not* scoped to the
+caller's ``sub``, unlike the rest of the API). Mutations (role grant/revoke,
+deactivate, force-logout) are written through the existing repositories and
+audited to ``auth_events`` with the acting admin recorded.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from application.api.oidc import denylist
+from application.api.user.authz import ROLE_ADMIN, admin_required
+from application.storage.db.repositories.admin_stats import AdminStatsRepository
+from application.storage.db.repositories.auth_events import AuthEventsRepository
+from application.storage.db.repositories.device_audit_log import DeviceAuditLogRepository
+from application.storage.db.repositories.token_usage import TokenUsageRepository
+from application.storage.db.repositories.user_roles import UserRolesRepository
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+admin_ns = Namespace("admin", description="Admin-only management endpoints", path="/api")
+
+_DEFAULT_PAGE_SIZE = 25
+_MAX_PAGE_SIZE = 100
+
+
+def _int_arg(name: str, default: int) -> int:
+    try:
+        return int(request.args.get(name, default))
+    except (TypeError, ValueError):
+        return default
+
+
+def _page() -> tuple[int, int, int]:
+    page = max(1, _int_arg("page", 1))
+    page_size = max(1, min(_MAX_PAGE_SIZE, _int_arg("page_size", _DEFAULT_PAGE_SIZE)))
+    return page, page_size, (page - 1) * page_size
+
+
+def _actor() -> str | None:
+    token = getattr(request, "decoded_token", None)
+    return token.get("sub") if isinstance(token, dict) else None
+
+
+def _since_arg(default_days: int) -> datetime | None:
+    """``?since_days=N`` → a UTC cutoff datetime, or None for 'all time'."""
+    days = _int_arg("since_days", default_days)
+    if days <= 0:
+        return None
+    return datetime.now(timezone.utc) - timedelta(days=days)
+
+
+@admin_ns.route("/admin/overview")
+class AdminOverviewResource(Resource):
+    @admin_required
+    def get(self):
+        """Top-line KPIs for the dashboard home."""
+        with db_readonly() as conn:
+            stats = AdminStatsRepository(conn).overview()
+        return make_response(jsonify({"success": True, **stats}), 200)
+
+
+@admin_ns.route("/admin/users")
+class AdminUsersResource(Resource):
+    @admin_required
+    def get(self):
+        """List users, paginated, most-recently-seen first.
+
+        Each row carries ``last_seen`` (max auth-event time). Admin status is
+        intentionally not joined here; the dashboard cross-references
+        GET /api/admin/admins for badges.
+        """
+        page, page_size, offset = _page()
+        user_id_filter = request.args.get("user_id") or None
+        with db_readonly() as conn:
+            total, rows = AdminStatsRepository(conn).list_users(
+                user_id_filter, offset, page_size
+            )
+        users = [
+            {
+                "user_id": row.get("user_id"),
+                "active": bool(row.get("active", True)),
+                "created_at": row.get("created_at"),
+                "last_seen": row.get("last_seen"),
+            }
+            for row in rows
+        ]
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "users": users,
+                    "page": page,
+                    "page_size": page_size,
+                    "total": total,
+                    "has_more": offset + len(rows) < total,
+                }
+            ),
+            200,
+        )
+
+
+@admin_ns.route("/admin/users/<string:user_id>")
+class AdminUserResource(Resource):
+    @admin_required
+    def get(self, user_id):
+        """Per-user drill-down: profile, roles, recent auth events, counts."""
+        with db_readonly() as conn:
+            user = UsersRepository(conn).get(user_id)
+            if user is None:
+                return make_response(jsonify({"success": False}), 404)
+            roles_repo = UserRolesRepository(conn)
+            body = {
+                "success": True,
+                "user": {
+                    "user_id": user.get("user_id"),
+                    "active": bool(user.get("active", True)),
+                    "created_at": user.get("created_at"),
+                    "updated_at": user.get("updated_at"),
+                },
+                "roles": sorted({"user", *roles_repo.role_names_for(user_id)}),
+                "grants": roles_repo.list_for(user_id),
+                "recent_events": AuthEventsRepository(conn).list_recent(
+                    user_id, limit=20
+                ),
+                "counts": AdminStatsRepository(conn).user_counts(user_id),
+            }
+        return make_response(jsonify(body), 200)
+
+    @admin_required
+    def patch(self, user_id):
+        """Activate/deactivate a user. Deactivation also revokes live sessions."""
+        actor = _actor()
+        data = request.get_json(silent=True) or {}
+        if "active" not in data or not isinstance(data["active"], bool):
+            return make_response(
+                jsonify({"success": False, "message": "Body requires boolean 'active'"}),
+                400,
+            )
+        active = data["active"]
+        if not active and user_id == actor:
+            return make_response(
+                jsonify(
+                    {"success": False, "message": "You cannot deactivate yourself"}
+                ),
+                409,
+            )
+        with db_session() as conn:
+            user = UsersRepository(conn).get(user_id)
+            if user is None:
+                return make_response(jsonify({"success": False}), 404)
+            updated = UsersRepository(conn).set_active(str(user["id"]), active)
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "admin_user_activated" if active else "admin_user_deactivated",
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata={"by": actor, "via": "admin_api"},
+            )
+        if not active:
+            # Best-effort live-session revocation (mirrors SCIM deactivation).
+            denylist.deny_user(user_id)
+        return make_response(
+            jsonify({"success": True, "active": bool(updated.get("active", active))}),
+            200,
+        )
+
+
+@admin_ns.route("/admin/users/<string:user_id>/role")
+class AdminUserRoleResource(Resource):
+    @admin_required
+    def post(self, user_id):
+        """Grant the admin role to ``user_id`` (manual source). Idempotent."""
+        actor = _actor()
+        with db_session() as conn:
+            inserted = UserRolesRepository(conn).grant(
+                user_id, ROLE_ADMIN, source="manual", granted_by=actor
+            )
+            if inserted:
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "role_granted",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={
+                        "role": ROLE_ADMIN,
+                        "source": "manual",
+                        "granted_by": actor,
+                        "via": "admin_api",
+                    },
+                )
+        return make_response(
+            jsonify({"success": True, "granted": inserted, "role": ROLE_ADMIN}), 200
+        )
+
+    @admin_required
+    def delete(self, user_id):
+        """Revoke the manual admin grant. Refuses to remove the last admin."""
+        actor = _actor()
+        with db_session() as conn:
+            roles_repo = UserRolesRepository(conn)
+            admins = roles_repo.list_admins()
+            if len(admins) <= 1 and any(a["user_id"] == user_id for a in admins):
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Cannot remove the last admin",
+                        }
+                    ),
+                    409,
+                )
+            removed = roles_repo.revoke(user_id, ROLE_ADMIN, source="manual")
+            if removed:
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "role_revoked",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={
+                        "role": ROLE_ADMIN,
+                        "source": "manual",
+                        "revoked_by": actor,
+                        "via": "admin_api",
+                    },
+                )
+        return make_response(jsonify({"success": True, "revoked": removed}), 200)
+
+
+@admin_ns.route("/admin/users/<string:user_id>/revoke-sessions")
+class AdminUserSessionsResource(Resource):
+    @admin_required
+    def post(self, user_id):
+        """Force-logout: revoke the user's live OIDC sessions (best-effort)."""
+        ok = denylist.deny_user(user_id)
+        with db_session() as conn:
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "admin_sessions_revoked",
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata={"by": _actor(), "via": "admin_api", "persisted": ok},
+            )
+        return make_response(jsonify({"success": True, "revoked": ok}), 200)
+
+
+@admin_ns.route("/admin/admins")
+class AdminAdminsResource(Resource):
+    @admin_required
+    def get(self):
+        """List all admins with earliest grant time + grant sources."""
+        with db_readonly() as conn:
+            admins = UserRolesRepository(conn).list_admins()
+        return make_response(jsonify({"success": True, "admins": admins}), 200)
+
+
+@admin_ns.route("/admin/usage")
+class AdminUsageResource(Resource):
+    _GROUP_BY = ("none", "model", "agent", "source")
+    _BUCKETS = ("day", "hour")
+
+    @admin_required
+    def get(self):
+        """Global token usage: time-bucketed series + total + top users."""
+        days = max(1, min(365, _int_arg("days", 30)))
+        bucket = request.args.get("bucket", "day")
+        group_by = request.args.get("group_by", "none")
+        if bucket not in self._BUCKETS or group_by not in self._GROUP_BY:
+            return make_response(jsonify({"success": False, "message": "Invalid option"}), 400)
+        start = datetime.now(timezone.utc) - timedelta(days=days)
+        with db_readonly() as conn:
+            usage_repo = TokenUsageRepository(conn)
+            series = usage_repo.bucketed_totals(
+                bucket_unit=bucket,
+                timestamp_gte=start,
+                group_by=None if group_by == "none" else group_by,
+            )
+            total = usage_repo.sum_tokens_in_range(
+                start=start, end=datetime.now(timezone.utc)
+            )
+            top_users = AdminStatsRepository(conn).top_token_users(since=start, limit=10)
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "days": days,
+                    "bucket": bucket,
+                    "group_by": group_by,
+                    "series": series,
+                    "total_tokens": int(total),
+                    "top_users": top_users,
+                }
+            ),
+            200,
+        )
+
+
+@admin_ns.route("/admin/audit")
+class AdminAuditResource(Resource):
+    @admin_required
+    def get(self):
+        """Global auth-events feed, newest first. Filter by event/user/since."""
+        page, page_size, offset = _page()
+        event = request.args.get("event") or None
+        user_id = request.args.get("user_id") or None
+        since = _since_arg(default_days=0)  # 0 = all time
+        with db_readonly() as conn:
+            repo = AuthEventsRepository(conn)
+            total = repo.count_all(event=event, user_id=user_id, since=since)
+            events = repo.list_all(
+                event=event,
+                user_id=user_id,
+                since=since,
+                limit=page_size,
+                offset=offset,
+            )
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "events": events,
+                    "page": page,
+                    "page_size": page_size,
+                    "total": total,
+                    "has_more": offset + len(events) < total,
+                }
+            ),
+            200,
+        )
+
+
+@admin_ns.route("/admin/devices/audit")
+class AdminDeviceAuditResource(Resource):
+    @admin_required
+    def get(self):
+        """Global remote-device command audit feed. Filter by decision/user/since."""
+        page, page_size, offset = _page()
+        decision = request.args.get("decision") or None
+        user_id = request.args.get("user_id") or None
+        since = _since_arg(default_days=0)
+        with db_readonly() as conn:
+            repo = DeviceAuditLogRepository(conn)
+            total = repo.count_global(decision=decision, user_id=user_id, since=since)
+            rows = repo.list_global(
+                decision=decision,
+                user_id=user_id,
+                since=since,
+                limit=page_size,
+                offset=offset,
+            )
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "invocations": rows,
+                    "page": page,
+                    "page_size": page_size,
+                    "total": total,
+                    "has_more": offset + len(rows) < total,
+                }
+            ),
+            200,
+        )

application/api/answer/__init__.py

@@ -0,0 +1,21 @@
+from flask import Blueprint
+
+from application.api import api
+from application.api.answer.routes.answer import AnswerResource
+from application.api.answer.routes.base import answer_ns
+from application.api.answer.routes.search import SearchResource
+from application.api.answer.routes.stream import StreamResource
+
+
+answer = Blueprint("answer", __name__)
+
+api.add_namespace(answer_ns)
+
+
+def init_answer_routes():
+    api.add_resource(StreamResource, "/stream")
+    api.add_resource(AnswerResource, "/api/answer")
+    api.add_resource(SearchResource, "/api/search")
+
+
+init_answer_routes()

application/api/answer/routes/answer.py

@@ -0,0 +1,173 @@
+import logging
+import traceback
+
+from flask import make_response, request
+from flask_restx import fields, Resource
+
+from application.api import api
+
+from application.api.answer.routes.base import answer_ns, BaseAnswerResource
+
+from application.api.answer.services.persistence_policy import resolve_persistence
+from application.api.answer.services.stream_processor import StreamProcessor
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/api/answer")
+class AnswerResource(Resource, BaseAnswerResource):
+    def __init__(self, *args, **kwargs):
+        Resource.__init__(self, *args, **kwargs)
+        BaseAnswerResource.__init__(self)
+
+    answer_model = answer_ns.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String,
+                required=False,
+                description="Conversation history (only for new conversations)",
+            ),
+            "conversation_id": fields.String(
+                required=False,
+                description="Existing conversation ID (loads history)",
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "agent_id": fields.String(required=False, description="Agent ID"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "save_conversation": fields.Boolean(
+                required=False,
+                description=(
+                    "Deprecated, no effect: conversations always persist. "
+                    "Use `visibility` to control sidebar listing."
+                ),
+            ),
+            "visibility": fields.String(
+                required=False,
+                default="hidden",
+                description=(
+                    "'listed' shows the conversation in the owner's sidebar; "
+                    "any other value (or omitting it) persists it hidden."
+                ),
+            ),
+            "model_id": fields.String(
+                required=False,
+                description="Model ID to use for this request",
+            ),
+            "passthrough": fields.Raw(
+                required=False,
+                description="Dynamic parameters to inject into prompt template",
+            ),
+        },
+    )
+
+    @api.expect(answer_model)
+    @api.doc(description="Provide a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        if error := self.validate_request(data):
+            return error
+        decoded_token = getattr(request, "decoded_token", None)
+        processor = StreamProcessor(data, decoded_token)
+        try:
+            # ---- Continuation mode ----
+            if data.get("tool_actions"):
+                (
+                    agent,
+                    messages,
+                    tools_dict,
+                    pending_tool_calls,
+                    tool_actions,
+                    reasoning_content,
+                ) = processor.resume_from_tool_actions(
+                    data["tool_actions"], data["conversation_id"]
+                )
+                if not processor.decoded_token:
+                    return make_response({"error": "Unauthorized"}, 401)
+                if error := self.check_usage(processor.agent_config):
+                    return error
+                stream = self.complete_stream(
+                    question="",
+                    agent=agent,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    agent_id=processor.agent_id,
+                    model_id=processor.model_id,
+                    _continuation={
+                        "messages": messages,
+                        "tools_dict": tools_dict,
+                        "pending_tool_calls": pending_tool_calls,
+                        "tool_actions": tool_actions,
+                        "reserved_message_id": processor.reserved_message_id,
+                        "request_id": processor.request_id,
+                        "reasoning_content": reasoning_content,
+                    },
+                )
+            else:
+                # ---- Normal mode ----
+                agent = processor.build_agent(data.get("question", ""))
+                if not processor.decoded_token:
+                    return make_response({"error": "Unauthorized"}, 401)
+
+                if error := self.check_usage(processor.agent_config):
+                    return error
+
+                should_persist, visibility = resolve_persistence(
+                    visibility_flag=data.get("visibility"),
+                    persist_flag=data.get("persist"),
+                )
+                stream = self.complete_stream(
+                    question=data["question"],
+                    agent=agent,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    isNoneDoc=data.get("isNoneDoc"),
+                    index=None,
+                    should_persist=should_persist,
+                    visibility=visibility,
+                    agent_id=processor.agent_id,
+                    is_shared_usage=processor.is_shared_usage,
+                    shared_token=processor.shared_token,
+                    model_id=processor.model_id,
+                )
+
+            stream_result = self.process_response_stream(stream)
+
+            if stream_result["error"]:
+                return make_response({"error": stream_result["error"]}, 400)
+
+            result = {
+                "conversation_id": stream_result["conversation_id"],
+                "answer": stream_result["answer"],
+                "sources": stream_result["sources"],
+                "tool_calls": stream_result["tool_calls"],
+                "thought": stream_result["thought"],
+            }
+
+            extra_info = stream_result.get("extra")
+            if extra_info:
+                result.update(extra_info)
+        except Exception as e:
+            logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return make_response({"error": "An error occurred processing your request"}, 500)
+        return make_response(result, 200)

application/api/answer/routes/search.py

@@ -0,0 +1,55 @@
+import logging
+
+from flask import make_response, request
+from flask_restx import fields, Resource
+
+from application.api.answer.routes.base import answer_ns
+from application.services.search_service import (
+    InvalidAPIKey,
+    SearchFailed,
+    search,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/api/search")
+class SearchResource(Resource):
+    """Fast search endpoint for retrieving relevant documents."""
+
+    search_model = answer_ns.model(
+        "SearchModel",
+        {
+            "question": fields.String(
+                required=True, description="Search query"
+            ),
+            "api_key": fields.String(
+                required=True, description="API key for authentication"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=5, description="Number of results to return"
+            ),
+        },
+    )
+
+    @answer_ns.expect(search_model)
+    @answer_ns.doc(description="Search for relevant documents based on query")
+    def post(self):
+        data = request.get_json() or {}
+
+        question = data.get("question")
+        api_key = data.get("api_key")
+        chunks = data.get("chunks", 5)
+
+        if not question:
+            return make_response({"error": "question is required"}, 400)
+        if not api_key:
+            return make_response({"error": "api_key is required"}, 400)
+
+        try:
+            return make_response(search(api_key, question, chunks), 200)
+        except InvalidAPIKey:
+            return make_response({"error": "Invalid API key"}, 401)
+        except SearchFailed:
+            logger.exception("/api/search failed")
+            return make_response({"error": "Search failed"}, 500)

application/api/answer/routes/stream.py

@@ -0,0 +1,193 @@
+import logging
+import traceback
+
+from flask import request, Response
+from flask_restx import fields, Resource
+
+from application.api import api
+
+from application.api.answer.routes.base import answer_ns, BaseAnswerResource
+
+from application.api.answer.services.persistence_policy import resolve_persistence
+from application.api.answer.services.stream_processor import StreamProcessor
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/stream")
+class StreamResource(Resource, BaseAnswerResource):
+    def __init__(self, *args, **kwargs):
+        Resource.__init__(self, *args, **kwargs)
+        BaseAnswerResource.__init__(self)
+
+    stream_model = answer_ns.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String,
+                required=False,
+                description="Conversation history (only for new conversations)",
+            ),
+            "conversation_id": fields.String(
+                required=False,
+                description="Existing conversation ID (loads history)",
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "agent_id": fields.String(required=False, description="Agent ID"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "index": fields.Integer(
+                required=False, description="Index of the query to update"
+            ),
+            "save_conversation": fields.Boolean(
+                required=False,
+                description=(
+                    "Deprecated, no effect: conversations always persist. "
+                    "Use `visibility` to control sidebar listing."
+                ),
+            ),
+            "visibility": fields.String(
+                required=False,
+                default="hidden",
+                description=(
+                    "'listed' shows the conversation in the owner's sidebar; "
+                    "any other value (or omitting it) persists it hidden."
+                ),
+            ),
+            "model_id": fields.String(
+                required=False,
+                description="Model ID to use for this request",
+            ),
+            "attachments": fields.List(
+                fields.String, required=False, description="List of attachment IDs"
+            ),
+            "passthrough": fields.Raw(
+                required=False,
+                description="Dynamic parameters to inject into prompt template",
+            ),
+        },
+    )
+
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        if error := self.validate_request(data, "index" in data):
+            return error
+        decoded_token = getattr(request, "decoded_token", None)
+        processor = StreamProcessor(data, decoded_token)
+
+        try:
+            # ---- Continuation mode ----
+            if data.get("tool_actions"):
+                (
+                    agent,
+                    messages,
+                    tools_dict,
+                    pending_tool_calls,
+                    tool_actions,
+                    reasoning_content,
+                ) = processor.resume_from_tool_actions(
+                    data["tool_actions"], data["conversation_id"]
+                )
+                if not processor.decoded_token:
+                    return Response(
+                        self.error_stream_generate("Unauthorized"),
+                        status=401,
+                        mimetype="text/event-stream",
+                    )
+                if error := self.check_usage(processor.agent_config):
+                    return error
+                return Response(
+                    self.complete_stream(
+                        question="",
+                        agent=agent,
+                        conversation_id=processor.conversation_id,
+                        user_api_key=processor.agent_config.get("user_api_key"),
+                        decoded_token=processor.decoded_token,
+                        agent_id=processor.agent_id,
+                        model_id=processor.model_id,
+                        model_user_id=processor.model_user_id,
+                        _continuation={
+                            "messages": messages,
+                            "tools_dict": tools_dict,
+                            "pending_tool_calls": pending_tool_calls,
+                            "tool_actions": tool_actions,
+                            "reserved_message_id": processor.reserved_message_id,
+                            "request_id": processor.request_id,
+                            "reasoning_content": reasoning_content,
+                        },
+                    ),
+                    mimetype="text/event-stream",
+                )
+
+            # ---- Normal mode ----
+            agent = processor.build_agent(data["question"])
+            if not processor.decoded_token:
+                return Response(
+                    self.error_stream_generate("Unauthorized"),
+                    status=401,
+                    mimetype="text/event-stream",
+                )
+
+            if error := self.check_usage(processor.agent_config):
+                return error
+            should_persist, visibility = resolve_persistence(
+                visibility_flag=data.get("visibility"),
+                persist_flag=data.get("persist"),
+            )
+            return Response(
+                self.complete_stream(
+                    question=data["question"],
+                    agent=agent,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    isNoneDoc=data.get("isNoneDoc"),
+                    index=data.get("index"),
+                    should_persist=should_persist,
+                    visibility=visibility,
+                    attachment_ids=data.get("attachments", []),
+                    agent_id=processor.agent_id,
+                    is_shared_usage=processor.is_shared_usage,
+                    shared_token=processor.shared_token,
+                    model_id=processor.model_id,
+                    model_user_id=processor.model_user_id,
+                ),
+                mimetype="text/event-stream",
+            )
+        except ValueError as e:
+            message = "Malformed request body"
+            logger.error(
+                f"/stream - error: {message} - specific error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return Response(
+                self.error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return Response(
+                self.error_stream_generate("Unknown error occurred"),
+                status=400,
+                mimetype="text/event-stream",
+            )

application/api/answer/services/compression/__init__.py

@@ -0,0 +1,20 @@
+"""
+Compression module for managing conversation context compression.
+
+"""
+
+from application.api.answer.services.compression.orchestrator import (
+    CompressionOrchestrator,
+)
+from application.api.answer.services.compression.service import CompressionService
+from application.api.answer.services.compression.types import (
+    CompressionResult,
+    CompressionMetadata,
+)
+
+__all__ = [
+    "CompressionOrchestrator",
+    "CompressionService",
+    "CompressionResult",
+    "CompressionMetadata",
+]

application/api/answer/services/compression/message_builder.py

@@ -0,0 +1,249 @@
+"""Message reconstruction utilities for compression."""
+
+import json
+import logging
+import uuid
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class MessageBuilder:
+    """Builds message arrays from compressed context."""
+
+    @staticmethod
+    def build_from_compressed_context(
+        system_prompt: str,
+        compressed_summary: Optional[str],
+        recent_queries: List[Dict],
+        include_tool_calls: bool = False,
+        context_type: str = "pre_request",
+    ) -> List[Dict]:
+        """
+        Build messages from compressed context.
+
+        Args:
+            system_prompt: Original system prompt
+            compressed_summary: Compressed summary (if any)
+            recent_queries: Recent uncompressed queries
+            include_tool_calls: Whether to include tool calls from history
+            context_type: Type of context ('pre_request' or 'mid_execution')
+
+        Returns:
+            List of message dicts ready for LLM
+        """
+        # Append compression summary to system prompt if present
+        if compressed_summary:
+            system_prompt = MessageBuilder._append_compression_context(
+                system_prompt, compressed_summary, context_type
+            )
+
+        messages = [{"role": "system", "content": system_prompt}]
+
+        # Add recent history
+        for query in recent_queries:
+            if "prompt" in query and "response" in query:
+                messages.append({"role": "user", "content": query["prompt"]})
+                messages.append({"role": "assistant", "content": query["response"]})
+
+            # Add tool calls from history if present
+            if include_tool_calls and "tool_calls" in query:
+                for tool_call in query["tool_calls"]:
+                    call_id = tool_call.get("call_id") or str(uuid.uuid4())
+                    args = tool_call.get("arguments")
+                    args_str = (
+                        json.dumps(args)
+                        if isinstance(args, dict)
+                        else (args or "{}")
+                    )
+                    messages.append({
+                        "role": "assistant",
+                        "content": None,
+                        "tool_calls": [{
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": tool_call.get("action_name", ""),
+                                "arguments": args_str,
+                            },
+                        }],
+                    })
+                    result = tool_call.get("result")
+                    result_str = (
+                        json.dumps(result)
+                        if not isinstance(result, str)
+                        else (result or "")
+                    )
+                    messages.append({
+                        "role": "tool",
+                        "tool_call_id": call_id,
+                        "content": result_str,
+                    })
+
+        # If no recent queries (everything was compressed), add a continuation user message
+        if len(recent_queries) == 0 and compressed_summary:
+            messages.append({
+                "role": "user",
+                "content": "Please continue with the remaining tasks based on the context above."
+            })
+            logger.info("Added continuation user message to maintain proper turn-taking after full compression")
+
+        return messages
+
+    @staticmethod
+    def _append_compression_context(
+        system_prompt: str, compressed_summary: str, context_type: str = "pre_request"
+    ) -> str:
+        """
+        Append compression context to system prompt.
+
+        Args:
+            system_prompt: Original system prompt
+            compressed_summary: Summary to append
+            context_type: Type of compression context
+
+        Returns:
+            Updated system prompt
+        """
+        # Remove existing compression context if present
+        if "This session is being continued" in system_prompt or "Context window limit reached" in system_prompt:
+            parts = system_prompt.split("\n\n---\n\n")
+            system_prompt = parts[0]
+
+        # Build appropriate context message based on type
+        if context_type == "mid_execution":
+            context_message = (
+                "\n\n---\n\n"
+                "Context window limit reached during execution. "
+                "Previous conversation has been compressed to fit within limits. "
+                "The conversation is summarized below:\n\n"
+                f"{compressed_summary}"
+            )
+        else:  # pre_request
+            context_message = (
+                "\n\n---\n\n"
+                "This session is being continued from a previous conversation that "
+                "has been compressed to fit within context limits. "
+                "The conversation is summarized below:\n\n"
+                f"{compressed_summary}"
+            )
+
+        return system_prompt + context_message
+
+    @staticmethod
+    def rebuild_messages_after_compression(
+        messages: List[Dict],
+        compressed_summary: Optional[str],
+        recent_queries: List[Dict],
+        include_current_execution: bool = False,
+        include_tool_calls: bool = False,
+    ) -> Optional[List[Dict]]:
+        """
+        Rebuild the message list after compression so tool execution can continue.
+
+        Args:
+            messages: Original message list
+            compressed_summary: Compressed summary
+            recent_queries: Recent uncompressed queries
+            include_current_execution: Whether to preserve current execution messages
+            include_tool_calls: Whether to include tool calls from history
+
+        Returns:
+            Rebuilt message list or None if failed
+        """
+        # Find the system message
+        system_message = next(
+            (msg for msg in messages if msg.get("role") == "system"), None
+        )
+        if not system_message:
+            logger.warning("No system message found in messages list")
+            return None
+
+        # Update system message with compressed summary
+        if compressed_summary:
+            content = system_message.get("content", "")
+            system_message["content"] = MessageBuilder._append_compression_context(
+                content, compressed_summary, "mid_execution"
+            )
+            logger.info(
+                "Appended compression summary to system prompt (truncated): %s",
+                (
+                    compressed_summary[:500] + "..."
+                    if len(compressed_summary) > 500
+                    else compressed_summary
+                ),
+            )
+
+        rebuilt_messages = [system_message]
+
+        # Add recent history from compressed context
+        for query in recent_queries:
+            if "prompt" in query and "response" in query:
+                rebuilt_messages.append({"role": "user", "content": query["prompt"]})
+                rebuilt_messages.append(
+                    {"role": "assistant", "content": query["response"]}
+                )
+
+            # Add tool calls from history if present
+            if include_tool_calls and "tool_calls" in query:
+                for tool_call in query["tool_calls"]:
+                    call_id = tool_call.get("call_id") or str(uuid.uuid4())
+                    args = tool_call.get("arguments")
+                    args_str = (
+                        json.dumps(args)
+                        if isinstance(args, dict)
+                        else (args or "{}")
+                    )
+                    rebuilt_messages.append({
+                        "role": "assistant",
+                        "content": None,
+                        "tool_calls": [{
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": tool_call.get("action_name", ""),
+                                "arguments": args_str,
+                            },
+                        }],
+                    })
+                    result = tool_call.get("result")
+                    result_str = (
+                        json.dumps(result)
+                        if not isinstance(result, str)
+                        else (result or "")
+                    )
+                    rebuilt_messages.append({
+                        "role": "tool",
+                        "tool_call_id": call_id,
+                        "content": result_str,
+                    })
+
+        # If no recent queries (everything was compressed), add a continuation user message
+        if len(recent_queries) == 0 and compressed_summary:
+            rebuilt_messages.append({
+                "role": "user",
+                "content": "Please continue with the remaining tasks based on the context above."
+            })
+            logger.info("Added continuation user message to maintain proper turn-taking after full compression")
+
+        if include_current_execution:
+            # Preserve any messages that were added during the current execution cycle
+            recent_msg_count = 1  # system message
+            for query in recent_queries:
+                if "prompt" in query and "response" in query:
+                    recent_msg_count += 2
+                if "tool_calls" in query:
+                    recent_msg_count += len(query["tool_calls"]) * 2
+
+            if len(messages) > recent_msg_count:
+                current_execution_messages = messages[recent_msg_count:]
+                rebuilt_messages.extend(current_execution_messages)
+                logger.info(
+                    f"Preserved {len(current_execution_messages)} messages from current execution cycle"
+                )
+
+        logger.info(
+            f"Messages rebuilt: {len(messages)} → {len(rebuilt_messages)} messages. "
+            f"Ready to continue tool execution."
+        )
+        return rebuilt_messages

application/api/answer/services/compression/orchestrator.py

@@ -0,0 +1,273 @@
+"""High-level compression orchestration."""
+
+import logging
+from typing import Any, Dict, Optional
+
+from application.api.answer.services.compression.service import CompressionService
+from application.api.answer.services.compression.threshold_checker import (
+    CompressionThresholdChecker,
+)
+from application.api.answer.services.compression.types import CompressionResult
+from application.api.answer.services.conversation_service import ConversationService
+from application.core.model_utils import (
+    get_api_key_for_provider,
+    get_provider_from_model_id,
+)
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionOrchestrator:
+    """
+    Facade for compression operations.
+
+    Coordinates between all compression components and provides
+    a simple interface for callers.
+    """
+
+    def __init__(
+        self,
+        conversation_service: ConversationService,
+        threshold_checker: Optional[CompressionThresholdChecker] = None,
+    ):
+        """
+        Initialize orchestrator.
+
+        Args:
+            conversation_service: Service for DB operations
+            threshold_checker: Custom threshold checker (optional)
+        """
+        self.conversation_service = conversation_service
+        self.threshold_checker = threshold_checker or CompressionThresholdChecker()
+
+    def compress_if_needed(
+        self,
+        conversation_id: str,
+        user_id: str,
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        current_query_tokens: int = 500,
+        model_user_id: Optional[str] = None,
+    ) -> CompressionResult:
+        """
+        Check if compression is needed and perform it if so.
+
+        This is the main entry point for compression operations.
+
+        Args:
+            conversation_id: Conversation ID
+            user_id: Caller's user id — used for conversation access checks
+            model_id: Model being used for conversation
+            decoded_token: User's decoded JWT token
+            current_query_tokens: Estimated tokens for current query
+            model_user_id: BYOM-resolution scope (model owner); defaults
+                to ``user_id`` for built-in / caller-owned models.
+
+        Returns:
+            CompressionResult with summary and recent queries
+        """
+        try:
+            # Conversation row is owned by the caller, not the model owner.
+            conversation = self.conversation_service.get_conversation(
+                conversation_id, user_id
+            )
+
+            if not conversation:
+                logger.warning(
+                    f"Conversation {conversation_id} not found for user {user_id}"
+                )
+                return CompressionResult.failure("Conversation not found")
+
+            # Use model-owner scope so per-user BYOM context windows
+            # (e.g. 8k) compute the threshold against the right limit.
+            registry_user_id = model_user_id or user_id
+            if not self.threshold_checker.should_compress(
+                conversation,
+                model_id,
+                current_query_tokens,
+                user_id=registry_user_id,
+            ):
+                # No compression needed, return full history
+                queries = conversation.get("queries", [])
+                return CompressionResult.success_no_compression(queries)
+
+            # Perform compression
+            return self._perform_compression(
+                conversation_id,
+                conversation,
+                model_id,
+                decoded_token,
+                user_id=user_id,
+                model_user_id=model_user_id,
+            )
+
+        except Exception as e:
+            logger.error(
+                f"Error in compress_if_needed: {str(e)}", exc_info=True
+            )
+            return CompressionResult.failure(str(e))
+
+    def _perform_compression(
+        self,
+        conversation_id: str,
+        conversation: Dict[str, Any],
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        user_id: Optional[str] = None,
+        model_user_id: Optional[str] = None,
+    ) -> CompressionResult:
+        """
+        Perform the actual compression operation.
+
+        Args:
+            conversation_id: Conversation ID
+            conversation: Conversation document
+            model_id: Model ID for conversation
+            decoded_token: User token
+            user_id: Caller's id (for conversation reload after compression)
+            model_user_id: BYOM-resolution scope (model owner)
+
+        Returns:
+            CompressionResult
+        """
+        try:
+            # Determine which model to use for compression
+            compression_model = (
+                settings.COMPRESSION_MODEL_OVERRIDE
+                if settings.COMPRESSION_MODEL_OVERRIDE
+                else model_id
+            )
+
+            # Use model-owner scope so provider/api_key resolves to the
+            # owner's BYOM record (shared-agent dispatch).
+            caller_user_id = user_id
+            if caller_user_id is None and isinstance(decoded_token, dict):
+                caller_user_id = decoded_token.get("sub")
+            registry_user_id = model_user_id or caller_user_id
+            provider = get_provider_from_model_id(
+                compression_model, user_id=registry_user_id
+            )
+            api_key = get_api_key_for_provider(provider)
+
+            compression_llm = LLMCreator.create_llm(
+                provider,
+                api_key=api_key,
+                user_api_key=None,
+                decoded_token=decoded_token,
+                model_id=compression_model,
+                agent_id=conversation.get("agent_id"),
+                model_user_id=registry_user_id,
+            )
+            # Side-channel LLM tag — distinguishes compression rows
+            # from primary stream rows for cost-attribution dashboards.
+            compression_llm._token_usage_source = "compression"
+
+            # Create compression service with DB update capability
+            compression_service = CompressionService(
+                llm=compression_llm,
+                model_id=compression_model,
+                conversation_service=self.conversation_service,
+            )
+
+            # Compress all queries up to the latest
+            queries_count = len(conversation.get("queries", []))
+            compress_up_to = queries_count - 1
+
+            if compress_up_to < 0:
+                logger.warning("No queries to compress")
+                return CompressionResult.success_no_compression([])
+
+            logger.info(
+                f"Initiating compression for conversation {conversation_id}: "
+                f"compressing all {queries_count} queries (0-{compress_up_to})"
+            )
+
+            # Perform compression and save to DB
+            metadata = compression_service.compress_and_save(
+                conversation_id, conversation, compress_up_to
+            )
+
+            logger.info(
+                f"Compression successful - ratio: {metadata.compression_ratio:.1f}x, "
+                f"saved {metadata.original_token_count - metadata.compressed_token_count} tokens"
+            )
+
+            # Reload under caller (conversation is owned by caller).
+            reload_user_id = caller_user_id
+            if reload_user_id is None and isinstance(decoded_token, dict):
+                reload_user_id = decoded_token.get("sub")
+            conversation = self.conversation_service.get_conversation(
+                conversation_id, user_id=reload_user_id
+            )
+
+            # Get compressed context
+            compressed_summary, recent_queries = (
+                compression_service.get_compressed_context(conversation)
+            )
+
+            return CompressionResult.success_with_compression(
+                compressed_summary, recent_queries, metadata
+            )
+
+        except Exception as e:
+            logger.error(f"Error performing compression: {str(e)}", exc_info=True)
+            return CompressionResult.failure(str(e))
+
+    def compress_mid_execution(
+        self,
+        conversation_id: str,
+        user_id: str,
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        current_conversation: Optional[Dict[str, Any]] = None,
+        model_user_id: Optional[str] = None,
+    ) -> CompressionResult:
+        """
+        Perform compression during tool execution.
+
+        Args:
+            conversation_id: Conversation ID
+            user_id: Caller's user id — used for conversation access checks
+            model_id: Model ID
+            decoded_token: User token
+            current_conversation: Pre-loaded conversation (optional)
+            model_user_id: BYOM-resolution scope (model owner). For
+                shared-agent dispatch this is the agent owner; defaults
+                to ``user_id`` so built-in / caller-owned models are
+                unaffected.
+
+        Returns:
+            CompressionResult
+        """
+        try:
+            # Load conversation if not provided
+            if current_conversation:
+                conversation = current_conversation
+            else:
+                conversation = self.conversation_service.get_conversation(
+                    conversation_id, user_id
+                )
+
+            if not conversation:
+                logger.warning(
+                    f"Could not load conversation {conversation_id} for mid-execution compression"
+                )
+                return CompressionResult.failure("Conversation not found")
+
+            # Perform compression
+            return self._perform_compression(
+                conversation_id,
+                conversation,
+                model_id,
+                decoded_token,
+                user_id=user_id,
+                model_user_id=model_user_id,
+            )
+
+        except Exception as e:
+            logger.error(
+                f"Error in mid-execution compression: {str(e)}", exc_info=True
+            )
+            return CompressionResult.failure(str(e))

application/api/answer/services/compression/prompt_builder.py

@@ -0,0 +1,149 @@
+"""Compression prompt building logic."""
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionPromptBuilder:
+    """Builds prompts for LLM compression calls."""
+
+    def __init__(self, version: str = "v1.0"):
+        """
+        Initialize prompt builder.
+
+        Args:
+            version: Prompt template version to use
+        """
+        self.version = version
+        self.system_prompt = self._load_prompt(version)
+
+    def _load_prompt(self, version: str) -> str:
+        """
+        Load prompt template from file.
+
+        Args:
+            version: Version string (e.g., 'v1.0')
+
+        Returns:
+            Prompt template content
+
+        Raises:
+            FileNotFoundError: If prompt template file doesn't exist
+        """
+        current_dir = Path(__file__).resolve().parents[4]
+        prompt_path = current_dir / "prompts" / "compression" / f"{version}.txt"
+
+        try:
+            with open(prompt_path, "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            logger.error(f"Compression prompt template not found: {prompt_path}")
+            raise FileNotFoundError(
+                f"Compression prompt template '{version}' not found at {prompt_path}. "
+                f"Please ensure the template file exists."
+            )
+
+    def build_prompt(
+        self,
+        queries: List[Dict[str, Any]],
+        existing_compressions: Optional[List[Dict[str, Any]]] = None,
+    ) -> List[Dict[str, str]]:
+        """
+        Build messages for compression LLM call.
+
+        Args:
+            queries: List of query objects to compress
+            existing_compressions: List of previous compression points
+
+        Returns:
+            List of message dicts for LLM
+        """
+        # Build conversation text
+        conversation_text = self._format_conversation(queries)
+
+        # Add existing compression context if present
+        existing_compression_context = ""
+        if existing_compressions and len(existing_compressions) > 0:
+            existing_compression_context = (
+                "\n\nIMPORTANT: This conversation has been compressed before. "
+                "Previous compression summaries:\n\n"
+            )
+            for i, comp in enumerate(existing_compressions):
+                existing_compression_context += (
+                    f"--- Compression {i + 1} (up to message {comp.get('query_index', 'unknown')}) ---\n"
+                    f"{comp.get('compressed_summary', '')}\n\n"
+                )
+            existing_compression_context += (
+                "Your task is to create a NEW summary that incorporates the context from "
+                "previous compressions AND the new messages below. The final summary should "
+                "be comprehensive and include all important information from both previous "
+                "compressions and new messages.\n\n"
+            )
+
+        user_prompt = (
+            f"{existing_compression_context}"
+            f"Here is the conversation to summarize:\n\n"
+            f"{conversation_text}"
+        )
+
+        messages = [
+            {"role": "system", "content": self.system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+
+        return messages
+
+    def _format_conversation(self, queries: List[Dict[str, Any]]) -> str:
+        """
+        Format conversation queries into readable text for compression.
+
+        Args:
+            queries: List of query objects
+
+        Returns:
+            Formatted conversation text
+        """
+        conversation_lines = []
+
+        for i, query in enumerate(queries):
+            conversation_lines.append(f"--- Message {i + 1} ---")
+            conversation_lines.append(f"User: {query.get('prompt', '')}")
+
+            # Add tool calls if present
+            tool_calls = query.get("tool_calls", [])
+            if tool_calls:
+                conversation_lines.append("\nTool Calls:")
+                for tc in tool_calls:
+                    tool_name = tc.get("tool_name", "unknown")
+                    action_name = tc.get("action_name", "unknown")
+                    arguments = tc.get("arguments", {})
+                    result = tc.get("result", "")
+                    if result is None:
+                        result = ""
+                    status = tc.get("status", "unknown")
+
+                    # Include full tool result for complete compression context
+                    conversation_lines.append(
+                        f"  - {tool_name}.{action_name}({arguments}) "
+                        f"[{status}] → {result}"
+                    )
+
+            # Add agent thought if present
+            thought = query.get("thought", "")
+            if thought:
+                conversation_lines.append(f"\nAgent Thought: {thought}")
+
+            # Add assistant response
+            conversation_lines.append(f"\nAssistant: {query.get('response', '')}")
+
+            # Add sources if present
+            sources = query.get("sources", [])
+            if sources:
+                conversation_lines.append(f"\nSources Used: {len(sources)} documents")
+
+            conversation_lines.append("")  # Empty line between messages
+
+        return "\n".join(conversation_lines)

application/api/answer/services/compression/service.py

@@ -0,0 +1,316 @@
+"""Core compression service with simplified responsibilities."""
+
+import logging
+import re
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from application.api.answer.services.compression.prompt_builder import (
+    CompressionPromptBuilder,
+)
+from application.api.answer.services.compression.token_counter import TokenCounter
+from application.api.answer.services.compression.types import (
+    CompressionMetadata,
+)
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionService:
+    """
+    Service for compressing conversation history.
+
+    Handles DB updates.
+    """
+
+    def __init__(
+        self,
+        llm,
+        model_id: str,
+        conversation_service=None,
+        prompt_builder: Optional[CompressionPromptBuilder] = None,
+    ):
+        """
+        Initialize compression service.
+
+        Args:
+            llm: LLM instance to use for compression
+            model_id: Model ID for compression
+            conversation_service: Service for DB operations (optional, for DB updates)
+            prompt_builder: Custom prompt builder (optional)
+        """
+        self.llm = llm
+        self.model_id = model_id
+        self.conversation_service = conversation_service
+        self.prompt_builder = prompt_builder or CompressionPromptBuilder(
+            version=settings.COMPRESSION_PROMPT_VERSION
+        )
+
+    def compress_conversation(
+        self,
+        conversation: Dict[str, Any],
+        compress_up_to_index: int,
+    ) -> CompressionMetadata:
+        """
+        Compress conversation history up to specified index.
+
+        Args:
+            conversation: Full conversation document
+            compress_up_to_index: Last query index to include in compression
+
+        Returns:
+            CompressionMetadata with compression details
+
+        Raises:
+            ValueError: If compress_up_to_index is invalid
+        """
+        try:
+            queries = conversation.get("queries", [])
+
+            if compress_up_to_index < 0 or compress_up_to_index >= len(queries):
+                raise ValueError(
+                    f"Invalid compress_up_to_index: {compress_up_to_index} "
+                    f"(conversation has {len(queries)} queries)"
+                )
+
+            # Get queries to compress
+            queries_to_compress = queries[: compress_up_to_index + 1]
+
+            # Check if there are existing compressions. ``compression_metadata``
+            # is a nullable JSONB column, so a never-compressed conversation
+            # reads back as None; ``get(key, {})`` would return that None (the
+            # default only applies to absent keys), so coalesce with ``or {}``.
+            existing_compressions = (conversation.get("compression_metadata") or {}).get(
+                "compression_points", []
+            )
+
+            if existing_compressions:
+                logger.info(
+                    f"Found {len(existing_compressions)} previous compression(s) - "
+                    f"will incorporate into new summary"
+                )
+
+            # Calculate original token count
+            original_tokens = TokenCounter.count_query_tokens(queries_to_compress)
+
+            # Log tool call stats
+            self._log_tool_call_stats(queries_to_compress)
+
+            # Build compression prompt
+            messages = self.prompt_builder.build_prompt(
+                queries_to_compress, existing_compressions
+            )
+
+            # Call LLM to generate compression
+            logger.info(
+                f"Starting compression: {len(queries_to_compress)} queries "
+                f"(messages 0-{compress_up_to_index}, {original_tokens} tokens) "
+                f"using model {self.model_id}"
+            )
+
+            # See note in conversation_service.py: ``self.model_id`` is
+            # the registry id (UUID for BYOM); the LLM's own model_id is
+            # what the provider's API actually expects.
+            response = self.llm.gen(
+                model=getattr(self.llm, "model_id", None) or self.model_id,
+                messages=messages,
+                max_tokens=4000,
+            )
+
+            # Extract summary from response
+            compressed_summary = self._extract_summary(response)
+
+            # Calculate compressed token count
+            compressed_tokens = TokenCounter.count_message_tokens(
+                [{"content": compressed_summary}]
+            )
+
+            # Calculate compression ratio
+            compression_ratio = (
+                original_tokens / compressed_tokens if compressed_tokens > 0 else 0
+            )
+
+            logger.info(
+                f"Compression complete: {original_tokens} → {compressed_tokens} tokens "
+                f"({compression_ratio:.1f}x compression)"
+            )
+
+            # Build compression metadata
+            compression_metadata = CompressionMetadata(
+                timestamp=datetime.now(timezone.utc),
+                query_index=compress_up_to_index,
+                compressed_summary=compressed_summary,
+                original_token_count=original_tokens,
+                compressed_token_count=compressed_tokens,
+                compression_ratio=compression_ratio,
+                model_used=self.model_id,
+                compression_prompt_version=self.prompt_builder.version,
+            )
+
+            return compression_metadata
+
+        except Exception as e:
+            logger.error(f"Error compressing conversation: {str(e)}", exc_info=True)
+            raise
+
+    def compress_and_save(
+        self,
+        conversation_id: str,
+        conversation: Dict[str, Any],
+        compress_up_to_index: int,
+    ) -> CompressionMetadata:
+        """
+        Compress conversation and save to database.
+
+        Args:
+            conversation_id: Conversation ID
+            conversation: Full conversation document
+            compress_up_to_index: Last query index to include
+
+        Returns:
+            CompressionMetadata
+
+        Raises:
+            ValueError: If conversation_service not provided or invalid index
+        """
+        if not self.conversation_service:
+            raise ValueError(
+                "conversation_service required for compress_and_save operation"
+            )
+
+        # Perform compression
+        metadata = self.compress_conversation(conversation, compress_up_to_index)
+
+        # Save to database
+        self.conversation_service.update_compression_metadata(
+            conversation_id, metadata.to_dict()
+        )
+
+        logger.info(f"Compression metadata saved to database for {conversation_id}")
+
+        return metadata
+
+    def get_compressed_context(
+        self, conversation: Dict[str, Any]
+    ) -> tuple[Optional[str], List[Dict[str, Any]]]:
+        """
+        Get compressed summary + recent uncompressed messages.
+
+        Args:
+            conversation: Full conversation document
+
+        Returns:
+            (compressed_summary, recent_messages)
+        """
+        try:
+            # ``or {}`` guards against a NULL ``compression_metadata`` column
+            # (reads back as None), which would crash the ``.get`` calls below.
+            compression_metadata = conversation.get("compression_metadata") or {}
+
+            if not compression_metadata.get("is_compressed"):
+                logger.debug("No compression metadata found - using full history")
+                queries = conversation.get("queries", [])
+                if queries is None:
+                    logger.error("Conversation queries is None - returning empty list")
+                    return None, []
+                return None, queries
+
+            compression_points = compression_metadata.get("compression_points", [])
+
+            if not compression_points:
+                logger.debug("No compression points found - using full history")
+                queries = conversation.get("queries", [])
+                if queries is None:
+                    logger.error("Conversation queries is None - returning empty list")
+                    return None, []
+                return None, queries
+
+            # Get the most recent compression point
+            latest_compression = compression_points[-1]
+            compressed_summary = latest_compression.get("compressed_summary")
+            last_compressed_index = latest_compression.get("query_index")
+            compressed_tokens = latest_compression.get("compressed_token_count", 0)
+            original_tokens = latest_compression.get("original_token_count", 0)
+
+            # Get only messages after compression point
+            queries = conversation.get("queries", [])
+            total_queries = len(queries)
+            recent_queries = queries[last_compressed_index + 1 :]
+
+            logger.info(
+                f"Using compressed context: summary ({compressed_tokens} tokens, "
+                f"compressed from {original_tokens}) + {len(recent_queries)} recent messages "
+                f"(messages {last_compressed_index + 1}-{total_queries - 1})"
+            )
+
+            return compressed_summary, recent_queries
+
+        except Exception as e:
+            logger.error(
+                f"Error getting compressed context: {str(e)}", exc_info=True
+            )
+            queries = conversation.get("queries", [])
+            if queries is None:
+                return None, []
+            return None, queries
+
+    def _extract_summary(self, llm_response: str) -> str:
+        """
+        Extract clean summary from LLM response.
+
+        Args:
+            llm_response: Raw LLM response
+
+        Returns:
+            Cleaned summary text
+        """
+        try:
+            # Try to extract content within <summary> tags
+            summary_match = re.search(
+                r"<summary>(.*?)</summary>", llm_response, re.DOTALL
+            )
+
+            if summary_match:
+                summary = summary_match.group(1).strip()
+            else:
+                # If no summary tags, remove analysis tags and use the rest
+                summary = re.sub(
+                    r"<analysis>.*?</analysis>", "", llm_response, flags=re.DOTALL
+                ).strip()
+
+            return summary
+
+        except Exception as e:
+            logger.warning(f"Error extracting summary: {str(e)}, using full response")
+            return llm_response
+
+    def _log_tool_call_stats(self, queries: List[Dict[str, Any]]) -> None:
+        """Log statistics about tool calls in queries."""
+        total_tool_calls = 0
+        total_tool_result_chars = 0
+        tool_call_breakdown = {}
+
+        for q in queries:
+            for tc in q.get("tool_calls", []):
+                total_tool_calls += 1
+                tool_name = tc.get("tool_name", "unknown")
+                action_name = tc.get("action_name", "unknown")
+                key = f"{tool_name}.{action_name}"
+                tool_call_breakdown[key] = tool_call_breakdown.get(key, 0) + 1
+
+                # Track total tool result size
+                result = tc.get("result", "")
+                if result:
+                    total_tool_result_chars += len(str(result))
+
+        if total_tool_calls > 0:
+            tool_breakdown_str = ", ".join(
+                f"{tool}({count})"
+                for tool, count in sorted(tool_call_breakdown.items())
+            )
+            tool_result_kb = total_tool_result_chars / 1024
+            logger.info(
+                f"Tool call breakdown: {tool_breakdown_str} "
+                f"(total result size: {tool_result_kb:.1f} KB, {total_tool_result_chars:,} chars)"
+            )

application/api/answer/services/compression/threshold_checker.py

@@ -0,0 +1,110 @@
+"""Compression threshold checking logic."""
+
+import logging
+from typing import Any, Dict
+
+from application.core.model_utils import get_token_limit
+from application.core.settings import settings
+from application.api.answer.services.compression.token_counter import TokenCounter
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionThresholdChecker:
+    """Determines if compression is needed based on token thresholds."""
+
+    def __init__(self, threshold_percentage: float = None):
+        """
+        Initialize threshold checker.
+
+        Args:
+            threshold_percentage: Percentage of context to use as threshold
+                                 (defaults to settings.COMPRESSION_THRESHOLD_PERCENTAGE)
+        """
+        self.threshold_percentage = (
+            threshold_percentage or settings.COMPRESSION_THRESHOLD_PERCENTAGE
+        )
+
+    def should_compress(
+        self,
+        conversation: Dict[str, Any],
+        model_id: str,
+        current_query_tokens: int = 500,
+        user_id: str | None = None,
+    ) -> bool:
+        """
+        Determine if compression is needed.
+
+        Args:
+            conversation: Full conversation document
+            model_id: Target model for this request
+            current_query_tokens: Estimated tokens for current query
+            user_id: Owner — needed so per-user BYOM custom-model UUIDs
+                resolve when looking up the context window.
+
+        Returns:
+            True if tokens >= threshold% of context window
+        """
+        try:
+            # Calculate total tokens in conversation
+            total_tokens = TokenCounter.count_conversation_tokens(conversation)
+            total_tokens += current_query_tokens
+
+            # Get context window limit for model
+            context_limit = get_token_limit(model_id, user_id=user_id)
+
+            # Calculate threshold
+            threshold = int(context_limit * self.threshold_percentage)
+
+            compression_needed = total_tokens >= threshold
+            percentage_used = (total_tokens / context_limit) * 100
+
+            if compression_needed:
+                logger.warning(
+                    f"COMPRESSION TRIGGERED: {total_tokens} tokens / {context_limit} limit "
+                    f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%)"
+                )
+            else:
+                logger.info(
+                    f"Compression check: {total_tokens}/{context_limit} tokens "
+                    f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%) - No compression needed"
+                )
+
+            return compression_needed
+
+        except Exception as e:
+            logger.error(f"Error checking compression need: {str(e)}", exc_info=True)
+            return False
+
+    def check_message_tokens(
+        self, messages: list, model_id: str, user_id: str | None = None
+    ) -> bool:
+        """
+        Check if message list exceeds threshold.
+
+        Args:
+            messages: List of message dicts
+            model_id: Target model
+            user_id: Owner — needed so per-user BYOM custom-model UUIDs
+                resolve when looking up the context window.
+
+        Returns:
+            True if at or above threshold
+        """
+        try:
+            current_tokens = TokenCounter.count_message_tokens(messages)
+            context_limit = get_token_limit(model_id, user_id=user_id)
+            threshold = int(context_limit * self.threshold_percentage)
+
+            if current_tokens >= threshold:
+                logger.warning(
+                    f"Message context limit approaching: {current_tokens}/{context_limit} tokens "
+                    f"({(current_tokens/context_limit)*100:.1f}%)"
+                )
+                return True
+
+            return False
+
+        except Exception as e:
+            logger.error(f"Error checking message tokens: {str(e)}", exc_info=True)
+            return False

application/api/answer/services/compression/token_counter.py

@@ -0,0 +1,133 @@
+"""Token counting utilities for compression."""
+
+import logging
+from typing import Any, Dict, List
+
+from application.utils import num_tokens_from_string
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class TokenCounter:
+    """Centralized token counting for conversations and messages."""
+
+    # Per-image token estimate. Provider tokenizers vary widely
+    # (Gemini ~258, GPT-4o 85-1500, Claude ~1500) and the actual cost
+    # depends on resolution/detail we can't see here. Errs slightly high
+    # so the threshold check stays conservative.
+    _IMAGE_PART_TOKEN_ESTIMATE = 1500
+
+    @staticmethod
+    def count_message_tokens(messages: List[Dict]) -> int:
+        """
+        Calculate total tokens in a list of messages.
+
+        Args:
+            messages: List of message dicts with 'content' field
+
+        Returns:
+            Total token count
+        """
+        total_tokens = 0
+        for message in messages:
+            content = message.get("content", "")
+            if isinstance(content, str):
+                total_tokens += num_tokens_from_string(content)
+            elif isinstance(content, list):
+                # Handle structured content (tool calls, image parts, etc.)
+                for item in content:
+                    if isinstance(item, dict):
+                        total_tokens += TokenCounter._count_content_part(item)
+        return total_tokens
+
+    @staticmethod
+    def _count_content_part(item: Dict) -> int:
+        # Image/file attachments are billed by the provider per image,
+        # not proportional to the inline bytes/base64 string.
+        # ``str(item)`` on a 1MB image inflates the count by ~10000x,
+        # which trips spurious compression and overflows downstream
+        # input limits.
+        item_type = item.get("type")
+
+        if "files" in item:
+            files = item.get("files")
+            count = len(files) if isinstance(files, list) and files else 1
+            return TokenCounter._IMAGE_PART_TOKEN_ESTIMATE * count
+
+        if "image_url" in item or item_type in {
+            "image",
+            "image_url",
+            "input_image",
+            "file",
+        }:
+            return TokenCounter._IMAGE_PART_TOKEN_ESTIMATE
+
+        return num_tokens_from_string(str(item))
+
+    @staticmethod
+    def count_query_tokens(
+        queries: List[Dict[str, Any]], include_tool_calls: bool = True
+    ) -> int:
+        """
+        Count tokens across multiple query objects.
+
+        Args:
+            queries: List of query objects from conversation
+            include_tool_calls: Whether to count tool call tokens
+
+        Returns:
+            Total token count
+        """
+        total_tokens = 0
+
+        for query in queries:
+            # Count prompt and response tokens
+            if "prompt" in query:
+                total_tokens += num_tokens_from_string(query["prompt"])
+            if "response" in query:
+                total_tokens += num_tokens_from_string(query["response"])
+            if "thought" in query:
+                total_tokens += num_tokens_from_string(query.get("thought", ""))
+
+            # Count tool call tokens
+            if include_tool_calls and "tool_calls" in query:
+                for tool_call in query["tool_calls"]:
+                    tool_call_string = (
+                        f"Tool: {tool_call.get('tool_name')} | "
+                        f"Action: {tool_call.get('action_name')} | "
+                        f"Args: {tool_call.get('arguments')} | "
+                        f"Response: {tool_call.get('result')}"
+                    )
+                    total_tokens += num_tokens_from_string(tool_call_string)
+
+        return total_tokens
+
+    @staticmethod
+    def count_conversation_tokens(
+        conversation: Dict[str, Any], include_system_prompt: bool = False
+    ) -> int:
+        """
+        Calculate total tokens in a conversation.
+
+        Args:
+            conversation: Conversation document
+            include_system_prompt: Whether to include system prompt in count
+
+        Returns:
+            Total token count
+        """
+        try:
+            queries = conversation.get("queries", [])
+            total_tokens = TokenCounter.count_query_tokens(queries)
+
+            # Add system prompt tokens if requested
+            if include_system_prompt:
+                # Rough estimate for system prompt
+                total_tokens += settings.RESERVED_TOKENS.get("system_prompt", 500)
+
+            return total_tokens
+
+        except Exception as e:
+            logger.error(f"Error calculating conversation tokens: {str(e)}")
+            return 0

application/api/answer/services/compression/types.py

@@ -0,0 +1,91 @@
+"""Type definitions for compression module."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class CompressionMetadata:
+    """Metadata about a compression operation."""
+
+    timestamp: datetime
+    query_index: int
+    compressed_summary: str
+    original_token_count: int
+    compressed_token_count: int
+    compression_ratio: float
+    model_used: str
+    compression_prompt_version: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for DB storage."""
+        return {
+            "timestamp": self.timestamp,
+            "query_index": self.query_index,
+            "compressed_summary": self.compressed_summary,
+            "original_token_count": self.original_token_count,
+            "compressed_token_count": self.compressed_token_count,
+            "compression_ratio": self.compression_ratio,
+            "model_used": self.model_used,
+            "compression_prompt_version": self.compression_prompt_version,
+        }
+
+
+@dataclass
+class CompressionResult:
+    """Result of a compression operation."""
+
+    success: bool
+    compressed_summary: Optional[str] = None
+    recent_queries: List[Dict[str, Any]] = field(default_factory=list)
+    metadata: Optional[CompressionMetadata] = None
+    error: Optional[str] = None
+    compression_performed: bool = False
+
+    @classmethod
+    def success_with_compression(
+        cls, summary: str, queries: List[Dict], metadata: CompressionMetadata
+    ) -> "CompressionResult":
+        """Create a successful result with compression."""
+        return cls(
+            success=True,
+            compressed_summary=summary,
+            recent_queries=queries,
+            metadata=metadata,
+            compression_performed=True,
+        )
+
+    @classmethod
+    def success_no_compression(cls, queries: List[Dict]) -> "CompressionResult":
+        """Create a successful result without compression needed."""
+        return cls(
+            success=True,
+            recent_queries=queries,
+            compression_performed=False,
+        )
+
+    @classmethod
+    def failure(cls, error: str) -> "CompressionResult":
+        """Create a failure result."""
+        return cls(success=False, error=error, compression_performed=False)
+
+    def as_history(self) -> List[Dict[str, str]]:
+        """
+        Convert recent queries to history format.
+
+        Returns:
+            List of prompt/response dicts (with thought when present so
+            DeepSeek-style providers can re-attach reasoning_content on
+            replay).
+        """
+        out: List[Dict[str, str]] = []
+        for q in self.recent_queries:
+            entry: Dict[str, str] = {
+                "prompt": q["prompt"],
+                "response": q["response"],
+            }
+            if q.get("thought"):
+                entry["thought"] = q["thought"]
+            out.append(entry)
+        return out

application/api/answer/services/continuation_service.py

@@ -0,0 +1,163 @@
+"""Service for saving and restoring tool-call continuation state.
+
+When a stream pauses (tool needs approval or client-side execution),
+the full execution state is persisted to Postgres so the client can
+resume later by sending tool_actions.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.conversations import ConversationsRepository
+from application.storage.db.repositories.pending_tool_state import (
+    PendingToolStateRepository,
+)
+from application.storage.db.serialization import coerce_pg_native as _make_serializable
+from application.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+# TTL for pending states — auto-cleaned after this period
+PENDING_STATE_TTL_SECONDS = 30 * 60  # 30 minutes
+
+# Re-export so the existing tests at tests/api/answer/services/test_continuation_service_pg.py
+# can keep importing ``_make_serializable`` from here.
+__all__ = ["_make_serializable", "ContinuationService", "PENDING_STATE_TTL_SECONDS"]
+
+
+class ContinuationService:
+    """Manages pending tool-call state in Postgres."""
+
+    def __init__(self):
+        # No-op constructor retained for call-site compatibility. State
+        # lives in Postgres now; each operation opens its own short-lived
+        # session rather than holding a connection on the service.
+        pass
+
+    def save_state(
+        self,
+        conversation_id: str,
+        user: str,
+        messages: List[Dict],
+        pending_tool_calls: List[Dict],
+        tools_dict: Dict,
+        tool_schemas: List[Dict],
+        agent_config: Dict,
+        client_tools: Optional[List[Dict]] = None,
+    ) -> str:
+        """Save execution state for later continuation.
+
+        ``conversation_id`` may be a Postgres UUID or the legacy Mongo
+        ``ObjectId`` string — the latter is resolved via
+        ``conversations.legacy_mongo_id`` to find the matching row.
+
+        Args:
+            conversation_id: The conversation this state belongs to.
+            user: Owner user ID.
+            messages: Full messages array at the pause point.
+            pending_tool_calls: Tool calls awaiting client action.
+            tools_dict: Serializable tools configuration dict.
+            tool_schemas: LLM-formatted tool schemas (agent.tools).
+            agent_config: Config needed to recreate the agent on resume.
+            client_tools: Client-provided tool schemas for client-side execution.
+
+        Returns:
+            The string ID (conversation_id as provided) of the saved state.
+        """
+        with db_session() as conn:
+            conv = ConversationsRepository(conn).get_by_legacy_id(conversation_id)
+            if conv is not None:
+                pg_conv_id = conv["id"]
+            elif looks_like_uuid(conversation_id):
+                pg_conv_id = conversation_id
+            else:
+                # Unresolvable legacy ObjectId — downstream ``CAST AS uuid``
+                # would raise and poison the save. Surface the mismatch so
+                # the caller can decide (the stream loop in routes/base.py
+                # already wraps this in try/except).
+                raise ValueError(
+                    f"Cannot save continuation state: conversation_id "
+                    f"{conversation_id!r} is neither a PG UUID nor a "
+                    f"backfilled legacy Mongo id."
+                )
+            PendingToolStateRepository(conn).save_state(
+                pg_conv_id,
+                user,
+                messages=_make_serializable(messages),
+                pending_tool_calls=_make_serializable(pending_tool_calls),
+                tools_dict=_make_serializable(tools_dict),
+                tool_schemas=_make_serializable(tool_schemas),
+                agent_config=_make_serializable(agent_config),
+                client_tools=_make_serializable(client_tools) if client_tools else None,
+            )
+
+        logger.info(
+            f"Saved continuation state for conversation {conversation_id} "
+            f"with {len(pending_tool_calls)} pending tool call(s)"
+        )
+        return conversation_id
+
+    def load_state(
+        self, conversation_id: str, user: str
+    ) -> Optional[Dict[str, Any]]:
+        """Load pending continuation state.
+
+        Returns:
+            The state dict, or None if no pending state exists.
+        """
+        with db_readonly() as conn:
+            conv = ConversationsRepository(conn).get_by_legacy_id(conversation_id)
+            if conv is not None:
+                pg_conv_id = conv["id"]
+            elif looks_like_uuid(conversation_id):
+                pg_conv_id = conversation_id
+            else:
+                # Unresolvable legacy ObjectId → no state can exist for it.
+                return None
+            doc = PendingToolStateRepository(conn).load_state(pg_conv_id, user)
+        if not doc:
+            return None
+        return doc
+
+    def delete_state(self, conversation_id: str, user: str) -> bool:
+        """Delete pending state after successful resumption.
+
+        Returns:
+            True if a row was deleted.
+        """
+        with db_session() as conn:
+            conv = ConversationsRepository(conn).get_by_legacy_id(conversation_id)
+            if conv is not None:
+                pg_conv_id = conv["id"]
+            elif looks_like_uuid(conversation_id):
+                pg_conv_id = conversation_id
+            else:
+                # Unresolvable legacy ObjectId → nothing to delete.
+                return False
+            deleted = PendingToolStateRepository(conn).delete_state(pg_conv_id, user)
+        if deleted:
+            logger.info(
+                f"Deleted continuation state for conversation {conversation_id}"
+            )
+        return deleted
+
+    def mark_resuming(self, conversation_id: str, user: str) -> bool:
+        """Flip the pending row to ``resuming`` so a crashed resume can be retried."""
+        with db_session() as conn:
+            conv = ConversationsRepository(conn).get_by_legacy_id(conversation_id)
+            if conv is not None:
+                pg_conv_id = conv["id"]
+            elif looks_like_uuid(conversation_id):
+                pg_conv_id = conversation_id
+            else:
+                return False
+            flipped = PendingToolStateRepository(conn).mark_resuming(
+                pg_conv_id, user
+            )
+        if flipped:
+            logger.info(
+                f"Marked continuation state as resuming for conversation "
+                f"{conversation_id}"
+            )
+        return flipped

application/api/answer/services/conversation_service.py

@@ -0,0 +1,546 @@
+"""Conversation persistence service backed by Postgres.
+
+Handles create / append / update / compression for conversations during
+the answer-streaming path. Connections are opened per-operation rather
+than held for the duration of a stream.
+"""
+
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from sqlalchemy import text as sql_text
+
+from application.core.settings import settings
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.conversations import (
+    ConversationsRepository,
+    MessageUpdateOutcome,
+)
+from application.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+# Shown to the user if the worker dies mid-stream and the response is never finalised.
+TERMINATED_RESPONSE_PLACEHOLDER = (
+    "Response was terminated prior to completion, try regenerating."
+)
+
+
+class ConversationService:
+    def get_conversation(
+        self, conversation_id: str, user_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """Retrieve a conversation with owner-or-shared access control.
+
+        Returns a dict in the legacy Mongo shape — ``queries`` is a list
+        of message dicts (prompt/response/...) — for compatibility with
+        the streaming pipeline that consumes this shape.
+        """
+        if not conversation_id or not user_id:
+            return None
+        try:
+            with db_readonly() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(conversation_id, user_id)
+                if conv is None:
+                    logger.warning(
+                        f"Conversation not found or unauthorized - ID: {conversation_id}, User: {user_id}"
+                    )
+                    return None
+                messages = repo.get_messages(str(conv["id"]))
+            conv["queries"] = messages
+            conv["_id"] = str(conv["id"])
+            return conv
+        except Exception as e:
+            logger.error(f"Error fetching conversation: {str(e)}", exc_info=True)
+            return None
+
+    def save_conversation(
+        self,
+        conversation_id: Optional[str],
+        question: str,
+        response: str,
+        thought: str,
+        sources: List[Dict[str, Any]],
+        tool_calls: List[Dict[str, Any]],
+        llm: Any,
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        index: Optional[int] = None,
+        api_key: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        is_shared_usage: bool = False,
+        shared_token: Optional[str] = None,
+        attachment_ids: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        visibility: str = "hidden",
+    ) -> str:
+        """Save or update a conversation in Postgres.
+
+        Returns the string conversation id (PG UUID as string, or the
+        caller-provided id if it was already a UUID).
+        """
+        if decoded_token is None:
+            raise ValueError("Invalid or missing authentication token")
+        user_id = decoded_token.get("sub")
+        if not user_id:
+            raise ValueError("User ID not found in token")
+        current_time = datetime.now(timezone.utc)
+
+        # Trim huge inline source text to a reasonable max before persist.
+        for source in sources:
+            if "text" in source and isinstance(source["text"], str):
+                source["text"] = source["text"][:1000]
+
+        message_payload = {
+            "prompt": question,
+            "response": response,
+            "thought": thought,
+            "sources": sources,
+            "tool_calls": tool_calls,
+            "attachments": attachment_ids,
+            "model_id": model_id,
+            "timestamp": current_time,
+        }
+        if metadata:
+            message_payload["metadata"] = metadata
+
+        if conversation_id is not None and index is not None:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(conversation_id, user_id)
+                if conv is None:
+                    raise ValueError("Conversation not found or unauthorized")
+                conv_pg_id = str(conv["id"])
+                repo.update_message_at(conv_pg_id, index, message_payload)
+                repo.truncate_after(conv_pg_id, index)
+            return conversation_id
+        elif conversation_id:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(conversation_id, user_id)
+                if conv is None:
+                    raise ValueError("Conversation not found or unauthorized")
+                conv_pg_id = str(conv["id"])
+                # append_message expects 'metadata' key either way; normalise.
+                append_payload = dict(message_payload)
+                append_payload.setdefault("metadata", metadata or {})
+                repo.append_message(conv_pg_id, append_payload)
+            return conversation_id
+        else:
+            messages_summary = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant that creates concise conversation titles. "
+                    "Summarize conversations in 3 words or less using the same language as the user.",
+                },
+                {
+                    "role": "user",
+                    "content": "Summarise following conversation in no more than 3 words, "
+                    "respond ONLY with the summary, use the same language as the "
+                    "user query \n\nUser: " + question + "\n\n" + "AI: " + response,
+                },
+            ]
+
+            # ``model_id`` here is the registry id (a UUID for BYOM
+            # records). The LLM's own ``model_id`` is the upstream name
+            # LLMCreator resolved at construction time — that's what
+            # the provider's API expects. Built-ins are unaffected.
+            completion = llm.gen(
+                model=getattr(llm, "model_id", None) or model_id,
+                messages=messages_summary,
+                # Reasoning-capable default models spend the whole budget inside
+                # reasoning_content before emitting any title, so 500 came back
+                # empty (finish_reason=length). Give enough room to finish
+                # thinking and still produce the 3-word title; non-reasoning
+                # models stop far short of this cap.
+                max_tokens=2000,
+            )
+
+            if not completion or not completion.strip():
+                completion = question[:50] if question else "New Conversation"
+
+            resolved_api_key: Optional[str] = None
+            resolved_agent_id: Optional[str] = None
+            if api_key:
+                with db_readonly() as conn:
+                    agent = AgentsRepository(conn).find_by_key(api_key)
+                if agent:
+                    resolved_api_key = agent.get("key")
+                if agent_id:
+                    resolved_agent_id = agent_id
+
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.create(
+                    user_id,
+                    completion,
+                    agent_id=resolved_agent_id,
+                    api_key=resolved_api_key,
+                    is_shared_usage=bool(resolved_agent_id and is_shared_usage),
+                    shared_token=(
+                        shared_token
+                        if (resolved_agent_id and is_shared_usage)
+                        else None
+                    ),
+                    visibility=visibility,
+                )
+                conv_pg_id = str(conv["id"])
+                append_payload = dict(message_payload)
+                append_payload.setdefault("metadata", metadata or {})
+                repo.append_message(conv_pg_id, append_payload)
+            return conv_pg_id
+
+    def save_user_question(
+        self,
+        conversation_id: Optional[str],
+        question: str,
+        decoded_token: Dict[str, Any],
+        *,
+        attachment_ids: Optional[List[str]] = None,
+        api_key: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        is_shared_usage: bool = False,
+        shared_token: Optional[str] = None,
+        model_id: Optional[str] = None,
+        request_id: Optional[str] = None,
+        visibility: str = "hidden",
+        status: str = "pending",
+        index: Optional[int] = None,
+    ) -> Dict[str, str]:
+        """Reserve the placeholder message row before the LLM call.
+
+        ``index`` triggers regenerate semantics: messages at
+        ``position >= index`` are truncated so the new placeholder
+        lands at ``position = index`` rather than appending.
+
+        Returns ``{"conversation_id", "message_id", "request_id"}``.
+        """
+        if decoded_token is None:
+            raise ValueError("Invalid or missing authentication token")
+        user_id = decoded_token.get("sub")
+        if not user_id:
+            raise ValueError("User ID not found in token")
+
+        request_id = request_id or str(uuid.uuid4())
+
+        resolved_api_key: Optional[str] = None
+        resolved_agent_id: Optional[str] = None
+        if api_key and not conversation_id:
+            with db_readonly() as conn:
+                agent = AgentsRepository(conn).find_by_key(api_key)
+            if agent:
+                resolved_api_key = agent.get("key")
+            if agent_id:
+                resolved_agent_id = agent_id
+
+        with db_session() as conn:
+            repo = ConversationsRepository(conn)
+            if conversation_id:
+                conv = repo.get_any(conversation_id, user_id)
+                if conv is None:
+                    raise ValueError("Conversation not found or unauthorized")
+                conv_pg_id = str(conv["id"])
+                # Regenerate / edit-prior-question: drop the message at
+                # ``index`` and everything after it so the new
+                # ``reserve_message`` lands at ``position=index`` rather
+                # than appending at the end of the conversation.
+                if isinstance(index, int) and index >= 0:
+                    repo.truncate_after(conv_pg_id, keep_up_to=index - 1)
+            else:
+                fallback_name = (question[:50] if question else "New Conversation")
+                conv = repo.create(
+                    user_id,
+                    fallback_name,
+                    agent_id=resolved_agent_id,
+                    api_key=resolved_api_key,
+                    is_shared_usage=bool(resolved_agent_id and is_shared_usage),
+                    shared_token=(
+                        shared_token
+                        if (resolved_agent_id and is_shared_usage)
+                        else None
+                    ),
+                    visibility=visibility,
+                )
+                conv_pg_id = str(conv["id"])
+
+            row = repo.reserve_message(
+                conv_pg_id,
+                prompt=question,
+                placeholder_response=TERMINATED_RESPONSE_PLACEHOLDER,
+                request_id=request_id,
+                status=status,
+                attachments=attachment_ids,
+                model_id=model_id,
+            )
+            message_id = str(row["id"])
+
+        return {
+            "conversation_id": conv_pg_id,
+            "message_id": message_id,
+            "request_id": request_id,
+        }
+
+    def update_message_status(self, message_id: str, status: str) -> bool:
+        """Cheap status-only transition (e.g. ``pending → streaming``)."""
+        if not message_id:
+            return False
+        with db_session() as conn:
+            return ConversationsRepository(conn).update_message_status(
+                message_id, status,
+            )
+
+    def heartbeat_message(self, message_id: str) -> bool:
+        """Bump ``message_metadata.last_heartbeat_at`` so the reconciler's
+        staleness sweep counts the row as alive. No-ops on terminal rows.
+        """
+        if not message_id:
+            return False
+        with db_session() as conn:
+            return ConversationsRepository(conn).heartbeat_message(message_id)
+
+    def finalize_message(
+        self,
+        message_id: str,
+        response: str,
+        *,
+        thought: str = "",
+        sources: Optional[List[Dict[str, Any]]] = None,
+        tool_calls: Optional[List[Dict[str, Any]]] = None,
+        model_id: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        status: str = "complete",
+        error: Optional[BaseException] = None,
+        title_inputs: Optional[Dict[str, Any]] = None,
+    ) -> MessageUpdateOutcome:
+        """Commit the response and tool_call confirms in one transaction.
+
+        The outcome propagates directly from ``update_message_by_id`` so
+        callers (notably the SSE abort handler) can tell a fresh
+        finalize from "the row was already terminal" — the latter must
+        still be treated as success when the prior state was
+        ``complete``.
+        """
+        if not message_id:
+            return MessageUpdateOutcome.INVALID
+        sources = sources or []
+        for source in sources:
+            if "text" in source and isinstance(source["text"], str):
+                source["text"] = source["text"][:1000]
+
+        merged_metadata: Dict[str, Any] = dict(metadata or {})
+        if status == "failed" and error is not None:
+            merged_metadata.setdefault(
+                "error", f"{type(error).__name__}: {str(error)}"
+            )
+
+        update_fields: Dict[str, Any] = {
+            "response": response,
+            "status": status,
+            "thought": thought,
+            "sources": sources,
+            "tool_calls": tool_calls or [],
+            "metadata": merged_metadata,
+        }
+        if model_id is not None:
+            update_fields["model_id"] = model_id
+
+        # Atomic message update + tool_call_attempts confirm; the
+        # ``only_if_non_terminal`` guard prevents a late stream from
+        # retracting a row the reconciler already escalated.
+        with db_session() as conn:
+            repo = ConversationsRepository(conn)
+            outcome = repo.update_message_by_id(
+                message_id, update_fields,
+                only_if_non_terminal=True,
+            )
+            if outcome is not MessageUpdateOutcome.UPDATED:
+                logger.warning(
+                    f"finalize_message: no row updated for message_id={message_id} "
+                    f"(outcome={outcome.value} — possibly already terminal)"
+                )
+                return outcome
+            repo.confirm_executed_tool_calls(message_id)
+
+        # Outside the txn — title-gen is a multi-second LLM round trip.
+        if title_inputs and status == "complete":
+            try:
+                with db_session() as conn:
+                    self._maybe_generate_title(conn, message_id, title_inputs)
+            except Exception as e:
+                logger.error(
+                    f"finalize_message title generation failed: {e}",
+                    exc_info=True,
+                )
+        return MessageUpdateOutcome.UPDATED
+
+    def _maybe_generate_title(
+        self,
+        conn,
+        message_id: str,
+        title_inputs: Dict[str, Any],
+    ) -> None:
+        """Generate an LLM-summarised conversation name if one isn't set yet."""
+        llm = title_inputs.get("llm")
+        question = title_inputs.get("question") or ""
+        response = title_inputs.get("response") or ""
+        fallback_name = title_inputs.get("fallback_name") or question[:50]
+        if llm is None:
+            return
+
+        row = conn.execute(
+            sql_text(
+                "SELECT c.id, c.name FROM conversation_messages m "
+                "JOIN conversations c ON c.id = m.conversation_id "
+                "WHERE m.id = CAST(:mid AS uuid)"
+            ),
+            {"mid": message_id},
+        ).fetchone()
+        if row is None:
+            return
+        conv_id, current_name = str(row[0]), row[1]
+        if current_name and current_name != fallback_name:
+            return
+
+        messages_summary = [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant that creates concise conversation titles. "
+                "Summarize conversations in 3 words or less using the same language as the user.",
+            },
+            {
+                "role": "user",
+                "content": "Summarise following conversation in no more than 3 words, "
+                "respond ONLY with the summary, use the same language as the "
+                "user query \n\nUser: " + question + "\n\n" + "AI: " + response,
+            },
+        ]
+        completion = llm.gen(
+            model=getattr(llm, "model_id", None) or title_inputs.get("model_id"),
+            messages=messages_summary,
+            # Reasoning-capable default models spend the whole budget inside
+            # reasoning_content before emitting any title, so 500 came back empty
+            # (finish_reason=length). Give room to finish and still produce the
+            # 3-word title; non-reasoning models stop far short of this cap.
+            max_tokens=2000,
+        )
+        if not completion or not completion.strip():
+            completion = fallback_name or "New Conversation"
+        conn.execute(
+            sql_text(
+                "UPDATE conversations SET name = :name, updated_at = now() "
+                "WHERE id = CAST(:id AS uuid)"
+            ),
+            {"id": conv_id, "name": completion.strip()},
+        )
+
+    def update_compression_metadata(
+        self, conversation_id: str, compression_metadata: Dict[str, Any]
+    ) -> None:
+        """Persist compression flags and append a compression point.
+
+        Mirrors the Mongo-era ``$set`` + ``$push $slice`` on
+        ``compression_metadata`` but goes through the PG repo API.
+        """
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                # conversation_id here comes from the streaming pipeline
+                # which has already resolved it; accept either UUID or
+                # legacy id for safety.
+                conv = repo.get_by_legacy_id(conversation_id)
+                conv_pg_id = (
+                    str(conv["id"]) if conv is not None else conversation_id
+                )
+                repo.set_compression_flags(
+                    conv_pg_id,
+                    is_compressed=True,
+                    last_compression_at=compression_metadata.get("timestamp"),
+                )
+                repo.append_compression_point(
+                    conv_pg_id,
+                    compression_metadata,
+                    max_points=settings.COMPRESSION_MAX_HISTORY_POINTS,
+                )
+            logger.info(
+                f"Updated compression metadata for conversation {conversation_id}"
+            )
+        except Exception as e:
+            logger.error(
+                f"Error updating compression metadata: {str(e)}", exc_info=True
+            )
+            raise
+
+    def append_compression_message(
+        self, conversation_id: str, compression_metadata: Dict[str, Any]
+    ) -> None:
+        """Append a synthetic compression summary message to the conversation."""
+        try:
+            summary = compression_metadata.get("compressed_summary", "")
+            if not summary:
+                return
+            timestamp = compression_metadata.get(
+                "timestamp", datetime.now(timezone.utc)
+            )
+
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_by_legacy_id(conversation_id)
+                conv_pg_id = (
+                    str(conv["id"]) if conv is not None else conversation_id
+                )
+                repo.append_message(conv_pg_id, {
+                    "prompt": "[Context Compression Summary]",
+                    "response": summary,
+                    "thought": "",
+                    "sources": [],
+                    "tool_calls": [],
+                    "attachments": [],
+                    "model_id": compression_metadata.get("model_used"),
+                    "timestamp": timestamp,
+                })
+            logger.info(
+                f"Appended compression summary to conversation {conversation_id}"
+            )
+        except Exception as e:
+            logger.error(
+                f"Error appending compression summary: {str(e)}", exc_info=True
+            )
+
+    def get_compression_metadata(
+        self, conversation_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """Fetch the stored compression metadata JSONB blob for a conversation."""
+        try:
+            with db_readonly() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_by_legacy_id(conversation_id)
+                if conv is None:
+                    # Fallback to UUID lookup without user scoping — the
+                    # caller already holds an authenticated conversation
+                    # id from the streaming path. Gate on id shape so a
+                    # non-UUID (legacy ObjectId that wasn't backfilled)
+                    # doesn't reach CAST — the cast raises and spams the
+                    # logs with a stack trace on every call.
+                    if not looks_like_uuid(conversation_id):
+                        return None
+                    result = conn.execute(
+                        sql_text(
+                            "SELECT compression_metadata FROM conversations "
+                            "WHERE id = CAST(:id AS uuid)"
+                        ),
+                        {"id": conversation_id},
+                    )
+                    row = result.fetchone()
+                    return row[0] if row is not None else None
+            return conv.get("compression_metadata") if conv else None
+        except Exception as e:
+            logger.error(
+                f"Error getting compression metadata: {str(e)}", exc_info=True
+            )
+            return None

application/api/answer/services/persistence_policy.py

@@ -0,0 +1,44 @@
+"""Resolve whether an answer is persisted and whether it lists in the sidebar.
+
+Persistence (is a row written at all?) and visibility (does it show in the
+owner's sidebar?) are separate decisions. Conversations persist by default
+everywhere, and visibility defaults to ``hidden`` for every caller: only an
+explicit request-level ``visibility: "listed"`` — which the first-party UI
+sends on normal chats — puts a conversation in the owner's sidebar. The
+legacy ``save_conversation`` flag no longer affects either decision, so
+API/OpenAI-compatible clients that still send it (its old meaning was
+"persist this conversation") can't list rows into the agent owner's sidebar.
+"""
+
+from typing import Any, Optional, Tuple
+
+VISIBILITY_LISTED = "listed"
+VISIBILITY_HIDDEN = "hidden"
+
+
+def resolve_persistence(
+    *,
+    visibility_flag: Optional[Any] = None,
+    persist_flag: Optional[bool] = None,
+) -> Tuple[bool, str]:
+    """Resolve ``(should_persist, visibility)`` for an answer request.
+
+    Args:
+        visibility_flag: Request-level ``visibility`` value. Only the exact
+            string ``"listed"`` opts the conversation into the owner's
+            sidebar; anything else (including ``None``) stays hidden.
+        persist_flag: Explicit persistence opt-out (``False`` to skip writing
+            a row, e.g. stateless tool rounds that would orphan one). ``None``
+            keeps the always-persist default.
+
+    Returns:
+        ``(should_persist, visibility)`` where ``visibility`` is
+        ``"listed"`` or ``"hidden"``.
+    """
+    should_persist = True if persist_flag is None else bool(persist_flag)
+    visibility = (
+        VISIBILITY_LISTED
+        if visibility_flag == VISIBILITY_LISTED
+        else VISIBILITY_HIDDEN
+    )
+    return should_persist, visibility

application/api/answer/services/prompt_renderer.py

@@ -0,0 +1,118 @@
+import logging
+from typing import Any, Dict, Optional
+
+from application.templates.namespaces import NamespaceManager
+
+from application.templates.template_engine import TemplateEngine, TemplateRenderError
+
+logger = logging.getLogger(__name__)
+
+
+def format_docs_for_prompt(docs: Optional[list]) -> Optional[str]:
+    """Format retrieved chunks as XML-tagged documents for prompt injection.
+
+    Each chunk is wrapped in a ``<document index="n">`` block with a
+    ``<source>`` subtag (when a filename/title is known) so the model can
+    tell chunks apart and cite them by name.
+    """
+    if not docs:
+        return None
+    parts = []
+    for i, doc in enumerate(docs, start=1):
+        source = doc.get("filename") or doc.get("title") or doc.get("source")
+        lines = [f'<document index="{i}">']
+        if source:
+            lines.append(f"<source>{source}</source>")
+        lines.append(f"<content>\n{doc.get('text', '')}\n</content>")
+        lines.append("</document>")
+        parts.append("\n".join(lines))
+    return "\n\n".join(parts)
+
+
+class PromptRenderer:
+    """Service for rendering prompts with dynamic context using namespaces"""
+
+    def __init__(self):
+        self.template_engine = TemplateEngine()
+        self.namespace_manager = NamespaceManager()
+
+    def render_prompt(
+        self,
+        prompt_content: str,
+        user_id: Optional[str] = None,
+        request_id: Optional[str] = None,
+        passthrough_data: Optional[Dict[str, Any]] = None,
+        docs: Optional[list] = None,
+        docs_together: Optional[str] = None,
+        tools_data: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> str:
+        """
+        Render prompt with full context from all namespaces.
+
+        Args:
+            prompt_content: Raw prompt template string
+            user_id: Current user identifier
+            request_id: Unique request identifier
+            passthrough_data: Parameters from web request
+            docs: RAG retrieved documents
+            docs_together: Concatenated document content
+            tools_data: Pre-fetched tool results organized by tool name
+            **kwargs: Additional parameters for namespace builders
+
+        Returns:
+            Rendered prompt string with all variables substituted
+
+        Raises:
+            TemplateRenderError: If template rendering fails
+        """
+        if not prompt_content:
+            return ""
+
+        uses_template = self._uses_template_syntax(prompt_content)
+
+        if not uses_template:
+            return self._apply_legacy_substitutions(prompt_content, docs_together)
+
+        try:
+            context = self.namespace_manager.build_context(
+                user_id=user_id,
+                request_id=request_id,
+                passthrough_data=passthrough_data,
+                docs=docs,
+                docs_together=docs_together,
+                tools_data=tools_data,
+                **kwargs,
+            )
+
+            return self.template_engine.render(prompt_content, context)
+        except TemplateRenderError:
+            raise
+        except Exception as e:
+            error_msg = f"Prompt rendering failed: {str(e)}"
+            logger.error(error_msg)
+            raise TemplateRenderError(error_msg) from e
+
+    def _uses_template_syntax(self, prompt_content: str) -> bool:
+        """Check if prompt uses Jinja2 template syntax"""
+        return "{{" in prompt_content and "}}" in prompt_content
+
+    def _apply_legacy_substitutions(
+        self, prompt_content: str, docs_together: Optional[str] = None
+    ) -> str:
+        """
+        Apply backward-compatible substitutions for old prompt format.
+
+        Handles the legacy {summaries} placeholder. When no documents were
+        retrieved the placeholder is removed so the model never sees the
+        raw template artifact.
+        """
+        return prompt_content.replace("{summaries}", docs_together or "")
+
+    def validate_template(self, prompt_content: str) -> bool:
+        """Validate prompt template syntax"""
+        return self.template_engine.validate_template(prompt_content)
+
+    def extract_variables(self, prompt_content: str) -> set[str]:
+        """Extract all variable names from prompt template"""
+        return self.template_engine.extract_variables(prompt_content)

application/api/async_sse.py

@@ -0,0 +1,248 @@
+"""Native-async (ASGI) SSE reader routes, mounted ahead of the Flask app.
+
+These Starlette routes serve the chat-stream *reconnect* path on the event
+loop, so a long-lived, mostly-idle tail costs a coroutine instead of one of
+the 32 a2wsgi threadpool slots (see ``application/asgi.py``). They are the
+sole reconnect reader — the old Flask blueprint has been removed. The heavy
+*producer* (``POST /api/answer/stream`` → agent → LLM) stays on the sync
+path untouched.
+
+Auth, message-id validation, ``Last-Event-ID`` parsing and ownership are
+done here; the snapshot/tail wire format is shared with the producer's
+journal via ``build_message_event_stream_async`` → ``format_sse_event``.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Optional
+
+import anyio
+from sqlalchemy import text
+from starlette.requests import Request
+from starlette.responses import JSONResponse, StreamingResponse
+from starlette.routing import Route
+
+from application.api.oidc.denylist import is_denied as oidc_session_denied
+from application.auth import handle_auth
+from application.core.settings import settings
+from application.events.keys import connection_counter_key
+from application.storage.db.session import db_readonly
+from application.streaming.async_event_replay import (
+    build_message_event_stream_async,
+)
+from application.streaming.async_redis import get_async_redis_instance
+from application.streaming.event_replay import (
+    DEFAULT_KEEPALIVE_SECONDS,
+    DEFAULT_POLL_TIMEOUT_SECONDS,
+)
+
+logger = logging.getLogger(__name__)
+
+# Per-user concurrent-connection counter TTL (seconds) — orphaned counts
+# from a hard crash self-heal after this window, mirroring the /api/events
+# notification stream. The reconnect reader shares the same counter key, so
+# the cap bounds a user's *total* live SSE footprint.
+_COUNTER_TTL_SECONDS = 3600
+
+# A message_id is the canonical UUID hex format. Reject anything else before
+# the SQL layer so a malformed cookie can't surface as a 500.
+_MESSAGE_ID_RE = re.compile(
+    r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
+    r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
+)
+# ``sequence_no`` is a non-negative decimal integer. Anything else is corrupt
+# client state — fall through to a fresh-replay cursor.
+_SEQUENCE_NO_RE = re.compile(r"^\d+$")
+
+
+def _normalise_last_event_id(raw: Optional[str]) -> Optional[int]:
+    """Parse a ``Last-Event-ID`` cursor; ``None`` for missing/invalid."""
+    if raw is None:
+        return None
+    raw = raw.strip()
+    if not raw or not _SEQUENCE_NO_RE.match(raw):
+        return None
+    return int(raw)
+
+
+def _user_owns_message(message_id: str, user_id: str) -> bool:
+    """Return True iff ``message_id`` belongs to ``user_id``."""
+    try:
+        with db_readonly() as conn:
+            row = conn.execute(
+                text(
+                    """
+                    SELECT 1 FROM conversation_messages
+                    WHERE id = CAST(:id AS uuid)
+                      AND user_id = :u
+                    LIMIT 1
+                    """
+                ),
+                {"id": message_id, "u": user_id},
+            ).first()
+        return row is not None
+    except Exception:
+        logger.exception(
+            "Ownership lookup failed for message_id=%s user_id=%s",
+            message_id,
+            user_id,
+        )
+        return False
+
+_SSE_HEADERS = {
+    "Cache-Control": "no-store",
+    "X-Accel-Buffering": "no",
+    "Connection": "keep-alive",
+    # Marks the response as served by the event-loop reader rather than the
+    # WSGI-threaded Flask fallback. Purely diagnostic — the frontend reads
+    # the body via fetch+getReader and ignores response headers.
+    "X-SSE-Transport": "async",
+}
+
+
+def _json(message: str, status_code: int) -> JSONResponse:
+    return JSONResponse(
+        {"success": False, "message": message}, status_code=status_code
+    )
+
+
+async def _acquire_stream_slot(user_id: str):
+    """Reserve a per-user connection slot; returns ``(redis, key)`` to release.
+
+    Mirrors the ``/api/events`` cap (INCR + safety TTL, reject when the
+    post-increment count exceeds ``SSE_MAX_CONCURRENT_PER_USER``). Returns
+    ``(None, None)`` when the cap is disabled or Redis is unavailable
+    (fail-open, like the notification stream). Raises ``_CapExceeded`` when
+    the user is over the cap so the caller can 429.
+    """
+    cap = int(getattr(settings, "SSE_MAX_CONCURRENT_PER_USER", 0))
+    if cap <= 0:
+        return None, None
+    redis = await get_async_redis_instance()
+    if redis is None:
+        return None, None
+    key = connection_counter_key(user_id)
+    try:
+        current = int(await redis.incr(key))
+    except Exception:
+        logger.debug("async SSE counter INCR failed for user=%s", user_id)
+        return None, None
+    # EXPIRE failure must not bypass the cap, so it's best-effort after INCR.
+    try:
+        await redis.expire(key, _COUNTER_TTL_SECONDS)
+    except Exception:
+        logger.debug("async SSE counter EXPIRE failed for user=%s", user_id)
+    if current > cap:
+        await _release_stream_slot(redis, key)
+        raise _CapExceeded()
+    return redis, key
+
+
+async def _release_stream_slot(redis, key) -> None:
+    if redis is None or key is None:
+        return
+    try:
+        await redis.decr(key)
+    except Exception:
+        logger.debug("async SSE counter DECR failed for key=%s", key)
+
+
+class _CapExceeded(Exception):
+    """Raised when a user is over their concurrent-stream cap."""
+
+
+async def _counted_stream(inner, redis, key):
+    """Wrap the reader so the per-user slot is released when it ends.
+
+    The slot is reserved before the response starts (so over-cap surfaces as
+    HTTP 429, not mid-stream); the release runs in ``finally`` on terminal
+    close, client disconnect, or error. Shielded so a disconnect-cancellation
+    can't skip the DECR and leak the count.
+    """
+    try:
+        async for line in inner:
+            yield line
+    finally:
+        with anyio.CancelScope(shield=True):
+            await _release_stream_slot(redis, key)
+
+
+async def stream_message_events(request: Request) -> JSONResponse | StreamingResponse:
+    """GET /api/messages/{message_id}/events — async reconnect tail.
+
+    Mirrors the Flask handler's gates (auth → id format → ownership →
+    cursor → per-user connection cap) then streams snapshot+tail off the
+    event loop.
+    """
+    # ``handle_auth`` only reads ``request.headers.get("Authorization")``;
+    # Starlette's headers are case-insensitive, so the Flask helper works
+    # verbatim. With AUTH_TYPE unset it returns ``{"sub": "local"}``.
+    decoded = handle_auth(request)
+    if isinstance(decoded, dict) and "error" in decoded:
+        return _json("Authentication error: invalid token", 401)
+    user_id = decoded.get("sub") if isinstance(decoded, dict) else None
+    if not user_id:
+        return _json("Authentication required", 401)
+    if settings.AUTH_TYPE == "oidc" and await anyio.to_thread.run_sync(
+        oidc_session_denied, decoded
+    ):
+        return _json("Authentication error: session revoked", 401)
+
+    message_id = request.path_params["message_id"]
+    if not _MESSAGE_ID_RE.match(message_id):
+        return _json("Invalid message id", 400)
+
+    # Ownership check is a sync DB read — push it off the loop.
+    owns = await anyio.to_thread.run_sync(_user_owns_message, message_id, user_id)
+    if not owns:
+        # Same opaque 404 as the Flask route — don't disclose existence.
+        return _json("Not found", 404)
+
+    # Per-user concurrent-connection cap — reserve before the response opens
+    # so an over-cap caller gets a clean 429 instead of a mid-stream cutoff.
+    try:
+        redis, counter_key = await _acquire_stream_slot(user_id)
+    except _CapExceeded:
+        logger.warning("sse.reconnect.rejected user_id=%s (over cap)", user_id)
+        return _json("Too many concurrent SSE connections", 429)
+
+    raw_cursor = request.headers.get("Last-Event-ID") or request.query_params.get(
+        "last_event_id"
+    )
+    last_event_id = _normalise_last_event_id(raw_cursor)
+    keepalive_seconds = float(
+        getattr(settings, "SSE_KEEPALIVE_SECONDS", DEFAULT_KEEPALIVE_SECONDS)
+    )
+
+    logger.info(
+        "message.event.connect.async message_id=%s user_id=%s last_event_id=%s",
+        message_id,
+        user_id,
+        last_event_id if last_event_id is not None else "-",
+    )
+
+    stream = build_message_event_stream_async(
+        message_id,
+        last_event_id=last_event_id,
+        user_id=user_id,
+        keepalive_seconds=keepalive_seconds,
+        poll_timeout_seconds=DEFAULT_POLL_TIMEOUT_SECONDS,
+    )
+    return StreamingResponse(
+        _counted_stream(stream, redis, counter_key),
+        media_type="text/event-stream",
+        headers=_SSE_HEADERS,
+    )
+
+
+# Mounted in ``application/asgi.py`` ahead of the Flask catch-all. Keep
+# each route's path identical to the Flask blueprint it shadows.
+async_sse_routes = [
+    Route(
+        "/api/messages/{message_id}/events",
+        stream_message_events,
+        methods=["GET"],
+    ),
+]

application/api/connector/routes.py

@@ -0,0 +1,551 @@
+import base64
+import html
+import json
+import uuid
+from urllib.parse import urlencode
+
+
+from flask import (
+    Blueprint,
+    current_app,
+    jsonify,
+    make_response,
+    request
+)
+from flask_restx import fields, Namespace, Resource
+
+
+from application.api import api
+from application.api.user.tasks import (
+    ingest_connector_task,
+)
+from application.parser.connectors.connector_creator import ConnectorCreator
+from application.storage.db.repositories.connector_sessions import (
+    ConnectorSessionsRepository,
+)
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+connector = Blueprint("connector", __name__)
+connectors_ns = Namespace("connectors", description="Connector operations", path="/")
+api.add_namespace(connectors_ns)
+
+# Fixed callback status path to prevent open redirect
+CALLBACK_STATUS_PATH = "/api/connectors/callback-status"
+
+
+def build_callback_redirect(params: dict) -> str:
+    """Build a safe redirect URL to the callback status page.
+
+    Uses a fixed path and properly URL-encodes all parameters
+    to prevent URL injection and open redirect vulnerabilities.
+    """
+    return f"{CALLBACK_STATUS_PATH}?{urlencode(params)}"
+
+
+
+@connectors_ns.route("/api/connectors/auth")
+class ConnectorAuth(Resource):
+    @api.doc(description="Get connector OAuth authorization URL", params={"provider": "Connector provider (e.g., google_drive)"})
+    def get(self):
+        try:
+            provider = request.args.get('provider') or request.args.get('source')
+            if not provider:
+                return make_response(jsonify({"success": False, "error": "Missing provider"}), 400)
+
+            if not ConnectorCreator.is_supported(provider):
+                return make_response(jsonify({"success": False, "error": f"Unsupported provider: {provider}"}), 400)
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user_id = decoded_token.get('sub')
+
+            with db_session() as conn:
+                session_row = ConnectorSessionsRepository(conn).upsert(
+                    user_id, provider, status="pending",
+                )
+            session_pg_id = str(session_row["id"])
+            state_dict = {
+                "provider": provider,
+                "object_id": session_pg_id,
+            }
+            state = base64.urlsafe_b64encode(json.dumps(state_dict).encode()).decode()
+
+            auth = ConnectorCreator.create_auth(provider)
+            authorization_url = auth.get_authorization_url(state=state)
+            return make_response(jsonify({
+                "success": True,
+                "authorization_url": authorization_url,
+                "state": state
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error generating connector auth URL: {e}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to generate authorization URL"}), 500)
+
+
+@connectors_ns.route("/api/connectors/callback")
+class ConnectorsCallback(Resource):
+    @api.doc(description="Handle OAuth callback for external connectors")
+    def get(self):
+        """Handle OAuth callback for external connectors"""
+        try:
+            from application.parser.connectors.connector_creator import ConnectorCreator
+            from flask import request, redirect
+
+            authorization_code = request.args.get('code')
+            state = request.args.get('state')
+            error = request.args.get('error')
+
+            state_dict = json.loads(base64.urlsafe_b64decode(state.encode()).decode())
+            provider = state_dict.get("provider")
+            state_object_id = state_dict.get("object_id")
+
+            # Validate provider
+            if not provider or not isinstance(provider, str) or not ConnectorCreator.is_supported(provider):
+                return redirect(build_callback_redirect({
+                    "status": "error",
+                    "message": "Invalid provider"
+                }))
+
+            if error:
+                if error == "access_denied":
+                    return redirect(build_callback_redirect({
+                        "status": "cancelled",
+                        "message": "Authentication was cancelled. You can try again if you'd like to connect your account.",
+                        "provider": provider
+                    }))
+                else:
+                    current_app.logger.warning(f"OAuth error in callback: {error}")
+                    return redirect(build_callback_redirect({
+                        "status": "error",
+                        "message": "Authentication failed. Please try again and make sure to grant all requested permissions.",
+                        "provider": provider
+                    }))
+
+            if not authorization_code:
+                return redirect(build_callback_redirect({
+                    "status": "error",
+                    "message": "Authentication failed. Please try again and make sure to grant all requested permissions.",
+                    "provider": provider
+                }))
+
+            try:
+                auth = ConnectorCreator.create_auth(provider)
+                token_info = auth.exchange_code_for_tokens(authorization_code)
+
+                session_token = str(uuid.uuid4())
+
+                try:
+                    if provider == "google_drive":
+                        credentials = auth.create_credentials_from_token_info(token_info)
+                        service = auth.build_drive_service(credentials)
+                        user_info = service.about().get(fields="user").execute()
+                        user_email = user_info.get('user', {}).get('emailAddress', 'Connected User')
+                    else:
+                        user_email = token_info.get('user_info', {}).get('email', 'Connected User')
+
+                except Exception as e:
+                    current_app.logger.warning(f"Could not get user info: {e}")
+                    user_email = 'Connected User'
+
+                sanitized_token_info = auth.sanitize_token_info(token_info)
+
+                # ``object_id`` in the OAuth state is the PG session row
+                # UUID (new flow) or a legacy Mongo ObjectId (pre-cutover
+                # issued state). Try UUID update first; fall back to
+                # legacy id path.
+                patch = {
+                    "session_token": session_token,
+                    "token_info": sanitized_token_info,
+                    "user_email": user_email,
+                    "status": "authorized",
+                }
+                with db_session() as conn:
+                    repo = ConnectorSessionsRepository(conn)
+                    if state_object_id:
+                        value = str(state_object_id)
+                        updated = False
+                        if len(value) == 36 and "-" in value:
+                            updated = repo.update(value, patch)
+                        if not updated:
+                            repo.update_by_legacy_id(value, patch)
+
+                # Redirect to success page with session token and user email
+                return redirect(build_callback_redirect({
+                    "status": "success",
+                    "message": "Authentication successful",
+                    "provider": provider,
+                    "session_token": session_token,
+                    "user_email": user_email
+                }))
+
+            except Exception as e:
+                current_app.logger.error(f"Error exchanging code for tokens: {str(e)}", exc_info=True)
+                return redirect(build_callback_redirect({
+                    "status": "error",
+                    "message": "Authentication failed. Please try again and make sure to grant all requested permissions.",
+                    "provider": provider
+                }))
+
+        except Exception as e:
+            current_app.logger.error(f"Error handling connector callback: {e}")
+            return redirect(build_callback_redirect({
+                "status": "error",
+                "message": "Authentication failed. Please try again and make sure to grant all requested permissions."
+            }))
+
+
+@connectors_ns.route("/api/connectors/files")
+class ConnectorFiles(Resource):
+    @api.expect(api.model("ConnectorFilesModel", {
+        "provider": fields.String(required=True),
+        "session_token": fields.String(required=True),
+        "folder_id": fields.String(required=False),
+        "limit": fields.Integer(required=False),
+        "page_token": fields.String(required=False),
+        "search_query": fields.String(required=False),
+    }))
+    @api.doc(description="List files from a connector provider (supports pagination and search)")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            limit = data.get('limit', 10)
+
+            if not provider or not session_token:
+                return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user = decoded_token.get('sub')
+            with db_readonly() as conn:
+                session = ConnectorSessionsRepository(conn).get_by_session_token(
+                    session_token,
+                )
+            if not session or session.get("user_id") != user:
+                return make_response(jsonify({"success": False, "error": "Invalid or unauthorized session"}), 401)
+
+            loader = ConnectorCreator.create_connector(provider, session_token)
+
+            generic_keys = {'provider', 'session_token'}
+            input_config = {
+                k: v for k, v in data.items() if k not in generic_keys
+            }
+            input_config['list_only'] = True
+                
+            documents = loader.load_data(input_config)
+
+            files = []
+            for doc in documents[:limit]:
+                metadata = doc.extra_info
+                modified_time = metadata.get('modified_time')
+                if modified_time:
+                    date_part = modified_time.split('T')[0]
+                    time_part = modified_time.split('T')[1].split('.')[0].split('Z')[0]
+                    formatted_time = f"{date_part} {time_part}"
+                else:
+                    formatted_time = None
+
+                files.append({
+                    'id': doc.doc_id,
+                    'name': metadata.get('file_name', 'Unknown File'),
+                    'type': metadata.get('mime_type', 'unknown'),
+                    'size': metadata.get('size', None),
+                    'modifiedTime': formatted_time,
+                    'isFolder': metadata.get('is_folder', False)
+                })
+
+            next_token = getattr(loader, 'next_page_token', None)
+            has_more = bool(next_token)
+
+            return make_response(jsonify({
+                "success": True, 
+                "files": files, 
+                "total": len(files), 
+                "next_page_token": next_token, 
+                "has_more": has_more
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error loading connector files: {e}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to load files"}), 500)
+
+
+@connectors_ns.route("/api/connectors/validate-session")
+class ConnectorValidateSession(Resource):
+    @api.expect(api.model("ConnectorValidateSessionModel", {"provider": fields.String(required=True), "session_token": fields.String(required=True)}))
+    @api.doc(description="Validate connector session token and return user info and access token")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            if not provider or not session_token:
+                return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user = decoded_token.get('sub')
+
+            with db_readonly() as conn:
+                session = ConnectorSessionsRepository(conn).get_by_session_token(
+                    session_token,
+                )
+            if not session or session.get("user_id") != user or not session.get("token_info"):
+                return make_response(jsonify({"success": False, "error": "Invalid or expired session"}), 401)
+
+            token_info = session["token_info"]
+            auth = ConnectorCreator.create_auth(provider)
+            is_expired = auth.is_token_expired(token_info)
+
+            if is_expired and token_info.get('refresh_token'):
+                try:
+                    refreshed_token_info = auth.refresh_access_token(token_info.get('refresh_token'))
+                    sanitized_token_info = auth.sanitize_token_info(refreshed_token_info)
+                    with db_session() as conn:
+                        repo = ConnectorSessionsRepository(conn)
+                        row = repo.get_by_session_token(session_token)
+                        if row:
+                            repo.update(str(row["id"]), {"token_info": sanitized_token_info})
+                    token_info = sanitized_token_info
+                    is_expired = False
+                except Exception as refresh_error:
+                    current_app.logger.error(f"Failed to refresh token: {refresh_error}")
+            
+            if is_expired:
+                return make_response(jsonify({
+                    "success": False,
+                    "expired": True,
+                    "error": "Session token has expired. Please reconnect."
+                }), 401)
+
+            _base_fields = {"access_token", "refresh_token", "token_uri", "expiry"}
+            provider_extras = {k: v for k, v in token_info.items() if k not in _base_fields}
+
+            response_data = {
+                "success": True,
+                "expired": False,
+                "user_email": session.get('user_email', 'Connected User'),
+                "access_token": token_info.get('access_token'),
+                **provider_extras,
+            }
+
+            return make_response(jsonify(response_data), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error validating connector session: {e}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to validate session"}), 500)
+
+
+@connectors_ns.route("/api/connectors/disconnect")
+class ConnectorDisconnect(Resource):
+    @api.expect(api.model("ConnectorDisconnectModel", {"provider": fields.String(required=True), "session_token": fields.String(required=False)}))
+    @api.doc(description="Disconnect a connector session")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            if not provider:
+                return make_response(jsonify({"success": False, "error": "provider is required"}), 400)
+
+
+            if session_token:
+                with db_session() as conn:
+                    ConnectorSessionsRepository(conn).delete_by_session_token(
+                        session_token,
+                    )
+
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error disconnecting connector session: {e}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to disconnect session"}), 500)
+
+
+@connectors_ns.route("/api/connectors/sync")
+class ConnectorSync(Resource):
+    @api.expect(
+        api.model(
+            "ConnectorSyncModel",
+            {
+                "source_id": fields.String(required=True, description="Source ID to sync"),
+                "session_token": fields.String(required=True, description="Authentication token")
+            },
+        )
+    )
+    @api.doc(description="Sync connector source to check for modifications")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+
+        try:
+            data = request.get_json()
+            source_id = data.get('source_id')
+            session_token = data.get('session_token')
+
+            if not all([source_id, session_token]):
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "source_id and session_token are required"
+                    }), 
+                    400
+                )
+            user_id = decoded_token.get('sub')
+            with db_readonly() as conn:
+                source = SourcesRepository(conn).get_any(source_id, user_id)
+            if not source:
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Source not found"
+                    }),
+                    404
+                )
+
+            # ``get_any`` already scopes by ``user_id``; an extra guard
+            # here would be dead code.
+
+            remote_data = source.get('remote_data') or {}
+            if isinstance(remote_data, str):
+                try:
+                    remote_data = json.loads(remote_data)
+                except json.JSONDecodeError:
+                    current_app.logger.error(f"Invalid remote_data format for source {source_id}")
+                    remote_data = {}
+
+            source_type = remote_data.get('provider')
+            if not source_type:
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Source provider not found in remote_data"
+                    }), 
+                    400
+                )
+
+            # Extract configuration from remote_data
+            file_ids = remote_data.get('file_ids', [])
+            folder_ids = remote_data.get('folder_ids', [])
+            recursive = remote_data.get('recursive', True)
+
+            # Start the sync task
+            task = ingest_connector_task.delay(
+                job_name=source.get('name'),
+                user=decoded_token.get('sub'),
+                source_type=source_type,
+                session_token=session_token,
+                file_ids=file_ids,
+                folder_ids=folder_ids,
+                recursive=recursive,
+                retriever=source.get('retriever', 'classic'),
+                operation_mode="sync",
+                doc_id=str(source.get('id') or source_id),
+                sync_frequency=source.get('sync_frequency', 'never')
+            )
+
+            return make_response(
+                jsonify({
+                    "success": True,
+                    "task_id": task.id
+                }), 
+                200
+            )
+
+        except Exception as err:
+            current_app.logger.error(
+                f"Error syncing connector source: {err}",
+                exc_info=True
+            )
+            return make_response(
+                jsonify({
+                    "success": False,
+                    "error": "Failed to sync connector source"
+                }),
+                400
+            )
+
+
+@connectors_ns.route("/api/connectors/callback-status")
+class ConnectorCallbackStatus(Resource):
+    @api.doc(description="Return HTML page with connector authentication status")
+    def get(self):
+        """Return HTML page with connector authentication status"""
+        try:
+            # Validate and sanitize status to a known value
+            status_raw = request.args.get('status', 'error')
+            status = status_raw if status_raw in ('success', 'error', 'cancelled') else 'error'
+
+            # Escape all user-controlled values for HTML context
+            message = html.escape(request.args.get('message', ''))
+            provider_raw = request.args.get('provider', 'connector')
+            provider = html.escape(provider_raw.replace('_', ' ').title())
+            session_token = request.args.get('session_token', '')
+            user_email = html.escape(request.args.get('user_email', ''))
+
+            def safe_js_string(value: str) -> str:
+                """Safely encode a string for embedding in inline JavaScript."""
+                js_encoded = json.dumps(value)
+                return js_encoded.replace('</', '<\\/').replace('<!--', '<\\!--')
+
+            js_status = safe_js_string(status)
+            js_session_token = safe_js_string(session_token)
+            js_user_email = safe_js_string(user_email)
+            js_provider_type = safe_js_string(provider_raw)
+
+            html_content = f"""
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <title>{provider} Authentication</title>
+                <style>
+                    body {{ font-family: Arial, sans-serif; text-align: center; padding: 40px; }}
+                    .container {{ max-width: 600px; margin: 0 auto; }}
+                    .success {{ color: #4CAF50; }}
+                    .error {{ color: #F44336; }}
+                    .cancelled {{ color: #FF9800; }}
+                </style>
+                <script>
+                    window.onload = function() {{
+                        const status = {js_status};
+                        const sessionToken = {js_session_token};
+                        const userEmail = {js_user_email};
+                        const providerType = {js_provider_type};
+
+                        if (status === "success" && window.opener) {{
+                            window.opener.postMessage({{
+                                type: providerType + '_auth_success',
+                                session_token: sessionToken,
+                                user_email: userEmail
+                            }}, '*');
+
+                            setTimeout(() => window.close(), 3000);
+                        }} else if (status === "cancelled" || status === "error") {{
+                            setTimeout(() => window.close(), 3000);
+                        }}
+                    }};
+                </script>
+            </head>
+            <body>
+                <div class="container">
+                    <h2>{provider} Authentication</h2>
+                    <div class="{status}">
+                        <p>{message}</p>
+                        {f'<p>Connected as: {user_email}</p>' if status == 'success' else ''}
+                    </div>
+                    <p><small>You can close this window. {f"Your {provider} is now connected and ready to use." if status == 'success' else "Feel free to close this window."}</small></p>
+                </div>
+            </body>
+            </html>
+            """
+
+            return make_response(html_content, 200, {'Content-Type': 'text/html'})
+        except Exception as e:
+            current_app.logger.error(f"Error rendering callback status page: {e}")
+            return make_response("Authentication error occurred", 500, {'Content-Type': 'text/html'})
+
+

application/api/devices/__init__.py

@@ -0,0 +1,11 @@
+"""Flask blueprint for the Remote Device feature."""
+
+from __future__ import annotations
+
+from flask import Blueprint
+
+from .routes import register as register_routes
+
+
+devices_bp = Blueprint("devices", __name__)
+register_routes(devices_bp)

application/api/devices/auth.py

@@ -0,0 +1,141 @@
+"""Device session-token verification + machine-key signature check."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import logging
+import time
+from typing import Optional, Tuple
+
+from flask import jsonify, make_response, request
+
+from application.core.settings import settings
+from application.storage.db.repositories.devices import DevicesRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+def hash_session_token(token: str) -> str:
+    """SHA-256 over the opaque session token. Stored in ``devices.token_hash``."""
+    return hashlib.sha256(token.encode("utf-8")).hexdigest()
+
+
+def fingerprint_pubkey(pubkey_b64: str) -> str:
+    """SHA-256 over the raw public-key bytes; stored as the fingerprint."""
+    raw = base64.b64decode(pubkey_b64)
+    return hashlib.sha256(raw).hexdigest()
+
+
+def _canonical_payload(method: str, path: str, ts: str, body: bytes) -> str:
+    """Canonical string the device signs / the server verifies.
+
+    Format: ``"{method} {path} {ts} {sha256_hex(body)}"``. The body hash
+    binds the request body into the signature so a captured signature can't
+    be replayed with a tampered body inside the timestamp window. For a GET
+    (empty body) this is the SHA-256 of the empty string.
+
+    KEEP IN SYNC with the DocsGPT-cli signer
+    (``internal/host/identity.go`` ``CanonicalPayload`` / ``SignRequest``).
+    The hex encoding and single-space separators must match byte-for-byte.
+    """
+    body_hash = hashlib.sha256(body or b"").hexdigest()
+    return f"{method} {path} {ts} {body_hash}"
+
+
+def verify_device_session(*, touch: bool = True) -> Tuple[Optional[dict], Optional[tuple]]:
+    """Validate the device session token on the current request.
+
+    Returns ``(device_row, None)`` on success or ``(None, response)`` on
+    failure, where ``response`` is a ``(json, status)`` Flask tuple.
+
+    Args:
+        touch: When True, bump ``last_seen_at`` on the device row.
+    """
+    auth = request.headers.get("Authorization", "")
+    if not auth.lower().startswith("bearer "):
+        return None, _error("missing_token", 401)
+    token = auth.split(" ", 1)[1].strip()
+    if not token:
+        return None, _error("missing_token", 401)
+
+    token_hash = hash_session_token(token)
+    with db_readonly() as conn:
+        device = DevicesRepository(conn).find_by_token_hash(token_hash)
+    if device is None:
+        return None, _error("invalid_token", 401)
+
+    if settings.REMOTE_DEVICE_REQUIRE_SIGNATURE:
+        sig_error = _verify_signature(device)
+        if sig_error is not None:
+            return None, sig_error
+
+    if touch:
+        try:
+            with db_session() as conn:
+                DevicesRepository(conn).touch_last_seen(device["id"])
+        except Exception:
+            logger.exception("touch_last_seen failed for device %s", device["id"])
+
+    return device, None
+
+
+def _verify_signature(device: dict) -> Optional[tuple]:
+    """Verify ``X-Device-Signature`` against the stored machine pubkey."""
+    sig_b64 = request.headers.get("X-Device-Signature")
+    ts = request.headers.get("X-Device-Timestamp")
+    fp = request.headers.get("X-Device-Machine-Key")
+    if not sig_b64 or not ts or not fp:
+        return _error("missing_signature", 401)
+
+    try:
+        ts_int = int(ts)
+    except (TypeError, ValueError):
+        return _error("invalid_timestamp", 401)
+    if abs(time.time() - ts_int) > 300:
+        return _error("timestamp_skew", 401)
+
+    if fp != device.get("machine_pubkey_fingerprint"):
+        return _error("fingerprint_mismatch", 401)
+
+    # Defer cryptography import to keep cold-start light when signatures
+    # are disabled (the dev default).
+    try:
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+            Ed25519PublicKey,
+        )
+        from cryptography.exceptions import InvalidSignature
+    except ImportError:
+        logger.error(
+            "Signature verification requested but ``cryptography`` is not installed."
+        )
+        return _error("signature_unsupported", 500)
+
+    # The full public key isn't stored — only the fingerprint. For signature
+    # verification we accept the pubkey in a header too; we trust it iff
+    # its fingerprint matches the stored one. This avoids a separate pubkey
+    # column for MVP.
+    pubkey_b64 = request.headers.get("X-Device-Machine-Pubkey")
+    if not pubkey_b64:
+        return _error("missing_pubkey", 401)
+    if fingerprint_pubkey(pubkey_b64) != device["machine_pubkey_fingerprint"]:
+        return _error("pubkey_fingerprint_mismatch", 401)
+
+    payload = _canonical_payload(
+        request.method, request.path, ts, request.get_data()
+    ).encode("utf-8")
+    try:
+        pubkey = Ed25519PublicKey.from_public_bytes(base64.b64decode(pubkey_b64))
+        pubkey.verify(base64.b64decode(sig_b64), payload)
+    except (InvalidSignature, ValueError):
+        return _error("invalid_signature", 401)
+
+    return None
+
+
+def _error(code: str, status: int) -> tuple:
+    return make_response(
+        jsonify({"success": False, "error": code}), status
+    )

application/api/devices/pairing.py

@@ -0,0 +1,518 @@
+"""Pairing flow (RFC 8628 device authorization grant)."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import secrets
+import time
+import uuid
+from typing import Optional
+
+from flask import jsonify, make_response, request
+from sqlalchemy.exc import IntegrityError
+
+from application.api.devices.auth import fingerprint_pubkey, hash_session_token
+from application.cache import get_redis_instance
+from application.core.settings import settings
+from application.storage.db.repositories.devices import DevicesRepository
+from application.storage.db.repositories.user_tools import UserToolsRepository
+from application.storage.db.session import db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+_USER_CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"
+_USER_CODE_LEN = 8  # ABCD-WXYZ (8 chars, displayed with a single dash)
+_PAIRING_REDIS_PREFIX = "docsgpt:device_pairing:"
+_ALLOWED_APPROVAL_MODES = {"ask", "full"}
+# Attempts to find a non-colliding device name before releasing the claim.
+_DEVICE_NAME_RETRIES = 5
+
+# Atomic redeem claim: read the pairing JSON, and only if its ``status`` is
+# ``pending`` flip it to ``redeemed`` and write it back (TTL preserved).
+# Returns 1 if this caller won the claim, 0 otherwise (missing/not pending).
+# Run server-side so two concurrent redeems of one code can't both win.
+_CLAIM_PAIRING_LUA = """
+local raw = redis.call('GET', KEYS[1])
+if not raw then
+    return 0
+end
+local ok, state = pcall(cjson.decode, raw)
+if not ok then
+    return 0
+end
+if state['status'] ~= 'pending' then
+    return 0
+end
+state['status'] = 'redeemed'
+local ttl = redis.call('TTL', KEYS[1])
+local encoded = cjson.encode(state)
+if ttl and ttl > 0 then
+    redis.call('SET', KEYS[1], encoded, 'EX', ttl)
+else
+    redis.call('SET', KEYS[1], encoded)
+end
+return 1
+"""
+
+
+def _redis():
+    return get_redis_instance()
+
+
+def _generate_user_code() -> str:
+    return "".join(secrets.choice(_USER_CODE_ALPHABET) for _ in range(_USER_CODE_LEN))
+
+
+def _format_user_code(code: str) -> str:
+    return f"{code[:4]}-{code[4:]}"
+
+
+def _normalize_user_code(code: str) -> str:
+    return code.replace("-", "").replace(" ", "").upper()
+
+
+def _user_code_index_key(user_code: str) -> str:
+    return f"docsgpt:device_pairing_code:{user_code}"
+
+
+def create_pairing() -> tuple:
+    """Create a new pairing code for the calling user.
+
+    Optional body fields ``name``, ``description``, and ``approval_mode``
+    are stashed in Redis alongside the pairing state and consumed by
+    ``redeem_pairing`` when the device row is created. Unknown values
+    fall back to: name -> CLI hostname; description -> empty; approval_mode
+    -> ``ask``.
+    """
+    decoded = getattr(request, "decoded_token", None)
+    user_id = decoded.get("sub") if isinstance(decoded, dict) else None
+    if not user_id:
+        return _error("authentication_required", 401)
+
+    body = request.get_json(silent=True) or {}
+    raw_name = body.get("name")
+    if raw_name is not None and not isinstance(raw_name, str):
+        return _error("invalid_name", 400)
+    requested_name = (raw_name or "").strip() or None
+    requested_description = body.get("description")
+    if isinstance(requested_description, str):
+        requested_description = requested_description.strip() or None
+    else:
+        requested_description = None
+    requested_mode = body.get("approval_mode")
+    if requested_mode is not None and requested_mode not in _ALLOWED_APPROVAL_MODES:
+        return _error("invalid_approval_mode", 400)
+
+    redis_client = _redis()
+    if redis_client is None:
+        return _error("redis_unavailable", 503)
+
+    device_code = f"dc_{uuid.uuid4().hex}"
+    user_code = _generate_user_code()
+    ttl = int(settings.REMOTE_DEVICE_PAIRING_TTL_SECONDS)
+
+    state = {
+        "device_code": device_code,
+        "user_code": _format_user_code(user_code),
+        "user_code_raw": user_code,
+        "user_id": user_id,
+        "status": "pending",
+        "created_at": int(time.time()),
+        "requested_name": requested_name,
+        "requested_description": requested_description,
+        "requested_approval_mode": requested_mode,
+    }
+    try:
+        redis_client.setex(
+            _PAIRING_REDIS_PREFIX + device_code, ttl, json.dumps(state)
+        )
+        redis_client.setex(_user_code_index_key(user_code), ttl, device_code)
+    except Exception:
+        logger.exception("redis setex failed during pairing create")
+        return _error("redis_unavailable", 503)
+
+    base_url = settings.API_URL or ""
+    return make_response(
+        jsonify(
+            {
+                "device_code": device_code,
+                "user_code": _format_user_code(user_code),
+                "verification_uri": base_url,
+                "expires_in": ttl,
+                "interval": 3,
+            }
+        ),
+        200,
+    )
+
+
+def get_pairing(device_code: str) -> tuple:
+    """Poll pairing status (UI side)."""
+    decoded = getattr(request, "decoded_token", None)
+    user_id = decoded.get("sub") if isinstance(decoded, dict) else None
+    if not user_id:
+        return _error("authentication_required", 401)
+
+    state = _load_pairing(device_code)
+    if state is None:
+        return _error("pairing_not_found", 404)
+    if state.get("user_id") != user_id:
+        return _error("pairing_not_found", 404)
+
+    return make_response(
+        jsonify(
+            {
+                "device_code": state["device_code"],
+                "user_code": state.get("user_code"),
+                "status": state.get("status", "pending"),
+                "device_id": state.get("device_id"),
+                "device_name": state.get("device_name"),
+            }
+        ),
+        200,
+    )
+
+
+def delete_pairing(device_code: str) -> tuple:
+    decoded = getattr(request, "decoded_token", None)
+    user_id = decoded.get("sub") if isinstance(decoded, dict) else None
+    if not user_id:
+        return _error("authentication_required", 401)
+    state = _load_pairing(device_code)
+    if state is None:
+        return _error("pairing_not_found", 404)
+    if state.get("user_id") != user_id:
+        return _error("pairing_not_found", 404)
+    redis_client = _redis()
+    if redis_client is None:
+        return _error("redis_unavailable", 503)
+    try:
+        redis_client.delete(_PAIRING_REDIS_PREFIX + device_code)
+        if state.get("user_code_raw"):
+            redis_client.delete(_user_code_index_key(state["user_code_raw"]))
+    except Exception:
+        logger.exception("redis delete failed during pairing cancel")
+    return make_response(jsonify({"success": True}), 200)
+
+
+def redeem_pairing() -> tuple:
+    """CLI submits the user_code and machine info to claim a device row.
+
+    Returns ``device_id`` + opaque ``session_token`` (shown once). The
+    token is stored only as ``token_hash`` so a Redis snapshot leak +
+    DB snapshot leak still can't reconstruct it.
+    """
+    body = request.get_json(silent=True) or {}
+    user_code_raw = _normalize_user_code(str(body.get("user_code", "")))
+    if not user_code_raw or len(user_code_raw) != _USER_CODE_LEN:
+        return _error("invalid_user_code", 400)
+    hostname = body.get("hostname") or "unknown"
+    os_name = body.get("os") or ""
+    arch = body.get("arch") or ""
+    cli_version = body.get("cli_version") or ""
+    pubkey_b64 = body.get("machine_pubkey")
+    if not pubkey_b64:
+        return _error("missing_machine_pubkey", 400)
+
+    redis_client = _redis()
+    if redis_client is None:
+        return _error("redis_unavailable", 503)
+    try:
+        device_code = redis_client.get(_user_code_index_key(user_code_raw))
+        if device_code is None:
+            return _error("pairing_not_found", 404)
+        if isinstance(device_code, (bytes, bytearray)):
+            device_code = device_code.decode("utf-8")
+    except Exception:
+        logger.exception("redis get failed during pairing redeem")
+        return _error("redis_unavailable", 503)
+
+    state = _load_pairing(device_code)
+    if state is None:
+        return _error("pairing_not_found", 404)
+    if state.get("status") != "pending":
+        return _error("pairing_already_redeemed", 409)
+
+    # Atomically claim the pairing before doing any work. Two concurrent
+    # redeems of the same code can both pass the check above; only the one
+    # that flips ``pending`` -> ``redeemed`` here proceeds to create a device.
+    claimed = _claim_pairing(redis_client, device_code)
+    if claimed is None:
+        return _error("redis_unavailable", 503)
+    if not claimed:
+        return _error("pairing_already_redeemed", 409)
+
+    user_id = state["user_id"]
+
+    try:
+        fingerprint = fingerprint_pubkey(pubkey_b64)
+    except Exception:
+        return _error("invalid_machine_pubkey", 400)
+    session_token = "tok_" + secrets.token_urlsafe(32)
+    token_hash = hash_session_token(session_token)
+    device_id = "dev_" + uuid.uuid4().hex
+
+    # Apply UI-side overrides stashed at pairing create time, with
+    # sensible fallbacks.
+    requested_name = state.get("requested_name") or None
+    description = state.get("requested_description") or None
+    approval_mode = state.get("requested_approval_mode") or "ask"
+    if approval_mode not in _ALLOWED_APPROVAL_MODES:
+        approval_mode = "ask"
+
+    name_source = requested_name or hostname
+    # The claim has already been consumed, so a failed device insert would
+    # strand a one-time code. ``_next_device_name`` only adds a 16-bit
+    # suffix, so a ``UNIQUE (user_id, name)`` collision is possible; retry
+    # with a fresh name a few times, and release the claim if every attempt
+    # collides so the user can redeem the code again.
+    name = None
+    created = False
+    for _attempt in range(_DEVICE_NAME_RETRIES):
+        candidate = _next_device_name(user_id, name_source)
+        try:
+            with db_session() as conn:
+                DevicesRepository(conn).create(
+                    device_id=device_id,
+                    user_id=user_id,
+                    name=candidate,
+                    machine_pubkey_fingerprint=fingerprint,
+                    token_hash=token_hash,
+                    hostname=hostname,
+                    os=os_name,
+                    arch=arch,
+                    cli_version=cli_version,
+                    approval_mode=approval_mode,
+                    description=description,
+                )
+                _upsert_remote_device_user_tool(
+                    conn, user_id=user_id, device_id=device_id,
+                    device_name=candidate, description=description,
+                )
+            name = candidate
+            created = True
+            break
+        except IntegrityError:
+            # Name collision on this attempt; the transaction rolled back.
+            # Loop to try a freshly-generated name.
+            logger.warning(
+                "device name collision on redeem (attempt %d); retrying",
+                _attempt + 1,
+            )
+            continue
+        except Exception:
+            logger.exception("failed to create device row during redeem")
+            _release_claim(redis_client, device_code, state, user_code_raw)
+            return _error("internal_error", 500)
+
+    if not created:
+        logger.error(
+            "device create exhausted name retries during redeem; "
+            "releasing claim",
+        )
+        _release_claim(redis_client, device_code, state, user_code_raw)
+        return _error("internal_error", 500)
+
+    state["status"] = "redeemed"
+    state["device_id"] = device_id
+    state["device_name"] = name
+    try:
+        redis_client.setex(
+            _PAIRING_REDIS_PREFIX + device_code, 300, json.dumps(state)
+        )
+        if state.get("user_code_raw"):
+            redis_client.delete(_user_code_index_key(state["user_code_raw"]))
+    except Exception:
+        logger.exception("redis update failed after redeem (non-fatal)")
+
+    return make_response(
+        jsonify(
+            {
+                "device_id": device_id,
+                "session_token": session_token,
+                "name": name,
+            }
+        ),
+        200,
+    )
+
+
+def _claim_pairing(redis_client, device_code: str) -> Optional[bool]:
+    """Atomically transition a pairing from ``pending`` to ``redeemed``.
+
+    Returns ``True`` if this caller won the claim, ``False`` if the pairing
+    is already non-pending or gone, and ``None`` if the atomic op itself
+    failed (caller should treat that as Redis-unavailable rather than
+    proceeding, to avoid a non-atomic double-redeem).
+    """
+    key = _PAIRING_REDIS_PREFIX + device_code
+    try:
+        result = redis_client.eval(_CLAIM_PAIRING_LUA, 1, key)
+    except Exception:
+        logger.exception("redis eval failed during pairing claim")
+        return None
+    return bool(result)
+
+
+def _release_claim(
+    redis_client, device_code: str, state: dict, user_code_raw: str,
+) -> None:
+    """Revert a consumed claim so the one-time code is redeemable again.
+
+    Called when the device row can't be created after the atomic claim
+    already flipped the pairing to ``redeemed``. Restores ``pending`` and
+    re-points the user-code index at the device code, preserving the
+    remaining TTL where possible. Best-effort: failures are logged, not
+    raised, since the caller is already returning an error.
+    """
+    key = _PAIRING_REDIS_PREFIX + device_code
+    try:
+        ttl = redis_client.ttl(key)
+        revert = dict(state)
+        revert["status"] = "pending"
+        revert.pop("device_id", None)
+        revert.pop("device_name", None)
+        encoded = json.dumps(revert)
+        if isinstance(ttl, int) and ttl > 0:
+            redis_client.setex(key, ttl, encoded)
+            if user_code_raw:
+                redis_client.setex(
+                    _user_code_index_key(user_code_raw), ttl, device_code
+                )
+        else:
+            redis_client.set(key, encoded)
+            if user_code_raw:
+                redis_client.set(_user_code_index_key(user_code_raw), device_code)
+    except Exception:
+        logger.exception("failed to release pairing claim for %s", device_code)
+
+
+def _load_pairing(device_code: str) -> Optional[dict]:
+    redis_client = _redis()
+    if redis_client is None:
+        return None
+    try:
+        raw = redis_client.get(_PAIRING_REDIS_PREFIX + device_code)
+    except Exception:
+        return None
+    if raw is None:
+        return None
+    if isinstance(raw, (bytes, bytearray)):
+        raw = raw.decode("utf-8")
+    try:
+        return json.loads(raw)
+    except Exception:
+        return None
+
+
+def _next_device_name(user_id: str, hostname: str) -> str:
+    """Return a candidate name for the new device row.
+
+    Adds a random 16-bit suffix to reduce collisions on the DB
+    ``UNIQUE (user_id, name)`` constraint. Collisions are still possible,
+    so ``redeem_pairing`` retries with a fresh candidate on conflict.
+    """
+    base = hostname or "device"
+    base = base.strip() or "device"
+    return f"{base}-{os.urandom(2).hex()}"
+
+
+def _upsert_remote_device_user_tool(
+    conn, *, user_id: str, device_id: str,
+    device_name: str, description: Optional[str],
+) -> None:
+    """Create or update the ``user_tools`` row that fronts this device."""
+    repo = UserToolsRepository(conn)
+    existing = _find_remote_device_tool_row(conn, user_id, device_id)
+    actions = [
+        {
+            "name": "run_command",
+            "description": (
+                f"Execute a shell command on the remote device '{device_name}'. "
+                f"{description or ''}".strip()
+            ),
+            "active": True,
+            # ``tool_executor`` consults ``RemoteDeviceTool.preview_requires_approval``
+            # live for ``remote_device`` calls, so the static snapshot must
+            # stay neutral. The tool's own ``execute_action`` still yields
+            # the awaiting-approval pause when the live decision says so.
+            "require_approval": False,
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "command": {
+                        "type": "string",
+                        "description": "Shell command to run.",
+                        "filled_by_llm": True,
+                        "value": "",
+                    },
+                    "working_directory": {
+                        "type": "string",
+                        "description": "Working directory on the remote.",
+                        "filled_by_llm": True,
+                        "value": "",
+                    },
+                    "timeout_ms": {
+                        "type": "integer",
+                        "description": "Timeout in milliseconds (max 600000).",
+                        "filled_by_llm": True,
+                        "value": "",
+                    },
+                },
+                "required": ["command"],
+            },
+        }
+    ]
+    config = {"device_id": device_id}
+    display_name = f"Remote device — {device_name}"
+    if existing:
+        repo.update(
+            str(existing["id"]),
+            user_id,
+            {
+                "display_name": display_name,
+                "description": description or "",
+                "config": config,
+                "actions": actions,
+                "status": True,
+            },
+        )
+    else:
+        repo.create(
+            user_id=user_id,
+            name="remote_device",
+            display_name=display_name,
+            description=description or "",
+            config=config,
+            actions=actions,
+            status=True,
+        )
+
+
+def _find_remote_device_tool_row(conn, user_id: str, device_id: str):
+    """Locate the user_tools row whose config.device_id matches."""
+    from sqlalchemy import text
+    row = conn.execute(
+        text(
+            """
+            SELECT *
+            FROM user_tools
+            WHERE user_id = :user_id
+              AND name    = 'remote_device'
+              AND config ->> 'device_id' = :device_id
+            LIMIT 1
+            """
+        ),
+        {"user_id": user_id, "device_id": device_id},
+    ).fetchone()
+    if row is None:
+        return None
+    return dict(row._mapping)
+
+
+def _error(code: str, status: int) -> tuple:
+    return make_response(jsonify({"success": False, "error": code}), status)

application/api/devices/routes.py

@@ -0,0 +1,327 @@
+"""Device CRUD + auto-approve pattern routes."""
+
+from __future__ import annotations
+
+import logging
+
+from flask import Blueprint, jsonify, make_response, request
+
+from application.api.devices.pairing import (
+    create_pairing,
+    delete_pairing,
+    get_pairing,
+    redeem_pairing,
+)
+from application.api.devices.session import (
+    ack_invocation,
+    me,
+    poll,
+    session_events,
+    submit_output,
+)
+from application.devices.normalizer import normalize_command
+from application.storage.db.repositories.device_audit_log import (
+    DeviceAuditLogRepository,
+)
+from application.storage.db.repositories.device_auto_approve_patterns import (
+    DeviceAutoApprovePatternsRepository,
+)
+from application.storage.db.repositories.devices import DevicesRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+_ALLOWED_APPROVAL_MODES = {"ask", "full"}
+
+
+def _authed_user_id():
+    decoded = getattr(request, "decoded_token", None)
+    if isinstance(decoded, dict):
+        return decoded.get("sub")
+    return None
+
+
+def _serialize_device(row: dict) -> dict:
+    """Strip server-internal fields before returning to the UI."""
+    out = {
+        "id": row.get("id"),
+        "name": row.get("name"),
+        "hostname": row.get("hostname"),
+        "os": row.get("os"),
+        "arch": row.get("arch"),
+        "cli_version": row.get("cli_version"),
+        "approval_mode": row.get("approval_mode"),
+        "description": row.get("description"),
+        "status": row.get("status"),
+        "paired_at": row.get("paired_at"),
+        "last_seen_at": row.get("last_seen_at"),
+        "revoked_at": row.get("revoked_at"),
+    }
+    # JSON-ify datetimes if SQLAlchemy returned them raw.
+    for key in ("paired_at", "last_seen_at", "revoked_at"):
+        value = out.get(key)
+        if value is not None and not isinstance(value, str):
+            out[key] = value.isoformat()
+    return out
+
+
+def list_devices():
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    with db_readonly() as conn:
+        rows = DevicesRepository(conn).list_for_user(user_id)
+    return make_response(
+        jsonify({"devices": [_serialize_device(r) for r in rows]}),
+        200,
+    )
+
+
+def get_device(device_id: str):
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    with db_readonly() as conn:
+        row = DevicesRepository(conn).get(device_id, user_id=user_id)
+    if row is None:
+        return make_response(jsonify({"success": False, "error": "not_found"}), 404)
+    return make_response(jsonify(_serialize_device(row)), 200)
+
+
+def update_device(device_id: str):
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    body = request.get_json(silent=True) or {}
+    update_fields: dict = {}
+    if "name" in body:
+        name = body["name"]
+        if not isinstance(name, str) or not name.strip():
+            return make_response(
+                jsonify({"success": False, "error": "invalid_name"}), 400
+            )
+        update_fields["name"] = name.strip()
+    if "description" in body:
+        description = body["description"]
+        if not isinstance(description, str):
+            return make_response(
+                jsonify({"success": False, "error": "invalid_description"}), 400
+            )
+        update_fields["description"] = description.strip()
+    if "approval_mode" in body:
+        mode = body["approval_mode"]
+        if mode not in _ALLOWED_APPROVAL_MODES:
+            return make_response(
+                jsonify({"success": False, "error": "invalid_approval_mode"}), 400
+            )
+        update_fields["approval_mode"] = mode
+    if not update_fields:
+        return make_response(jsonify({"success": False, "error": "no_fields"}), 400)
+    with db_session() as conn:
+        ok = DevicesRepository(conn).update(device_id, user_id, update_fields)
+        if not ok:
+            return make_response(
+                jsonify({"success": False, "error": "not_found"}), 404
+            )
+        # Reflect name/description into the user_tools row that fronts this device.
+        if "name" in update_fields or "description" in update_fields:
+            from application.api.devices.pairing import (
+                _upsert_remote_device_user_tool,
+            )
+            row = DevicesRepository(conn).get(device_id, user_id=user_id)
+            if row is not None:
+                _upsert_remote_device_user_tool(
+                    conn,
+                    user_id=user_id,
+                    device_id=device_id,
+                    device_name=row.get("name") or "device",
+                    description=row.get("description"),
+                )
+        row = DevicesRepository(conn).get(device_id, user_id=user_id)
+    return make_response(jsonify(_serialize_device(row)), 200)
+
+
+def delete_device(device_id: str):
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    with db_session() as conn:
+        ok = DevicesRepository(conn).revoke(device_id, user_id)
+        if not ok:
+            return make_response(
+                jsonify({"success": False, "error": "not_found"}), 404
+            )
+        # Drop the corresponding user_tools row so the tool picker stops showing it.
+        from sqlalchemy import text
+        conn.execute(
+            text(
+                """
+                DELETE FROM user_tools
+                WHERE user_id = :user_id
+                  AND name    = 'remote_device'
+                  AND config ->> 'device_id' = :device_id
+                """
+            ),
+            {"user_id": user_id, "device_id": device_id},
+        )
+        DeviceAutoApprovePatternsRepository(conn).clear_for_device(device_id)
+    return make_response(jsonify({"success": True}), 200)
+
+
+def add_auto_approve_pattern(device_id: str):
+    """Server-side normalization: client sends the raw command, we store the pattern."""
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    body = request.get_json(silent=True) or {}
+    command = body.get("command") or body.get("pattern")
+    if not command:
+        return make_response(
+            jsonify({"success": False, "error": "missing_command"}), 400
+        )
+    if not isinstance(command, str):
+        return make_response(
+            jsonify({"success": False, "error": "invalid_command"}), 400
+        )
+    pattern = normalize_command(command)
+    if not pattern:
+        return make_response(
+            jsonify({"success": False, "error": "invalid_command"}), 400
+        )
+    with db_session() as conn:
+        if DevicesRepository(conn).get(device_id, user_id=user_id) is None:
+            return make_response(
+                jsonify({"success": False, "error": "not_found"}), 404
+            )
+        DeviceAutoApprovePatternsRepository(conn).add(device_id, user_id, pattern)
+    return make_response(jsonify({"pattern": pattern}), 200)
+
+
+def list_auto_approve_patterns(device_id: str):
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    with db_readonly() as conn:
+        if DevicesRepository(conn).get(device_id, user_id=user_id) is None:
+            return make_response(
+                jsonify({"success": False, "error": "not_found"}), 404
+            )
+        patterns = DeviceAutoApprovePatternsRepository(conn).list_for_device(
+            device_id, user_id
+        )
+    return make_response(jsonify({"patterns": patterns}), 200)
+
+
+def delete_auto_approve_pattern(device_id: str):
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    body = request.get_json(silent=True) or {}
+    pattern = body.get("pattern")
+    if not pattern:
+        return make_response(
+            jsonify({"success": False, "error": "missing_pattern"}), 400
+        )
+    with db_session() as conn:
+        ok = DeviceAutoApprovePatternsRepository(conn).remove(
+            device_id, user_id, pattern
+        )
+    return make_response(jsonify({"success": ok}), 200)
+
+
+def list_audit(device_id: str):
+    user_id = _authed_user_id()
+    if not user_id:
+        return make_response(jsonify({"success": False, "error": "auth"}), 401)
+    with db_readonly() as conn:
+        if DevicesRepository(conn).get(device_id, user_id=user_id) is None:
+            return make_response(
+                jsonify({"success": False, "error": "not_found"}), 404
+            )
+        rows = DeviceAuditLogRepository(conn).list_for_device(device_id, user_id)
+    # Normalize datetimes to ISO strings.
+    serialized = []
+    for row in rows:
+        out = dict(row)
+        for key in ("issued_at", "started_at", "finished_at", "created_at"):
+            v = out.get(key)
+            if v is not None and not isinstance(v, str):
+                out[key] = v.isoformat()
+        serialized.append(out)
+    return make_response(jsonify({"entries": serialized}), 200)
+
+
+def register(bp: Blueprint) -> None:
+    """Attach all device routes to ``bp``."""
+    bp.add_url_rule(
+        "/api/devices", view_func=list_devices, methods=["GET"], endpoint="list_devices",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>", view_func=get_device, methods=["GET"],
+        endpoint="get_device",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>", view_func=update_device, methods=["PATCH"],
+        endpoint="update_device",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>", view_func=delete_device, methods=["DELETE"],
+        endpoint="delete_device",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>/auto-approve",
+        view_func=add_auto_approve_pattern, methods=["POST"],
+        endpoint="add_auto_approve_pattern",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>/auto-approve",
+        view_func=list_auto_approve_patterns, methods=["GET"],
+        endpoint="list_auto_approve_patterns",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>/auto-approve",
+        view_func=delete_auto_approve_pattern, methods=["DELETE"],
+        endpoint="delete_auto_approve_pattern",
+    )
+    bp.add_url_rule(
+        "/api/devices/<device_id>/audit", view_func=list_audit, methods=["GET"],
+        endpoint="list_audit",
+    )
+    # Pairing endpoints
+    bp.add_url_rule(
+        "/api/devices/pairings", view_func=create_pairing, methods=["POST"],
+        endpoint="create_pairing",
+    )
+    bp.add_url_rule(
+        "/api/devices/pairings/redeem", view_func=redeem_pairing, methods=["POST"],
+        endpoint="redeem_pairing",
+    )
+    bp.add_url_rule(
+        "/api/devices/pairings/<device_code>", view_func=get_pairing, methods=["GET"],
+        endpoint="get_pairing",
+    )
+    bp.add_url_rule(
+        "/api/devices/pairings/<device_code>", view_func=delete_pairing,
+        methods=["DELETE"], endpoint="delete_pairing",
+    )
+    # Session endpoints (device-token auth)
+    bp.add_url_rule(
+        "/api/devices/poll", view_func=poll, methods=["GET"], endpoint="device_poll",
+    )
+    bp.add_url_rule(
+        "/api/devices/me", view_func=me, methods=["GET"], endpoint="device_me",
+    )
+    bp.add_url_rule(
+        "/api/devices/sessions/<session_id>/events", view_func=session_events,
+        methods=["GET"], endpoint="device_session_events",
+    )
+    bp.add_url_rule(
+        "/api/devices/sessions/<session_id>/invocations/<invocation_id>/ack",
+        view_func=ack_invocation, methods=["POST"], endpoint="device_ack",
+    )
+    bp.add_url_rule(
+        "/api/devices/sessions/<session_id>/invocations/<invocation_id>/output",
+        view_func=submit_output, methods=["POST"], endpoint="device_output",
+    )

application/api/devices/session.py

@@ -0,0 +1,272 @@
+"""Device session endpoints: poll, SSE, ack, output."""
+
+from __future__ import annotations
+
+import gzip
+import io
+import json
+import logging
+import time
+from typing import Iterator
+
+from flask import Response, jsonify, make_response, request, stream_with_context
+
+from application.api.devices.auth import verify_device_session
+from application.core.settings import settings
+from application.core.shutdown import is_shutting_down
+from application.devices.broker import get_broker
+from application.storage.db.repositories.device_audit_log import (
+    DeviceAuditLogRepository,
+)
+from application.storage.db.session import db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+# Window (seconds) the CLI has to upgrade a poll-issued ticket to an SSE
+# stream. Advertised to the CLI as ``expires_in`` and used as the broker
+# ticket TTL so the two never drift.
+_SESSION_TICKET_TTL_SECONDS = 30
+
+
+def poll() -> Response:
+    """``GET /api/devices/poll`` — long-poll for queued invocations.
+
+    Returns ``202`` with empty body when nothing is queued and ``200`` with
+    a session ticket payload when work is waiting.
+    """
+    device, err = verify_device_session()
+    if err is not None:
+        return err
+    broker = get_broker()
+    ticket = broker.claim_ticket(device["id"], _SESSION_TICKET_TTL_SECONDS)
+    if ticket is None:
+        return make_response("", 202)
+    return make_response(
+        jsonify(
+            {
+                "session_ticket": ticket,
+                "session_url": f"/api/devices/sessions/{ticket}/events",
+                "expires_in": _SESSION_TICKET_TTL_SECONDS,
+            }
+        ),
+        200,
+    )
+
+
+def me() -> Response:
+    """``GET /api/devices/me`` — return the calling device's own record.
+
+    Auth: device session token (same as ``/poll``). Used by ``docsgpt-cli
+    host status`` to show live server state.
+    """
+    device, err = verify_device_session()
+    if err is not None:
+        return err
+    out = {
+        "id": device.get("id"),
+        "name": device.get("name"),
+        "hostname": device.get("hostname"),
+        "os": device.get("os"),
+        "status": device.get("status"),
+        "approval_mode": device.get("approval_mode"),
+        "description": device.get("description"),
+        "paired_at": device.get("paired_at"),
+        "last_seen_at": device.get("last_seen_at"),
+    }
+    for key in ("paired_at", "last_seen_at"):
+        value = out.get(key)
+        if value is not None and not isinstance(value, str):
+            out[key] = value.isoformat()
+    return make_response(jsonify(out), 200)
+
+
+def session_events(session_id: str) -> Response:
+    """``GET /api/devices/sessions/{id}/events`` — SSE invocation stream.
+
+    The ``session_id`` must be the ``session_ticket`` the device's own
+    ``/poll`` just issued (the path it was handed as ``session_url``). A
+    stale, mismatched, or fabricated ticket is rejected with ``410 Gone``
+    before any stream is opened.
+    """
+    device, err = verify_device_session()
+    if err is not None:
+        return err
+    broker = get_broker()
+    if not broker.validate_ticket(device["id"], session_id):
+        return make_response(
+            jsonify({"success": False, "error": "session_ticket_invalid"}), 410
+        )
+    sess = broker.register_session(device["id"], device["user_id"])
+
+    keepalive_interval = float(settings.SSE_KEEPALIVE_SECONDS)
+    idle_seconds = float(settings.REMOTE_DEVICE_SESSION_IDLE_SECONDS)
+
+    @stream_with_context
+    def generate() -> Iterator[str]:
+        try:
+            last_keepalive = time.time()
+            while not sess.closed.is_set():
+                # Break promptly on shutdown (see application/core/shutdown.py).
+                if is_shutting_down():
+                    break
+                now = time.time()
+                if now - sess.last_activity_at > idle_seconds:
+                    yield _sse_event(
+                        "session_end",
+                        {"reason": "inactivity_timeout"},
+                        sess.last_event_id + 1,
+                    )
+                    sess.last_event_id += 1
+                    broker.close_session(sess.session_id, reason="idle")
+                    return
+                envelope = broker.next_command(sess, timeout=1.0)
+                if envelope is None:
+                    if time.time() - last_keepalive >= keepalive_interval:
+                        last_keepalive = time.time()
+                        yield ": heartbeat\n\n"
+                    continue
+                sess.last_event_id += 1
+                sess.last_activity_at = time.time()
+                yield _sse_event("invocation", envelope, sess.last_event_id)
+                last_keepalive = time.time()
+        except GeneratorExit:
+            logger.debug("device SSE generator exiting for session %s", sess.session_id)
+            raise
+        finally:
+            broker.close_session(sess.session_id, reason="generator_exit")
+
+    return Response(
+        generate(),
+        mimetype="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+def ack_invocation(session_id: str, invocation_id: str) -> Response:
+    """CLI acks (accepted / denied / auto_approved) the invocation."""
+    device, err = verify_device_session()
+    if err is not None:
+        return err
+    body = request.get_json(silent=True) or {}
+    decision = body.get("decision")
+    reason = body.get("reason")
+    if decision not in {"accepted", "denied", "auto_approved"}:
+        return make_response(
+            jsonify({"success": False, "error": "invalid_decision"}), 400
+        )
+    broker = get_broker()
+    inv = broker.get_invocation(invocation_id)
+    if inv is None or inv.device_id != device["id"]:
+        return make_response(
+            jsonify({"success": False, "error": "invocation_not_found"}), 404
+        )
+    broker.submit_ack(invocation_id, decision, reason)
+    if decision == "denied":
+        # A denial is terminal and produces no device output, so submit_output's
+        # audit write is never reached. Record the outcome here from locally
+        # known facts (not re-read Redis state the agent's drain races to clean
+        # up), so the audit row reflects the denial instead of staying
+        # "dispatched". Accepted/auto_approved runs record via submit_output.
+        from datetime import datetime, timezone
+        try:
+            with db_session() as conn:
+                DeviceAuditLogRepository(conn).record_result(
+                    invocation_id,
+                    finished_at=datetime.now(timezone.utc),
+                    error="denied",
+                )
+        except Exception:
+            logger.exception("audit record_result (denied) failed for %s", invocation_id)
+    return make_response(jsonify({"success": True}), 200)
+
+
+def submit_output(session_id: str, invocation_id: str) -> Response:
+    """CLI streams stdout/stderr/control chunks (NDJSON, gzip-aware)."""
+    device, err = verify_device_session()
+    if err is not None:
+        return err
+    broker = get_broker()
+    inv = broker.get_invocation(invocation_id)
+    if inv is None or inv.device_id != device["id"]:
+        return make_response(
+            jsonify({"success": False, "error": "invocation_not_found"}), 404
+        )
+
+    body = request.get_data() or b""
+    if request.headers.get("Content-Encoding", "").lower() == "gzip":
+        try:
+            body = gzip.decompress(body)
+        except OSError:
+            return make_response(
+                jsonify({"success": False, "error": "invalid_gzip"}), 400
+            )
+
+    received = 0
+    control_chunk = None
+    for line in io.BytesIO(body):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            chunk = json.loads(line.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            continue
+        if not isinstance(chunk, dict):
+            continue
+        if chunk.get("stream") == "control":
+            control_chunk = chunk
+        broker.submit_output_chunk(invocation_id, chunk)
+        received += 1
+
+    # Persist the outcome when the closing control chunk arrived in this POST.
+    # Its fields are captured locally so the audit write survives the draining
+    # (worker) process racing to delete the invocation's Redis state. Byte
+    # totals / started_at live in the hash, read best-effort (the functional
+    # exit_code/error/duration still land even if the hash is already gone).
+    if control_chunk is not None:
+        from datetime import datetime, timezone
+        snap = broker.get_invocation(invocation_id)
+        try:
+            with db_session() as conn:
+                DeviceAuditLogRepository(conn).record_result(
+                    invocation_id,
+                    started_at=(
+                        datetime.fromtimestamp(snap.started_at, tz=timezone.utc)
+                        if snap is not None and snap.started_at else None
+                    ),
+                    finished_at=datetime.now(timezone.utc),
+                    exit_code=_as_opt_int(control_chunk.get("exit_code")),
+                    duration_ms=_as_opt_int(control_chunk.get("duration_ms")),
+                    stdout_bytes=(snap.stdout_bytes if snap is not None else 0),
+                    stderr_bytes=(snap.stderr_bytes if snap is not None else 0),
+                    error=control_chunk.get("error"),
+                )
+        except Exception:
+            logger.exception("audit record_result failed for %s", invocation_id)
+
+    return make_response(
+        jsonify({"success": True, "received": received}), 200
+    )
+
+
+def _as_opt_int(value) -> int | None:
+    """Coerce a CLI-supplied JSON value to int for an INTEGER audit column."""
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _sse_event(name: str, payload: dict, event_id: int) -> str:
+    return (
+        f"event: {name}\n"
+        f"id: {event_id}\n"
+        f"data: {json.dumps(payload)}\n\n"
+    )

application/api/events/routes.py

@@ -0,0 +1,537 @@
+"""GET /api/events — user-scoped Server-Sent Events endpoint.
+
+Subscribe-then-snapshot pattern: subscribe to ``user:{user_id}``
+pub/sub, snapshot the Redis Streams backlog past ``Last-Event-ID``
+inside the SUBSCRIBE-ack callback, flush snapshot, then tail live
+events (dedup'd by stream id). See ``docs/runbooks/sse-notifications.md``.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from typing import Iterator, Optional
+
+from flask import Blueprint, Response, jsonify, make_response, request, stream_with_context
+
+from application.cache import get_redis_instance
+from application.core.settings import settings
+from application.core.shutdown import is_shutting_down
+from application.events.keys import (
+    connection_counter_key,
+    replay_budget_key,
+    stream_id_compare,
+    stream_key,
+    topic_name,
+)
+from application.streaming.broadcast_channel import Topic
+
+logger = logging.getLogger(__name__)
+
+events = Blueprint("event_stream", __name__)
+
+SUBSCRIBE_POLL_INTERVAL_SECONDS = 1.0
+
+# WHATWG SSE treats CRLF, CR, and LF equivalently as line terminators.
+_SSE_LINE_SPLIT = re.compile(r"\r\n|\r|\n")
+
+# Redis Streams ids are ``ms`` or ``ms-seq`` where both halves are decimal.
+# Anything else is a corrupted client cookie / IndexedDB residue and must
+# not be passed to XRANGE — Redis would reject it and our truncation gate
+# would silently fail.
+_STREAM_ID_RE = re.compile(r"^\d+(-\d+)?$")
+
+# Only emitted at most once per process so a misconfigured deployment
+# doesn't drown the logs.
+_local_user_warned = False
+
+
+def _format_sse(data: str, *, event_id: Optional[str] = None) -> str:
+    """Encode a payload as one SSE message terminated by a blank line.
+
+    Splits on any line-terminator variant (``\\r\\n``, ``\\r``, ``\\n``)
+    so a stray CR in upstream content can't smuggle a premature line
+    boundary into the wire format.
+    """
+    lines: list[str] = []
+    if event_id:
+        lines.append(f"id: {event_id}")
+    for line in _SSE_LINE_SPLIT.split(data):
+        lines.append(f"data: {line}")
+    return "\n".join(lines) + "\n\n"
+
+
+def _decode(value) -> Optional[str]:
+    if value is None:
+        return None
+    if isinstance(value, (bytes, bytearray)):
+        try:
+            return value.decode("utf-8")
+        except Exception:
+            return None
+    return str(value)
+
+
+def _oldest_retained_id(redis_client, user_id: str) -> Optional[str]:
+    """Return the id of the oldest entry still in the stream, or ``None``.
+
+    Used to detect ``Last-Event-ID`` having slid off the back of the
+    MAXLEN'd window.
+    """
+    try:
+        info = redis_client.xinfo_stream(stream_key(user_id))
+    except Exception:
+        return None
+    if not isinstance(info, dict):
+        return None
+    # redis-py 7.4 returns str-keyed dicts here; the bytes-key probe is
+    # defence in depth in case ``decode_responses`` is ever flipped.
+    first_entry = info.get("first-entry") or info.get(b"first-entry")
+    if not first_entry:
+        return None
+    # XINFO STREAM returns first-entry as [id, [field, value, ...]]
+    try:
+        return _decode(first_entry[0])
+    except Exception:
+        return None
+
+
+def _allow_replay(
+    redis_client, user_id: str, last_event_id: Optional[str]
+) -> bool:
+    """Per-user sliding-window snapshot-replay budget.
+
+    Fails open on Redis errors or when the budget is disabled. No-cursor
+    connects never consume budget: fresh sessions start live and do no
+    snapshot work, so counting them only starved real reconnects (a burst
+    of fresh tabs could 429 a user's cursor-bearing reconnect).
+    """
+    budget = int(settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW)
+    if budget <= 0:
+        return True
+    if redis_client is None:
+        return True
+    if last_event_id is None:
+        return True
+
+    window = max(1, int(settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS))
+    key = replay_budget_key(user_id)
+    try:
+        used = int(redis_client.incr(key))
+        # Always (re)seed the TTL. Gating on ``used == 1`` would wedge
+        # the counter forever if INCR succeeds but EXPIRE raises on
+        # the seeding call. EXPIRE on an existing key resets the TTL
+        # to ``window`` — within ±1s of the per-window budget semantic.
+        redis_client.expire(key, window)
+    except Exception:
+        logger.debug(
+            "replay budget probe failed for user=%s; failing open",
+            user_id,
+        )
+        return True
+    return used <= budget
+
+
+def _normalize_last_event_id(raw: Optional[str]) -> Optional[str]:
+    """Validate the ``Last-Event-ID`` header / query param.
+
+    Returns the value unchanged when it parses as a Redis Streams id,
+    otherwise ``None`` — callers treat ``None`` as "client has nothing"
+    and start the stream live (no snapshot). Invalid ids would
+    otherwise pass straight to XRANGE and surface as a quiet replay
+    failure plus broken truncation detection.
+    """
+    if raw is None:
+        return None
+    raw = raw.strip()
+    if not raw or not _STREAM_ID_RE.match(raw):
+        return None
+    return raw
+
+
+def _replay_floor_id() -> Optional[str]:
+    """Oldest stream id the snapshot replay may reach, or ``None``.
+
+    Streams ids are millisecond timestamps, so an age ceiling maps
+    directly to an id floor. MAXLEN caps the stream by count, not time —
+    for a low-traffic user 1000 entries can span weeks, and shipping
+    that on reconnect helps no one.
+    """
+    max_age_hours = int(settings.EVENTS_REPLAY_MAX_AGE_HOURS)
+    if max_age_hours <= 0:
+        return None
+    floor_ms = int(time.time() * 1000) - max_age_hours * 3600 * 1000
+    return f"{floor_ms}-0"
+
+
+def _replay_backlog(
+    redis_client, user_id: str, last_event_id: Optional[str], max_count: int
+) -> Iterator[tuple[str, str]]:
+    """Yield ``(entry_id, sse_line)`` for backlog entries past ``last_event_id``.
+
+    Capped at ``max_count`` rows; clients catch up across reconnects.
+    Parse failures are skipped; the Streams id is injected into the
+    envelope so replay matches live-tail shape.
+    """
+    floor = _replay_floor_id()
+    if last_event_id is None:
+        start = floor or "-"
+    elif floor and stream_id_compare(last_event_id, floor) < 0:
+        # Cursor is past the age ceiling: clamp to the floor (inclusive).
+        # The caller emits ``backlog.truncated`` for this case.
+        start = floor
+    else:
+        # Exclusive start: '(<id>' skips the already-delivered entry.
+        start = f"({last_event_id}"
+    try:
+        entries = redis_client.xrange(
+            stream_key(user_id), min=start, max="+", count=max_count
+        )
+    except Exception as exc:
+        logger.warning(
+            "xrange replay failed for user=%s last_id=%s err=%s",
+            user_id,
+            last_event_id or "-",
+            exc,
+        )
+        return
+
+    for entry_id, fields in entries:
+        entry_id_str = _decode(entry_id)
+        if not entry_id_str:
+            continue
+        # decode_responses=False on the cache client ⇒ field keys/values
+        # are bytes. The string-key fallback covers a future flip of that
+        # default without a forced refactor here.
+        raw_event = None
+        if isinstance(fields, dict):
+            raw_event = fields.get(b"event")
+            if raw_event is None:
+                raw_event = fields.get("event")
+        event_str = _decode(raw_event)
+        if not event_str:
+            continue
+        try:
+            envelope = json.loads(event_str)
+            if isinstance(envelope, dict):
+                envelope["id"] = entry_id_str
+                event_str = json.dumps(envelope)
+        except Exception:
+            logger.debug(
+                "Replay envelope parse failed for entry %s; passing through raw",
+                entry_id_str,
+            )
+        yield entry_id_str, _format_sse(event_str, event_id=entry_id_str)
+
+
+def _truncation_notice_line(oldest_id: str) -> str:
+    """SSE event the frontend can react to with a full-state refetch."""
+    return _format_sse(
+        json.dumps(
+            {
+                "type": "backlog.truncated",
+                "payload": {"oldest_retained_id": oldest_id},
+            }
+        )
+    )
+
+
+@events.route("/api/events", methods=["GET"])
+def stream_events() -> Response:
+    decoded = getattr(request, "decoded_token", None)
+    user_id = decoded.get("sub") if isinstance(decoded, dict) else None
+    if not user_id:
+        return make_response(
+            jsonify({"success": False, "message": "Authentication required"}),
+            401,
+        )
+
+    # In dev deployments without AUTH_TYPE configured, every request
+    # resolves to user_id="local" and shares one stream. Surface this so
+    # an accidentally-multi-user dev box doesn't silently cross-stream.
+    global _local_user_warned
+    if user_id == "local" and not _local_user_warned:
+        logger.warning(
+            "SSE serving user_id='local' (AUTH_TYPE not set). "
+            "All clients on this deployment will share one event stream."
+        )
+        _local_user_warned = True
+
+    raw_last_event_id = request.headers.get("Last-Event-ID") or request.args.get(
+        "last_event_id"
+    )
+    last_event_id = _normalize_last_event_id(raw_last_event_id)
+    last_event_id_invalid = raw_last_event_id is not None and last_event_id is None
+
+    keepalive_seconds = float(settings.SSE_KEEPALIVE_SECONDS)
+    push_enabled = settings.ENABLE_SSE_PUSH
+    cap = int(settings.SSE_MAX_CONCURRENT_PER_USER)
+
+    redis_client = get_redis_instance()
+    counter_key = connection_counter_key(user_id)
+    counted = False
+
+    if push_enabled and redis_client is not None and cap > 0:
+        try:
+            current = int(redis_client.incr(counter_key))
+            counted = True
+        except Exception:
+            current = 0
+            logger.debug(
+                "SSE connection counter INCR failed for user=%s", user_id
+            )
+        if counted:
+            # 1h safety TTL — orphaned counts from hard crashes self-heal.
+            # EXPIRE failure must NOT clobber ``current`` and bypass the cap.
+            try:
+                redis_client.expire(counter_key, 3600)
+            except Exception:
+                logger.debug(
+                    "SSE connection counter EXPIRE failed for user=%s", user_id
+                )
+            if current > cap:
+                try:
+                    redis_client.decr(counter_key)
+                except Exception:
+                    logger.debug(
+                        "SSE connection counter DECR failed for user=%s",
+                        user_id,
+                    )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Too many concurrent SSE connections",
+                        }
+                    ),
+                    429,
+                )
+
+    # Replay budget is checked here, before the generator opens the
+    # stream, so a denial can surface as HTTP 429 instead of a silent
+    # snapshot skip. The earlier in-generator skip lost events between
+    # the client's cursor and the first live-tailed entry: the live
+    # tail still carried ``id:`` headers, the frontend advanced
+    # ``lastEventId`` to one of those ids, and the events in between
+    # were never reachable on the next reconnect. 429 keeps the
+    # cursor pinned and lets the frontend back off until the window
+    # slides (eventStreamClient.ts treats 429 as escalated backoff).
+    if push_enabled and redis_client is not None and not _allow_replay(
+        redis_client, user_id, last_event_id
+    ):
+        if counted:
+            try:
+                redis_client.decr(counter_key)
+            except Exception:
+                logger.debug(
+                    "SSE connection counter DECR failed for user=%s",
+                    user_id,
+                )
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": "Replay budget exhausted",
+                }
+            ),
+            429,
+        )
+
+    @stream_with_context
+    def generate() -> Iterator[str]:
+        connect_ts = time.monotonic()
+        replayed_count = 0
+        try:
+            # First frame primes intermediaries (Cloudflare, nginx) so they
+            # don't sit on a buffer waiting for body bytes.
+            yield ": connected\n\n"
+
+            if not push_enabled:
+                yield ": push_disabled\n\n"
+                return
+
+            replay_lines: list[str] = []
+            max_replayed_id: Optional[str] = None
+            replay_done = False
+
+            # If the client sent a malformed Last-Event-ID, surface the
+            # truncation notice synchronously *before* the subscribe
+            # loop. Buffering it into ``replay_lines`` would lose it
+            # when ``Topic.subscribe`` returns immediately (Redis down)
+            # — the loop body never runs, and the flush at line ~335
+            # never fires.
+            if last_event_id_invalid:
+                yield _truncation_notice_line("")
+                replayed_count += 1
+
+            def _on_subscribe_callback() -> None:
+                # Runs synchronously inside Topic.subscribe after the
+                # SUBSCRIBE is acked. By doing XRANGE here, any publisher
+                # firing between SUBSCRIBE-send and SUBSCRIBE-ack has its
+                # XADD captured by XRANGE *and* its PUBLISH buffered at
+                # the connection layer until we read it — closing the
+                # replay/subscribe race the design doc warns about.
+                #
+                # Truncation contract: ``backlog.truncated`` is emitted
+                # ONLY when the client's ``Last-Event-ID`` has slid off
+                # the MAXLEN'd window — that's the case where the
+                # journal is genuinely gone past the cursor and the
+                # frontend should clear its slice cursor and refetch
+                # state. Cap-hit skips the snapshot silently: the
+                # cursor advances via the per-entry ``id:`` headers
+                # and the frontend's slice keeps the latest id so the
+                # next reconnect resumes from there. Budget-exhausted
+                # never reaches this callback — the route 429s before
+                # opening the stream, keeping the cursor pinned.
+                # Conflating these with stale-cursor truncation would
+                # tell the client to clear its cursor and re-receive
+                # the same oldest-N entries on every reconnect —
+                # locking the user out of entries past N.
+                nonlocal max_replayed_id, replay_done
+                try:
+                    if redis_client is None:
+                        return
+                    if last_event_id is None:
+                        # Fresh session: start live. A no-cursor connect has
+                        # no state to catch up on — replaying the whole
+                        # retained window shipped weeks-old entries on every
+                        # tab-open (MAXLEN caps by count, not age). Clients
+                        # that had state present a cursor; the malformed-
+                        # cursor case already got its truncation notice
+                        # before the subscribe loop.
+                        return
+                    oldest = _oldest_retained_id(redis_client, user_id)
+                    floor = _replay_floor_id()
+                    # The snapshot can't reach past whichever is newer:
+                    # the MAXLEN'd window edge or the age floor.
+                    effective_oldest = oldest
+                    if floor and (
+                        effective_oldest is None
+                        or stream_id_compare(floor, effective_oldest) > 0
+                    ):
+                        effective_oldest = floor
+                    if (
+                        effective_oldest
+                        and stream_id_compare(last_event_id, effective_oldest) < 0
+                    ):
+                        # The Last-Event-ID has slid off the replayable
+                        # window. Tell the client so it can fetch full state.
+                        replay_lines.append(
+                            _truncation_notice_line(effective_oldest)
+                        )
+                    replay_cap = int(settings.EVENTS_REPLAY_MAX_PER_REQUEST)
+                    for entry_id, sse_line in _replay_backlog(
+                        redis_client, user_id, last_event_id, replay_cap
+                    ):
+                        replay_lines.append(sse_line)
+                        max_replayed_id = entry_id
+                finally:
+                    # Always flip the flag — even on partial-replay failure
+                    # the outer loop must reach the flush step so we don't
+                    # silently strand whatever entries did land.
+                    replay_done = True
+
+            topic = Topic(topic_name(user_id))
+            last_keepalive = time.monotonic()
+            for payload in topic.subscribe(
+                on_subscribe=_on_subscribe_callback,
+                poll_timeout=SUBSCRIBE_POLL_INTERVAL_SECONDS,
+            ):
+                # Break promptly on shutdown — this a2wsgi thread can't be
+                # cancelled by asyncio (see application/core/shutdown.py).
+                if is_shutting_down():
+                    break
+
+                # Flush snapshot on the first iteration after the SUBSCRIBE
+                # callback ran. This runs at most once per connection.
+                if replay_done and replay_lines:
+                    for line in replay_lines:
+                        yield line
+                        replayed_count += 1
+                    replay_lines.clear()
+
+                now = time.monotonic()
+                if payload is None:
+                    if now - last_keepalive >= keepalive_seconds:
+                        yield ": keepalive\n\n"
+                        last_keepalive = now
+                    continue
+
+                event_str = _decode(payload) or ""
+                event_id: Optional[str] = None
+                try:
+                    envelope = json.loads(event_str)
+                    if isinstance(envelope, dict):
+                        candidate = envelope.get("id")
+                        # Only trust ids that look like real Redis Streams
+                        # ids (``ms`` or ``ms-seq``). A malformed or
+                        # adversarial publisher could otherwise pin
+                        # dedupe forever — a lex-greater bogus id would
+                        # make every legitimate later id compare ``<=``
+                        # and get dropped silently.
+                        if isinstance(candidate, str) and _STREAM_ID_RE.match(
+                            candidate
+                        ):
+                            event_id = candidate
+                except Exception:
+                    pass
+
+                # Dedupe: if this id was already covered by replay, drop it.
+                if (
+                    event_id is not None
+                    and max_replayed_id is not None
+                    and stream_id_compare(event_id, max_replayed_id) <= 0
+                ):
+                    continue
+
+                yield _format_sse(event_str, event_id=event_id)
+                last_keepalive = now
+
+            # Topic.subscribe exited before the first yield (transient
+            # Redis hiccup between SUBSCRIBE-ack and the first poll, or
+            # an immediate Redis-down return). The callback may already
+            # have populated the snapshot — flush it so the client gets
+            # the backlog instead of a silent drop. Safe no-op when the
+            # in-loop flush ran (it clear()'d the buffer) and when the
+            # callback never fired (replay_done stays False).
+            if replay_done and replay_lines:
+                for line in replay_lines:
+                    yield line
+                    replayed_count += 1
+                replay_lines.clear()
+        except GeneratorExit:
+            return
+        except Exception:
+            logger.exception(
+                "SSE event-stream generator crashed for user=%s", user_id
+            )
+        finally:
+            duration_s = time.monotonic() - connect_ts
+            logger.info(
+                "event.disconnect user=%s duration_s=%.1f replayed=%d",
+                user_id,
+                duration_s,
+                replayed_count,
+            )
+            if counted and redis_client is not None:
+                try:
+                    redis_client.decr(counter_key)
+                except Exception:
+                    logger.debug(
+                        "SSE connection counter DECR failed for user=%s on disconnect",
+                        user_id,
+                    )
+
+    response = Response(generate(), mimetype="text/event-stream")
+    response.headers["Cache-Control"] = "no-store"
+    response.headers["X-Accel-Buffering"] = "no"
+    response.headers["Connection"] = "keep-alive"
+    logger.info(
+        "event.connect user=%s last_event_id=%s%s",
+        user_id,
+        last_event_id or "-",
+        " (rejected_invalid)" if last_event_id_invalid else "",
+    )
+    return response

application/api/internal/routes.py

@@ -0,0 +1,172 @@
+import os
+import datetime
+import json
+from flask import Blueprint, request, send_from_directory, jsonify
+from werkzeug.utils import secure_filename
+import logging
+
+from application.core.settings import settings
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.session import db_session
+from application.storage.storage_creator import StorageCreator
+
+
+logger = logging.getLogger(__name__)
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
+
+
+@internal.before_request
+def verify_internal_key():
+    """Verify INTERNAL_KEY for all internal endpoint requests.
+
+    Deny by default: if INTERNAL_KEY is not configured, reject all requests.
+    """
+    if not settings.INTERNAL_KEY:
+        logger.warning(
+            f"Internal API request rejected from {request.remote_addr}: "
+            "INTERNAL_KEY is not configured"
+        )
+        return jsonify({"error": "Unauthorized", "message": "Internal API is not configured"}), 401
+    internal_key = request.headers.get("X-Internal-Key")
+    if not internal_key or internal_key != settings.INTERNAL_KEY:
+        logger.warning(f"Unauthorized internal API access attempt from {request.remote_addr}")
+        return jsonify({"error": "Unauthorized", "message": "Invalid or missing internal key"}), 401
+
+
+@internal.route("/api/download", methods=["get"])
+def download_file():
+    user = secure_filename(request.args.get("user"))
+    job_name = secure_filename(request.args.get("name"))
+    filename = secure_filename(request.args.get("file"))
+    save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
+    return send_from_directory(save_dir, filename, as_attachment=True)
+
+
+@internal.route("/api/upload_index", methods=["POST"])
+def upload_index_files():
+    """Upload two files(index.faiss, index.pkl) to the user's folder."""
+    if "user" not in request.form:
+        return {"status": "no user"}
+    user = request.form["user"]
+    if "name" not in request.form:
+        return {"status": "no name"}
+    job_name = request.form["name"]
+    tokens = request.form["tokens"]
+    retriever = request.form["retriever"]
+    source_id = request.form["id"]
+    type = request.form["type"]
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = request.form["sync_frequency"] if "sync_frequency" in request.form else None
+
+    file_path = request.form.get("file_path")
+    directory_structure = request.form.get("directory_structure")
+    file_name_map = request.form.get("file_name_map")
+    config = request.form.get("config")
+
+    if config:
+        try:
+            config = json.loads(config)
+        except Exception:
+            logger.error("Error parsing config")
+            config = None
+    else:
+        config = None
+
+    if directory_structure:
+        try:
+            directory_structure = json.loads(directory_structure)
+        except Exception:
+            logger.error("Error parsing directory_structure")
+            directory_structure = {}
+    else:
+        directory_structure = {}
+    if file_name_map:
+        try:
+            file_name_map = json.loads(file_name_map)
+        except Exception:
+            logger.error("Error parsing file_name_map")
+            file_name_map = None
+    else:
+        file_name_map = None
+
+    storage = StorageCreator.get_storage()
+    index_base_path = f"indexes/{source_id}"
+
+    if settings.VECTOR_STORE == "faiss":
+        if "file_faiss" not in request.files:
+            logger.error("No file_faiss part")
+            return {"status": "no file"}
+        file_faiss = request.files["file_faiss"]
+        if file_faiss.filename == "":
+            return {"status": "no file name"}
+        if "file_pkl" not in request.files:
+            logger.error("No file_pkl part")
+            return {"status": "no file"}
+        file_pkl = request.files["file_pkl"]
+        if file_pkl.filename == "":
+            return {"status": "no file name"}
+
+        # Save index files to storage
+        faiss_storage_path = f"{index_base_path}/index.faiss"
+        pkl_storage_path = f"{index_base_path}/index.pkl"
+        storage.save_file(file_faiss, faiss_storage_path)
+        storage.save_file(file_pkl, pkl_storage_path)
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    update_fields = {
+        "name": job_name,
+        "type": type,
+        "language": job_name,
+        "date": now,
+        "model": settings.EMBEDDINGS_NAME,
+        "tokens": tokens,
+        "retriever": retriever,
+        "remote_data": remote_data,
+        "sync_frequency": sync_frequency,
+        "file_path": file_path,
+        "directory_structure": directory_structure,
+    }
+    if file_name_map is not None:
+        update_fields["file_name_map"] = file_name_map
+    # Only persist ``config`` when supplied so a re-ingest that omits it
+    # doesn't clobber an existing source's config back to ``{}``. Config is
+    # treated as immutable for the ingest dedup window (see upload.py).
+    if config is not None:
+        update_fields["config"] = config
+
+    with db_session() as conn:
+        repo = SourcesRepository(conn)
+        existing = None
+        if looks_like_uuid(source_id):
+            existing = repo.get(source_id, user)
+        if existing is None:
+            existing = repo.get_by_legacy_id(source_id, user)
+        if existing is not None:
+            repo.update(str(existing["id"]), user, update_fields)
+        else:
+            repo.create(
+                job_name,
+                source_id=source_id if looks_like_uuid(source_id) else None,
+                user_id=user,
+                type=type,
+                config=config,
+                tokens=tokens,
+                retriever=retriever,
+                remote_data=remote_data,
+                sync_frequency=sync_frequency,
+                file_path=file_path,
+                directory_structure=directory_structure,
+                file_name_map=file_name_map,
+                language=job_name,
+                model=settings.EMBEDDINGS_NAME,
+                date=now,
+                legacy_mongo_id=None if looks_like_uuid(source_id) else str(source_id),
+            )
+    return {"status": "ok"}

application/api/oidc/__init__.py

@@ -0,0 +1,11 @@
+"""Flask blueprint for OIDC SSO (AUTH_TYPE=oidc)."""
+
+from __future__ import annotations
+
+from flask import Blueprint
+
+from .routes import register as register_routes
+
+
+oidc_bp = Blueprint("oidc", __name__)
+register_routes(oidc_bp)

application/api/oidc/denylist.py

@@ -0,0 +1,107 @@
+"""Redis-backed session denylist for OIDC revocation.
+
+Back-channel logout and SCIM deactivation drop identifiers here; the
+request path refuses any session token whose identifiers match. Each entry
+stores a revocation *watermark* (a Unix timestamp): a session is denied
+only when it was issued (``iat``) at or before the watermark. Storing a
+watermark instead of a boolean is what lets a fresh login self-supersede a
+prior revocation — its newer ``iat`` simply sits above the watermark —
+without deleting the entry and thereby resurrecting still-live sessions
+that were revoked on other devices.
+
+Entries live slightly longer than ``OIDC_SESSION_LIFETIME_SECONDS`` —
+every session issued at or before the watermark expires before the entry
+does, so nothing needs to be stored durably.
+
+Revocation is best-effort by design: if Redis is unreachable the check
+fails open (sessions keep working) rather than taking the whole API down.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+
+from application.cache import get_redis_instance
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+_USER_PREFIX = "oidc:deny:user:"
+_SUB_PREFIX = "oidc:deny:sub:"
+_SID_PREFIX = "oidc:deny:sid:"
+
+
+def _ttl_seconds() -> int:
+    return settings.OIDC_SESSION_LIFETIME_SECONDS + 60
+
+
+def _set(key: str) -> bool:
+    redis = get_redis_instance()
+    if redis is None:
+        logger.error("Redis unavailable — could not denylist %s", key)
+        return False
+    try:
+        # Store the revocation instant; existing entries are overwritten with a
+        # newer watermark (revoking again only ever moves it forward in time).
+        redis.set(key, str(int(time.time())), ex=_ttl_seconds())
+        return True
+    except Exception:
+        logger.error("Failed to denylist %s", key, exc_info=True)
+        return False
+
+
+def deny_user(user_id: str) -> bool:
+    """Revoke every live session of the DocsGPT user ``user_id``."""
+    return _set(_USER_PREFIX + user_id)
+
+
+def deny_idp_sub(sub: str) -> bool:
+    """Revoke sessions by IdP ``sub`` (back-channel logout tokens carry this)."""
+    return _set(_SUB_PREFIX + sub)
+
+
+def deny_sid(sid: str) -> bool:
+    """Revoke sessions of one IdP session id (``sid``-only logout tokens)."""
+    return _set(_SID_PREFIX + sid)
+
+
+def _watermark(value) -> float:
+    """Parse a stored watermark to a float; unparseable values deny everything."""
+    if isinstance(value, bytes):
+        value = value.decode("utf-8", "ignore")
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        # Corrupt/legacy entry: treat as "deny" (a watermark far in the future).
+        return float("inf")
+
+
+def is_denied(decoded_token: dict) -> bool:
+    """True when the token was issued at/before a matching revocation watermark."""
+    keys = []
+    if decoded_token.get("sub"):
+        keys.append(_USER_PREFIX + str(decoded_token["sub"]))
+    if decoded_token.get("oidc_sub"):
+        keys.append(_SUB_PREFIX + str(decoded_token["oidc_sub"]))
+    if decoded_token.get("oidc_sid"):
+        keys.append(_SID_PREFIX + str(decoded_token["oidc_sid"]))
+    if not keys:
+        return False
+    redis = get_redis_instance()
+    if redis is None:
+        return False
+    try:
+        values = redis.mget(keys)
+    except Exception:
+        logger.warning("Denylist check failed — allowing request", exc_info=True)
+        return False
+    try:
+        iat = float(decoded_token.get("iat"))
+    except (TypeError, ValueError):
+        # No usable issue time — if any revocation exists for this identity we
+        # cannot prove the token post-dates it, so deny.
+        return any(value is not None for value in values)
+    # Strict ``<``: a session issued in the same second as (or after) the
+    # revocation — e.g. an immediate re-login — is allowed.
+    return any(value is not None and iat < _watermark(value) for value in values)

application/api/oidc/provider.py

@@ -0,0 +1,270 @@
+"""OIDC provider client: discovery, JWKS, token grants, ID/logout-token validation."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+
+import requests
+from jose import jwt
+from jose.exceptions import ExpiredSignatureError, JWTClaimsError
+
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+DISCOVERY_TTL_SECONDS = 3600
+FORCE_REFETCH_COOLDOWN_SECONDS = 10
+LEEWAY_SECONDS = 60
+HTTP_TIMEOUT_SECONDS = 10
+BACKCHANNEL_LOGOUT_EVENT = "http://schemas.openid.net/event/backchannel-logout"
+# Asymmetric algorithms only: a symmetric alg here would let an attacker
+# forge ID tokens signed with the (public) JWKS material.
+ALLOWED_ID_TOKEN_ALGS = [
+    "RS256", "RS384", "RS512",
+    "ES256", "ES384", "ES512",
+    "PS256", "PS384", "PS512",
+]
+
+_lock = threading.Lock()
+_cache: dict = {
+    "discovery": None,
+    "discovery_at": 0.0,
+    "jwks": None,
+    "jwks_at": 0.0,
+    "jwks_force_at": 0.0,
+}
+
+
+class OIDCError(Exception):
+    """Raised when an OIDC flow step fails."""
+
+
+class OIDCTransientError(OIDCError):
+    """Raised when the IdP was unreachable or returned 5xx — the call can be retried.
+
+    Subclasses OIDCError so existing ``except OIDCError`` handlers still catch
+    it; callers that can retry (e.g. session refresh) catch this first.
+    """
+
+
+def reset_cache() -> None:
+    """Clear the cached discovery document and JWKS (used by tests)."""
+    with _lock:
+        _cache.update(
+            {"discovery": None, "discovery_at": 0.0, "jwks": None, "jwks_at": 0.0, "jwks_force_at": 0.0}
+        )
+
+
+def _fetch_json(url: str) -> dict:
+    try:
+        response = requests.get(url, timeout=HTTP_TIMEOUT_SECONDS)
+    except requests.RequestException as exc:
+        raise OIDCError(f"OIDC request to {url} failed: {exc}") from exc
+    if response.status_code != 200:
+        raise OIDCError(f"OIDC request to {url} returned {response.status_code}")
+    return response.json()
+
+
+def get_discovery() -> dict:
+    """Return the IdP discovery document, fetching/caching it per process."""
+    with _lock:
+        if _cache["discovery"] is not None and time.time() - _cache["discovery_at"] < DISCOVERY_TTL_SECONDS:
+            return _cache["discovery"]
+    url = settings.OIDC_ISSUER.rstrip("/") + "/.well-known/openid-configuration"
+    document = _fetch_json(url)
+    with _lock:
+        _cache["discovery"] = document
+        _cache["discovery_at"] = time.time()
+    return document
+
+
+def get_jwks(force: bool = False) -> dict:
+    """Return the IdP JWKS; ``force=True`` bypasses the cache (key rotation)."""
+    with _lock:
+        fresh = (
+            _cache["jwks"] is not None
+            and time.time() - _cache["jwks_at"] < DISCOVERY_TTL_SECONDS
+        )
+        if not force and fresh:
+            return _cache["jwks"]
+        if force and fresh:
+            # Rate-limit forced refetches: unauthenticated callers (back-channel
+            # logout) must not be able to hammer the IdP through us.
+            if time.time() - _cache["jwks_force_at"] < FORCE_REFETCH_COOLDOWN_SECONDS:
+                return _cache["jwks"]
+            _cache["jwks_force_at"] = time.time()
+    jwks = _fetch_json(get_discovery()["jwks_uri"])
+    with _lock:
+        _cache["jwks"] = jwks
+        _cache["jwks_at"] = time.time()
+    return jwks
+
+
+def _find_key(kid: str | None) -> dict | None:
+    keys = get_jwks().get("keys", [])
+    if kid is None:
+        return keys[0] if len(keys) == 1 else None
+    return next((key for key in keys if key.get("kid") == kid), None)
+
+
+def _resolve_signing_key(token: str) -> dict:
+    """Return the JWKS key matching the token header, refetching once on unknown kid."""
+    try:
+        header = jwt.get_unverified_header(token)
+    except Exception as exc:
+        raise OIDCError(f"Malformed token: {exc}") from exc
+    if header.get("alg") not in ALLOWED_ID_TOKEN_ALGS:
+        raise OIDCError(f"Disallowed token alg: {header.get('alg')}")
+
+    key = _find_key(header.get("kid"))
+    if key is None:
+        get_jwks(force=True)
+        key = _find_key(header.get("kid"))
+    if key is None:
+        raise OIDCError("No matching key in IdP JWKS")
+    return key
+
+
+def _decode_verified(token: str, options: dict) -> dict:
+    """Decode ``token`` against the JWKS, retrying once if the IdP re-keyed.
+
+    A signature failure can mean the IdP replaced its signing key while
+    reusing the same kid — the kid-miss refetch never triggers then, so
+    retry once against a freshly fetched JWKS (rate-limited in get_jwks).
+    """
+    key = _resolve_signing_key(token)
+    decode_kwargs = {
+        "algorithms": ALLOWED_ID_TOKEN_ALGS,
+        "audience": settings.OIDC_CLIENT_ID,
+        # Compare against the discovery document's own issuer value —
+        # some IdPs (Authentik) use a trailing slash the operator may
+        # not have typed into OIDC_ISSUER.
+        "issuer": get_discovery()["issuer"],
+        "options": options,
+    }
+    try:
+        return jwt.decode(token, key, **decode_kwargs)
+    except (ExpiredSignatureError, JWTClaimsError) as exc:
+        raise OIDCError(f"token validation failed: {exc}") from exc
+    except Exception:
+        get_jwks(force=True)
+        key = _find_key(jwt.get_unverified_header(token).get("kid"))
+        if key is None:
+            raise OIDCError("No matching key in IdP JWKS")
+        try:
+            return jwt.decode(token, key, **decode_kwargs)
+        except Exception as exc:
+            raise OIDCError(f"token validation failed: {exc}") from exc
+
+
+def validate_id_token(id_token: str, nonce: str | None = None) -> dict:
+    """Verify the ID token's signature, iss, aud, exp, and (when given) nonce; return claims."""
+    claims = _decode_verified(
+        id_token,
+        options={
+            "verify_at_hash": False,
+            "leeway": LEEWAY_SECONDS,
+            "require_iss": True,
+            "require_aud": True,
+            "require_exp": True,
+            "require_sub": True,
+        },
+    )
+    # Refresh-issued id_tokens carry no nonce; callers pass None to skip the check.
+    if nonce is not None and claims.get("nonce") != nonce:
+        raise OIDCError("nonce mismatch")
+    return claims
+
+
+def validate_logout_token(logout_token: str) -> dict:
+    """Verify a back-channel logout token per OIDC Back-Channel Logout 1.0; return claims."""
+    claims = _decode_verified(
+        logout_token,
+        options={
+            "verify_at_hash": False,
+            "leeway": LEEWAY_SECONDS,
+            "require_iss": True,
+            "require_aud": True,
+            "require_iat": True,
+            # jti is REQUIRED by OIDC Back-Channel Logout 1.0 and underpins
+            # replay protection — reject tokens that omit it. exp is not a
+            # required logout-token claim, so it stays optional (the caller
+            # bounds replay via iat freshness instead).
+            "require_jti": True,
+            "require_exp": False,
+        },
+    )
+    events = claims.get("events")
+    if not isinstance(events, dict) or BACKCHANNEL_LOGOUT_EVENT not in events:
+        raise OIDCError("logout_token missing the backchannel-logout event")
+    if "nonce" in claims:
+        raise OIDCError("logout_token must not contain a nonce")
+    if not claims.get("sub") and not claims.get("sid"):
+        raise OIDCError("logout_token must contain sub or sid")
+    return claims
+
+
+def _token_request(data: dict) -> dict:
+    """POST to the token endpoint using the discovery-advertised client auth method."""
+    discovery = get_discovery()
+    data = {**data, "client_id": settings.OIDC_CLIENT_ID}
+    post_kwargs: dict = {"data": data, "timeout": HTTP_TIMEOUT_SECONDS}
+    if settings.OIDC_CLIENT_SECRET:
+        # Absent metadata means the RFC 8414 default, client_secret_basic.
+        methods = discovery.get("token_endpoint_auth_methods_supported") or ["client_secret_basic"]
+        if "client_secret_post" in methods:
+            data["client_secret"] = settings.OIDC_CLIENT_SECRET
+        else:
+            post_kwargs["auth"] = (settings.OIDC_CLIENT_ID, settings.OIDC_CLIENT_SECRET)
+    try:
+        response = requests.post(discovery["token_endpoint"], **post_kwargs)
+    except requests.RequestException as exc:
+        # Network failure — retryable.
+        raise OIDCTransientError(f"Token request failed: {exc}") from exc
+    if response.status_code != 200:
+        logger.error(
+            "OIDC token request failed (%s): %s", response.status_code, response.text[:500]
+        )
+        # 5xx is an IdP-side hiccup (retryable); 4xx (e.g. invalid_grant) is a
+        # definitive rejection the caller must not retry.
+        if response.status_code >= 500:
+            raise OIDCTransientError(f"Token endpoint returned {response.status_code}")
+        raise OIDCError(f"Token endpoint returned {response.status_code}")
+    return response.json()
+
+
+def exchange_code(code: str, code_verifier: str, redirect_uri: str) -> dict:
+    """Exchange the authorization code at the IdP token endpoint."""
+    return _token_request(
+        {
+            "grant_type": "authorization_code",
+            "code": code,
+            "redirect_uri": redirect_uri,
+            "code_verifier": code_verifier,
+        }
+    )
+
+
+def refresh_grant(refresh_token: str) -> dict:
+    """Redeem a refresh token at the IdP token endpoint."""
+    return _token_request({"grant_type": "refresh_token", "refresh_token": refresh_token})
+
+
+def fetch_userinfo(access_token: str) -> dict:
+    """Fetch claims from the IdP userinfo endpoint with a Bearer access token."""
+    endpoint = get_discovery().get("userinfo_endpoint")
+    if not endpoint:
+        raise OIDCError("No userinfo_endpoint in discovery document")
+    try:
+        response = requests.get(
+            endpoint,
+            headers={"Authorization": f"Bearer {access_token}"},
+            timeout=HTTP_TIMEOUT_SECONDS,
+        )
+    except requests.RequestException as exc:
+        raise OIDCError(f"userinfo request failed: {exc}") from exc
+    if response.status_code != 200:
+        raise OIDCError(f"userinfo endpoint returned {response.status_code}")
+    return response.json()

application/api/oidc/routes.py

@@ -0,0 +1,670 @@
+"""Login, callback, session-token, logout, and refresh endpoints for AUTH_TYPE=oidc.
+
+Flow: the backend redirects the browser to the IdP (Authorization Code +
+PKCE), validates the ID token at the callback, mints a local HS256 session
+JWT, and hands it to the SPA via a short-lived single-use code in the URL
+fragment. Browser redirects only ever target the configured
+``OIDC_FRONTEND_URL`` or IdP endpoints taken from discovery.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import logging
+import secrets
+import time
+import uuid
+from urllib.parse import quote, urlencode
+
+from flask import Blueprint, Response, jsonify, make_response, redirect, request
+from jose import jwt
+
+from application.api.oidc import denylist, provider
+from application.auth import handle_auth
+from application.cache import get_redis_instance
+from application.core.settings import settings
+from application.storage.db.repositories.auth_events import AuthEventsRepository
+from application.storage.db.repositories.user_roles import UserRolesRepository
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+STATE_TTL_SECONDS = 600
+HANDOFF_TTL_SECONDS = 60
+LOGOUT_JTI_TTL_SECONDS = 600
+MAX_PICTURE_CLAIM_CHARS = 2048
+# Browser-bound CSRF guard for the login flow: the callback requires this
+# cookie to echo the ``state`` it received. Scoped to the oidc paths so it is
+# only ever sent on the callback.
+STATE_COOKIE_NAME = "oidc_state"
+STATE_COOKIE_PATH = "/api/auth/oidc/"
+
+
+def _state_key(state: str) -> str:
+    return f"oidc:state:{state}"
+
+
+def _handoff_key(code: str) -> str:
+    return f"oidc:handoff:{code}"
+
+
+def _refresh_key(jti: str) -> str:
+    return f"oidc:refresh:{jti}"
+
+
+def _logout_jti_key(jti: str) -> str:
+    return f"oidc:bcl:jti:{jti}"
+
+
+def _pkce_challenge(verifier: str) -> str:
+    digest = hashlib.sha256(verifier.encode("ascii")).digest()
+    return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")
+
+
+def _redirect_uri() -> str:
+    return settings.OIDC_REDIRECT_URI or request.host_url.rstrip("/") + "/api/auth/oidc/callback"
+
+
+def _frontend_url() -> str:
+    return (settings.OIDC_FRONTEND_URL or "").rstrip("/") or "/"
+
+
+def _state_cookie_secure() -> bool:
+    """Mark the state cookie ``Secure`` when the app is served over HTTPS."""
+    return (settings.OIDC_FRONTEND_URL or "").lower().startswith("https")
+
+
+def _frontend_redirect(fragment: str):
+    base = (settings.OIDC_FRONTEND_URL or "").rstrip("/")
+    return redirect(f"{base}/#{fragment}", code=302)
+
+
+def _no_store(payload, status: int = 200) -> Response:
+    """Build a response marked non-cacheable (back-channel logout requirement)."""
+    response = make_response(payload, status)
+    response.headers["Cache-Control"] = "no-store"
+    return response
+
+
+def _allowed_groups() -> list[str]:
+    """Parse the comma-separated group allowlist; empty/unset means everyone."""
+    raw = settings.OIDC_ALLOWED_GROUPS or ""
+    return [group.strip() for group in raw.split(",") if group.strip()]
+
+
+def _admin_groups() -> list[str]:
+    """Parse the comma-separated admin-group list; empty/unset disables OIDC admin mapping."""
+    raw = settings.OIDC_ADMIN_GROUPS or ""
+    return [group.strip() for group in raw.split(",") if group.strip()]
+
+
+def _reconcile_oidc_admin(user_id: str, groups: list[str] | None) -> None:
+    """Sync the OIDC-group-derived admin grant for ``user_id``; best-effort, never raises.
+
+    Empty/unset ``OIDC_ADMIN_GROUPS`` disables the mapping entirely (no-op), so a
+    blank env var never mass-revokes. ``groups`` is ``None`` when the IdP asserted
+    no groups claim at all (absent from both id_token and userinfo); admin-ness is
+    then unknowable, so we skip rather than revoke — otherwise a silent refresh
+    whose token omits groups would demote a still-eligible admin. Only the
+    ``oidc_group`` grant source is touched — manual grants are left intact. A
+    change is audited.
+    """
+    admin_groups = _admin_groups()
+    if not admin_groups:
+        return
+    if groups is None:
+        return
+    is_admin = bool(set(groups) & set(admin_groups))
+    try:
+        with db_session() as conn:
+            change = UserRolesRepository(conn).reconcile_oidc_admin(user_id, is_admin)
+            if change:
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "role_granted" if change == "granted" else "role_revoked",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={"role": "admin", "source": "oidc_group"},
+                )
+    except Exception:
+        logger.error("OIDC admin reconcile failed for %s", user_id, exc_info=True)
+
+
+def _claim_groups(claims: dict) -> list[str]:
+    """Read the groups claim as a list of strings; missing means no groups."""
+    value = claims.get(settings.OIDC_GROUPS_CLAIM)
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, (list, tuple)):
+        return [str(member) for member in value]
+    return [str(value)]
+
+
+def _effective_claims(tokens: dict, claims: dict) -> dict:
+    """Merge userinfo into the id_token claims when required claims are missing."""
+    effective = dict(claims)
+    need_user_id = not effective.get(settings.OIDC_USER_ID_CLAIM)
+    # Fetch userinfo to backfill groups when EITHER a login allowlist or an admin
+    # mapping is configured — many IdPs (Okta/Auth0/Azure AD) only expose groups
+    # via userinfo, and without this the admin reconcile would see no groups.
+    need_groups = (
+        bool(_allowed_groups()) or bool(_admin_groups())
+    ) and settings.OIDC_GROUPS_CLAIM not in effective
+    if not (need_user_id or need_groups) or not tokens.get("access_token"):
+        return effective
+    try:
+        userinfo = provider.fetch_userinfo(tokens["access_token"])
+    except provider.OIDCError:
+        logger.warning("OIDC userinfo fetch failed; continuing with id_token claims", exc_info=True)
+        return effective
+    if userinfo.get("sub") != claims.get("sub"):
+        raise provider.OIDCError("userinfo sub does not match id_token sub")
+    for key, value in userinfo.items():
+        effective.setdefault(key, value)
+    return effective
+
+
+def _mint_session_token(identity: dict) -> tuple[str, str]:
+    """Mint the local HS256 session JWT for ``identity``; returns (token, jti)."""
+    now = int(time.time())
+    jti = str(uuid.uuid4())
+    payload = {
+        "sub": str(identity["sub"]),
+        "jti": jti,
+        "iat": now,
+        "exp": now + settings.OIDC_SESSION_LIFETIME_SECONDS,
+    }
+    if identity.get("oidc_sub"):
+        payload["oidc_sub"] = str(identity["oidc_sub"])
+    if identity.get("oidc_sid"):
+        payload["oidc_sid"] = str(identity["oidc_sid"])
+    for claim in ("email", "name"):
+        if identity.get(claim):
+            payload[claim] = identity[claim]
+    picture = identity.get("picture")
+    if picture and isinstance(picture, str) and len(picture) < MAX_PICTURE_CLAIM_CHARS:
+        payload["picture"] = picture
+    return jwt.encode(payload, settings.JWT_SECRET_KEY, algorithm="HS256"), jti
+
+
+def _record_login_denied(user_id: str, metadata: dict) -> None:
+    """Best-effort audit of a denied login; never raises."""
+    try:
+        with db_session() as conn:
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "oidc_login_denied",
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata=metadata,
+            )
+    except Exception:
+        logger.warning("Failed to record oidc_login_denied for %s", user_id, exc_info=True)
+
+
+def _gate_and_audit_login(user_id: str, effective: dict, groups: list[str]) -> bool:
+    """Reject disabled users, provision new ones, audit the login.
+
+    Returns False only when the user row was readable and marked inactive;
+    a DB outage logs an error and lets the login proceed.
+    """
+    disabled = False
+    try:
+        with db_session() as conn:
+            users = UsersRepository(conn)
+            row = users.get(user_id)
+            if row is not None and row.get("active") is False:
+                disabled = True
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "oidc_login_denied",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={"reason": "account_disabled"},
+                )
+            else:
+                if row is None:
+                    # New user: provision with the email claim so a team admin
+                    # can later add them by email.
+                    users.upsert(user_id, email=effective.get("email"))
+                else:
+                    # Existing user: targeted email backfill only (don't re-run
+                    # the full upsert — preserves the no-upsert-on-login path).
+                    users.set_email(user_id, effective.get("email"))
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "oidc_login",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={"email": effective.get("email"), "groups": groups or None},
+                )
+    except Exception:
+        logger.error(
+            "OIDC provisioning/audit failed for %s%s",
+            user_id,
+            "" if disabled else "; continuing login",
+            exc_info=True,
+        )
+    return not disabled
+
+
+def oidc_login():
+    """Start the Authorization Code + PKCE flow with a 302 to the IdP."""
+    redis = get_redis_instance()
+    if redis is None:
+        return make_response(jsonify({"error": "redis_unavailable"}), 503)
+    try:
+        authorization_endpoint = provider.get_discovery()["authorization_endpoint"]
+    except (provider.OIDCError, KeyError):
+        logger.error("OIDC discovery failed during login", exc_info=True)
+        return make_response(jsonify({"error": "idp_unavailable"}), 503)
+
+    state = secrets.token_urlsafe(32)
+    nonce = secrets.token_urlsafe(32)
+    code_verifier = secrets.token_urlsafe(64)
+    redis.set(
+        _state_key(state),
+        json.dumps({"code_verifier": code_verifier, "nonce": nonce}),
+        ex=STATE_TTL_SECONDS,
+        nx=True,
+    )
+    params = {
+        "response_type": "code",
+        "client_id": settings.OIDC_CLIENT_ID,
+        "redirect_uri": _redirect_uri(),
+        "scope": settings.OIDC_SCOPES,
+        "state": state,
+        "nonce": nonce,
+        "code_challenge": _pkce_challenge(code_verifier),
+        "code_challenge_method": "S256",
+    }
+    response = redirect(f"{authorization_endpoint}?{urlencode(params)}", code=302)
+    # Bind this login to the browser: the callback rejects any state that
+    # isn't echoed by this cookie, so a code+state captured from another
+    # browser (login CSRF / session fixation) can't complete the flow.
+    response.set_cookie(
+        STATE_COOKIE_NAME,
+        state,
+        max_age=STATE_TTL_SECONDS,
+        httponly=True,
+        secure=_state_cookie_secure(),
+        samesite="Lax",
+        path=STATE_COOKIE_PATH,
+    )
+    return response
+
+
+def oidc_callback():
+    """Validate the IdP response, mint a session JWT, redirect with a handoff code."""
+    if request.args.get("error"):
+        return _frontend_redirect("oidc_error=" + quote(request.args["error"]))
+    state = request.args.get("state")
+    code = request.args.get("code")
+    if not state or not code:
+        return _frontend_redirect("oidc_error=invalid_state")
+
+    # Browser binding: the state must match the cookie set at login. Without
+    # this, an attacker could feed a victim a code+state from the attacker's
+    # own login and silently sign the victim into the attacker's account.
+    cookie_state = request.cookies.get(STATE_COOKIE_NAME)
+    if not cookie_state or not hmac.compare_digest(cookie_state, state):
+        logger.warning("OIDC callback rejected: state cookie missing or mismatched")
+        return _frontend_redirect("oidc_error=invalid_state")
+
+    redis = get_redis_instance()
+    if redis is None:
+        logger.error("Redis unavailable during OIDC callback")
+        return _frontend_redirect("oidc_error=auth_failed")
+    raw_state = redis.getdel(_state_key(state))
+    if raw_state is None:
+        return _frontend_redirect("oidc_error=invalid_state")
+    stored = json.loads(raw_state)
+
+    try:
+        tokens = provider.exchange_code(code, stored["code_verifier"], _redirect_uri())
+        claims = provider.validate_id_token(tokens["id_token"], stored["nonce"])
+        effective = _effective_claims(tokens, claims)
+    except (provider.OIDCError, KeyError):
+        logger.error("OIDC callback failed", exc_info=True)
+        return _frontend_redirect("oidc_error=auth_failed")
+
+    user_id = effective.get(settings.OIDC_USER_ID_CLAIM)
+    if not user_id:
+        logger.error("OIDC id_token missing user id claim %r", settings.OIDC_USER_ID_CLAIM)
+        return _frontend_redirect("oidc_error=missing_claim")
+    user_id = str(user_id)
+
+    allowed = _allowed_groups()
+    groups = _claim_groups(effective)
+    if allowed and not set(groups) & set(allowed):
+        logger.info("OIDC login denied for %s: groups %s not in allowlist", user_id, groups)
+        _record_login_denied(user_id, {"reason": "not_authorized", "groups": groups})
+        return _frontend_redirect("oidc_error=not_authorized")
+
+    if not _gate_and_audit_login(user_id, effective, groups):
+        return _frontend_redirect("oidc_error=account_disabled")
+
+    # Reconcile OIDC-group-derived admin against the fresh group claims so
+    # adds/removals take effect at login (manual grants are untouched). Pass
+    # None when the claim is absent entirely so a missing claim never revokes.
+    groups_present = settings.OIDC_GROUPS_CLAIM in effective
+    _reconcile_oidc_admin(user_id, groups if groups_present else None)
+
+    # No need to lift prior revocations here: the denylist keys on a revocation
+    # timestamp and the session minted below carries a newer ``iat``, so it is
+    # allowed automatically while still-live sessions revoked on other devices
+    # stay denied.
+    session_token, jti = _mint_session_token(
+        {
+            "sub": user_id,
+            "email": effective.get("email"),
+            "name": effective.get("name"),
+            "picture": effective.get("picture"),
+            "oidc_sub": claims["sub"],
+            "oidc_sid": claims.get("sid"),
+        }
+    )
+
+    refresh_token = tokens.get("refresh_token")
+    if refresh_token:
+        try:
+            redis.set(_refresh_key(jti), refresh_token, ex=settings.OIDC_SESSION_LIFETIME_SECONDS)
+        except Exception:
+            logger.warning("Failed to store OIDC refresh token", exc_info=True)
+
+    handoff = secrets.token_urlsafe(32)
+    redis.set(_handoff_key(handoff), session_token, ex=HANDOFF_TTL_SECONDS, nx=True)
+    return _frontend_redirect("oidc_code=" + handoff)
+
+
+def oidc_token():
+    """Redeem a single-use handoff code for the minted session JWT."""
+    body = request.get_json(silent=True) or {}
+    code = body.get("code")
+    if not code or not isinstance(code, str):
+        return make_response(jsonify({"error": "invalid_code"}), 401)
+    redis = get_redis_instance()
+    if redis is None:
+        return make_response(jsonify({"error": "redis_unavailable"}), 503)
+    raw = redis.getdel(_handoff_key(code))
+    if raw is None:
+        return make_response(jsonify({"error": "invalid_code"}), 401)
+    token = raw.decode("utf-8") if isinstance(raw, bytes) else raw
+    return jsonify({"token": token})
+
+
+def _is_account_disabled(user_id: str) -> bool:
+    """True only when a readable user row is marked inactive (DB outage fails open)."""
+    try:
+        with db_readonly() as conn:
+            row = UsersRepository(conn).get(user_id)
+    except Exception:
+        logger.error("User lookup failed during OIDC refresh", exc_info=True)
+        return False
+    return bool(row is not None and row.get("active") is False)
+
+
+def oidc_refresh():
+    """Rotate the stored IdP refresh token and mint a fresh session JWT."""
+    decoded = handle_auth(request)
+    if (
+        not isinstance(decoded, dict)
+        or "error" in decoded
+        or not decoded.get("sub")
+        or not decoded.get("jti")
+    ):
+        error = "invalid_token"
+        if isinstance(decoded, dict) and decoded.get("error") == "token_expired":
+            error = "token_expired"
+        return make_response(jsonify({"error": error}), 401)
+
+    if denylist.is_denied(decoded):
+        return make_response(jsonify({"error": "token_revoked"}), 401)
+
+    # Gate the current session identity before spending the refresh token.
+    if _is_account_disabled(str(decoded["sub"])):
+        return make_response(jsonify({"error": "account_disabled"}), 401)
+
+    redis = get_redis_instance()
+    if redis is None:
+        return make_response(jsonify({"error": "redis_unavailable"}), 503)
+    raw = redis.getdel(_refresh_key(str(decoded["jti"])))
+    if raw is None:
+        return make_response(jsonify({"error": "no_refresh_token"}), 404)
+    refresh_token = raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)
+
+    try:
+        tokens = provider.refresh_grant(refresh_token)
+    except provider.OIDCTransientError:
+        # IdP unreachable / 5xx — the refresh token is still valid. Put it back
+        # and tell the client to retry instead of killing a live session over a
+        # transient blip (the frontend reschedules a renewal on 503).
+        try:
+            redis.set(
+                _refresh_key(str(decoded["jti"])),
+                refresh_token,
+                ex=settings.OIDC_SESSION_LIFETIME_SECONDS,
+            )
+        except Exception:
+            logger.warning("Failed to restore refresh token after transient error", exc_info=True)
+        logger.warning("OIDC refresh grant failed transiently", exc_info=True)
+        return make_response(jsonify({"error": "refresh_unavailable"}), 503)
+    except provider.OIDCError:
+        # invalid_grant / 4xx — the refresh token is spent or revoked; leave it
+        # consumed so the client falls back to a fresh login.
+        logger.warning("OIDC refresh grant rejected", exc_info=True)
+        return make_response(jsonify({"error": "refresh_failed"}), 401)
+
+    identity = {
+        "sub": str(decoded["sub"]),
+        "email": decoded.get("email"),
+        "name": decoded.get("name"),
+        "picture": decoded.get("picture"),
+        "oidc_sub": decoded.get("oidc_sub"),
+        "oidc_sid": decoded.get("oidc_sid"),
+    }
+    id_token = tokens.get("id_token")
+    if id_token:
+        try:
+            claims = provider.validate_id_token(id_token, nonce=None)
+            effective = _effective_claims(tokens, claims)
+        except provider.OIDCError:
+            logger.warning("Refresh-issued id_token failed validation", exc_info=True)
+            return make_response(jsonify({"error": "refresh_failed"}), 401)
+        # Re-gate group membership on every renewal that carries fresh
+        # claims — otherwise removal from the allowlist would never bite
+        # while silent renewal keeps extending the session.
+        allowed = _allowed_groups()
+        groups = _claim_groups(effective)
+        if allowed and not (set(groups) & set(allowed)):
+            denied_user = str(effective.get(settings.OIDC_USER_ID_CLAIM) or decoded["sub"])
+            logger.info("OIDC refresh denied for %s: groups %s not allowed", denied_user, groups)
+            _record_login_denied(
+                denied_user,
+                {"reason": "not_authorized", "via": "refresh", "groups": groups},
+            )
+            return make_response(jsonify({"error": "not_authorized"}), 401)
+        user_id = effective.get(settings.OIDC_USER_ID_CLAIM)
+        if user_id:
+            user_id = str(user_id)
+            # The pre-grant gate only saw the old sub. If the refreshed identity
+            # maps to a different user id, re-check that account is enabled
+            # before minting a session for it.
+            if user_id != str(decoded["sub"]) and _is_account_disabled(user_id):
+                return make_response(jsonify({"error": "account_disabled"}), 401)
+            identity["sub"] = user_id
+        for claim in ("email", "name", "picture"):
+            if effective.get(claim):
+                identity[claim] = effective[claim]
+        identity["oidc_sub"] = effective.get("sub") or identity["oidc_sub"]
+        if effective.get("sid"):
+            identity["oidc_sid"] = effective["sid"]
+        # Re-reconcile admin on every renewal carrying fresh group claims, so an
+        # OIDC group add/remove takes effect without waiting for a full re-login.
+        # None when the claim is absent so a missing claim never revokes admin.
+        groups_present = settings.OIDC_GROUPS_CLAIM in effective
+        _reconcile_oidc_admin(identity["sub"], groups if groups_present else None)
+
+    # Re-check revocation right before minting, against the (possibly remapped)
+    # identity but with the ORIGINAL session's iat: a back-channel logout or
+    # SCIM deny that landed during the IdP grant — or one targeting the
+    # refreshed sub/sid — sets a watermark newer than this iat and must block
+    # renewal. The renewed token's own (fresh) iat would post-date the
+    # watermark and slip past the per-request check, so anchor on the old iat.
+    if denylist.is_denied(
+        {
+            "sub": identity["sub"],
+            "oidc_sub": identity.get("oidc_sub"),
+            "oidc_sid": identity.get("oidc_sid"),
+            "iat": decoded.get("iat"),
+        }
+    ):
+        return make_response(jsonify({"error": "token_revoked"}), 401)
+
+    new_token, new_jti = _mint_session_token(identity)
+    new_refresh = tokens.get("refresh_token") or refresh_token
+    try:
+        redis.set(_refresh_key(new_jti), new_refresh, ex=settings.OIDC_SESSION_LIFETIME_SECONDS)
+    except Exception:
+        logger.warning("Failed to store rotated OIDC refresh token", exc_info=True)
+
+    try:
+        with db_session() as conn:
+            AuthEventsRepository(conn).insert(
+                identity["sub"],
+                "oidc_refresh",
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+            )
+    except Exception:
+        logger.warning("Failed to record oidc_refresh for %s", identity["sub"], exc_info=True)
+
+    return jsonify({"token": new_token})
+
+
+def oidc_backchannel_logout():
+    """Revoke sessions named by a signed IdP back-channel logout token."""
+    logout_token = request.form.get("logout_token")
+    if not logout_token:
+        body = request.get_json(silent=True)
+        if isinstance(body, dict):
+            logout_token = body.get("logout_token")
+    if not logout_token or not isinstance(logout_token, str):
+        return _no_store(jsonify({"error": "missing_logout_token"}), 400)
+
+    try:
+        claims = provider.validate_logout_token(logout_token)
+    except provider.OIDCError:
+        logger.warning("Rejected OIDC back-channel logout token", exc_info=True)
+        return _no_store(jsonify({"error": "invalid_logout_token"}), 400)
+
+    # Reject stale tokens: past the jti replay-cache window we can no longer
+    # detect replays by jti, so bound acceptance to that window (logout tokens
+    # are short-lived). Combined with the always-on jti check below, a captured
+    # token can be replayed neither within the window (jti dedupe) nor after it
+    # (iat too old).
+    now = int(time.time())
+    iat = claims.get("iat")
+    if not isinstance(iat, (int, float)) or now - iat > LOGOUT_JTI_TTL_SECONDS:
+        logger.warning("Rejected stale OIDC back-channel logout token")
+        return _no_store(jsonify({"error": "invalid_logout_token"}), 400)
+
+    jti = claims.get("jti")
+    redis = get_redis_instance()
+    if redis is not None and jti:
+        try:
+            fresh = redis.set(_logout_jti_key(str(jti)), "1", ex=LOGOUT_JTI_TTL_SECONDS, nx=True)
+        except Exception:
+            logger.warning("Logout-token jti replay check failed; accepting token", exc_info=True)
+            fresh = True
+        if not fresh:
+            return _no_store(jsonify({"error": "invalid_logout_token"}), 400)
+
+    sub = claims.get("sub")
+    sid = claims.get("sid")
+    revoked = True
+    if sub:
+        revoked = bool(denylist.deny_idp_sub(str(sub))) and revoked
+    if sid:
+        revoked = bool(denylist.deny_sid(str(sid))) and revoked
+    if not revoked:
+        # The denylist write failed (Redis down). Report failure so the IdP
+        # retries rather than recording a logout that never took effect.
+        logger.error("Back-channel logout could not persist the revocation")
+        return _no_store(jsonify({"error": "revocation_unavailable"}), 502)
+
+    user_id = str(sub) if sub else f"sid:{sid}"
+    try:
+        with db_session() as conn:
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "backchannel_logout",
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata={"sid": str(sid)} if sid else None,
+            )
+    except Exception:
+        logger.warning("Failed to record backchannel_logout for %s", user_id, exc_info=True)
+
+    return _no_store("", 200)
+
+
+def oidc_logout():
+    """Redirect to the IdP end-session endpoint, falling back to the frontend."""
+    frontend = _frontend_url()
+    try:
+        end_session = provider.get_discovery().get("end_session_endpoint")
+    except provider.OIDCError:
+        end_session = None
+    if not end_session:
+        return redirect(frontend, code=302)
+    params = {
+        "post_logout_redirect_uri": frontend,
+        "client_id": settings.OIDC_CLIENT_ID,
+    }
+    return redirect(f"{end_session}?{urlencode(params)}", code=302)
+
+
+def _require_oidc_enabled() -> Response | None:
+    """404 every oidc route unless OIDC is the active auth mode.
+
+    Registration is unconditional (so the import-time auth mode doesn't matter),
+    but the endpoints only work under ``AUTH_TYPE=oidc`` — otherwise OIDC_ISSUER
+    is unset and ``get_discovery`` would dereference ``None`` and 500. Mirrors
+    SCIM's ``SCIM_ENABLED`` gate.
+    """
+    if settings.AUTH_TYPE != "oidc":
+        return make_response(jsonify({"error": "oidc_not_enabled"}), 404)
+    return None
+
+
+def register(bp: Blueprint) -> None:
+    """Attach the oidc auth routes to ``bp``."""
+    bp.before_request(_require_oidc_enabled)
+    bp.add_url_rule(
+        "/api/auth/oidc/login", view_func=oidc_login, methods=["GET"], endpoint="login"
+    )
+    bp.add_url_rule(
+        "/api/auth/oidc/callback", view_func=oidc_callback, methods=["GET"], endpoint="callback"
+    )
+    bp.add_url_rule(
+        "/api/auth/oidc/token", view_func=oidc_token, methods=["POST"], endpoint="token"
+    )
+    bp.add_url_rule(
+        "/api/auth/oidc/refresh", view_func=oidc_refresh, methods=["POST"], endpoint="refresh"
+    )
+    bp.add_url_rule(
+        "/api/auth/oidc/backchannel-logout",
+        view_func=oidc_backchannel_logout,
+        methods=["POST"],
+        endpoint="backchannel_logout",
+    )
+    bp.add_url_rule(
+        "/api/auth/oidc/logout", view_func=oidc_logout, methods=["GET"], endpoint="logout"
+    )

application/api/scim/__init__.py

@@ -0,0 +1,11 @@
+"""Flask blueprint for SCIM 2.0 user provisioning (/scim/v2)."""
+
+from __future__ import annotations
+
+from flask import Blueprint
+
+from .routes import register as register_routes
+
+
+scim_bp = Blueprint("scim", __name__)
+register_routes(scim_bp)

application/api/scim/routes.py

@@ -0,0 +1,452 @@
+"""SCIM 2.0 user-provisioning endpoints (RFC 7643/7644 subset for IdP clients).
+
+IdPs (Okta, Authentik, Entra) push user lifecycle into DocsGPT through
+``/scim/v2``: create users ahead of first login and deactivate them on
+offboarding. Deactivation also revokes live sessions via the Redis
+denylist; login refuses inactive users elsewhere. Only ``userName`` and
+``active`` are honored — everything else IdPs send is ignored.
+"""
+
+from __future__ import annotations
+
+import hmac
+import json
+import logging
+import re
+from typing import Any, Optional
+
+from flask import Blueprint, Response, request
+from sqlalchemy import Connection
+
+from application.api.oidc.denylist import deny_user
+from application.core.settings import settings
+from application.storage.db.repositories.auth_events import AuthEventsRepository
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+_SCIM_MEDIA_TYPE = "application/scim+json"
+_ERROR_URN = "urn:ietf:params:scim:api:messages:2.0:Error"
+_LIST_RESPONSE_URN = "urn:ietf:params:scim:api:messages:2.0:ListResponse"
+_USER_URN = "urn:ietf:params:scim:schemas:core:2.0:User"
+
+_DEFAULT_COUNT = 100
+_MAX_COUNT = 200
+
+# The only filter IdPs need for provisioning: exact userName lookup.
+_USERNAME_EQ_FILTER = re.compile(r'^\s*userName\s+eq\s+"([^"]*)"\s*$', re.IGNORECASE)
+
+_SERVICE_PROVIDER_CONFIG = {
+    "schemas": ["urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig"],
+    "patch": {"supported": True},
+    "bulk": {"supported": False},
+    "filter": {"supported": True, "maxResults": _MAX_COUNT},
+    "changePassword": {"supported": False},
+    "sort": {"supported": False},
+    "etag": {"supported": False},
+    "authenticationSchemes": [
+        {
+            "type": "oauthbearertoken",
+            "name": "Bearer Token",
+            "description": "Authorization header carrying the configured SCIM bearer token",
+        }
+    ],
+}
+
+_USER_RESOURCE_TYPE = {
+    "schemas": ["urn:ietf:params:scim:schemas:core:2.0:ResourceType"],
+    "id": "User",
+    "name": "User",
+    "endpoint": "/scim/v2/Users",
+    "schema": _USER_URN,
+    "meta": {"resourceType": "ResourceType", "location": "/scim/v2/ResourceTypes/User"},
+}
+
+_USER_SCHEMA = {
+    "id": _USER_URN,
+    "name": "User",
+    "description": "DocsGPT user account",
+    "attributes": [
+        {
+            "name": "userName",
+            "type": "string",
+            "multiValued": False,
+            "required": True,
+            # RFC 7643 §8.7.1 defines userName as caseExact=false; matching is
+            # case-insensitive (see UsersRepository.list_paginated / create).
+            "caseExact": False,
+            "mutability": "immutable",
+            "returned": "default",
+            "uniqueness": "server",
+        },
+        {
+            "name": "active",
+            "type": "boolean",
+            "multiValued": False,
+            "required": False,
+            "mutability": "readWrite",
+            "returned": "default",
+        },
+    ],
+    "meta": {"resourceType": "Schema", "location": f"/scim/v2/Schemas/{_USER_URN}"},
+}
+
+
+# ----------------------------------------------------------------------
+# Response helpers
+# ----------------------------------------------------------------------
+def _scim_response(payload: Optional[dict], status: int, headers: Optional[dict] = None) -> Response:
+    """Build a response with the SCIM media type; ``None`` payload means empty body."""
+    body = "" if payload is None else json.dumps(payload)
+    response = Response(body, status=status, mimetype=_SCIM_MEDIA_TYPE)
+    for key, value in (headers or {}).items():
+        response.headers[key] = value
+    return response
+
+
+def _scim_error(status: int, detail: str, scim_type: Optional[str] = None) -> Response:
+    """Build an RFC 7644 error response."""
+    payload: dict = {"schemas": [_ERROR_URN], "status": str(status), "detail": detail}
+    if scim_type:
+        payload["scimType"] = scim_type
+    return _scim_response(payload, status)
+
+
+def _static_list_response(resources: list) -> dict:
+    """Wrap fixed resources in a SCIM ListResponse."""
+    return {
+        "schemas": [_LIST_RESPONSE_URN],
+        "totalResults": len(resources),
+        "startIndex": 1,
+        "itemsPerPage": len(resources),
+        "Resources": resources,
+    }
+
+
+def _iso(value: Any) -> Optional[str]:
+    """Return an ISO-8601 string (repository rows may carry str or datetime)."""
+    if value is None:
+        return None
+    return value if isinstance(value, str) else value.isoformat()
+
+
+def _serialize_user(row: dict) -> dict:
+    """Map a ``users`` row to a SCIM User resource."""
+    pk = str(row["id"])
+    user_name = row["user_id"]
+    resource = {
+        "schemas": [_USER_URN],
+        "id": pk,
+        "userName": user_name,
+        "active": bool(row["active"]),
+    }
+    if "@" in user_name:
+        resource["emails"] = [{"value": user_name, "primary": True}]
+    resource["meta"] = {
+        "resourceType": "User",
+        "created": _iso(row.get("created_at")),
+        "lastModified": _iso(row.get("updated_at")),
+        "location": f"/scim/v2/Users/{pk}",
+    }
+    return resource
+
+
+# ----------------------------------------------------------------------
+# Request parsing helpers
+# ----------------------------------------------------------------------
+def _coerce_active(value: Any) -> Optional[bool]:
+    """Coerce a SCIM ``active`` value to bool; ``None`` when invalid (Okta sends strings)."""
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str) and value.lower() in ("true", "false"):
+        return value.lower() == "true"
+    return None
+
+
+def _int_arg(name: str, default: int) -> int:
+    """Read an integer query parameter, falling back to ``default``."""
+    raw = request.args.get(name)
+    if raw is None:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+
+def _parse_filter(raw: Optional[str]) -> tuple[Optional[str], Optional[Response]]:
+    """Parse the ``filter`` query param; only ``userName eq "value"`` is supported."""
+    if raw is None or not raw.strip():
+        return None, None
+    match = _USERNAME_EQ_FILTER.match(raw)
+    if match is None:
+        return None, _scim_error(400, 'Only the filter userName eq "value" is supported', "invalidFilter")
+    return match.group(1), None
+
+
+# ----------------------------------------------------------------------
+# Side effects
+# ----------------------------------------------------------------------
+class _RevocationUnavailable(Exception):
+    """A SCIM deactivation could not persist its session revocation (Redis down)."""
+
+
+def _revocation_unavailable(_exc: _RevocationUnavailable) -> Response:
+    return _scim_error(503, "Session revocation backend unavailable; retry")
+
+
+def _audit(conn: Connection, user_id: str, event: str) -> None:
+    """Best-effort audit insert in a savepoint; failure never fails the request."""
+    try:
+        with conn.begin_nested():
+            AuthEventsRepository(conn).insert(user_id, event, metadata={"via": "scim"})
+    except Exception:
+        logger.error("SCIM audit insert failed for user %s event %s", user_id, event, exc_info=True)
+
+
+def _apply_active(conn: Connection, row: dict, desired: bool) -> dict:
+    """Apply an ``active`` transition; side effects run only when the value changes."""
+    if bool(row["active"]) == desired:
+        return row
+    updated = UsersRepository(conn).set_active(str(row["id"]), desired) or row
+    user_id = row["user_id"]
+    if desired:
+        # Reactivation needs no denylist write: deactivation set a revocation
+        # watermark that already lets a fresh login through (newer iat) while
+        # keeping the pre-deactivation sessions revoked.
+        _audit(conn, user_id, "scim_reactivated")
+    else:
+        if not deny_user(user_id):
+            # Redis is down — we cannot revoke live sessions. Fail the request
+            # (the surrounding transaction rolls the deactivation back) so the
+            # IdP retries instead of recording a deprovision that didn't revoke.
+            raise _RevocationUnavailable(user_id)
+        _audit(conn, user_id, "scim_deactivated")
+    return updated
+
+
+# ----------------------------------------------------------------------
+# Bearer-token gate
+# ----------------------------------------------------------------------
+def _enforce_scim_auth() -> Optional[Response]:
+    """Gate every SCIM request on SCIM_ENABLED and the shared bearer token."""
+    if not settings.SCIM_ENABLED:
+        return _scim_error(404, "SCIM provisioning is not enabled")
+    token = settings.SCIM_TOKEN
+    if not token:
+        logger.error("SCIM is enabled but SCIM_TOKEN is not configured — rejecting request")
+        return _scim_error(503, "SCIM is enabled but no SCIM_TOKEN is configured")
+    scheme, _, presented = request.headers.get("Authorization", "").partition(" ")
+    if scheme.lower() != "bearer" or not hmac.compare_digest(
+        presented.strip().encode("utf-8"), token.encode("utf-8")
+    ):
+        return _scim_error(401, "Invalid or missing bearer token")
+    return None
+
+
+# ----------------------------------------------------------------------
+# Discovery endpoints
+# ----------------------------------------------------------------------
+def service_provider_config():
+    """Static service-provider capabilities document."""
+    return _scim_response(_SERVICE_PROVIDER_CONFIG, 200)
+
+
+def resource_types():
+    """Advertise the User resource type."""
+    return _scim_response(_static_list_response([_USER_RESOURCE_TYPE]), 200)
+
+
+def schemas():
+    """Advertise the User schema."""
+    return _scim_response(_static_list_response([_USER_SCHEMA]), 200)
+
+
+# ----------------------------------------------------------------------
+# Users
+# ----------------------------------------------------------------------
+def list_users():
+    """List users with optional exact userName filter and 1-based pagination."""
+    user_name, error = _parse_filter(request.args.get("filter"))
+    if error is not None:
+        return error
+    start_index = max(1, _int_arg("startIndex", 1))
+    count = min(max(0, _int_arg("count", _DEFAULT_COUNT)), _MAX_COUNT)
+    with db_readonly() as conn:
+        total, rows = UsersRepository(conn).list_paginated(user_name, start_index - 1, count)
+    return _scim_response(
+        {
+            "schemas": [_LIST_RESPONSE_URN],
+            "totalResults": total,
+            "startIndex": start_index,
+            "itemsPerPage": len(rows),
+            "Resources": [_serialize_user(row) for row in rows],
+        },
+        200,
+    )
+
+
+def create_user():
+    """Create a user from ``userName`` (+ optional ``active``); 409 on duplicates."""
+    body = request.get_json(force=True, silent=True)
+    if not isinstance(body, dict):
+        return _scim_error(400, "Request body must be a JSON object", "invalidValue")
+    user_name = body.get("userName")
+    if not isinstance(user_name, str) or not user_name.strip():
+        return _scim_error(400, "userName is required", "invalidValue")
+    active = _coerce_active(body.get("active", True))
+    if active is None:
+        return _scim_error(400, "active must be a boolean", "invalidValue")
+    with db_session() as conn:
+        row = UsersRepository(conn).create(user_name, active=active)
+        if row is None:
+            return _scim_error(409, f"User {user_name} already exists", "uniqueness")
+        _audit(conn, user_name, "scim_created")
+    resource = _serialize_user(row)
+    return _scim_response(resource, 201, headers={"Location": resource["meta"]["location"]})
+
+
+def get_user(user_pk: str):
+    """Fetch one user by primary key."""
+    with db_readonly() as conn:
+        row = UsersRepository(conn).get_by_pk(user_pk)
+    if not row:
+        return _scim_error(404, "User not found")
+    return _scim_response(_serialize_user(row), 200)
+
+
+def replace_user(user_pk: str):
+    """Full replace; only ``active`` is honored and ``userName`` is immutable."""
+    body = request.get_json(force=True, silent=True)
+    if not isinstance(body, dict):
+        return _scim_error(400, "Request body must be a JSON object", "invalidValue")
+    desired: Optional[bool] = None
+    if "active" in body:
+        desired = _coerce_active(body["active"])
+        if desired is None:
+            return _scim_error(400, "active must be a boolean", "invalidValue")
+    with db_session() as conn:
+        row = UsersRepository(conn).get_by_pk(user_pk)
+        if not row:
+            return _scim_error(404, "User not found")
+        if "userName" in body and str(body["userName"]).lower() != str(row["user_id"]).lower():
+            # userName is case-insensitive (caseExact=false), so a PUT that
+            # echoes the stored name under different casing is NOT a change —
+            # rejecting it would block deprovision for case-divergent IdPs.
+            return _scim_error(400, "userName is immutable", "mutability")
+        if desired is not None:
+            row = _apply_active(conn, row, desired)
+    return _scim_response(_serialize_user(row), 200)
+
+
+def patch_user(user_pk: str):
+    """Apply PatchOp replace operations targeting ``active``."""
+    body = request.get_json(force=True, silent=True)
+    if not isinstance(body, dict):
+        return _scim_error(400, "Request body must be a JSON object", "invalidValue")
+    operations = body.get("Operations")
+    if not isinstance(operations, list) or not operations:
+        return _scim_error(400, "PatchOp body with Operations is required", "invalidValue")
+    desired: Optional[bool] = None
+    for operation in operations:
+        if not isinstance(operation, dict) or str(operation.get("op", "")).strip().lower() != "replace":
+            return _scim_error(400, "Only the replace operation is supported", "invalidPath")
+        path = str(operation.get("path") or "").strip()
+        if not path:
+            value = operation.get("value")
+            if not isinstance(value, dict):
+                return _scim_error(400, "replace without path requires an object value", "invalidValue")
+            if "active" not in value:
+                continue  # Only "active" is honored; other attributes are ignored.
+            candidate = value["active"]
+        elif path.lower() == "active":
+            candidate = operation.get("value")
+        else:
+            return _scim_error(400, f"Unsupported path: {path}", "invalidPath")
+        coerced = _coerce_active(candidate)
+        if coerced is None:
+            return _scim_error(400, "active must be a boolean", "invalidValue")
+        desired = coerced
+    with db_session() as conn:
+        row = UsersRepository(conn).get_by_pk(user_pk)
+        if not row:
+            return _scim_error(404, "User not found")
+        if desired is not None:
+            row = _apply_active(conn, row, desired)
+    return _scim_response(_serialize_user(row), 200)
+
+
+def delete_user(user_pk: str):
+    """Soft delete: deactivate the user and revoke live sessions."""
+    with db_session() as conn:
+        row = UsersRepository(conn).get_by_pk(user_pk)
+        if not row:
+            return _scim_error(404, "User not found")
+        _apply_active(conn, row, False)
+    return _scim_response(None, 204)
+
+
+# ----------------------------------------------------------------------
+# Groups (not supported — answered so IdP probes don't hard-fail)
+# ----------------------------------------------------------------------
+def list_groups():
+    """Groups are not provisioned; always an empty ListResponse."""
+    return _scim_response(_static_list_response([]), 200)
+
+
+def create_group():
+    """Group creation is not supported."""
+    return _scim_error(501, "Group provisioning is not supported")
+
+
+def group_detail(group_id: str):
+    """Individual groups never exist; mutations are unsupported."""
+    if request.method == "GET":
+        return _scim_error(404, "Group not found")
+    return _scim_error(501, "Group provisioning is not supported")
+
+
+def register(bp: Blueprint) -> None:
+    """Attach the SCIM routes and bearer-token gate to ``bp``."""
+    bp.before_request(_enforce_scim_auth)
+    bp.register_error_handler(_RevocationUnavailable, _revocation_unavailable)
+    bp.add_url_rule(
+        "/scim/v2/ServiceProviderConfig", view_func=service_provider_config, methods=["GET"],
+        endpoint="service_provider_config",
+    )
+    bp.add_url_rule(
+        "/scim/v2/ResourceTypes", view_func=resource_types, methods=["GET"], endpoint="resource_types",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Schemas", view_func=schemas, methods=["GET"], endpoint="schemas",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Users", view_func=list_users, methods=["GET"], endpoint="list_users",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Users", view_func=create_user, methods=["POST"], endpoint="create_user",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Users/<user_pk>", view_func=get_user, methods=["GET"], endpoint="get_user",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Users/<user_pk>", view_func=replace_user, methods=["PUT"], endpoint="replace_user",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Users/<user_pk>", view_func=patch_user, methods=["PATCH"], endpoint="patch_user",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Users/<user_pk>", view_func=delete_user, methods=["DELETE"], endpoint="delete_user",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Groups", view_func=list_groups, methods=["GET"], endpoint="list_groups",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Groups", view_func=create_group, methods=["POST"], endpoint="create_group",
+    )
+    bp.add_url_rule(
+        "/scim/v2/Groups/<group_id>", view_func=group_detail,
+        methods=["GET", "PUT", "PATCH", "DELETE"], endpoint="group_detail",
+    )

application/api/user/__init__.py

@@ -0,0 +1,5 @@
+"""User API module - provides all user-related API endpoints"""
+
+from .routes import user
+
+__all__ = ["user"]

application/api/user/agents/__init__.py

@@ -0,0 +1,15 @@
+"""Agents module."""
+
+from .routes import agents_ns
+from .sharing import agents_sharing_ns
+from .webhooks import agents_webhooks_ns
+from .folders import agents_folders_ns
+from .portability import agents_portability_ns
+
+__all__ = [
+    "agents_ns",
+    "agents_sharing_ns",
+    "agents_webhooks_ns",
+    "agents_folders_ns",
+    "agents_portability_ns",
+]

application/api/user/agents/folders.py

@@ -0,0 +1,366 @@
+"""
+Agent folders management routes.
+Provides virtual folder organization for agents (Google Drive-like structure).
+"""
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource, fields
+from sqlalchemy import text as _sql_text
+
+from application.api import api
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agent_folders import AgentFoldersRepository
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+agents_folders_ns = Namespace(
+    "agents_folders", description="Agent folder management", path="/api/agents/folders"
+)
+
+
+def _resolve_folder_id(repo: AgentFoldersRepository, folder_id: str, user: str):
+    """Resolve a folder id that may be either a UUID or legacy Mongo ObjectId."""
+    if not folder_id:
+        return None
+    if looks_like_uuid(folder_id):
+        row = repo.get(folder_id, user)
+        if row is not None:
+            return row
+    return repo.get_by_legacy_id(folder_id, user)
+
+
+def _folder_error_response(message: str, err: Exception):
+    current_app.logger.error(f"{message}: {err}", exc_info=True)
+    return make_response(jsonify({"success": False, "message": message}), 400)
+
+
+def _serialize_folder(f: dict) -> dict:
+    created_at = f.get("created_at")
+    updated_at = f.get("updated_at")
+    return {
+        "id": str(f["id"]),
+        "name": f.get("name"),
+        "parent_id": str(f["parent_id"]) if f.get("parent_id") else None,
+        "created_at": created_at.isoformat() if hasattr(created_at, "isoformat") else created_at,
+        "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at,
+    }
+
+
+@agents_folders_ns.route("/")
+class AgentFolders(Resource):
+    @api.doc(description="Get all folders for the user")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                folders = AgentFoldersRepository(conn).list_for_user(user)
+            result = [_serialize_folder(f) for f in folders]
+            return make_response(jsonify({"folders": result}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to fetch folders", err)
+
+    @api.doc(description="Create a new folder")
+    @api.expect(
+        api.model(
+            "CreateFolder",
+            {
+                "name": fields.String(required=True, description="Folder name"),
+                "parent_id": fields.String(required=False, description="Parent folder ID"),
+            },
+        )
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data or not data.get("name"):
+            return make_response(jsonify({"success": False, "message": "Folder name is required"}), 400)
+
+        parent_id_input = data.get("parent_id")
+        description = data.get("description")
+
+        try:
+            with db_session() as conn:
+                repo = AgentFoldersRepository(conn)
+                pg_parent_id = None
+                if parent_id_input:
+                    parent = _resolve_folder_id(repo, parent_id_input, user)
+                    if not parent:
+                        return make_response(
+                            jsonify({"success": False, "message": "Parent folder not found"}),
+                            404,
+                        )
+                    pg_parent_id = str(parent["id"])
+                folder = repo.create(
+                    user, data["name"],
+                    description=description,
+                    parent_id=pg_parent_id,
+                )
+            return make_response(
+                jsonify(
+                    {
+                        "id": str(folder["id"]),
+                        "name": folder["name"],
+                        "parent_id": pg_parent_id,
+                    }
+                ),
+                201,
+            )
+        except Exception as err:
+            return _folder_error_response("Failed to create folder", err)
+
+
+@agents_folders_ns.route("/<string:folder_id>")
+class AgentFolder(Resource):
+    @api.doc(description="Get a specific folder with its agents")
+    def get(self, folder_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                folders_repo = AgentFoldersRepository(conn)
+                folder = _resolve_folder_id(folders_repo, folder_id, user)
+                if not folder:
+                    return make_response(
+                        jsonify({"success": False, "message": "Folder not found"}),
+                        404,
+                    )
+                pg_folder_id = str(folder["id"])
+
+                agents_rows = conn.execute(
+                    _sql_text(
+                        "SELECT id, name, description FROM agents "
+                        "WHERE user_id = :user_id AND folder_id = CAST(:fid AS uuid) "
+                        "ORDER BY created_at DESC"
+                    ),
+                    {"user_id": user, "fid": pg_folder_id},
+                ).fetchall()
+                agents_list = [
+                    {
+                        "id": str(row._mapping["id"]),
+                        "name": row._mapping["name"],
+                        "description": row._mapping.get("description", "") or "",
+                    }
+                    for row in agents_rows
+                ]
+
+                subfolders = folders_repo.list_children(pg_folder_id, user)
+                subfolders_list = [
+                    {"id": str(sf["id"]), "name": sf["name"]}
+                    for sf in subfolders
+                ]
+
+            return make_response(
+                jsonify(
+                    {
+                        "id": pg_folder_id,
+                        "name": folder["name"],
+                        "parent_id": (
+                            str(folder["parent_id"]) if folder.get("parent_id") else None
+                        ),
+                        "agents": agents_list,
+                        "subfolders": subfolders_list,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            return _folder_error_response("Failed to fetch folder", err)
+
+    @api.doc(description="Update a folder")
+    def put(self, folder_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data:
+            return make_response(jsonify({"success": False, "message": "No data provided"}), 400)
+
+        try:
+            with db_session() as conn:
+                repo = AgentFoldersRepository(conn)
+                folder = _resolve_folder_id(repo, folder_id, user)
+                if not folder:
+                    return make_response(
+                        jsonify({"success": False, "message": "Folder not found"}),
+                        404,
+                    )
+                pg_folder_id = str(folder["id"])
+
+                update_fields: dict = {}
+                if "name" in data:
+                    update_fields["name"] = data["name"]
+                if "description" in data:
+                    update_fields["description"] = data["description"]
+                if "parent_id" in data:
+                    parent_input = data.get("parent_id")
+                    if parent_input:
+                        if parent_input == folder_id or parent_input == pg_folder_id:
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": "Cannot set folder as its own parent",
+                                    }
+                                ),
+                                400,
+                            )
+                        parent = _resolve_folder_id(repo, parent_input, user)
+                        if not parent:
+                            return make_response(
+                                jsonify({"success": False, "message": "Parent folder not found"}),
+                                404,
+                            )
+                        if str(parent["id"]) == pg_folder_id:
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": "Cannot set folder as its own parent",
+                                    }
+                                ),
+                                400,
+                            )
+                        update_fields["parent_id"] = str(parent["id"])
+                    else:
+                        update_fields["parent_id"] = None
+
+                if update_fields:
+                    repo.update(pg_folder_id, user, update_fields)
+
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to update folder", err)
+
+    @api.doc(description="Delete a folder")
+    def delete(self, folder_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                repo = AgentFoldersRepository(conn)
+                folder = _resolve_folder_id(repo, folder_id, user)
+                if not folder:
+                    return make_response(
+                        jsonify({"success": False, "message": "Folder not found"}),
+                        404,
+                    )
+                pg_folder_id = str(folder["id"])
+                # Clear folder assignments from agents; self-FK
+                # ``ON DELETE SET NULL`` handles child folders.
+                AgentsRepository(conn).clear_folder_for_all(pg_folder_id, user)
+                deleted = repo.delete(pg_folder_id, user)
+            if not deleted:
+                return make_response(
+                    jsonify({"success": False, "message": "Folder not found"}),
+                    404,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to delete folder", err)
+
+
+@agents_folders_ns.route("/move_agent")
+class MoveAgentToFolder(Resource):
+    @api.doc(description="Move an agent to a folder or remove from folder")
+    @api.expect(
+        api.model(
+            "MoveAgent",
+            {
+                "agent_id": fields.String(required=True, description="Agent ID to move"),
+                "folder_id": fields.String(required=False, description="Target folder ID (null to remove from folder)"),
+            },
+        )
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data or not data.get("agent_id"):
+            return make_response(jsonify({"success": False, "message": "Agent ID is required"}), 400)
+
+        agent_id_input = data["agent_id"]
+        folder_id_input = data.get("folder_id")
+
+        try:
+            with db_session() as conn:
+                agents_repo = AgentsRepository(conn)
+                agent = agents_repo.get_any(agent_id_input, user)
+                if not agent:
+                    return make_response(
+                        jsonify({"success": False, "message": "Agent not found"}),
+                        404,
+                    )
+                pg_folder_id = None
+                if folder_id_input:
+                    folders_repo = AgentFoldersRepository(conn)
+                    folder = _resolve_folder_id(folders_repo, folder_id_input, user)
+                    if not folder:
+                        return make_response(
+                            jsonify({"success": False, "message": "Folder not found"}),
+                            404,
+                        )
+                    pg_folder_id = str(folder["id"])
+                agents_repo.set_folder(str(agent["id"]), user, pg_folder_id)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to move agent", err)
+
+
+@agents_folders_ns.route("/bulk_move")
+class BulkMoveAgents(Resource):
+    @api.doc(description="Move multiple agents to a folder")
+    @api.expect(
+        api.model(
+            "BulkMoveAgents",
+            {
+                "agent_ids": fields.List(fields.String, required=True, description="List of agent IDs"),
+                "folder_id": fields.String(required=False, description="Target folder ID"),
+            },
+        )
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data or not data.get("agent_ids"):
+            return make_response(jsonify({"success": False, "message": "Agent IDs are required"}), 400)
+
+        agent_ids = data["agent_ids"]
+        folder_id_input = data.get("folder_id")
+
+        try:
+            with db_session() as conn:
+                agents_repo = AgentsRepository(conn)
+                pg_folder_id = None
+                if folder_id_input:
+                    folders_repo = AgentFoldersRepository(conn)
+                    folder = _resolve_folder_id(folders_repo, folder_id_input, user)
+                    if not folder:
+                        return make_response(
+                            jsonify({"success": False, "message": "Folder not found"}),
+                            404,
+                        )
+                    pg_folder_id = str(folder["id"])
+                for agent_id_input in agent_ids:
+                    agent = agents_repo.get_any(agent_id_input, user)
+                    if agent is not None:
+                        agents_repo.set_folder(str(agent["id"]), user, pg_folder_id)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to move agents", err)

application/api/user/agents/portability.py

@@ -0,0 +1,949 @@
+"""Agent YAML export / import.
+
+Agents are mostly references (sources, tools, prompt, models). Export
+translates those into portable identifiers and never emits a secret;
+import resolves them back against the importing user — matching existing
+resources or creating them from the file plus user-supplied tokens.
+
+Re-import is idempotent at every level: the agent is matched by
+``metadata.id`` then ``metadata.slug``, tools by ``(type, name)``,
+prompts by ``(name, content)``, custom models by
+``(display_name, upstream_model_id, base_url)`` — so the same file
+applied twice updates rather than duplicating.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Optional
+
+import yaml
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from application.agents.default_tools import (
+    default_tool_id,
+    is_synthesized_tool_id,
+    synthesize_tool_by_name,
+    synthesized_tool_name_for_id,
+)
+from application.api import api
+from application.core.model_utils import validate_model_id
+from application.core.url_validation import SSRFError, validate_url
+from application.security.safe_url import UnsafeUserUrlError, validate_user_base_url
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.prompts import PromptsRepository
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.repositories.user_custom_models import (
+    UserCustomModelsRepository,
+)
+from application.storage.db.repositories.user_tools import UserToolsRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+API_VERSION = "docsgpt.arc53.com/v1"
+KIND = "Agent"
+
+# Import safety bounds (DoS): cap the document size and the number of
+# child resources a single import may reference / create.
+MAX_IMPORT_BYTES = 512 * 1024
+MAX_LIST_ITEMS = 100
+
+
+class AgentImportError(Exception):
+    """Raised when an agent YAML document is malformed or unsupported."""
+
+
+class _SafeNoAliasLoader(yaml.SafeLoader):
+    """SafeLoader that also rejects anchors/aliases (billion-laughs guard)."""
+
+    def compose_node(self, parent, index):
+        if self.check_event(yaml.events.AliasEvent):
+            raise AgentImportError("YAML anchors/aliases are not allowed")
+        return super().compose_node(parent, index)
+
+
+# ---------------------------------------------------------------------------
+# Slug helpers
+# ---------------------------------------------------------------------------
+
+
+def slugify(name: Optional[str]) -> str:
+    """Turn an agent name into a url-safe slug; never empty."""
+    s = (name or "").strip().lower()
+    s = re.sub(r"[^a-z0-9]+", "-", s)
+    s = re.sub(r"-{2,}", "-", s).strip("-")
+    return s or "agent"
+
+
+def _unique_slug(
+    repo: AgentsRepository,
+    user: str,
+    base: str,
+    *,
+    exclude_id: Optional[str] = None,
+) -> str:
+    """Return ``base`` (slugified) or the first ``-N`` variant free for ``user``."""
+    base = slugify(base)
+    candidate = base
+    n = 2
+    while True:
+        found = repo.find_by_slug(user, candidate)
+        if found is None or (exclude_id and str(found["id"]) == str(exclude_id)):
+            return candidate
+        candidate = f"{base}-{n}"
+        n += 1
+
+
+def ensure_agent_slug(conn, agent: dict, user: str) -> str:
+    """Return the agent's slug, lazily assigning and persisting one if absent."""
+    existing = agent.get("slug")
+    if existing:
+        return str(existing)
+    repo = AgentsRepository(conn)
+    slug = _unique_slug(repo, user, agent.get("name"), exclude_id=str(agent["id"]))
+    repo.update(str(agent["id"]), user, {"slug": slug})
+    return slug
+
+
+# ---------------------------------------------------------------------------
+# Tool-manager access (lazy — avoids importing the tool registry at module load)
+# ---------------------------------------------------------------------------
+
+
+def _tool_manager():
+    from application.api.user.tools.routes import tool_manager
+
+    return tool_manager
+
+
+def _tool_instance(tool_type: str):
+    return _tool_manager().tools.get(tool_type)
+
+
+def _secret_field_names(config_requirements: dict, *, required_only: bool = False) -> list:
+    out = []
+    for key, spec in (config_requirements or {}).items():
+        if not isinstance(spec, dict) or not spec.get("secret"):
+            continue
+        if required_only and not spec.get("required"):
+            continue
+        out.append(key)
+    return out
+
+
+def _live_requires_secrets(tool_type: str) -> list:
+    inst = _tool_instance(tool_type)
+    if inst is None:
+        return []
+    return _secret_field_names(inst.get_config_requirements() or {})
+
+
+def _safe_export_config(stored_config: dict, config_requirements: dict) -> dict:
+    """Return only config values provably non-secret (allowlist).
+
+    Export must never leak a credential. A key is emitted ONLY if
+    ``config_requirements`` declares it and does not mark it secret.
+    Anything not described by requirements — e.g. ``api_tool``/MCP free-form
+    ``headers``, ``url`` query strings, ``server_url`` that routinely embed
+    tokens, or tools with empty ``config_requirements`` — is dropped. Reuse
+    on import matches the existing tool; create re-collects config from the
+    user. This blocklist-free approach can't be defeated by a stale or empty
+    ``config_requirements``.
+    """
+    requirements = config_requirements or {}
+    safe: dict[str, Any] = {}
+    for key, value in (stored_config or {}).items():
+        spec = requirements.get(key)
+        if isinstance(spec, dict) and not spec.get("secret"):
+            safe[key] = value
+    return safe
+
+
+# ---------------------------------------------------------------------------
+# Export
+# ---------------------------------------------------------------------------
+
+
+def _serialize_prompt(conn, agent: dict, user: str):
+    prompt_id = agent.get("prompt_id")
+    if not prompt_id:
+        return "default"
+    row = PromptsRepository(conn).get(str(prompt_id), user)
+    if not row:
+        return "default"
+    return {"name": row.get("name") or "", "content": row.get("content") or ""}
+
+
+def _serialize_sources(conn, agent: dict, user: str) -> list:
+    ids: list[str] = []
+    if agent.get("source_id"):
+        ids.append(str(agent["source_id"]))
+    for sid in agent.get("extra_source_ids") or []:
+        s = str(sid)
+        if s and s not in ids:
+            ids.append(s)
+    repo = SourcesRepository(conn)
+    out = []
+    for sid in ids:
+        row = repo.get(sid, user)
+        if not row:
+            continue
+        out.append(
+            {
+                "name": row.get("name") or "",
+                "type": row.get("type") or "",
+                "ref": sid,
+            }
+        )
+    return out
+
+
+def _serialize_tools(conn, agent: dict, user: str) -> list:
+    repo = UserToolsRepository(conn)
+    out = []
+    for tid in agent.get("tools") or []:
+        tid_str = str(tid)
+        if is_synthesized_tool_id(tid_str):
+            out.append({"type": synthesized_tool_name_for_id(tid_str), "builtin": True})
+            continue
+        row = repo.get_any(tid_str, user)
+        if not row:
+            continue
+        requirements = row.get("config_requirements") or {}
+        requires_secrets = _secret_field_names(requirements)
+        entry: dict[str, Any] = {
+            "type": row.get("name") or "",
+            # Raw custom_name (may be "") so import matching is symmetric and
+            # idempotent — see _find_user_tool.
+            "name": row.get("custom_name") or "",
+            "display_name": row.get("display_name") or "",
+            "description": row.get("description") or "",
+            "config": _safe_export_config(row.get("config") or {}, requirements),
+            "ref": tid_str,
+        }
+        if requires_secrets:
+            entry["requires_secrets"] = requires_secrets
+        out.append(entry)
+    return out
+
+
+def _serialize_models(conn, agent: dict, user: str) -> dict:
+    repo = UserCustomModelsRepository(conn)
+
+    def describe(model_id: str):
+        mid = str(model_id)
+        if looks_like_uuid(mid):
+            row = repo.get(mid, user)
+            if not row:
+                return None
+            return {
+                "type": "custom",
+                "display_name": row.get("display_name") or "",
+                "upstream_model_id": row.get("upstream_model_id") or "",
+                "base_url": row.get("base_url") or "",
+                "capabilities": row.get("capabilities") or {},
+                "requires_secrets": ["api_key"],
+                "ref": mid,
+            }
+        return mid
+
+    available = []
+    for m in agent.get("models") or []:
+        described = describe(m)
+        if described is not None:
+            available.append(described)
+
+    default_value = ""
+    raw_default = agent.get("default_model_id") or ""
+    if raw_default:
+        described = describe(raw_default)
+        if isinstance(described, dict):
+            default_value = described["display_name"]
+        elif isinstance(described, str):
+            default_value = described
+    return {"default": default_value, "available": available}
+
+
+def serialize_agent(conn, agent: dict, user: str) -> dict:
+    """Build the portable export document for an agent row."""
+    spec = {
+        "name": agent.get("name") or "",
+        "description": agent.get("description") or "",
+        "agent_type": agent.get("agent_type") or "classic",
+        "retriever": agent.get("retriever") or "classic",
+        "chunks": int(agent["chunks"]) if agent.get("chunks") is not None else None,
+        "prompt": _serialize_prompt(conn, agent, user),
+        "model": _serialize_models(conn, agent, user),
+        "sources": _serialize_sources(conn, agent, user),
+        "tools": _serialize_tools(conn, agent, user),
+        "limits": {
+            "limited_token_mode": bool(agent.get("limited_token_mode", False)),
+            "token_limit": agent.get("token_limit"),
+            "limited_request_mode": bool(agent.get("limited_request_mode", False)),
+            "request_limit": agent.get("request_limit"),
+        },
+        "json_schema": agent.get("json_schema"),
+        "allow_system_prompt_override": bool(agent.get("allow_system_prompt_override", False)),
+    }
+    return {
+        "apiVersion": API_VERSION,
+        "kind": KIND,
+        "metadata": {"id": str(agent["id"]), "slug": agent.get("slug") or ""},
+        "spec": spec,
+    }
+
+
+class _AgentYamlDumper(yaml.SafeDumper):
+    """SafeDumper that renders multi-line strings as block scalars."""
+
+
+def _str_representer(dumper, data):
+    style = "|" if "\n" in data else None
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style)
+
+
+_AgentYamlDumper.add_representer(str, _str_representer)
+
+
+def agent_to_yaml(export: dict) -> str:
+    """Serialize an export document to YAML, preserving key order."""
+    return yaml.dump(
+        export,
+        Dumper=_AgentYamlDumper,
+        sort_keys=False,
+        default_flow_style=False,
+        allow_unicode=True,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Parse
+# ---------------------------------------------------------------------------
+
+
+def parse_agent_yaml(text: str) -> dict:
+    """Parse and shallow-validate an agent YAML document."""
+    if text and len(text) > MAX_IMPORT_BYTES:
+        raise AgentImportError("Import document too large")
+    try:
+        doc = yaml.load(text, Loader=_SafeNoAliasLoader)
+    except yaml.YAMLError as exc:
+        raise AgentImportError(f"Invalid YAML: {exc}") from exc
+    if not isinstance(doc, dict):
+        raise AgentImportError("Top-level YAML must be a mapping")
+    if doc.get("kind") != KIND:
+        raise AgentImportError(f"Unsupported kind {doc.get('kind')!r}; expected {KIND!r}")
+    if not str(doc.get("apiVersion") or "").startswith("docsgpt."):
+        raise AgentImportError(f"Unsupported apiVersion {doc.get('apiVersion')!r}")
+    spec = doc.get("spec")
+    if not isinstance(spec, dict):
+        raise AgentImportError("Missing or invalid 'spec'")
+    if not spec.get("name"):
+        raise AgentImportError("spec.name is required")
+    if spec.get("agent_type") == "workflow":
+        raise AgentImportError("Workflow agents are not supported for import yet")
+    for key in ("sources", "tools"):
+        value = spec.get(key)
+        if isinstance(value, list) and len(value) > MAX_LIST_ITEMS:
+            raise AgentImportError(f"Too many {key} (max {MAX_LIST_ITEMS})")
+    available = (spec.get("model") or {}).get("available")
+    if isinstance(available, list) and len(available) > MAX_LIST_ITEMS:
+        raise AgentImportError(f"Too many models (max {MAX_LIST_ITEMS})")
+    return doc
+
+
+# ---------------------------------------------------------------------------
+# Reference resolution (shared by plan + apply)
+# ---------------------------------------------------------------------------
+
+
+def _resolve_target(conn, user: str, metadata: dict) -> dict:
+    repo = AgentsRepository(conn)
+    metadata = metadata or {}
+    agent_id = metadata.get("id")
+    if agent_id and looks_like_uuid(str(agent_id)):
+        row = repo.get(str(agent_id), user)
+        if row:
+            return {
+                "action": "update",
+                "agent_id": str(row["id"]),
+                "matched_by": "id",
+                "status": row.get("status"),
+            }
+    slug = metadata.get("slug")
+    if slug:
+        row = repo.find_by_slug(user, str(slug))
+        if row:
+            return {
+                "action": "update",
+                "agent_id": str(row["id"]),
+                "matched_by": "slug",
+                "status": row.get("status"),
+            }
+    return {"action": "create", "agent_id": None, "matched_by": None, "status": None}
+
+
+def _find_user_tool(user_tools: list, tool_type: str, custom_name: str):
+    """Match a stored tool by ``(type, custom_name)`` for idempotent reuse.
+
+    Compares against the stored ``custom_name`` directly (export emits the
+    raw value), so a named tool matches exactly and an unnamed one (``""``)
+    reuses the first same-type tool rather than spawning a duplicate.
+    """
+    target = custom_name or ""
+    for row in user_tools:
+        if (row.get("name") or "") != tool_type:
+            continue
+        if (row.get("custom_name") or "") == target:
+            return row
+    return None
+
+
+def _find_custom_model(customs: list, spec_model: dict):
+    for row in customs:
+        if (
+            (row.get("display_name") or "") == (spec_model.get("display_name") or "")
+            and (row.get("upstream_model_id") or "") == (spec_model.get("upstream_model_id") or "")
+            and (row.get("base_url") or "") == (spec_model.get("base_url") or "")
+        ):
+            return row
+    return None
+
+
+def _tool_key(index: int) -> str:
+    return f"tool-{index}"
+
+
+# ---------------------------------------------------------------------------
+# Plan (dry run)
+# ---------------------------------------------------------------------------
+
+
+def plan_import(conn, user: str, doc: dict) -> dict:
+    """Resolve every reference without writing; returns a resolution report."""
+    spec = doc["spec"]
+    target = _resolve_target(conn, user, doc.get("metadata") or {})
+
+    sources_repo = SourcesRepository(conn)
+    sources = []
+    for src in spec.get("sources") or []:
+        name = src.get("name") or ""
+        match = sources_repo.find_by_name(user, name)
+        sources.append(
+            {
+                "name": name,
+                "type": src.get("type") or "",
+                "status": "matched" if match else "missing",
+                "target_id": str(match["id"]) if match else None,
+            }
+        )
+
+    user_tools = UserToolsRepository(conn).list_for_user(user)
+    tools = []
+    for index, tool in enumerate(spec.get("tools") or []):
+        tool_type = tool.get("type") or ""
+        if tool.get("builtin"):
+            available = synthesize_tool_by_name(tool_type) is not None
+            tools.append(
+                {
+                    "key": _tool_key(index),
+                    "type": tool_type,
+                    "builtin": True,
+                    "status": "builtin" if available else "unavailable",
+                    "target_id": default_tool_id(tool_type) if available else None,
+                }
+            )
+            continue
+        custom_name = tool.get("name") or ""
+        match = _find_user_tool(user_tools, tool_type, custom_name)
+        if match:
+            tools.append(
+                {
+                    "key": _tool_key(index),
+                    "type": tool_type,
+                    "name": custom_name,
+                    "status": "reuse",
+                    "target_id": str(match["id"]),
+                }
+            )
+            continue
+        available = _tool_instance(tool_type) is not None
+        tools.append(
+            {
+                "key": _tool_key(index),
+                "type": tool_type,
+                "name": custom_name,
+                "status": "create" if available else "unavailable",
+                "requires_secrets": tool.get("requires_secrets") or _live_requires_secrets(tool_type),
+            }
+        )
+
+    prompt_spec = spec.get("prompt")
+    if isinstance(prompt_spec, dict):
+        existing = PromptsRepository(conn).find(
+            user, prompt_spec.get("name") or "", prompt_spec.get("content") or ""
+        )
+        prompt = {"status": "reuse" if existing else "create", "name": prompt_spec.get("name") or ""}
+    else:
+        prompt = {"status": "default"}
+
+    customs = UserCustomModelsRepository(conn).list_for_user(user)
+    models = []
+    for entry in (spec.get("model") or {}).get("available") or []:
+        if isinstance(entry, str):
+            models.append(
+                {
+                    "id": entry,
+                    "status": "matched" if validate_model_id(entry, user) else "unavailable",
+                }
+            )
+        elif isinstance(entry, dict):
+            match = _find_custom_model(customs, entry)
+            models.append(
+                {
+                    "display_name": entry.get("display_name") or "",
+                    "status": "reuse" if match else "create",
+                    "requires_secrets": ["api_key"] if not match else [],
+                }
+            )
+
+    return {"target": target, "sources": sources, "tools": tools, "prompt": prompt, "models": models}
+
+
+# ---------------------------------------------------------------------------
+# Apply
+# ---------------------------------------------------------------------------
+
+
+def _apply_prompt(conn, user: str, spec: dict) -> Optional[str]:
+    prompt_spec = spec.get("prompt")
+    if not isinstance(prompt_spec, dict):
+        return None
+    row = PromptsRepository(conn).find_or_create(
+        user, prompt_spec.get("name") or "Imported prompt", prompt_spec.get("content") or ""
+    )
+    return str(row["id"])
+
+
+def _apply_sources(conn, user: str, spec: dict, resolution: dict, warnings: list) -> list:
+    repo = SourcesRepository(conn)
+    mapping = resolution.get("sources")
+    if not isinstance(mapping, dict):
+        mapping = {}
+    resolved: list[str] = []
+    for src in spec.get("sources") or []:
+        name = src.get("name") or ""
+        mapped = mapping.get(name)
+        if mapped:
+            # Ownership-check the client-supplied id before linking (IDOR guard).
+            owned = repo.get_any(str(mapped), user)
+            if owned:
+                resolved.append(str(owned["id"]))
+            else:
+                warnings.append(f"Source mapping for '{name}' is not yours; ignored")
+            continue
+        match = repo.find_by_name(user, name)
+        if match:
+            resolved.append(str(match["id"]))
+        else:
+            warnings.append(f"Source '{name}' not found; left unattached")
+    seen: set = set()
+    out = []
+    for sid in resolved:
+        if sid not in seen:
+            seen.add(sid)
+            out.append(sid)
+    return out
+
+
+def _validate_tool_urls(tool_type: str, config: dict) -> Optional[str]:
+    """SSRF-guard imported tool config to parity with the create routes.
+
+    Returns an error message if a URL is unsafe, else None.
+    """
+    try:
+        if tool_type == "mcp_tool":
+            server_url = (config.get("server_url") or "").strip()
+            if server_url:
+                validate_url(server_url)
+        elif tool_type == "api_tool":
+            url = (config.get("url") or "").strip()
+            if url:
+                validate_url(url)
+    except SSRFError:
+        return f"Tool '{tool_type}' has an unsafe URL; not created"
+    return None
+
+
+def _create_tool_from_spec(conn, user: str, tool: dict, secrets: dict, warnings: list) -> Optional[str]:
+    from application.api.user.tools.routes import _encrypt_secret_fields, transform_actions
+
+    tool_type = tool.get("type") or ""
+    inst = _tool_instance(tool_type)
+    if inst is None:
+        warnings.append(f"Tool type '{tool_type}' not available on this instance; skipped")
+        return None
+    config_requirements = inst.get_config_requirements() or {}
+    config = dict(tool.get("config") or {})
+    config.update(secrets or {})
+    missing = [k for k in _secret_field_names(config_requirements, required_only=True) if not config.get(k)]
+    if missing:
+        warnings.append(f"Tool '{tool_type}' needs secret(s) {missing}; not created")
+        return None
+    url_error = _validate_tool_urls(tool_type, config)
+    if url_error:
+        warnings.append(url_error)
+        return None
+    provided_secrets = [k for k in _secret_field_names(config_requirements) if config.get(k)]
+    storage_config = _encrypt_secret_fields(config, config_requirements, user)
+    if provided_secrets and not storage_config.get("encrypted_credentials"):
+        warnings.append(f"Tool '{tool_type}' secret encryption failed; not created")
+        return None
+    actions = transform_actions(inst.get_actions_metadata() or [])
+    created = UserToolsRepository(conn).create(
+        user,
+        tool_type,
+        config=storage_config,
+        custom_name=tool.get("name") or "",
+        display_name=tool.get("display_name") or tool_type,
+        description=tool.get("description") or "",
+        config_requirements=config_requirements,
+        actions=actions,
+        status=True,
+    )
+    return str(created["id"])
+
+
+def _apply_tools(conn, user: str, spec: dict, resolution: dict, warnings: list) -> list:
+    repo = UserToolsRepository(conn)
+    user_tools = repo.list_for_user(user)
+    decisions = resolution.get("tools")
+    if not isinstance(decisions, dict):
+        decisions = {}
+    out: list[str] = []
+    for index, tool in enumerate(spec.get("tools") or []):
+        tool_type = tool.get("type") or ""
+        decision = decisions.get(_tool_key(index))
+        if not isinstance(decision, dict):
+            decision = {}
+        choice = decision.get("decision")
+
+        if tool.get("builtin"):
+            if synthesize_tool_by_name(tool_type) is None:
+                warnings.append(f"Built-in tool '{tool_type}' unavailable; skipped")
+                continue
+            out.append(default_tool_id(tool_type))
+            continue
+
+        if choice == "skip":
+            continue
+        if choice == "reuse" and decision.get("tool_id"):
+            # Ownership-check the client-supplied id before linking (IDOR guard).
+            tid = str(decision["tool_id"])
+            if is_synthesized_tool_id(tid) or repo.get_any(tid, user):
+                out.append(tid)
+            else:
+                warnings.append(f"Tool reuse id '{tid}' is not yours; ignored")
+            continue
+
+        custom_name = tool.get("name") or ""
+        match = _find_user_tool(user_tools, tool_type, custom_name)
+        if match and choice != "create":
+            out.append(str(match["id"]))
+            continue
+
+        created_id = _create_tool_from_spec(conn, user, tool, decision.get("secrets") or {}, warnings)
+        if created_id:
+            out.append(created_id)
+            # Record so a later identical spec entry reuses rather than re-creates.
+            user_tools.append({"id": created_id, "name": tool_type, "custom_name": custom_name})
+    return out
+
+
+def _apply_models(conn, user: str, spec: dict, resolution: dict, warnings: list):
+    model_spec = spec.get("model") or {}
+    repo = UserCustomModelsRepository(conn)
+    customs = repo.list_for_user(user)
+    decisions = resolution.get("models")
+    if not isinstance(decisions, dict):
+        decisions = {}
+    name_to_id: dict[str, str] = {}
+    models: list[str] = []
+
+    for entry in model_spec.get("available") or []:
+        if isinstance(entry, str):
+            if validate_model_id(entry, user):
+                models.append(entry)
+                name_to_id[entry] = entry
+            else:
+                warnings.append(f"Model '{entry}' not available; skipped")
+            continue
+        if not isinstance(entry, dict):
+            continue
+        display_name = entry.get("display_name") or ""
+        match = _find_custom_model(customs, entry)
+        if match:
+            models.append(str(match["id"]))
+            name_to_id[display_name] = str(match["id"])
+            continue
+        decision = decisions.get(display_name)
+        api_key = decision.get("api_key") if isinstance(decision, dict) else None
+        if not api_key:
+            warnings.append(f"Custom model '{display_name}' needs an API key; skipped")
+            continue
+        base_url = entry.get("base_url") or ""
+        try:
+            validate_user_base_url(base_url)
+        except UnsafeUserUrlError:
+            warnings.append(f"Custom model '{display_name}' has an unsafe base URL; skipped")
+            continue
+        created = repo.create(
+            user,
+            entry.get("upstream_model_id") or "",
+            display_name,
+            base_url,
+            api_key,
+            capabilities=entry.get("capabilities") or {},
+        )
+        if not created.get("api_key_encrypted"):
+            warnings.append(f"Custom model '{display_name}' key encryption failed; skipped")
+            continue
+        models.append(str(created["id"]))
+        name_to_id[display_name] = str(created["id"])
+
+    default_id = ""
+    raw_default = model_spec.get("default") or ""
+    if raw_default:
+        if raw_default in name_to_id:
+            default_id = name_to_id[raw_default]
+        elif validate_model_id(raw_default, user):
+            default_id = raw_default
+        else:
+            # Custom-model default carried as its display_name — resolve against
+            # the user's own models even if it wasn't in `available`.
+            owned = next(
+                (c for c in customs if (c.get("display_name") or "") == raw_default), None
+            )
+            if owned:
+                default_id = str(owned["id"])
+            else:
+                warnings.append(f"Default model '{raw_default}' not resolved")
+    return (models or None), (default_id or None)
+
+
+def _limit(spec: dict, key: str):
+    return (spec.get("limits") or {}).get(key)
+
+
+def apply_import(conn, user: str, doc: dict, resolution: Optional[dict] = None) -> dict:
+    """Create or update an agent from a parsed YAML doc.
+
+    A newly created agent lands as a draft. An update to an existing agent
+    (matched by id or slug) preserves that agent's current status, so
+    re-importing over a published agent keeps it live — its API key and any
+    active users are unaffected — rather than silently reverting it to draft.
+
+    On update the YAML is authoritative: fields it specifies are written even
+    when that clears a value (e.g. removing all models, dropping ``json_schema``,
+    or switching ``prompt`` back to default), so re-importing an edited file is
+    a true sync rather than an additive merge.
+    """
+    if not isinstance(resolution, dict):
+        resolution = {}
+    spec = doc["spec"]
+    metadata = doc.get("metadata") or {}
+    warnings: list[str] = []
+
+    target = _resolve_target(conn, user, metadata)
+
+    prompt_id = _apply_prompt(conn, user, spec)
+    source_ids = _apply_sources(conn, user, spec, resolution, warnings)
+    tool_ids = _apply_tools(conn, user, spec, resolution, warnings)
+    models, default_model_id = _apply_models(conn, user, spec, resolution, warnings)
+
+    agents_repo = AgentsRepository(conn)
+    is_update = bool(target.get("action") == "update" and target.get("agent_id"))
+    exclude_id = str(target["agent_id"]) if is_update else None
+    # Slug is recomputed from the file, not preserved from the matched row:
+    # a round-tripped export carries metadata.slug so it stays stable, but an
+    # update-by-id whose file omits the slug and renames the agent rewrites the
+    # slug to follow the new name (excluding the row's own slug from collision
+    # checks). Intended — the slug tracks the file.
+    slug = _unique_slug(agents_repo, user, metadata.get("slug") or spec.get("name"), exclude_id=exclude_id)
+
+    try:
+        chunks_value = int(spec["chunks"]) if spec.get("chunks") is not None else 2
+    except (TypeError, ValueError):
+        chunks_value = 2
+
+    # YAML-authoritative fields — written even when the resolved value is None,
+    # so a re-import can CLEAR models / json_schema / prompt-to-default. On
+    # create, AgentsRepository.create() skips None kwargs, so Nones become
+    # column defaults.
+    authoritative: dict[str, Any] = {
+        "description": spec.get("description") or "",
+        "agent_type": spec.get("agent_type") or "classic",
+        "chunks": chunks_value,
+        "prompt_id": prompt_id,
+        "tools": tool_ids,
+        "json_schema": spec.get("json_schema"),
+        "models": models,
+        "default_model_id": default_model_id,
+        "extra_source_ids": source_ids,
+        "limited_token_mode": bool(_limit(spec, "limited_token_mode")),
+        "limited_request_mode": bool(_limit(spec, "limited_request_mode")),
+        "allow_system_prompt_override": bool(spec.get("allow_system_prompt_override")),
+        "slug": slug,
+    }
+    # Optional fields — applied only when present so a partial file doesn't wipe them.
+    optional = {
+        "retriever": spec.get("retriever") or ("classic" if not source_ids else None),
+        "token_limit": _limit(spec, "token_limit"),
+        "request_limit": _limit(spec, "request_limit"),
+    }
+    optional = {k: v for k, v in optional.items() if v is not None}
+
+    if is_update:
+        # Status is intentionally omitted so an update never changes it:
+        # re-importing over a published agent keeps it live (its API key and
+        # any active users are unaffected) instead of reverting it to draft.
+        # Only a brand-new agent (the create path below) starts as a draft.
+        fields = {**authoritative, **optional, "name": spec.get("name")}
+        if agents_repo.update(str(target["agent_id"]), user, fields):
+            return {
+                "agent_id": str(target["agent_id"]),
+                "action": "updated",
+                "status": target.get("status") or "draft",
+                "slug": slug,
+                "warnings": warnings,
+            }
+        # Row vanished between resolve and write — fall through to create.
+
+    row = agents_repo.create(user, spec.get("name"), "draft", **{**authoritative, **optional})
+    return {
+        "agent_id": str(row["id"]),
+        "action": "created",
+        "status": "draft",
+        "slug": slug,
+        "warnings": warnings,
+    }
+
+
+# ---------------------------------------------------------------------------
+# HTTP surface
+# ---------------------------------------------------------------------------
+
+agents_portability_ns = Namespace(
+    "agents", description="Agent import/export operations", path="/api"
+)
+
+
+def _read_import_payload(req):
+    """Pull (yaml_text, resolution) from a JSON body or uploaded file.
+
+    Enforces a body-size cap before reading so a giant document can't be
+    buffered into memory.
+    """
+    if req.content_length and req.content_length > MAX_IMPORT_BYTES:
+        raise AgentImportError("Import document too large")
+    content_type = req.content_type or ""
+    if "application/json" in content_type:
+        data = req.get_json(silent=True) or {}
+        resolution = data.get("resolution")
+        return (
+            data.get("yaml") or data.get("content") or "",
+            resolution if isinstance(resolution, dict) else {},
+        )
+    if "file" in req.files:
+        raw = req.files["file"].read(MAX_IMPORT_BYTES + 1)
+        if len(raw) > MAX_IMPORT_BYTES:
+            raise AgentImportError("Import document too large")
+        return raw.decode("utf-8", "replace"), {}
+    # Raw body: read from the stream with a hard cap so a chunked request
+    # (no Content-Length) can't be buffered unbounded into memory.
+    raw = req.stream.read(MAX_IMPORT_BYTES + 1)
+    if len(raw) > MAX_IMPORT_BYTES:
+        raise AgentImportError("Import document too large")
+    return raw.decode("utf-8", "replace"), {}
+
+
+@agents_portability_ns.route("/export_agent")
+class ExportAgent(Resource):
+    @api.doc(params={"id": "Agent ID"}, description="Export an agent as YAML")
+    def get(self):
+        if not (decoded_token := request.decoded_token):
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        agent_id = request.args.get("id")
+        if not agent_id:
+            return make_response(jsonify({"success": False, "message": "id is required"}), 400)
+        with db_session() as conn:
+            repo = AgentsRepository(conn)
+            agent = repo.get_any(agent_id, user)
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Agent not found"}), 404
+                )
+            if (agent.get("agent_type") or "") == "workflow":
+                return make_response(
+                    jsonify(
+                        {"success": False, "message": "Workflow agents can't be exported yet"}
+                    ),
+                    400,
+                )
+            agent["slug"] = ensure_agent_slug(conn, agent, user)
+            export = serialize_agent(conn, agent, user)
+        body = agent_to_yaml(export)
+        filename = f"{export['metadata'].get('slug') or 'agent'}.agent.yaml"
+        response = make_response(body, 200)
+        response.headers["Content-Type"] = "application/x-yaml; charset=utf-8"
+        response.headers["Content-Disposition"] = f'attachment; filename="{filename}"'
+        return response
+
+
+@agents_portability_ns.route("/import_agent/plan")
+class ImportAgentPlan(Resource):
+    @api.doc(description="Dry-run an agent YAML import and return the resolution plan")
+    def post(self):
+        if not (decoded_token := request.decoded_token):
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            yaml_text, _ = _read_import_payload(request)
+            doc = parse_agent_yaml(yaml_text)
+        except AgentImportError as exc:
+            return make_response(jsonify({"success": False, "message": str(exc)}), 400)
+        try:
+            with db_readonly() as conn:
+                plan = plan_import(conn, user, doc)
+        except Exception:
+            current_app.logger.error("Agent import plan failed", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "message": "Could not analyze the agent file"}), 500
+            )
+        return make_response(jsonify({"success": True, "plan": plan}), 200)
+
+
+@agents_portability_ns.route("/import_agent")
+class ImportAgent(Resource):
+    @api.doc(description="Import an agent from YAML (created as a draft)")
+    def post(self):
+        if not (decoded_token := request.decoded_token):
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            yaml_text, resolution = _read_import_payload(request)
+            doc = parse_agent_yaml(yaml_text)
+        except AgentImportError as exc:
+            return make_response(jsonify({"success": False, "message": str(exc)}), 400)
+        try:
+            with db_session() as conn:
+                result = apply_import(conn, user, doc, resolution)
+        except Exception:
+            current_app.logger.error("Agent import failed", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "message": "Import failed"}), 500
+            )
+        return make_response(jsonify({"success": True, **result}), 200)

application/api/user/agents/sharing.py

@@ -0,0 +1,274 @@
+"""Agent management sharing functionality."""
+
+import datetime
+import secrets
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+from sqlalchemy import text as _sql_text
+
+from application.api import api
+from application.core.settings import settings
+from application.api.user.base import resolve_tool_details
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+from application.utils import generate_image_url
+
+agents_sharing_ns = Namespace(
+    "agents", description="Agent management operations", path="/api"
+)
+
+
+def _serialize_agent_basic(agent: dict) -> dict:
+    """Shape a PG agent row into the API response dict."""
+    source_id = agent.get("source_id")
+    return {
+        "id": str(agent["id"]),
+        "user": agent.get("user_id", ""),
+        "name": agent.get("name", ""),
+        "image": (
+            generate_image_url(agent["image"]) if agent.get("image") else ""
+        ),
+        "description": agent.get("description", ""),
+        "source": str(source_id) if source_id else "",
+        "chunks": str(agent["chunks"]) if agent.get("chunks") is not None else "0",
+        "retriever": agent.get("retriever", "classic") or "classic",
+        "prompt_id": str(agent["prompt_id"]) if agent.get("prompt_id") else "default",
+        "tools": agent.get("tools", []) or [],
+        "tool_details": resolve_tool_details(agent.get("tools", []) or []),
+        "agent_type": agent.get("agent_type", "") or "",
+        "status": agent.get("status", "") or "",
+        "json_schema": agent.get("json_schema"),
+        "limited_token_mode": agent.get("limited_token_mode", False),
+        "token_limit": agent.get("token_limit") or settings.DEFAULT_AGENT_LIMITS["token_limit"],
+        "limited_request_mode": agent.get("limited_request_mode", False),
+        "request_limit": agent.get("request_limit") or settings.DEFAULT_AGENT_LIMITS["request_limit"],
+        "created_at": agent.get("created_at", ""),
+        "updated_at": agent.get("updated_at", ""),
+        "shared": bool(agent.get("shared", False)),
+        "shared_token": agent.get("shared_token", "") or "",
+        "shared_metadata": agent.get("shared_metadata", {}) or {},
+    }
+
+
+@agents_sharing_ns.route("/shared_agent")
+class SharedAgent(Resource):
+    @api.doc(
+        params={
+            "token": "Shared token of the agent",
+        },
+        description="Get a shared agent by token or ID",
+    )
+    def get(self):
+        shared_token = request.args.get("token")
+
+        if not shared_token:
+            return make_response(
+                jsonify({"success": False, "message": "Token or ID is required"}), 400
+            )
+        try:
+            with db_readonly() as conn:
+                shared_agent = AgentsRepository(conn).find_by_shared_token(
+                    shared_token,
+                )
+            if not shared_agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Shared agent not found"}),
+                    404,
+                )
+            agent_id = str(shared_agent["id"])
+            data = _serialize_agent_basic(shared_agent)
+
+            if data["tools"]:
+                enriched_tools = []
+                for detail in data["tool_details"]:
+                    enriched_tools.append(detail.get("name", ""))
+                data["tools"] = enriched_tools
+            decoded_token = getattr(request, "decoded_token", None)
+            if decoded_token:
+                user_id = decoded_token.get("sub")
+                owner_id = shared_agent.get("user_id")
+
+                if user_id != owner_id:
+                    with db_session() as conn:
+                        users_repo = UsersRepository(conn)
+                        users_repo.upsert(user_id)
+                        users_repo.add_shared(user_id, agent_id)
+            return make_response(jsonify(data), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving shared agent: {err}")
+            return make_response(jsonify({"success": False}), 400)
+
+
+@agents_sharing_ns.route("/shared_agents")
+class SharedAgents(Resource):
+    @api.doc(description="Get shared agents explicitly shared with the user")
+    def get(self):
+        try:
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False}), 401)
+            user_id = decoded_token.get("sub")
+
+            with db_session() as conn:
+                users_repo = UsersRepository(conn)
+                user_doc = users_repo.upsert(user_id)
+                shared_with_ids = (
+                    user_doc.get("agent_preferences", {}).get("shared_with_me", [])
+                    if isinstance(user_doc.get("agent_preferences"), dict)
+                    else []
+                )
+                # Keep only UUID-shaped ids; ObjectId leftovers are stripped below.
+                uuid_ids = [sid for sid in shared_with_ids if looks_like_uuid(sid)]
+                non_uuid_ids = [sid for sid in shared_with_ids if not looks_like_uuid(sid)]
+
+                if uuid_ids:
+                    result = conn.execute(
+                        _sql_text(
+                            "SELECT * FROM agents "
+                            "WHERE id = ANY(CAST(:ids AS uuid[])) "
+                            "AND shared = true"
+                        ),
+                        {"ids": uuid_ids},
+                    )
+                    shared_agents = [dict(row._mapping) for row in result.fetchall()]
+                else:
+                    shared_agents = []
+
+                found_ids_set = {str(agent["id"]) for agent in shared_agents}
+                stale_ids = [sid for sid in uuid_ids if sid not in found_ids_set]
+                stale_ids.extend(non_uuid_ids)
+                if stale_ids:
+                    users_repo.remove_shared_bulk(user_id, stale_ids)
+
+                pinned_ids = set(
+                    user_doc.get("agent_preferences", {}).get("pinned", [])
+                    if isinstance(user_doc.get("agent_preferences"), dict)
+                    else []
+                )
+
+            list_shared_agents = []
+            for agent in shared_agents:
+                agent_id_str = str(agent["id"])
+                list_shared_agents.append(
+                    {
+                        "id": agent_id_str,
+                        "name": agent.get("name", ""),
+                        "description": agent.get("description", ""),
+                        "image": (
+                            generate_image_url(agent["image"]) if agent.get("image") else ""
+                        ),
+                        "tools": agent.get("tools", []) or [],
+                        "tool_details": resolve_tool_details(
+                            agent.get("tools", []) or []
+                        ),
+                        "agent_type": agent.get("agent_type", "") or "",
+                        "status": agent.get("status", "") or "",
+                        "json_schema": agent.get("json_schema"),
+                        "limited_token_mode": agent.get("limited_token_mode", False),
+                        "token_limit": agent.get("token_limit") or settings.DEFAULT_AGENT_LIMITS["token_limit"],
+                        "limited_request_mode": agent.get("limited_request_mode", False),
+                        "request_limit": agent.get("request_limit") or settings.DEFAULT_AGENT_LIMITS["request_limit"],
+                        "created_at": agent.get("created_at", ""),
+                        "updated_at": agent.get("updated_at", ""),
+                        "pinned": agent_id_str in pinned_ids,
+                        "shared": bool(agent.get("shared", False)),
+                        "shared_token": agent.get("shared_token", "") or "",
+                        "shared_metadata": agent.get("shared_metadata", {}) or {},
+                    }
+                )
+
+            return make_response(jsonify(list_shared_agents), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving shared agents: {err}")
+            return make_response(jsonify({"success": False}), 400)
+
+
+@agents_sharing_ns.route("/share_agent")
+class ShareAgent(Resource):
+    @api.expect(
+        api.model(
+            "ShareAgentModel",
+            {
+                "id": fields.String(required=True, description="ID of the agent"),
+                "shared": fields.Boolean(
+                    required=True, description="Share or unshare the agent"
+                ),
+                "username": fields.String(
+                    required=False, description="Name of the user"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Share or unshare an agent")
+    def put(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+
+        data = request.get_json()
+        if not data:
+            return make_response(
+                jsonify({"success": False, "message": "Missing JSON body"}), 400
+            )
+        agent_id = data.get("id")
+        shared = data.get("shared")
+        username = data.get("username", "")
+
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        if shared is None:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Shared parameter is required and must be true or false",
+                    }
+                ),
+                400,
+            )
+        shared_token = None
+        try:
+            with db_session() as conn:
+                repo = AgentsRepository(conn)
+                agent = repo.get_any(agent_id, user)
+                if not agent:
+                    return make_response(
+                        jsonify({"success": False, "message": "Agent not found"}), 404
+                    )
+                if shared:
+                    shared_metadata = {
+                        "shared_by": username,
+                        "shared_at": datetime.datetime.now(
+                            datetime.timezone.utc
+                        ).isoformat(),
+                    }
+                    shared_token = secrets.token_urlsafe(32)
+                    repo.update(
+                        str(agent["id"]), user,
+                        {
+                            "shared": True,
+                            "shared_token": shared_token,
+                            "shared_metadata": shared_metadata,
+                        },
+                    )
+                else:
+                    repo.update(
+                        str(agent["id"]), user,
+                        {
+                            "shared": False,
+                            "shared_token": None,
+                            "shared_metadata": None,
+                        },
+                    )
+        except Exception as err:
+            current_app.logger.error(f"Error sharing/unsharing agent: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to update agent sharing status"}), 400)
+        return make_response(
+            jsonify({"success": True, "shared_token": shared_token}), 200
+        )

application/api/user/agents/webhooks.py

@@ -0,0 +1,232 @@
+"""Agent management webhook handlers."""
+
+import secrets
+import uuid
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource
+from sqlalchemy import text as sql_text
+
+from application.api import api
+from application.api.user.base import require_agent
+from application.api.user.tasks import process_agent_webhook
+from application.core.settings import settings
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.idempotency import IdempotencyRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+agents_webhooks_ns = Namespace(
+    "agents", description="Agent management operations", path="/api"
+)
+
+
+_IDEMPOTENCY_KEY_MAX_LEN = 256
+
+
+def _read_idempotency_key():
+    """Return (key, error_response). Empty header → (None, None); oversized → (None, 400)."""
+    key = request.headers.get("Idempotency-Key")
+    if not key:
+        return None, None
+    if len(key) > _IDEMPOTENCY_KEY_MAX_LEN:
+        return None, make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": (
+                        f"Idempotency-Key exceeds maximum length of "
+                        f"{_IDEMPOTENCY_KEY_MAX_LEN} characters"
+                    ),
+                }
+            ),
+            400,
+        )
+    return key, None
+
+
+def _scoped_idempotency_key(idempotency_key, scope):
+    """``{scope}:{key}`` so different agents can't collide on the same key."""
+    if not idempotency_key or not scope:
+        return None
+    return f"{scope}:{idempotency_key}"
+
+
+@agents_webhooks_ns.route("/agent_webhook")
+class AgentWebhook(Resource):
+    @api.doc(
+        params={"id": "ID of the agent"},
+        description="Generate webhook URL for the agent",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        agent_id = request.args.get("id")
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            with db_readonly() as conn:
+                agent = AgentsRepository(conn).get_any(agent_id, user)
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Agent not found"}), 404
+                )
+            webhook_token = agent.get("incoming_webhook_token")
+            if not webhook_token:
+                webhook_token = secrets.token_urlsafe(32)
+                with db_session() as conn:
+                    AgentsRepository(conn).update(
+                        str(agent["id"]), user,
+                        {"incoming_webhook_token": webhook_token},
+                    )
+            base_url = settings.API_URL.rstrip("/")
+            full_webhook_url = f"{base_url}/api/webhooks/agents/{webhook_token}"
+        except Exception as err:
+            current_app.logger.error(
+                f"Error generating webhook URL: {err}", exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Error generating webhook URL"}),
+                400,
+            )
+        return make_response(
+            jsonify({"success": True, "webhook_url": full_webhook_url}), 200
+        )
+
+
+@agents_webhooks_ns.route("/webhooks/agents/<string:webhook_token>")
+class AgentWebhookListener(Resource):
+    method_decorators = [require_agent]
+
+    def _enqueue_webhook_task(self, agent_id_str, payload, source_method, agent=None):
+        if not payload:
+            current_app.logger.warning(
+                f"Webhook ({source_method}) received for agent {agent_id_str} with empty payload."
+            )
+        current_app.logger.info(
+            f"Incoming {source_method} webhook for agent {agent_id_str}. Enqueuing task with payload: {payload}"
+        )
+
+        idempotency_key, key_error = _read_idempotency_key()
+        if key_error is not None:
+            return key_error
+        # Resolve to PG UUID first so dedup writes don't crash on legacy ids.
+        agent_uuid = None
+        if agent is not None:
+            candidate = str(agent.get("id") or "")
+            if looks_like_uuid(candidate):
+                agent_uuid = candidate
+        if idempotency_key and agent_uuid is None:
+            current_app.logger.warning(
+                "Skipping webhook idempotency dedup: agent %s has non-UUID id",
+                agent_id_str,
+            )
+            idempotency_key = None
+        # Agent-scoped (webhooks have no user_id).
+        scoped_key = _scoped_idempotency_key(idempotency_key, agent_uuid)
+        # Claim before enqueue; the loser returns the winner's task_id.
+        predetermined_task_id = None
+        if scoped_key:
+            predetermined_task_id = str(uuid.uuid4())
+            with db_session() as conn:
+                claimed = IdempotencyRepository(conn).record_webhook(
+                    key=scoped_key,
+                    agent_id=agent_uuid,
+                    task_id=predetermined_task_id,
+                    response_json={
+                        "success": True, "task_id": predetermined_task_id,
+                    },
+                )
+            if claimed is None:
+                with db_readonly() as conn:
+                    cached = IdempotencyRepository(conn).get_webhook(scoped_key)
+                if cached is not None:
+                    return make_response(jsonify(cached["response_json"]), 200)
+                return make_response(
+                    jsonify({"success": True, "task_id": "deduplicated"}), 200
+                )
+
+        try:
+            apply_kwargs = dict(
+                kwargs={
+                    "agent_id": agent_id_str,
+                    "payload": payload,
+                    # Scoped so the worker dedup row matches the HTTP claim.
+                    "idempotency_key": scoped_key or idempotency_key,
+                },
+            )
+            if predetermined_task_id is not None:
+                apply_kwargs["task_id"] = predetermined_task_id
+            task = process_agent_webhook.apply_async(**apply_kwargs)
+            current_app.logger.info(
+                f"Task {task.id} enqueued for agent {agent_id_str} ({source_method})."
+            )
+            response_payload = {"success": True, "task_id": task.id}
+            return make_response(jsonify(response_payload), 200)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error enqueuing webhook task ({source_method}) for agent {agent_id_str}: {err}",
+                exc_info=True,
+            )
+            if scoped_key:
+                # Roll back the claim so a retry can succeed.
+                try:
+                    with db_session() as conn:
+                        conn.execute(
+                            sql_text(
+                                "DELETE FROM webhook_dedup "
+                                "WHERE idempotency_key = :k"
+                            ),
+                            {"k": scoped_key},
+                        )
+                except Exception:
+                    current_app.logger.exception(
+                        "Failed to release webhook_dedup claim for key=%s",
+                        scoped_key,
+                    )
+            return make_response(
+                jsonify({"success": False, "message": "Error processing webhook"}), 500
+            )
+
+    @api.doc(
+        description=(
+            "Webhook listener for agent events (POST). Expects JSON payload, which "
+            "is used to trigger processing. Honors an optional ``Idempotency-Key`` "
+            "header: a repeat request with the same key within 24h returns the "
+            "original cached response and does not re-enqueue the task."
+        ),
+    )
+    def post(self, webhook_token, agent, agent_id_str):
+        payload = request.get_json()
+        if payload is None:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Invalid or missing JSON data in request body",
+                    }
+                ),
+                400,
+            )
+        return self._enqueue_webhook_task(
+            agent_id_str, payload, source_method="POST", agent=agent,
+        )
+
+    @api.doc(
+        description=(
+            "Webhook listener for agent events (GET). Uses URL query parameters as "
+            "payload to trigger processing. Honors an optional ``Idempotency-Key`` "
+            "header: a repeat request with the same key within 24h returns the "
+            "original cached response and does not re-enqueue the task."
+        ),
+    )
+    def get(self, webhook_token, agent, agent_id_str):
+        payload = request.args.to_dict(flat=True)
+        return self._enqueue_webhook_task(
+            agent_id_str, payload, source_method="GET", agent=agent,
+        )

application/api/user/analytics/__init__.py

@@ -0,0 +1,5 @@
+"""Analytics module."""
+
+from .routes import analytics_ns
+
+__all__ = ["analytics_ns"]

application/api/user/artifacts/__init__.py

@@ -0,0 +1,5 @@
+"""Artifacts module."""
+
+from .routes import artifacts_ns
+
+__all__ = ["artifacts_ns"]

application/api/user/artifacts/authz.py

@@ -0,0 +1,168 @@
+"""Parent-derived authorization helpers for artifact access."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from flask import request
+
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.artifacts import ArtifactsRepository
+from application.storage.db.repositories.conversations import ConversationsRepository
+from application.storage.db.repositories.shared_conversations import (
+    SharedConversationsRepository,
+)
+from application.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+from application.storage.db.session import db_readonly
+
+
+@dataclass(frozen=True)
+class Principal:
+    """The resolved caller of an artifact route.
+
+    A decoded JWT is a full *owner* session (``agent_id`` is ``None``). An
+    ``api_key`` is a low-trust, publicly-distributed *agent* credential -- it is
+    embedded client-side in the widget and accepted from the query string -- so
+    it resolves to the owning ``user_id`` but stays **agent-scoped**: it may reach
+    an artifact only when the request also carries that artifact's parent
+    ``conversation_id`` (the unguessable per-visitor secret), confining it to a
+    single conversation rather than the agent's whole corpus, and may not perform
+    mutating operations.
+    """
+
+    user_id: Optional[str] = None
+    agent_id: Optional[str] = None
+
+    @property
+    def is_agent_scoped(self) -> bool:
+        """True for an api_key (agent) principal that must be confined to its agent."""
+        return self.agent_id is not None
+
+
+def resolve_principal() -> Principal:
+    """Resolve the caller to a :class:`Principal`.
+
+    Priority: a decoded JWT (owner session) first, then an ``api_key`` query/form
+    param (agent-scoped). An unresolvable caller yields an anonymous principal
+    (both fields ``None``) that only a valid ``share_token`` can authorize.
+    """
+    decoded_token = getattr(request, "decoded_token", None)
+    if decoded_token:
+        return Principal(user_id=decoded_token.get("sub"))
+
+    api_key = request.args.get("api_key") or request.form.get("api_key")
+    if api_key:
+        with db_readonly() as conn:
+            agent = AgentsRepository(conn).find_by_key(api_key)
+        if agent and agent.get("user_id") and agent.get("id"):
+            return Principal(user_id=str(agent["user_id"]), agent_id=str(agent["id"]))
+    return Principal()
+
+
+def _shared_row_for(conn, conversation_id, share_token):
+    """Return the shared_conversations row iff share_token grants this conversation, else None."""
+    if not share_token:
+        return None
+    shared = SharedConversationsRepository(conn).find_by_uuid(share_token)
+    if shared and str(shared.get("conversation_id")) == str(conversation_id):
+        return shared
+    return None
+
+
+def user_can_access_conversation(
+    conn, conversation_id: str, user_id: Optional[str], share_token: Optional[str]
+) -> bool:
+    """Allow if the caller owns/shares the conversation, or holds a valid share token.
+
+    Conversation-level gate only. Reuses ``ConversationsRepository.get`` (owner OR
+    ``shared_with``) so artifact access tracks message access, and honours a valid
+    share token. A share-token caller reaches the conversation here but must still
+    be snapshot-scoped per-artifact by the caller (see ``authorize_artifact``): a
+    valid token does NOT imply access to every artifact in the conversation.
+    """
+    if user_id:
+        if ConversationsRepository(conn).get(conversation_id, user_id) is not None:
+            return True
+    return _shared_row_for(conn, conversation_id, share_token) is not None
+
+
+def authorize_artifact(conn, artifact: dict, principal: Principal) -> bool:
+    """Authorize a READ of ``artifact`` for ``principal``; missing parent fails closed.
+
+    A low-trust agent api_key is confined to a single conversation it proves by
+    carrying that artifact's parent ``conversation_id`` query param (owner match +
+    matching conversation + agent scope) and never inherits share-link access. A
+    JWT owner or ``shared_with`` collaborator gets full access to every artifact of
+    the parent; a share-token holder is confined to the shared ``first_n_queries``
+    snapshot (an artifact whose ``message_id`` is outside it, or NULL, is denied).
+    """
+    if principal.is_agent_scoped:
+        # An agent key is not the owner's session and is embedded in public widget
+        # JS, so require the artifact to belong to the owner AND the request to
+        # carry the parent conversation_id (the unguessable per-visitor secret):
+        # the key alone, without that id, cannot download a known artifact.
+        if str(artifact.get("user_id")) != str(principal.user_id):
+            return False
+        req_conv = request.args.get("conversation_id")
+        parent = artifact.get("conversation_id")
+        if not req_conv or parent is None or str(parent) != str(req_conv):
+            return False
+        return ArtifactsRepository(conn).artifact_in_agent_scope(
+            str(artifact.get("id")), str(principal.agent_id)
+        )
+
+    conversation_id = artifact.get("conversation_id")
+    workflow_run_id = artifact.get("workflow_run_id")
+    share_token = request.args.get("share_token")
+
+    if conversation_id is not None:
+        # Owner or shared_with collaborator: full access to every artifact.
+        if principal.user_id and ConversationsRepository(conn).get(
+            str(conversation_id), principal.user_id
+        ) is not None:
+            return True
+        # Share-token holder: only artifacts inside the shared first_n_queries snapshot.
+        shared = _shared_row_for(conn, str(conversation_id), share_token)
+        if shared is None:
+            return False
+        message_id = artifact.get("message_id")
+        if not message_id:
+            return False
+        first_n = int(shared.get("first_n_queries") or 0)
+        return ConversationsRepository(conn).message_in_first_n(
+            str(conversation_id), str(message_id), first_n
+        )
+    if workflow_run_id is not None:
+        if not principal.user_id:
+            return False
+        run = WorkflowRunsRepository(conn).get(str(workflow_run_id))
+        return run is not None and run.get("user_id") == principal.user_id
+    # No parent row reachable -> deny (e.g. deleted conversation/run).
+    return False
+
+
+def authorize_artifact_write(conn, artifact: dict, principal: Principal) -> bool:
+    """Authorize a *mutating* artifact operation (e.g. delete / restore).
+
+    Stricter than :func:`authorize_artifact`: a write requires an authenticated
+    *owner* of the parent. Low-trust agent api_keys, share links, and
+    ``shared_with`` collaborators inherit read/download access only, so an
+    agent-scoped, anonymous, or share-token-only caller is denied -- they can read
+    an artifact but never mutate it.
+    """
+    if principal.is_agent_scoped or not principal.user_id:
+        return False
+    conversation_id = artifact.get("conversation_id")
+    workflow_run_id = artifact.get("workflow_run_id")
+
+    if conversation_id is not None:
+        return (
+            ConversationsRepository(conn).get_owned(str(conversation_id), principal.user_id)
+            is not None
+        )
+    if workflow_run_id is not None:
+        run = WorkflowRunsRepository(conn).get(str(workflow_run_id))
+        return run is not None and run.get("user_id") == principal.user_id
+    # No parent row reachable -> deny (e.g. deleted conversation/run).
+    return False

application/api/user/artifacts/routes.py

@@ -0,0 +1,402 @@
+"""Artifact metadata and download routes (parent-derived authz)."""
+
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from flask import (
+    Response,
+    current_app,
+    jsonify,
+    make_response,
+    redirect,
+    request,
+    stream_with_context,
+)
+from flask_restx import Namespace, Resource
+
+from application.api import api
+from application.api.user.artifacts.authz import (
+    _shared_row_for,
+    authorize_artifact,
+    authorize_artifact_write,
+    resolve_principal,
+    user_can_access_conversation,
+)
+from application.core.settings import settings
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.artifacts import ArtifactsRepository
+from application.storage.db.repositories.conversations import ConversationsRepository
+from application.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+from application.storage.db.session import db_readonly, db_session
+from application.storage.storage_creator import StorageCreator
+
+artifacts_ns = Namespace("artifacts", description="Artifact operations", path="/api")
+
+# Presigned-URL TTL for private S3 artifact downloads (seconds).
+_PRESIGNED_URL_TTL = 300
+
+
+_ARTIFACT_URL_ENVELOPE_MIME = "application/vnd.docsgpt.artifact-url+json"
+
+
+def _sanitize_header_filename(filename: Optional[str], fallback: str) -> str:
+    """Strip CRLF / quotes from a display filename for a Content-Disposition header."""
+    if not filename:
+        return fallback
+    cleaned = re.sub(r'[\r\n"]', "", str(filename)).strip()
+    return cleaned or fallback
+
+
+def _artifact_summary(artifact: dict) -> dict:
+    """Project an artifact identity row to its API metadata shape (owner id withheld)."""
+    return {
+        "id": str(artifact.get("id")),
+        "conversation_id": (str(artifact["conversation_id"]) if artifact.get("conversation_id") is not None else None),
+        "workflow_run_id": (str(artifact["workflow_run_id"]) if artifact.get("workflow_run_id") is not None else None),
+        "message_id": (str(artifact["message_id"]) if artifact.get("message_id") is not None else None),
+        "kind": artifact.get("kind"),
+        "title": artifact.get("title"),
+        "metadata": artifact.get("metadata"),
+        "current_version": artifact.get("current_version"),
+        "created_at": _iso(artifact.get("created_at")),
+        "updated_at": _iso(artifact.get("updated_at")),
+    }
+
+
+def _version_summary(version: dict, *, include_spec: bool = False) -> dict:
+    """Project a version row to its API metadata shape (storage_path withheld)."""
+    out = {
+        "version": version.get("version"),
+        "mime_type": version.get("mime_type"),
+        "filename": version.get("filename"),
+        "size": version.get("size"),
+        "sha256": version.get("sha256"),
+        "preview_text": version.get("preview_text"),
+        "produced_by": version.get("produced_by"),
+        "created_at": _iso(version.get("created_at")),
+    }
+    if include_spec:
+        out["spec"] = version.get("spec")
+    return out
+
+
+def _iso(value):
+    """ISO-format a datetime, passing through other values unchanged."""
+    return value.isoformat() if hasattr(value, "isoformat") else value
+
+
+@artifacts_ns.route("/artifacts")
+class ListArtifacts(Resource):
+    @api.doc(description="List artifacts for a conversation, workflow run, or the caller")
+    def get(self):
+        principal = resolve_principal()
+        user_id = principal.user_id
+        conversation_id = request.args.get("conversation_id")
+        workflow_run_id = request.args.get("workflow_run_id")
+        share_token = request.args.get("share_token")
+
+        if not user_id and not share_token:
+            return make_response(jsonify({"success": False, "message": "Authentication required"}), 401)
+
+        # Gate UUID-shape before any CAST(:id AS uuid) reaches the repo, so a
+        # malformed id is rejected cleanly instead of poisoning the transaction.
+        if conversation_id and not looks_like_uuid(conversation_id):
+            return make_response(jsonify({"success": False, "message": "Invalid conversation_id"}), 400)
+        if workflow_run_id and not looks_like_uuid(workflow_run_id):
+            return make_response(jsonify({"success": False, "message": "Invalid workflow_run_id"}), 400)
+
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                if principal.is_agent_scoped:
+                    # A low-trust agent api_key is embedded in public widget JS and
+                    # shared by every anonymous visitor, so it may NOT enumerate the
+                    # agent's artifacts. The unguessable conversation_id (UUIDv4) is
+                    # the only per-visitor secret: require it as a bearer capability
+                    # and scope the query to it. Workflow runs are not agent-scoped,
+                    # so reject that filter.
+                    if workflow_run_id:
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    if not conversation_id:
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    rows = repo.list_artifacts_for_agent(principal.agent_id, user_id, conversation_id=conversation_id)
+                elif conversation_id:
+                    if not user_can_access_conversation(conn, conversation_id, user_id, share_token):
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    rows = repo.list_artifacts(conversation_id=conversation_id)
+                    # Owner / shared_with collaborator sees every artifact; a
+                    # share-token caller is confined to the shared first_n_queries
+                    # snapshot (drop artifacts whose message is outside it or NULL).
+                    conv_repo = ConversationsRepository(conn)
+                    is_owner = bool(user_id and conv_repo.get(conversation_id, user_id) is not None)
+                    if not is_owner:
+                        shared = _shared_row_for(conn, conversation_id, share_token)
+                        first_n = int(shared.get("first_n_queries") or 0) if shared else 0
+                        snapshot_ids = conv_repo.first_n_message_ids(conversation_id, first_n)
+                        rows = [
+                            r
+                            for r in rows
+                            if r.get("message_id") is not None and str(r.get("message_id")) in snapshot_ids
+                        ]
+                elif workflow_run_id:
+                    run = WorkflowRunsRepository(conn).get(workflow_run_id)
+                    if run is None or run.get("user_id") != user_id:
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    rows = repo.list_artifacts(workflow_run_id=workflow_run_id)
+                else:
+                    if not user_id:
+                        return make_response(
+                            jsonify({"success": False, "message": "Authentication required"}),
+                            401,
+                        )
+                    rows = repo.list_artifacts(user_id=user_id)
+
+            return make_response(
+                jsonify({"success": True, "artifacts": [_artifact_summary(r) for r in rows]}),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error listing artifacts: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>")
+class GetArtifact(Resource):
+    @api.doc(description="Get an artifact's metadata, version list, and current spec")
+    def get(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                if not authorize_artifact(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                versions = repo.list_versions(artifact_id)
+                current = repo.get_version(artifact_id, artifact.get("current_version"))
+
+            payload = _artifact_summary(artifact)
+            payload["versions"] = [_version_summary(v) for v in versions]
+            payload["spec"] = current.get("spec") if current else None
+            return make_response(jsonify({"success": True, "artifact": payload}), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+    @api.doc(description="Delete an artifact and all its versions (owner only)")
+    def delete(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        try:
+            with db_session() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                # Delete is a WRITE: only the parent owner may delete; share
+                # links / read-only collaborators are denied (read access only).
+                if not authorize_artifact_write(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                storage_paths = repo.delete_artifact(artifact_id)
+            # Reap the bytes best-effort AFTER the row delete commits.
+            _reap_storage(storage_paths)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error deleting artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+def _reap_storage(paths: list) -> None:
+    """Best-effort delete artifact bytes after the DB rows are gone; never raises."""
+    if not paths:
+        return
+    try:
+        storage = StorageCreator.get_storage()
+    except Exception:
+        current_app.logger.warning("artifact delete: storage unavailable", exc_info=True)
+        return
+    for path in paths:
+        try:
+            storage.delete_file(path)
+        except Exception:
+            current_app.logger.warning("artifact delete: failed to delete bytes %s", path, exc_info=True)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>/versions/<int:version>")
+class GetArtifactVersion(Resource):
+    @api.doc(description="Get a single artifact version's metadata and spec")
+    def get(self, artifact_id: str, version: int):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                if not authorize_artifact(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                version_row = repo.get_version(artifact_id, version)
+            if version_row is None:
+                return make_response(jsonify({"success": False, "message": "Version not found"}), 404)
+            return make_response(
+                jsonify({"success": True, "version": _version_summary(version_row, include_spec=True)}),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving artifact version: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>/download")
+class DownloadArtifact(Resource):
+    @api.doc(description="Download an artifact's bytes (302 to a presigned URL on S3)")
+    def get(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        version_arg = request.args.get("version")
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                if not authorize_artifact(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                version = artifact.get("current_version")
+                if version_arg is not None:
+                    try:
+                        version = int(version_arg)
+                    except ValueError:
+                        return make_response(jsonify({"success": False, "message": "Invalid version"}), 400)
+                version_row = repo.get_version(artifact_id, version)
+
+            if version_row is None:
+                return make_response(jsonify({"success": False, "message": "Version not found"}), 404)
+            # The object key is derived only from the stored path, never client input.
+            storage_path = version_row.get("storage_path")
+            if not storage_path:
+                return make_response(jsonify({"success": False, "message": "No file for this version"}), 404)
+
+            filename = _sanitize_header_filename(version_row.get("filename"), f"artifact-{artifact_id}")
+            mime_type = version_row.get("mime_type") or "application/octet-stream"
+            storage = StorageCreator.get_storage()
+
+            # With URL_STRATEGY=="s3" the contract is to hand back a presigned
+            # URL. If the active backend can't mint one, that's a config error:
+            # surface a 500 rather than silently proxying bytes from a backend
+            # the operator expected to be off the hot path.
+            if getattr(settings, "URL_STRATEGY", "backend") == "s3":
+                try:
+                    url = storage.generate_presigned_url(storage_path, expires_in=_PRESIGNED_URL_TTL)
+                except NotImplementedError:
+                    current_app.logger.error(
+                        "URL_STRATEGY=s3 but %s cannot mint presigned URLs",
+                        type(storage).__name__,
+                    )
+                    return make_response(
+                        jsonify({"success": False, "message": "Storage misconfigured"}),
+                        500,
+                    )
+                # A 302 to a cross-origin S3 URL can't be read by the app's authed
+                # fetch (the bucket has no CORS grant for the app origin). When the
+                # client opts in via ?disposition=url (or Accept: application/json)
+                # hand the presigned URL back as JSON so it can navigate to it
+                # top-level (no CORS). Default stays a 302 so nothing else breaks.
+                if request.args.get("disposition") == "url" or (
+                    "application/json" in request.headers.get("Accept", "")
+                ):
+                    resp = make_response(jsonify({"success": True, "url": url}), 200)
+                    # Tag the envelope with a distinctive media type so the client
+                    # keys off a server-set signal, not the JSON body shape. A
+                    # ``data`` artifact whose bytes happen to be
+                    # ``{"success":true,"url":"..."}`` is streamed under the backend
+                    # strategy with its own (``application/json``) content-type and
+                    # can never carry this vendor type, so it can't be mistaken for a
+                    # redirect (open-redirect gadget). Content-Type is CORS-safelisted,
+                    # so this stays readable cross-origin without expose-headers.
+                    resp.headers["Content-Type"] = _ARTIFACT_URL_ENVELOPE_MIME
+                    return resp
+                return redirect(url, code=302)
+
+            # Stream the bytes in chunks instead of buffering the whole object in
+            # worker memory (artifacts can be many MB); close the handle when done.
+            file_obj = storage.get_file(storage_path)
+
+            def _stream():
+                try:
+                    for chunk in iter(lambda: file_obj.read(65536), b""):
+                        yield chunk
+                finally:
+                    close = getattr(file_obj, "close", None)
+                    if callable(close):
+                        close()
+
+            return Response(
+                stream_with_context(_stream()),
+                mimetype=mime_type,
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+            )
+        except FileNotFoundError:
+            return make_response(jsonify({"success": False, "message": "File not found"}), 404)
+        except Exception as err:
+            current_app.logger.error(f"Error downloading artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>/restore")
+class RestoreArtifact(Resource):
+    @api.doc(description="Restore a prior version by appending it as the new current version")
+    def post(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        data = request.get_json(silent=True) or {}
+        target_version = data.get("version")
+        if target_version is None:
+            return make_response(jsonify({"success": False, "message": "Missing version"}), 400)
+        try:
+            target_version = int(target_version)
+        except (ValueError, TypeError):
+            return make_response(jsonify({"success": False, "message": "Invalid version"}), 400)
+
+        try:
+            with db_session() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                # Restore is a WRITE (it appends a new current version); share
+                # links / shared_with collaborators inherit read access only, so
+                # gate on the stricter owner-required write check.
+                if not authorize_artifact_write(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                source = repo.get_version(artifact_id, target_version)
+                if source is None:
+                    return make_response(jsonify({"success": False, "message": "Version not found"}), 404)
+                new_version = repo.append_version(
+                    artifact_id,
+                    mime_type=source.get("mime_type"),
+                    filename=source.get("filename"),
+                    storage_path=source.get("storage_path"),
+                    size=source.get("size"),
+                    sha256=source.get("sha256"),
+                    spec=source.get("spec"),
+                    preview_text=source.get("preview_text"),
+                    produced_by=source.get("produced_by"),
+                )
+            return make_response(
+                jsonify({"success": True, "version": _version_summary(new_version, include_spec=True)}),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error restoring artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)

application/api/user/attachments/__init__.py

@@ -0,0 +1,5 @@
+"""Attachments module."""
+
+from .routes import attachments_ns
+
+__all__ = ["attachments_ns"]

application/api/user/attachments/routes.py

@@ -0,0 +1,700 @@
+"""File attachments and media routes."""
+
+import os
+import tempfile
+from pathlib import Path
+
+import uuid
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.cache import get_redis_instance
+from application.core.settings import settings
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.session import db_readonly
+from application.stt.constants import (
+    SUPPORTED_AUDIO_EXTENSIONS,
+    SUPPORTED_AUDIO_MIME_TYPES,
+)
+from application.stt.upload_limits import (
+    AudioFileTooLargeError,
+    build_stt_file_size_limit_message,
+    enforce_audio_file_size_limit,
+    is_audio_filename,
+)
+from application.stt.live_session import (
+    apply_live_stt_hypothesis,
+    create_live_stt_session,
+    delete_live_stt_session,
+    finalize_live_stt_session,
+    get_live_stt_transcript_text,
+    load_live_stt_session,
+    save_live_stt_session,
+)
+from application.stt.stt_creator import STTCreator
+from application.tts.tts_creator import TTSCreator
+from application.utils import safe_filename
+
+
+attachments_ns = Namespace(
+    "attachments", description="File attachments and media operations", path="/api"
+)
+
+
+def _resolve_authenticated_user():
+    decoded_token = getattr(request, "decoded_token", None)
+    api_key = request.form.get("api_key") or request.args.get("api_key")
+
+    # Return the RAW user identity (not safe_filename'd): the attachment row's
+    # ``user_id`` is an identity key that must match the raw ``sub`` used by
+    # /stream (initial_user_id) and the artifact authz gate. safe_filename is
+    # applied only to the storage path component at write time. (An email-style
+    # sub like ``a@b.com`` was previously sanitized to ``abcom``, so the
+    # uploaded attachment became unreadable by its own owner on /stream.)
+    if decoded_token:
+        return decoded_token.get("sub")
+
+    if api_key:
+        with db_readonly() as conn:
+            agent = AgentsRepository(conn).find_by_key(api_key)
+        if not agent:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid API key"}), 401
+            )
+        return agent.get("user_id")
+
+    return None
+
+
+def _get_uploaded_file_size(file) -> int:
+    try:
+        current_position = file.stream.tell()
+        file.stream.seek(0, os.SEEK_END)
+        size_bytes = file.stream.tell()
+        file.stream.seek(current_position)
+        return size_bytes
+    except Exception:
+        return 0
+
+
+def _is_supported_audio_mimetype(mimetype: str) -> bool:
+    if not mimetype:
+        return True
+    normalized = mimetype.split(";")[0].strip().lower()
+    return normalized.startswith("audio/") or normalized in SUPPORTED_AUDIO_MIME_TYPES
+
+
+def _enforce_uploaded_audio_size_limit(file, filename: str) -> None:
+    if not is_audio_filename(filename):
+        return
+    size_bytes = _get_uploaded_file_size(file)
+    if size_bytes:
+        enforce_audio_file_size_limit(size_bytes)
+
+
+def _get_store_attachment_user_error(exc: Exception) -> str:
+    if isinstance(exc, AudioFileTooLargeError):
+        return build_stt_file_size_limit_message()
+    return "Failed to process file"
+
+
+def _require_live_stt_redis():
+    redis_client = get_redis_instance()
+    if redis_client:
+        return redis_client
+    return make_response(
+        jsonify({"success": False, "message": "Live transcription is unavailable"}),
+        503,
+    )
+
+
+def _parse_bool_form_value(value: str | None) -> bool:
+    if value is None:
+        return False
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+@attachments_ns.route("/store_attachment")
+class StoreAttachment(Resource):
+    @api.expect(
+        api.model(
+            "AttachmentModel",
+            {
+                "file": fields.Raw(required=True, description="File(s) to upload"),
+                "api_key": fields.String(
+                    required=False, description="API key (optional)"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Stores one or multiple attachments without vectorization or training. Supports user or API key authentication."
+    )
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        
+        files = request.files.getlist("file")
+        if not files:
+            single_file = request.files.get("file")
+            if single_file:
+                files = [single_file]
+        
+        if not files or all(f.filename == "" for f in files):
+            return make_response(
+                jsonify({"status": "error", "message": "Missing file(s)"}),
+                400,
+            )
+        
+        user = auth_user
+        if not user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}), 401
+            )
+        
+        try:
+            from application.api.user.tasks import store_attachment
+            from application.api.user.base import storage
+
+            tasks = []
+            errors = []
+            original_file_count = len(files)
+            
+            for idx, file in enumerate(files):
+                try:
+                    attachment_id = uuid.uuid4()
+                    original_filename = safe_filename(os.path.basename(file.filename))
+                    _enforce_uploaded_audio_size_limit(file, original_filename)
+                    # safe_filename only the path component; the DB user_id stays raw.
+                    path_user = safe_filename(user)
+                    relative_path = (
+                        f"{settings.UPLOAD_FOLDER}/{path_user}/attachments/"
+                        f"{str(attachment_id)}/{original_filename}"
+                    )
+
+                    metadata = storage.save_file(file, relative_path)
+                    file_info = {
+                        "filename": original_filename,
+                        "attachment_id": str(attachment_id),
+                        "path": relative_path,
+                        "metadata": metadata,
+                    }
+
+                    task = store_attachment.delay(file_info, user)
+                    tasks.append({
+                        "task_id": task.id,
+                        "filename": original_filename,
+                        "attachment_id": str(attachment_id),
+                        "upload_index": idx,
+                    })
+                except Exception as file_err:
+                    current_app.logger.error(f"Error processing file {idx} ({file.filename}): {file_err}", exc_info=True)
+                    errors.append({
+                        "upload_index": idx,
+                        "filename": file.filename,
+                        "error": _get_store_attachment_user_error(file_err),
+                    })
+            
+            if not tasks:
+                if errors and all(
+                    error.get("error") == build_stt_file_size_limit_message()
+                    for error in errors
+                ):
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "message": build_stt_file_size_limit_message(),
+                                "errors": errors,
+                            }
+                        ),
+                        413,
+                    )
+                return make_response(
+                    jsonify({"status": "error", "message": "No valid files to upload"}),
+                    400,
+                )
+            
+            if original_file_count == 1 and len(tasks) == 1:
+                current_app.logger.info("Returning single task_id response")
+                return make_response(
+                    jsonify(
+                        {
+                            "success": True,
+                            "task_id": tasks[0]["task_id"],
+                            # Surface the attachment_id so the frontend
+                            # can correlate ``attachment.*`` SSE events
+                            # to this row and skip the polling fallback.
+                            "attachment_id": tasks[0]["attachment_id"],
+                            "message": "File uploaded successfully. Processing started.",
+                        }
+                    ),
+                    200,
+                )
+            else:
+                response_data = {
+                    "success": True,
+                    "tasks": tasks,
+                    "message": f"{len(tasks)} file(s) uploaded successfully. Processing started.",
+                }
+                if errors:
+                    response_data["errors"] = errors
+                    response_data["message"] += f" {len(errors)} file(s) failed."
+                
+                return make_response(
+                    jsonify(response_data),
+                    200,
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error storing attachment: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to store attachment"}), 400)
+
+
+@attachments_ns.route("/stt")
+class SpeechToText(Resource):
+    @api.expect(
+        api.model(
+            "SpeechToTextModel",
+            {
+                "file": fields.Raw(required=True, description="Audio file"),
+                "language": fields.String(
+                    required=False, description="Optional transcription language hint"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Transcribe an uploaded audio file")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        file = request.files.get("file")
+        if not file or file.filename == "":
+            return make_response(
+                jsonify({"success": False, "message": "Missing file"}),
+                400,
+            )
+
+        filename = safe_filename(os.path.basename(file.filename))
+        suffix = Path(filename).suffix.lower()
+        if suffix not in SUPPORTED_AUDIO_EXTENSIONS:
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio format"}),
+                400,
+            )
+
+        if not _is_supported_audio_mimetype(file.mimetype or ""):
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio MIME type"}),
+                400,
+            )
+
+        try:
+            _enforce_uploaded_audio_size_limit(file, filename)
+        except AudioFileTooLargeError:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": build_stt_file_size_limit_message(),
+                    }
+                ),
+                413,
+            )
+
+        temp_path = None
+        try:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+                file.save(temp_file.name)
+                temp_path = Path(temp_file.name)
+
+            stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)
+            transcript = stt_instance.transcribe(
+                temp_path,
+                language=request.form.get("language") or settings.STT_LANGUAGE,
+                timestamps=settings.STT_ENABLE_TIMESTAMPS,
+                diarize=settings.STT_ENABLE_DIARIZATION,
+            )
+            return make_response(jsonify({"success": True, **transcript}), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error transcribing audio: {err}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "message": "Failed to transcribe audio"}),
+                400,
+            )
+        finally:
+            if temp_path and temp_path.exists():
+                temp_path.unlink()
+
+
+@attachments_ns.route("/stt/live/start")
+class LiveSpeechToTextStart(Resource):
+    @api.doc(description="Start a live speech-to-text session")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        redis_client = _require_live_stt_redis()
+        if hasattr(redis_client, "status_code"):
+            return redis_client
+
+        payload = request.get_json(silent=True) or {}
+        session_state = create_live_stt_session(
+            user=auth_user,
+            language=payload.get("language") or settings.STT_LANGUAGE,
+        )
+        save_live_stt_session(redis_client, session_state)
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "session_id": session_state["session_id"],
+                    "language": session_state.get("language"),
+                    "committed_text": "",
+                    "mutable_text": "",
+                    "previous_hypothesis": "",
+                    "latest_hypothesis": "",
+                    "finalized_text": "",
+                    "pending_text": "",
+                    "transcript_text": "",
+                }
+            ),
+            200,
+        )
+
+
+@attachments_ns.route("/stt/live/chunk")
+class LiveSpeechToTextChunk(Resource):
+    @api.expect(
+        api.model(
+            "LiveSpeechToTextChunkModel",
+            {
+                "session_id": fields.String(
+                    required=True, description="Live transcription session ID"
+                ),
+                "chunk_index": fields.Integer(
+                    required=True, description="Sequential chunk index"
+                ),
+                "is_silence": fields.Boolean(
+                    required=False,
+                    description="Whether the latest capture window was mostly silence",
+                ),
+                "file": fields.Raw(required=True, description="Audio chunk"),
+            },
+        )
+    )
+    @api.doc(description="Transcribe a chunk for a live speech-to-text session")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        redis_client = _require_live_stt_redis()
+        if hasattr(redis_client, "status_code"):
+            return redis_client
+
+        session_id = request.form.get("session_id", "").strip()
+        if not session_id:
+            return make_response(
+                jsonify({"success": False, "message": "Missing session_id"}),
+                400,
+            )
+
+        session_state = load_live_stt_session(redis_client, session_id)
+        if not session_state:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Live transcription session not found",
+                    }
+                ),
+                404,
+            )
+
+        # The stored ``user`` is the RAW sub (see _resolve_authenticated_user), and
+        # auth_user is also raw, so compare raw-vs-raw -- never safe_filename'd, or an
+        # email-style sub (a@b.com) would 403 the owner on their own session.
+        if str(session_state.get("user", "")) != auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Forbidden"}),
+                403,
+            )
+
+        chunk_index_raw = request.form.get("chunk_index", "").strip()
+        if chunk_index_raw == "":
+            return make_response(
+                jsonify({"success": False, "message": "Missing chunk_index"}),
+                400,
+            )
+
+        try:
+            chunk_index = int(chunk_index_raw)
+        except ValueError:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid chunk_index"}),
+                400,
+            )
+        is_silence = _parse_bool_form_value(request.form.get("is_silence"))
+
+        file = request.files.get("file")
+        if not file or file.filename == "":
+            return make_response(
+                jsonify({"success": False, "message": "Missing file"}),
+                400,
+            )
+
+        filename = safe_filename(os.path.basename(file.filename))
+        suffix = Path(filename).suffix.lower()
+        if suffix not in SUPPORTED_AUDIO_EXTENSIONS:
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio format"}),
+                400,
+            )
+
+        if not _is_supported_audio_mimetype(file.mimetype or ""):
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio MIME type"}),
+                400,
+            )
+
+        try:
+            _enforce_uploaded_audio_size_limit(file, filename)
+        except AudioFileTooLargeError:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": build_stt_file_size_limit_message(),
+                    }
+                ),
+                413,
+            )
+
+        temp_path = None
+        try:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+                file.save(temp_file.name)
+                temp_path = Path(temp_file.name)
+
+            session_language = session_state.get("language") or settings.STT_LANGUAGE
+            stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)
+            transcript = stt_instance.transcribe(
+                temp_path,
+                language=session_language,
+                timestamps=False,
+                diarize=False,
+            )
+            if not session_state.get("language") and transcript.get("language"):
+                session_state["language"] = transcript["language"]
+
+            try:
+                apply_live_stt_hypothesis(
+                    session_state,
+                    str(transcript.get("text", "")),
+                    chunk_index,
+                    is_silence=is_silence,
+                )
+            except ValueError:
+                current_app.logger.warning(
+                    "Invalid live transcription chunk",
+                    exc_info=True,
+                )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Invalid live transcription chunk",
+                        }
+                    ),
+                    409,
+                )
+            save_live_stt_session(redis_client, session_state)
+
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "session_id": session_id,
+                        "chunk_index": chunk_index,
+                        "chunk_text": transcript.get("text", ""),
+                        "is_silence": is_silence,
+                        "language": session_state.get("language"),
+                        "committed_text": session_state.get("committed_text", ""),
+                        "mutable_text": session_state.get("mutable_text", ""),
+                        "previous_hypothesis": session_state.get(
+                            "previous_hypothesis", ""
+                        ),
+                        "latest_hypothesis": session_state.get(
+                            "latest_hypothesis", ""
+                        ),
+                        "finalized_text": session_state.get("committed_text", ""),
+                        "pending_text": session_state.get("mutable_text", ""),
+                        "transcript_text": get_live_stt_transcript_text(session_state),
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error transcribing live audio chunk: {err}", exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Failed to transcribe audio"}),
+                400,
+            )
+        finally:
+            if temp_path and temp_path.exists():
+                temp_path.unlink()
+
+
+@attachments_ns.route("/stt/live/finish")
+class LiveSpeechToTextFinish(Resource):
+    @api.doc(description="Finish a live speech-to-text session")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        redis_client = _require_live_stt_redis()
+        if hasattr(redis_client, "status_code"):
+            return redis_client
+
+        payload = request.get_json(silent=True) or {}
+        session_id = str(payload.get("session_id", "")).strip()
+        if not session_id:
+            return make_response(
+                jsonify({"success": False, "message": "Missing session_id"}),
+                400,
+            )
+
+        session_state = load_live_stt_session(redis_client, session_id)
+        if not session_state:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Live transcription session not found",
+                    }
+                ),
+                404,
+            )
+
+        # Stored ``user`` and auth_user are both RAW subs; compare raw-vs-raw so an
+        # email-style sub owner is not 403'd on their own finish call.
+        if str(session_state.get("user", "")) != auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Forbidden"}),
+                403,
+            )
+
+        final_text = finalize_live_stt_session(session_state)
+        delete_live_stt_session(redis_client, session_id)
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "session_id": session_id,
+                    "language": session_state.get("language"),
+                    "text": final_text,
+                }
+            ),
+            200,
+        )
+
+
+@attachments_ns.route("/images/<path:image_path>")
+class ServeImage(Resource):
+    @api.doc(description="Serve an image from storage")
+    def get(self, image_path):
+        if ".." in image_path or image_path.startswith("/") or "\x00" in image_path:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid image path"}), 400
+            )
+        try:
+            from application.api.user.base import storage
+
+            file_obj = storage.get_file(image_path)
+            extension = image_path.split(".")[-1].lower()
+            content_type = f"image/{extension}"
+            if extension == "jpg":
+                content_type = "image/jpeg"
+            response = make_response(file_obj.read())
+            response.headers.set("Content-Type", content_type)
+            response.headers.set("Cache-Control", "max-age=86400")
+
+            return response
+        except FileNotFoundError:
+            return make_response(
+                jsonify({"success": False, "message": "Image not found"}), 404
+            )
+        except ValueError:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid image path"}), 400
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error serving image: {e}")
+            return make_response(
+                jsonify({"success": False, "message": "Error retrieving image"}), 500
+            )
+
+
+@attachments_ns.route("/tts")
+class TextToSpeech(Resource):
+    tts_model = api.model(
+        "TextToSpeechModel",
+        {
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
+        },
+    )
+
+    @api.expect(tts_model)
+    @api.doc(description="Synthesize audio speech from text")
+    def post(self):
+        data = request.get_json()
+        text = data["text"]
+        try:
+            tts_instance = TTSCreator.create_tts(settings.TTS_PROVIDER)
+            audio_base64, detected_language = tts_instance.text_to_speech(text)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error synthesizing audio: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)

application/api/user/authz.py

@@ -0,0 +1,97 @@
+"""Role resolution and authorization helpers (admin/user RBAC).
+
+Roles are resolved per-request from the database plus computed overlays and are
+NEVER read from the inbound JWT: ``simple_jwt``/``session_jwt`` tokens are
+self-mintable, so trusting a ``roles`` claim would be a trivial privilege
+escalation. ``resolve_roles`` always rebuilds the set from scratch.
+
+Policy (see ``rbac-spec.md``):
+- Persisted RBAC (``user_roles``) applies only under ``AUTH_TYPE=oidc``.
+- ``AUTH_TYPE=None`` (no-auth self-host) grants admin only when
+  ``LOCAL_MODE_ADMIN`` is enabled (default off).
+- ``simple_jwt`` / ``session_jwt`` can never be admin (shared/throwaway ``sub``).
+- ``/v1`` agent keys, device tokens, and pre-auth requests are role-less.
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import wraps
+
+from flask import jsonify, make_response, request
+
+from application.core.settings import settings
+from application.storage.db.repositories.user_roles import UserRolesRepository
+from application.storage.db.session import db_readonly
+
+logger = logging.getLogger(__name__)
+
+ROLE_USER = "user"
+ROLE_ADMIN = "admin"
+
+
+def resolve_roles(token: dict | None) -> list[str]:
+    """Compute the sorted role set for a request principal.
+
+    Fails open to less privilege: a DB error while reading grants demotes
+    DB-backed admins to ``user`` rather than raising (which would turn a roles
+    outage into a total-auth outage on the universal before-request path).
+    """
+    if not token:
+        return [ROLE_USER]
+    roles = {ROLE_USER}
+    sub = token.get("sub")
+    if settings.AUTH_TYPE == "oidc":
+        if sub:
+            try:
+                with db_readonly() as conn:
+                    roles.update(UserRolesRepository(conn).role_names_for(sub))
+            except Exception:
+                logger.error(
+                    "resolve_roles: user_roles read failed for sub=%s", sub, exc_info=True
+                )
+    elif settings.AUTH_TYPE is None and settings.LOCAL_MODE_ADMIN:
+        roles.add(ROLE_ADMIN)
+    # simple_jwt / session_jwt: never admin.
+    return sorted(roles)
+
+
+def has_role(token: dict | None, name: str) -> bool:
+    """True if the principal holds ``name``.
+
+    ``user`` is implicit and always true. Tolerates a ``None`` token or a token
+    that never went through ``resolve_roles`` (e.g. async/SSE or /v1 paths) —
+    those are treated as role-less and only satisfy the implicit ``user`` role.
+    """
+    if name == ROLE_USER:
+        return True
+    roles = (token or {}).get("roles") or []
+    return name in roles
+
+
+def require_role(name: str):
+    """Decorator factory: 401 when unauthenticated, 403 when lacking ``name``.
+
+    Fails closed — a missing/None token or absent ``roles`` key never raises and
+    never passes through. The frontend route guard is cosmetic; this is the
+    security boundary.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            token = getattr(request, "decoded_token", None)
+            if not token:
+                return make_response(
+                    jsonify({"success": False, "message": "Authentication required"}), 401
+                )
+            if not has_role(token, name):
+                return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+admin_required = require_role(ROLE_ADMIN)

application/api/user/base.py

@@ -0,0 +1,308 @@
+"""
+Shared utilities, database connections, and helper functions for user API routes.
+"""
+
+import datetime
+import os
+import uuid
+from functools import wraps
+from typing import Optional, Tuple
+
+from flask import current_app, jsonify, make_response, Response
+from werkzeug.utils import secure_filename
+
+from sqlalchemy import text as _sql_text
+
+from application.core.settings import settings
+from application.storage.db.base_repository import looks_like_uuid, row_to_dict
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+from application.storage.storage_creator import StorageCreator
+from application.vectorstore.vector_creator import VectorCreator
+
+
+storage = StorageCreator.get_storage()
+
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+def generate_minute_range(start_date, end_date):
+    """Generate a dictionary with minute-level time ranges."""
+    return {
+        (start_date + datetime.timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 60) + 1)
+    }
+
+
+def generate_hourly_range(start_date, end_date):
+    """Generate a dictionary with hourly time ranges."""
+    return {
+        (start_date + datetime.timedelta(hours=i)).strftime("%Y-%m-%d %H:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 3600) + 1)
+    }
+
+
+def generate_date_range(start_date, end_date):
+    """Generate a dictionary with daily date ranges."""
+    return {
+        (start_date + datetime.timedelta(days=i)).strftime("%Y-%m-%d"): 0
+        for i in range((end_date - start_date).days + 1)
+    }
+
+
+def ensure_user_doc(user_id):
+    """
+    Ensure a Postgres ``users`` row exists for ``user_id``.
+
+    Returns the row as a dict with the shape legacy callers expect — in
+    particular ``user_id`` and ``agent_preferences`` (with ``pinned`` and
+    ``shared_with_me`` list keys always present).
+
+    Args:
+        user_id: The user ID to ensure
+
+    Returns:
+        The user document as a dict.
+    """
+    with db_session() as conn:
+        user_doc = UsersRepository(conn).upsert(user_id)
+
+    prefs = user_doc.get("agent_preferences") or {}
+    if not isinstance(prefs, dict):
+        prefs = {}
+    prefs.setdefault("pinned", [])
+    prefs.setdefault("shared_with_me", [])
+    user_doc["agent_preferences"] = prefs
+    return user_doc
+
+
+def resolve_tool_details(tool_ids):
+    """
+    Resolve tool IDs to their display details.
+
+    Accepts Postgres UUIDs, legacy Mongo ObjectId strings, or the
+    synthetic ids of default chat tools / agent-selectable builtins
+    (mixed lists are supported). Synthetic ids are resolved in memory;
+    real ids are looked up via ``get_any``. Unknown ids are silently
+    skipped.
+
+    Args:
+        tool_ids: List of tool IDs (UUIDs, legacy ObjectId strings, or
+            synthetic default-tool / builtin ids).
+
+    Returns:
+        List of tool details with ``id``, ``name``, and ``display_name``.
+    """
+    if not tool_ids:
+        return []
+
+    from application.agents.default_tools import (
+        is_synthesized_tool_id,
+        synthesize_tool_by_name,
+        synthesized_tool_name_for_id,
+    )
+
+    uuid_ids: list[str] = []
+    legacy_ids: list[str] = []
+    default_details: list[dict] = []
+    for tid in tool_ids:
+        if not tid:
+            continue
+        tid_str = str(tid)
+        if is_synthesized_tool_id(tid_str):
+            synth = synthesize_tool_by_name(synthesized_tool_name_for_id(tid_str))
+            if synth is not None:
+                default_details.append(
+                    {
+                        "id": tid_str,
+                        "name": synth.get("name", ""),
+                        "display_name": synth.get("display_name", ""),
+                    }
+                )
+            continue
+        if looks_like_uuid(tid_str):
+            uuid_ids.append(tid_str)
+        else:
+            legacy_ids.append(tid_str)
+
+    if not uuid_ids and not legacy_ids:
+        return default_details
+
+    rows: list[dict] = []
+    with db_readonly() as conn:
+        if uuid_ids:
+            result = conn.execute(
+                _sql_text(
+                    "SELECT * FROM user_tools "
+                    "WHERE id = ANY(CAST(:ids AS uuid[]))"
+                ),
+                {"ids": uuid_ids},
+            )
+            rows.extend(row_to_dict(r) for r in result.fetchall())
+        if legacy_ids:
+            result = conn.execute(
+                _sql_text(
+                    "SELECT * FROM user_tools "
+                    "WHERE legacy_mongo_id = ANY(:ids)"
+                ),
+                {"ids": legacy_ids},
+            )
+            rows.extend(row_to_dict(r) for r in result.fetchall())
+
+    return default_details + [
+        {
+            "id": str(tool.get("id") or tool.get("legacy_mongo_id") or ""),
+            "name": tool.get("name", "") or "",
+            "display_name": (
+                tool.get("custom_name")
+                or tool.get("display_name")
+                or tool.get("name", "")
+                or ""
+            ),
+        }
+        for tool in rows
+    ]
+
+
+_BUILTIN_PROMPT_NAMES = {
+    "default": "Default",
+    "creative": "Creative",
+    "strict": "Strict",
+}
+
+
+def resolve_prompt_name(prompt_id) -> Optional[str]:
+    """Resolve a prompt id to its display name by id (owner-agnostic).
+
+    Mirrors ``resolve_tool_details``: looks the prompt up by id without user
+    scoping, so a team member viewing a shared agent sees the OWNER's prompt
+    name rather than nothing. Built-in prompt sentinels map to friendly labels.
+    Returns None when the prompt is missing/unknown.
+    """
+    if not prompt_id:
+        return None
+    pid = str(prompt_id)
+    if pid in _BUILTIN_PROMPT_NAMES:
+        return _BUILTIN_PROMPT_NAMES[pid]
+    if not looks_like_uuid(pid):
+        return None
+    with db_readonly() as conn:
+        result = conn.execute(
+            _sql_text("SELECT name FROM prompts WHERE id = CAST(:id AS uuid)"),
+            {"id": pid},
+        )
+        row = result.fetchone()
+    return row[0] if row is not None else None
+
+
+def resolve_source_details(source_ids) -> list[dict]:
+    """Resolve source ids to ``[{"id", "name"}]`` by id (owner-agnostic).
+
+    Order-preserving; an id with no matching source row yields ``name: None`` so
+    the client can fall back. Lets a team member viewing a shared agent see the
+    owner's source names instead of a raw id / "External KB".
+    """
+    ids = [str(s) for s in (source_ids or []) if s and looks_like_uuid(str(s))]
+    if not ids:
+        return []
+    with db_readonly() as conn:
+        result = conn.execute(
+            _sql_text(
+                "SELECT id, name FROM sources WHERE id = ANY(CAST(:ids AS uuid[]))"
+            ),
+            {"ids": ids},
+        )
+        by_id = {str(r[0]): r[1] for r in result.fetchall()}
+    return [{"id": sid, "name": by_id.get(sid)} for sid in ids]
+
+
+def get_vector_store(source_id):
+    """
+    Get the Vector Store for a given source ID.
+
+    Args:
+        source_id (str): source id of the document
+
+    Returns:
+        Vector store instance
+    """
+    store = VectorCreator.create_vectorstore(
+        settings.VECTOR_STORE,
+        source_id=source_id,
+        embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+    )
+    return store
+
+
+def handle_image_upload(
+    request, existing_url: str, user: str, storage, base_path: str = "attachments/"
+) -> Tuple[str, Optional[Response]]:
+    """
+    Handle image file upload from request.
+
+    Args:
+        request: Flask request object
+        existing_url: Existing image URL (fallback)
+        user: User ID
+        storage: Storage instance
+        base_path: Base path for upload
+
+    Returns:
+        Tuple of (image_url, error_response)
+    """
+    image_url = existing_url
+
+    if "image" in request.files:
+        file = request.files["image"]
+        if file.filename != "":
+            filename = secure_filename(file.filename)
+            upload_path = f"{settings.UPLOAD_FOLDER.rstrip('/')}/{user}/{base_path.rstrip('/')}/{uuid.uuid4()}_{filename}"
+            try:
+                storage.save_file(file, upload_path, storage_class="STANDARD")
+                image_url = upload_path
+            except Exception as e:
+                current_app.logger.error(f"Error uploading image: {e}")
+                return None, make_response(
+                    jsonify({"success": False, "message": "Image upload failed"}),
+                    400,
+                )
+    return image_url, None
+
+
+def require_agent(func):
+    """
+    Decorator to require valid agent webhook token.
+
+    Args:
+        func: Function to decorate
+
+    Returns:
+        Wrapped function
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        from application.storage.db.repositories.agents import AgentsRepository
+
+        webhook_token = kwargs.get("webhook_token")
+        if not webhook_token:
+            return make_response(
+                jsonify({"success": False, "message": "Webhook token missing"}), 400
+            )
+        with db_readonly() as conn:
+            agent = AgentsRepository(conn).find_by_webhook_token(webhook_token)
+        if not agent:
+            current_app.logger.warning(
+                f"Webhook attempt with invalid token: {webhook_token}"
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Agent not found"}), 404
+            )
+        kwargs["agent"] = agent
+        kwargs["agent_id_str"] = str(agent["id"])
+        return func(*args, **kwargs)
+
+    return wrapper

application/api/user/conversations/__init__.py

@@ -0,0 +1,5 @@
+"""Conversation management module."""
+
+from .routes import conversations_ns
+
+__all__ = ["conversations_ns"]

application/api/user/conversations/routes.py

@@ -0,0 +1,475 @@
+"""Conversation management routes."""
+
+import datetime
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+from sqlalchemy import text as sql_text
+
+from application.api import api
+from application.api.answer.services.conversation_service import (
+    TERMINATED_RESPONSE_PLACEHOLDER,
+)
+from application.storage.db.base_repository import looks_like_uuid, row_to_dict
+from application.storage.db.repositories.attachments import AttachmentsRepository
+from application.storage.db.repositories.conversations import ConversationsRepository
+from application.storage.db.repositories.message_events import MessageEventsRepository
+from application.storage.db.session import db_readonly, db_session
+from application.utils import check_required_fields
+
+conversations_ns = Namespace(
+    "conversations", description="Conversation management operations", path="/api"
+)
+
+
+@conversations_ns.route("/delete_conversation")
+class DeleteConversation(Resource):
+    @api.doc(
+        description="Deletes a conversation by ID",
+        params={"id": "The ID of the conversation to delete"},
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        conversation_id = request.args.get("id")
+        if not conversation_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        user_id = decoded_token["sub"]
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(conversation_id, user_id)
+                if conv is not None:
+                    repo.delete(str(conv["id"]), user_id)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/delete_all_conversations")
+class DeleteAllConversations(Resource):
+    @api.doc(
+        description="Deletes all conversations for a specific user",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user_id = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                ConversationsRepository(conn).delete_all_for_user(user_id)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting all conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/get_conversations")
+class GetConversations(Resource):
+    @api.doc(
+        description="Retrieve a list of the latest 30 sidebar conversations (visibility = listed)",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                conversations = ConversationsRepository(conn).list_for_user(
+                    user_id, limit=30
+                )
+            list_conversations = [
+                {
+                    "id": str(conversation["id"]),
+                    "name": conversation["name"],
+                    "agent_id": (
+                        str(conversation["agent_id"])
+                        if conversation.get("agent_id")
+                        else None
+                    ),
+                    "is_shared_usage": conversation.get("is_shared_usage", False),
+                    "shared_token": conversation.get("shared_token", None),
+                }
+                for conversation in conversations
+            ]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_conversations), 200)
+
+
+@conversations_ns.route("/search_conversations")
+class SearchConversations(Resource):
+    @staticmethod
+    def _build_match_snippet(text_value: str, query: str, radius: int = 60) -> str:
+        if not text_value:
+            return ""
+        idx = text_value.lower().find(query.lower())
+        if idx == -1:
+            snippet = text_value[: radius * 2]
+            return snippet + ("…" if len(text_value) > len(snippet) else "")
+        start = max(0, idx - radius)
+        end = min(len(text_value), idx + len(query) + radius)
+        snippet = text_value[start:end]
+        if start > 0:
+            snippet = "…" + snippet
+        if end < len(text_value):
+            snippet = snippet + "…"
+        return snippet
+
+    @api.doc(
+        description=(
+            "Search the authenticated user's conversations by name or "
+            "message content (case-insensitive substring match). Mirrors "
+            "the visibility filter and response shape of /get_conversations, "
+            "and additionally returns ``match_field`` (``name``, ``prompt`` "
+            "or ``response``) and ``match_snippet`` (a short excerpt of the "
+            "matched text centered on the query) for each result."
+        ),
+        params={
+            "q": "Search term (required)",
+            "limit": "Maximum number of results to return (default 30, max 100)",
+        },
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        query = (request.args.get("q") or "").strip()
+        if not query:
+            return make_response(
+                jsonify({"success": False, "message": "q is required"}), 400
+            )
+        try:
+            limit = int(request.args.get("limit", 30))
+        except (TypeError, ValueError):
+            limit = 30
+        limit = max(1, min(limit, 100))
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                conversations = ConversationsRepository(conn).search_for_user(
+                    user_id, query, limit=limit
+                )
+            list_conversations = [
+                {
+                    "id": str(conversation["id"]),
+                    "name": conversation["name"],
+                    "agent_id": (
+                        str(conversation["agent_id"])
+                        if conversation.get("agent_id")
+                        else None
+                    ),
+                    "is_shared_usage": conversation.get("is_shared_usage", False),
+                    "shared_token": conversation.get("shared_token", None),
+                    "match_field": conversation.get("match_field"),
+                    "match_snippet": self._build_match_snippet(
+                        conversation.get("match_text") or "", query
+                    ),
+                }
+                for conversation in conversations
+            ]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error searching conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_conversations), 200)
+
+
+@conversations_ns.route("/get_single_conversation")
+class GetSingleConversation(Resource):
+    @api.doc(
+        description="Retrieve a single conversation by ID",
+        params={"id": "The conversation ID"},
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        conversation_id = request.args.get("id")
+        if not conversation_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                repo = ConversationsRepository(conn)
+                conversation = repo.get_any(conversation_id, user_id)
+                if not conversation:
+                    return make_response(jsonify({"status": "not found"}), 404)
+                conv_pg_id = str(conversation["id"])
+                messages = repo.get_messages(conv_pg_id)
+
+                # Resolve attachment details (id, fileName) for each message.
+                attachments_repo = AttachmentsRepository(conn)
+                queries = []
+                for msg in messages:
+                    metadata = msg.get("metadata") or {}
+                    query = {
+                        "prompt": msg.get("prompt"),
+                        "response": msg.get("response"),
+                        "thought": msg.get("thought"),
+                        "sources": msg.get("sources") or [],
+                        "tool_calls": msg.get("tool_calls") or [],
+                        "timestamp": msg.get("timestamp"),
+                        "model_id": msg.get("model_id"),
+                        # Lets the client distinguish placeholder rows from
+                        # finalised answers and tail-poll in-flight ones.
+                        "message_id": str(msg["id"]) if msg.get("id") else None,
+                        "status": msg.get("status"),
+                        "request_id": msg.get("request_id"),
+                        "last_heartbeat_at": metadata.get("last_heartbeat_at"),
+                        # Surfaced from metadata so the chat can render a
+                        # workflow run's produced artifacts on reload.
+                        "workflow_run_id": metadata.get("workflow_run_id"),
+                    }
+                    if metadata:
+                        query["metadata"] = metadata
+                    # Feedback on conversation_messages is a JSONB blob with
+                    # shape {"text": <str>, "timestamp": <iso>}. The legacy
+                    # frontend consumed a flat scalar feedback string, so
+                    # unwrap the ``text`` field for compat.
+                    feedback = msg.get("feedback")
+                    if feedback is not None:
+                        if isinstance(feedback, dict):
+                            query["feedback"] = feedback.get("text")
+                            if feedback.get("timestamp"):
+                                query["feedback_timestamp"] = feedback["timestamp"]
+                        else:
+                            query["feedback"] = feedback
+                    attachments = msg.get("attachments") or []
+                    if attachments:
+                        attachment_details = []
+                        for attachment_id in attachments:
+                            try:
+                                att = attachments_repo.get_any(
+                                    str(attachment_id), user_id
+                                )
+                                if att:
+                                    attachment_details.append(
+                                        {
+                                            "id": str(att["id"]),
+                                            "fileName": att.get(
+                                                "filename", "Unknown file"
+                                            ),
+                                        }
+                                    )
+                            except Exception as e:
+                                current_app.logger.error(
+                                    f"Error retrieving attachment {attachment_id}: {e}",
+                                    exc_info=True,
+                                )
+                        query["attachments"] = attachment_details
+                    queries.append(query)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        data = {
+            "queries": queries,
+            "agent_id": (
+                str(conversation["agent_id"]) if conversation.get("agent_id") else None
+            ),
+            "is_shared_usage": conversation.get("is_shared_usage", False),
+            "shared_token": conversation.get("shared_token", None),
+        }
+        return make_response(jsonify(data), 200)
+
+
+@conversations_ns.route("/update_conversation_name")
+class UpdateConversationName(Resource):
+    @api.expect(
+        api.model(
+            "UpdateConversationModel",
+            {
+                "id": fields.String(required=True, description="Conversation ID"),
+                "name": fields.String(
+                    required=True, description="New name of the conversation"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Updates the name of a conversation",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["id", "name"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user_id = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(data["id"], user_id)
+                if conv is not None:
+                    repo.rename(str(conv["id"]), user_id, data["name"])
+        except Exception as err:
+            current_app.logger.error(
+                f"Error updating conversation name: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/feedback")
+class SubmitFeedback(Resource):
+    @api.expect(
+        api.model(
+            "FeedbackModel",
+            {
+                "question": fields.String(
+                    required=False, description="The user question"
+                ),
+                "answer": fields.String(required=False, description="The AI answer"),
+                "feedback": fields.String(required=True, description="User feedback"),
+                "question_index": fields.Integer(
+                    required=True,
+                    description="The question number in that particular conversation",
+                ),
+                "conversation_id": fields.String(
+                    required=True, description="id of the particular conversation"
+                ),
+                "api_key": fields.String(description="Optional API key"),
+            },
+        )
+    )
+    @api.doc(
+        description="Submit feedback for a conversation",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["feedback", "conversation_id", "question_index"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user_id = decoded_token.get("sub")
+        feedback_value = data["feedback"]
+        question_index = int(data["question_index"])
+        # Normalize string feedback to lowercase so analytics queries
+        # (which match 'like'/'dislike') count rows correctly. Tolerate
+        # legacy uppercase clients on ingest. Non-string values pass through.
+        if isinstance(feedback_value, str):
+            feedback_value = feedback_value.lower()
+        feedback_payload = (
+            None
+            if feedback_value is None
+            else {
+                "text": feedback_value,
+                "timestamp": datetime.datetime.now(
+                    datetime.timezone.utc
+                ).isoformat(),
+            }
+        )
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(data["conversation_id"], user_id)
+                if conv is None:
+                    return make_response(
+                        jsonify({"success": False, "message": "Not found"}), 404
+                    )
+                repo.set_feedback(str(conv["id"]), question_index, feedback_payload)
+        except Exception as err:
+            current_app.logger.error(f"Error submitting feedback: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/messages/<string:message_id>/tail")
+class GetMessageTail(Resource):
+    @api.doc(
+        description=(
+            "Current state of one conversation_messages row, scoped to the "
+            "authenticated user. Used to reconnect to an in-flight stream "
+            "after a refresh."
+        ),
+        params={"message_id": "Message UUID"},
+    )
+    def get(self, message_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        if not looks_like_uuid(message_id):
+            return make_response(
+                jsonify({"success": False, "message": "Invalid message id"}), 400
+            )
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                # Owner-or-shared, matching ``ConversationsRepository.get``.
+                row = conn.execute(
+                    sql_text(
+                        "SELECT m.* FROM conversation_messages m "
+                        "JOIN conversations c ON c.id = m.conversation_id "
+                        "WHERE m.id = CAST(:mid AS uuid) "
+                        "AND (c.user_id = :uid OR :uid = ANY(c.shared_with))"
+                    ),
+                    {"mid": message_id, "uid": user_id},
+                ).fetchone()
+                if row is None:
+                    return make_response(jsonify({"status": "not found"}), 404)
+                msg = row_to_dict(row)
+                # Mid-stream the row's response is the placeholder; rebuild
+                # the live partial from the journal so /tail mirrors SSE.
+                status = msg.get("status")
+                response = msg.get("response")
+                thought = msg.get("thought")
+                sources = msg.get("sources") or []
+                tool_calls = msg.get("tool_calls") or []
+                if status in ("pending", "streaming") and (
+                    response == TERMINATED_RESPONSE_PLACEHOLDER
+                ):
+                    partial = MessageEventsRepository(conn).reconstruct_partial(
+                        message_id
+                    )
+                    response = partial["response"]
+                    thought = partial["thought"] or thought
+                    if partial["sources"]:
+                        sources = partial["sources"]
+                    if partial["tool_calls"]:
+                        tool_calls = partial["tool_calls"]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error tailing message {message_id}: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        metadata = msg.get("message_metadata") or {}
+        return make_response(
+            jsonify(
+                {
+                    "message_id": str(msg["id"]),
+                    "status": status,
+                    "response": response,
+                    "thought": thought,
+                    "sources": sources,
+                    "tool_calls": tool_calls,
+                    "request_id": msg.get("request_id"),
+                    "last_heartbeat_at": metadata.get("last_heartbeat_at"),
+                    "error": metadata.get("error"),
+                }
+            ),
+            200,
+        )

application/api/user/idempotency.py

@@ -0,0 +1,294 @@
+"""Per-Celery-task idempotency wrapper backed by ``task_dedup``."""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import logging
+import threading
+import uuid
+from typing import Any, Callable, Optional
+
+from application.storage.db.repositories.idempotency import IdempotencyRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+# Poison-loop cap; transient-failure headroom without infinite retry.
+MAX_TASK_ATTEMPTS = 5
+
+# 30s heartbeat / 60s TTL → ~2 missed ticks of slack before reclaim.
+LEASE_TTL_SECONDS = 60
+LEASE_HEARTBEAT_INTERVAL = 30
+
+# 10 × 60s ≈ 5 min of deferral before giving up on a held lease.
+LEASE_RETRY_MAX = 10
+
+
+def with_idempotency(
+    task_name: str,
+    *,
+    on_poison: Optional[Callable[[str, dict], None]] = None,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    """Short-circuit on completed key; gate concurrent runs via a lease.
+
+    The guard key is the caller's ``idempotency_key``, or one synthesized
+    from ``source_id`` so a keyless dispatch is still poison-guarded.
+
+    Entry short-circuits:
+      - completed row → return cached result
+      - live lease held → retry(countdown=LEASE_TTL_SECONDS)
+      - attempt_count > MAX_TASK_ATTEMPTS → poison alert; ``on_poison`` fires
+    Success writes ``completed``; exceptions leave ``pending`` for
+    autoretry until the poison-loop guard trips.
+    """
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        @functools.wraps(fn)
+        def wrapper(self, *args: Any, idempotency_key: Any = None, **kwargs: Any) -> Any:
+            explicit_key = (
+                idempotency_key
+                if isinstance(idempotency_key, str) and idempotency_key
+                else None
+            )
+            # A keyless dispatch still gets the guard via a synthesized key;
+            # None means no anchor exists — run unguarded, as before.
+            key = explicit_key or _synthesize_guard_key(task_name, kwargs)
+            if key is None:
+                return fn(self, *args, idempotency_key=idempotency_key, **kwargs)
+
+            cached = _lookup_completed(key)
+            if cached is not None:
+                logger.info(
+                    "idempotency hit for task=%s key=%s — returning cached result",
+                    task_name, key,
+                )
+                return cached
+
+            owner_id = str(uuid.uuid4())
+            attempt = _try_claim_lease(
+                key, task_name, _safe_task_id(self), owner_id,
+            )
+            if attempt is None:
+                # Live lease held by another worker. Re-queue and bail
+                # quickly — by the time the retry fires (LEASE_TTL
+                # seconds), Worker 1 has either finalised (we'll hit
+                # ``_lookup_completed`` and return cached) or its lease
+                # has expired and we can claim.
+                logger.info(
+                    "idempotency: live lease held; deferring task=%s key=%s",
+                    task_name, key,
+                )
+                raise self.retry(
+                    countdown=LEASE_TTL_SECONDS,
+                    max_retries=LEASE_RETRY_MAX,
+                )
+
+            if attempt > MAX_TASK_ATTEMPTS:
+                logger.error(
+                    "idempotency poison-loop guard: task=%s key=%s attempts=%s",
+                    task_name, key, attempt,
+                    extra={
+                        "alert": "idempotency_poison_loop",
+                        "task_name": task_name,
+                        "idempotency_key": key,
+                        "attempts": attempt,
+                    },
+                )
+                poisoned = {
+                    "success": False,
+                    "error": "idempotency poison-loop guard tripped",
+                    "attempts": attempt,
+                }
+                _finalize(key, poisoned, status="failed")
+                _run_poison_hook(
+                    on_poison, task_name, fn, self, args, kwargs, idempotency_key,
+                )
+                return poisoned
+
+            heartbeat_thread, heartbeat_stop = _start_lease_heartbeat(
+                key, owner_id,
+            )
+            try:
+                result = fn(self, *args, idempotency_key=idempotency_key, **kwargs)
+                _finalize(key, result, status="completed")
+                return result
+            except Exception:
+                # Drop the lease so the next retry doesn't wait LEASE_TTL.
+                _release_lease(key, owner_id)
+                raise
+            finally:
+                _stop_lease_heartbeat(heartbeat_thread, heartbeat_stop)
+
+        return wrapper
+
+    return decorator
+
+
+def _synthesize_guard_key(task_name: str, kwargs: dict) -> Optional[str]:
+    """Derive a deterministic guard key from ``source_id`` for a keyless dispatch.
+
+    ``source_id`` is stable across broker redeliveries and unique per
+    upload, so the poison-loop counter survives an OOM SIGKILL. Returns
+    ``None`` when absent — the dispatch then runs unguarded as before.
+    """
+    source_id = kwargs.get("source_id")
+    if source_id:
+        return f"auto:{task_name}:{source_id}"
+    return None
+
+
+def _run_poison_hook(
+    on_poison: Optional[Callable[[str, dict], None]],
+    task_name: str,
+    fn: Callable[..., Any],
+    task_self: Any,
+    args: tuple,
+    kwargs: dict,
+    idempotency_key: Any,
+) -> None:
+    """Invoke a task's poison-path hook with named call args; swallow failures.
+
+    A hook failure must never change the poison-guard outcome.
+    """
+    if on_poison is None:
+        return
+    try:
+        bound = inspect.signature(fn).bind_partial(
+            task_self, *args, idempotency_key=idempotency_key, **kwargs,
+        )
+        on_poison(task_name, dict(bound.arguments))
+    except Exception:
+        logger.exception(
+            "idempotency: poison hook failed for task=%s", task_name,
+        )
+
+
+def _lookup_completed(key: str) -> Any:
+    """Return cached ``result_json`` if a completed row exists for ``key``, else None."""
+    with db_readonly() as conn:
+        row = IdempotencyRepository(conn).get_task(key)
+    if row is None:
+        return None
+    if row.get("status") != "completed":
+        return None
+    return row.get("result_json")
+
+
+def _try_claim_lease(
+    key: str, task_name: str, task_id: str, owner_id: str,
+) -> Optional[int]:
+    """Atomic CAS; returns ``attempt_count`` or ``None`` when held.
+
+    DB outage → treated as ``attempt=1`` so transient failures don't
+    block all task execution; reconciler repairs the lease columns.
+    """
+    try:
+        with db_session() as conn:
+            return IdempotencyRepository(conn).try_claim_lease(
+                key=key,
+                task_name=task_name,
+                task_id=task_id,
+                owner_id=owner_id,
+                ttl_seconds=LEASE_TTL_SECONDS,
+            )
+    except Exception:
+        logger.exception(
+            "idempotency lease-claim failed for key=%s task=%s", key, task_name,
+        )
+        return 1
+
+
+def _finalize(key: str, result_json: Any, *, status: str) -> None:
+    """Best-effort terminal write. Never let DB outage fail the task."""
+    try:
+        with db_session() as conn:
+            IdempotencyRepository(conn).finalize_task(
+                key=key, result_json=result_json, status=status,
+            )
+    except Exception:
+        logger.exception(
+            "idempotency finalize failed for key=%s status=%s", key, status,
+        )
+
+
+def _release_lease(key: str, owner_id: str) -> None:
+    """Best-effort lease release on the wrapper's exception path."""
+    try:
+        with db_session() as conn:
+            IdempotencyRepository(conn).release_lease(key, owner_id)
+    except Exception:
+        logger.exception("idempotency release-lease failed for key=%s", key)
+
+
+def _start_lease_heartbeat(
+    key: str, owner_id: str,
+) -> tuple[threading.Thread, threading.Event]:
+    """Spawn a daemon thread that bumps ``lease_expires_at`` every
+    :data:`LEASE_HEARTBEAT_INTERVAL` seconds until ``stop_event`` fires.
+
+    Mirrors ``application.worker._start_ingest_heartbeat`` so the two
+    durability heartbeats share shape and cadence.
+    """
+    stop_event = threading.Event()
+    thread = threading.Thread(
+        target=_lease_heartbeat_loop,
+        args=(key, owner_id, stop_event, LEASE_HEARTBEAT_INTERVAL),
+        daemon=True,
+        name=f"idempotency-lease-heartbeat:{key[:32]}",
+    )
+    thread.start()
+    return thread, stop_event
+
+
+def _stop_lease_heartbeat(
+    thread: threading.Thread, stop_event: threading.Event,
+) -> None:
+    """Signal the heartbeat thread to exit and join with a short timeout."""
+    stop_event.set()
+    thread.join(timeout=10)
+
+
+def _lease_heartbeat_loop(
+    key: str,
+    owner_id: str,
+    stop_event: threading.Event,
+    interval: int,
+) -> None:
+    """Refresh the lease until ``stop_event`` is set or ownership is lost.
+
+    A failed refresh (rowcount 0) means another worker stole the lease
+    after expiry — at that point the damage is already possible, so we
+    log and keep ticking. Don't escalate to thread death; the main task
+    body needs to keep running so its outcome is at least *recorded*.
+    """
+    while not stop_event.wait(interval):
+        try:
+            with db_session() as conn:
+                still_owned = IdempotencyRepository(conn).refresh_lease(
+                    key=key, owner_id=owner_id, ttl_seconds=LEASE_TTL_SECONDS,
+                )
+            if not still_owned:
+                logger.warning(
+                    "idempotency lease lost mid-task for key=%s "
+                    "(another worker may have taken over)",
+                    key,
+                )
+        except Exception:
+            logger.exception(
+                "idempotency lease-heartbeat tick failed for key=%s", key,
+            )
+
+
+def _safe_task_id(task_self: Any) -> str:
+    """Best-effort extraction of ``self.request.id`` from a Celery task."""
+    try:
+        request = getattr(task_self, "request", None)
+        task_id: Optional[str] = (
+            getattr(request, "id", None) if request is not None else None
+        )
+    except Exception:
+        task_id = None
+    return task_id or "unknown"

application/api/user/me/__init__.py

@@ -0,0 +1,3 @@
+from .routes import me_ns
+
+__all__ = ["me_ns"]

application/api/user/me/routes.py

@@ -0,0 +1,34 @@
+"""Current-principal endpoint.
+
+``GET /api/user/me`` returns the caller's user id and resolved roles, sourced
+only from ``request.decoded_token`` (already populated and role-resolved by the
+auth chokepoint in ``app.py``). Auth-mode-agnostic. ``email``/``name``/
+``picture`` are OIDC-only and optional — they are echoed from the token and are
+never present for ``simple_jwt``/``session_jwt``/no-auth modes.
+"""
+
+from __future__ import annotations
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+me_ns = Namespace("me", description="Current user identity and roles", path="/api")
+
+
+@me_ns.route("/user/me")
+class MeResource(Resource):
+    def get(self):
+        """Return ``{user_id, roles, email?, name?, picture?}`` for the caller."""
+        decoded_token = getattr(request, "decoded_token", None)
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        body = {
+            "success": True,
+            "user_id": decoded_token.get("sub"),
+            "roles": decoded_token.get("roles") or ["user"],
+        }
+        for field in ("email", "name", "picture"):
+            value = decoded_token.get(field)
+            if value:
+                body[field] = value
+        return make_response(jsonify(body), 200)

application/api/user/models/__init__.py

@@ -0,0 +1,3 @@
+from .routes import models_ns
+
+__all__ = ["models_ns"]

application/api/user/models/routes.py

@@ -0,0 +1,521 @@
+"""Model routes.
+
+- ``GET /api/models`` — list available models for the current user.
+  Combines the built-in catalog with the user's BYOM records.
+- ``GET/POST/PATCH/DELETE /api/user/models[/<id>]`` — CRUD for the
+  user's own OpenAI-compatible model registrations (BYOM).
+- ``POST /api/user/models/<id>/test`` — sanity-check the upstream
+  endpoint with a tiny request.
+
+Every BYOM endpoint is user-scoped at the repository layer
+(every query filters on ``user_id`` from ``request.decoded_token``).
+"""
+
+from __future__ import annotations
+
+import logging
+
+import requests
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from application.api import api
+from application.core.model_registry import ModelRegistry
+from application.security.safe_url import (
+    UnsafeUserUrlError,
+    pinned_post,
+    validate_user_base_url,
+)
+from application.storage.db.repositories.user_custom_models import (
+    UserCustomModelsRepository,
+)
+from application.storage.db.session import db_readonly, db_session
+from application.utils import check_required_fields
+
+
+logger = logging.getLogger(__name__)
+
+
+models_ns = Namespace("models", description="Available models", path="/api")
+
+
+_CONTEXT_WINDOW_MIN = 1_000
+_CONTEXT_WINDOW_MAX = 10_000_000
+
+
+def _user_id_or_401():
+    decoded_token = request.decoded_token
+    if not decoded_token:
+        return None, make_response(jsonify({"success": False}), 401)
+    user_id = decoded_token.get("sub")
+    if not user_id:
+        return None, make_response(jsonify({"success": False}), 401)
+    return user_id, None
+
+
+def _normalize_capabilities(raw) -> dict:
+    """Coerce + bound the user-supplied capabilities payload."""
+    raw = raw or {}
+    out = {}
+    if "supports_tools" in raw:
+        out["supports_tools"] = bool(raw["supports_tools"])
+    if "supports_structured_output" in raw:
+        out["supports_structured_output"] = bool(raw["supports_structured_output"])
+    if "supports_streaming" in raw:
+        out["supports_streaming"] = bool(raw["supports_streaming"])
+    if "attachments" in raw:
+        atts = raw["attachments"] or []
+        if not isinstance(atts, list):
+            raise ValueError("'capabilities.attachments' must be a list")
+        coerced = [str(a) for a in atts]
+        # Reject unknown aliases at the API boundary so bad payloads
+        # never reach the registry layer (where lenient expansion just
+        # drops them). Raw MIME types (containing ``/``) pass through
+        # unchanged for parity with the built-in YAML schema.
+        from application.core.model_yaml import builtin_attachment_aliases
+
+        aliases = builtin_attachment_aliases()
+        for entry in coerced:
+            if "/" in entry:
+                continue
+            if entry not in aliases:
+                valid = ", ".join(sorted(aliases.keys())) or "<none defined>"
+                raise ValueError(
+                    f"unknown attachment alias '{entry}' in "
+                    f"'capabilities.attachments'. Valid aliases: {valid}, "
+                    f"or use a raw MIME type like 'image/png'."
+                )
+        out["attachments"] = coerced
+    if "context_window" in raw:
+        try:
+            cw = int(raw["context_window"])
+        except (TypeError, ValueError):
+            raise ValueError("'capabilities.context_window' must be an integer")
+        if not (_CONTEXT_WINDOW_MIN <= cw <= _CONTEXT_WINDOW_MAX):
+            raise ValueError(
+                f"'capabilities.context_window' must be between "
+                f"{_CONTEXT_WINDOW_MIN} and {_CONTEXT_WINDOW_MAX}"
+            )
+        out["context_window"] = cw
+    return out
+
+
+def _row_to_response(row: dict) -> dict:
+    """Wire-format projection — never includes the API key."""
+    return {
+        "id": str(row["id"]),
+        "upstream_model_id": row["upstream_model_id"],
+        "display_name": row["display_name"],
+        "description": row.get("description") or "",
+        "base_url": row["base_url"],
+        "capabilities": row.get("capabilities") or {},
+        "enabled": bool(row.get("enabled", True)),
+        "source": "user",
+    }
+
+
+@models_ns.route("/models")
+class ModelsListResource(Resource):
+    def get(self):
+        """Get list of available models with their capabilities.
+
+        When the request is authenticated, the response includes the
+        user's own BYOM registrations alongside the built-in catalog.
+        """
+        try:
+            user_id = None
+            decoded_token = getattr(request, "decoded_token", None)
+            if decoded_token:
+                user_id = decoded_token.get("sub")
+
+            registry = ModelRegistry.get_instance()
+            models = registry.get_enabled_models(user_id=user_id)
+
+            response = {
+                "models": [model.to_dict() for model in models],
+                "default_model_id": registry.default_model_id,
+                "count": len(models),
+            }
+        except Exception as err:
+            current_app.logger.error(f"Error fetching models: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+        return make_response(jsonify(response), 200)
+
+
+@models_ns.route("/user/models")
+class UserModelsCollectionResource(Resource):
+    @api.doc(description="List the current user's BYOM custom models")
+    def get(self):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+        try:
+            with db_readonly() as conn:
+                rows = UserCustomModelsRepository(conn).list_for_user(user_id)
+            return make_response(
+                jsonify({"models": [_row_to_response(r) for r in rows]}), 200
+            )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error listing user custom models: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+
+    @api.doc(description="Register a new BYOM custom model")
+    def post(self):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+        missing = check_required_fields(
+            data,
+            ["upstream_model_id", "display_name", "base_url", "api_key"],
+        )
+        if missing:
+            return missing
+
+        # SECURITY: reject blank api_key — would leak instance API key
+        # to the user-supplied base_url via LLMCreator fallback.
+        for required_nonblank in (
+            "upstream_model_id",
+            "display_name",
+            "base_url",
+            "api_key",
+        ):
+            value = data.get(required_nonblank)
+            if not isinstance(value, str) or not value.strip():
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "error": f"'{required_nonblank}' must be a non-empty string",
+                        }
+                    ),
+                    400,
+                )
+
+        # SSRF guard at create time. Re-runs at dispatch time (LLMCreator)
+        # as defense in depth against DNS rebinding and pre-guard rows.
+        try:
+            validate_user_base_url(data["base_url"])
+        except UnsafeUserUrlError as e:
+            return make_response(
+                jsonify({"success": False, "error": str(e)}), 400
+            )
+
+        try:
+            capabilities = _normalize_capabilities(data.get("capabilities"))
+        except ValueError as e:
+            return make_response(
+                jsonify({"success": False, "error": str(e)}), 400
+            )
+
+        try:
+            with db_session() as conn:
+                row = UserCustomModelsRepository(conn).create(
+                    user_id=user_id,
+                    upstream_model_id=data["upstream_model_id"],
+                    display_name=data["display_name"],
+                    description=data.get("description") or "",
+                    base_url=data["base_url"],
+                    api_key_plaintext=data["api_key"],
+                    capabilities=capabilities,
+                    enabled=bool(data.get("enabled", True)),
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error creating user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+
+        ModelRegistry.invalidate_user(user_id)
+        return make_response(jsonify(_row_to_response(row)), 201)
+
+
+@models_ns.route("/user/models/<string:model_id>")
+class UserModelResource(Resource):
+    @api.doc(description="Get one BYOM custom model")
+    def get(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+        try:
+            with db_readonly() as conn:
+                row = UserCustomModelsRepository(conn).get(model_id, user_id)
+        except Exception as e:
+            current_app.logger.error(
+                f"Error fetching user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+        if row is None:
+            return make_response(jsonify({"success": False}), 404)
+        return make_response(jsonify(_row_to_response(row)), 200)
+
+    @api.doc(description="Update a BYOM custom model (partial)")
+    def patch(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+
+        # Reject present-but-blank values for fields where blank doesn't
+        # mean "no change". (The api_key special case — blank means "keep
+        # existing" — is handled below.)
+        for required_nonblank in (
+            "upstream_model_id",
+            "display_name",
+            "base_url",
+        ):
+            if required_nonblank in data:
+                value = data[required_nonblank]
+                if not isinstance(value, str) or not value.strip():
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": f"'{required_nonblank}' cannot be blank",
+                            }
+                        ),
+                        400,
+                    )
+
+        if "base_url" in data and data["base_url"]:
+            try:
+                validate_user_base_url(data["base_url"])
+            except UnsafeUserUrlError as e:
+                return make_response(
+                    jsonify({"success": False, "error": str(e)}), 400
+                )
+
+        update_fields: dict = {}
+        for k in (
+            "upstream_model_id",
+            "display_name",
+            "description",
+            "base_url",
+            "enabled",
+        ):
+            if k in data:
+                update_fields[k] = data[k]
+
+        if "capabilities" in data:
+            try:
+                update_fields["capabilities"] = _normalize_capabilities(
+                    data["capabilities"]
+                )
+            except ValueError as e:
+                return make_response(
+                    jsonify({"success": False, "error": str(e)}), 400
+                )
+
+        # PATCH semantics: blank/missing api_key → keep the existing
+        # ciphertext; non-empty api_key → re-encrypt and replace.
+        if data.get("api_key"):
+            update_fields["api_key_plaintext"] = data["api_key"]
+
+        if not update_fields:
+            return make_response(
+                jsonify({"success": False, "error": "no updatable fields"}), 400
+            )
+
+        try:
+            with db_session() as conn:
+                ok = UserCustomModelsRepository(conn).update(
+                    model_id, user_id, update_fields
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error updating user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+
+        if not ok:
+            return make_response(jsonify({"success": False}), 404)
+
+        ModelRegistry.invalidate_user(user_id)
+        with db_readonly() as conn:
+            row = UserCustomModelsRepository(conn).get(model_id, user_id)
+        return make_response(jsonify(_row_to_response(row)), 200)
+
+    @api.doc(description="Delete a BYOM custom model")
+    def delete(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+        try:
+            with db_session() as conn:
+                ok = UserCustomModelsRepository(conn).delete(model_id, user_id)
+        except Exception as e:
+            current_app.logger.error(
+                f"Error deleting user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+        if not ok:
+            return make_response(jsonify({"success": False}), 404)
+
+        ModelRegistry.invalidate_user(user_id)
+        return make_response(jsonify({"success": True}), 200)
+
+
+def _run_connection_test(
+    base_url: str, api_key: str, upstream_model_id: str
+):
+    """Send a 1-token chat-completion to verify a BYOM endpoint.
+
+    Returns ``(body, http_status)``. Upstream errors return 200 with
+    ``ok=False`` so the UI can render inline errors; only local SSRF
+    rejection returns 400.
+    """
+    url = base_url.rstrip("/") + "/chat/completions"
+    payload = {
+        "model": upstream_model_id,
+        "messages": [{"role": "user", "content": "hi"}],
+        "max_tokens": 1,
+        "stream": False,
+    }
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    try:
+        # pinned_post closes the DNS-rebinding window. Redirects off
+        # because 3xx could bounce to an internal address (the SSRF
+        # guard only validates the supplied URL).
+        resp = pinned_post(
+            url,
+            json=payload,
+            headers=headers,
+            timeout=5,
+            allow_redirects=False,
+        )
+    except UnsafeUserUrlError as e:
+        return {"ok": False, "error": str(e)}, 400
+    except requests.RequestException as e:
+        return {"ok": False, "error": f"connection error: {e}"}, 200
+
+    if 300 <= resp.status_code < 400:
+        return (
+            {
+                "ok": False,
+                "error": (
+                    f"upstream returned HTTP {resp.status_code} "
+                    "redirect; refusing to follow"
+                ),
+            },
+            200,
+        )
+
+    if resp.status_code >= 400:
+        # Cap and only reflect JSON to avoid body-exfil via non-API responses.
+        content_type = (resp.headers.get("Content-Type") or "").lower()
+        if "application/json" in content_type:
+            text = (resp.text or "")[:500]
+            error_msg = f"upstream returned HTTP {resp.status_code}: {text}"
+        else:
+            error_msg = f"upstream returned HTTP {resp.status_code}"
+        return {"ok": False, "error": error_msg}, 200
+
+    return {"ok": True}, 200
+
+
+@models_ns.route("/user/models/test")
+class UserModelTestPayloadResource(Resource):
+    @api.doc(
+        description=(
+            "Test an arbitrary BYOM payload (display_name / model id / "
+            "base_url / api_key) without saving. Used by the UI's 'Test "
+            "connection' button so the user can validate before they "
+            "Save. Same SSRF guard, same 1-token request, same 5s "
+            "timeout as the by-id variant."
+        )
+    )
+    def post(self):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+        missing = check_required_fields(
+            data, ["base_url", "api_key", "upstream_model_id"]
+        )
+        if missing:
+            return missing
+
+        body, status = _run_connection_test(
+            data["base_url"], data["api_key"], data["upstream_model_id"]
+        )
+        return make_response(jsonify(body), status)
+
+
+@models_ns.route("/user/models/<string:model_id>/test")
+class UserModelTestResource(Resource):
+    @api.doc(
+        description=(
+            "Test a saved BYOM record. Defaults to the stored "
+            "base_url / upstream_model_id / encrypted api_key, but "
+            "any of those can be overridden via the request body so "
+            "the UI can test in-flight edits before saving. Used by "
+            "the 'Test connection' button in edit mode."
+        )
+    )
+    def post(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+        # Per-field overrides; blank/missing falls back to stored value.
+        override_base_url = (data.get("base_url") or "").strip() or None
+        override_upstream_model_id = (
+            data.get("upstream_model_id") or ""
+        ).strip() or None
+        override_api_key = (data.get("api_key") or "").strip() or None
+
+        try:
+            with db_readonly() as conn:
+                repo = UserCustomModelsRepository(conn)
+                row = repo.get(model_id, user_id)
+                if row is None:
+                    return make_response(jsonify({"success": False}), 404)
+                stored_api_key = (
+                    repo._decrypt_api_key(
+                        row.get("api_key_encrypted", ""), user_id
+                    )
+                    if not override_api_key
+                    else None
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error loading user custom model for test: {e}", exc_info=True
+            )
+            return make_response(
+                jsonify({"ok": False, "error": "internal error loading model"}),
+                500,
+            )
+
+        api_key = override_api_key or stored_api_key
+        if not api_key:
+            return make_response(
+                jsonify(
+                    {
+                        "ok": False,
+                        "error": (
+                            "Stored API key could not be decrypted. The "
+                            "encryption secret may have rotated. Re-save "
+                            "the model with the API key to recover."
+                        ),
+                    }
+                ),
+                400,
+            )
+
+        base_url = override_base_url or row["base_url"]
+        upstream_model_id = (
+            override_upstream_model_id or row["upstream_model_id"]
+        )
+        body, status = _run_connection_test(
+            base_url, api_key, upstream_model_id
+        )
+        return make_response(jsonify(body), status)

application/api/user/prompts/__init__.py

@@ -0,0 +1,5 @@
+"""Prompts module."""
+
+from .routes import prompts_ns
+
+__all__ = ["prompts_ns"]

application/api/user/prompts/routes.py

@@ -0,0 +1,250 @@
+"""Prompt management routes."""
+
+import os
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import current_dir
+from application.api.user.team_sharing import team_access_for, visible_with_access
+from application.storage.db.repositories.prompts import PromptsRepository
+from application.storage.db.session import db_readonly, db_session
+from application.utils import check_required_fields
+
+prompts_ns = Namespace(
+    "prompts", description="Prompt management operations", path="/api"
+)
+
+
+@prompts_ns.route("/create_prompt")
+class CreatePrompt(Resource):
+    create_prompt_model = api.model(
+        "CreatePromptModel",
+        {
+            "content": fields.String(
+                required=True, description="Content of the prompt"
+            ),
+            "name": fields.String(required=True, description="Name of the prompt"),
+        },
+    )
+
+    @api.expect(create_prompt_model)
+    @api.doc(description="Create a new prompt")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["content", "name"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                prompt = PromptsRepository(conn).create(user, data["name"], data["content"])
+            new_id = str(prompt["id"])
+        except Exception as err:
+            current_app.logger.error(f"Error creating prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"id": new_id}), 200)
+
+
+@prompts_ns.route("/get_prompts")
+class GetPrompts(Resource):
+    @api.doc(description="Get all prompts for the user")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                repo = PromptsRepository(conn)
+                prompts = repo.list_for_user(user)
+                owned_ids = {str(p["id"]) for p in prompts}
+                team_shared = visible_with_access(conn, user, "prompt")
+                shared_ids = [pid for pid in team_shared if pid not in owned_ids]
+                shared_prompts = repo.list_by_ids(shared_ids)
+            list_prompts = [
+                {"id": "default", "name": "default", "type": "public"},
+                {"id": "creative", "name": "creative", "type": "public"},
+                {"id": "strict", "name": "strict", "type": "public"},
+            ]
+            for prompt in prompts:
+                list_prompts.append(
+                    {
+                        "id": str(prompt["id"]),
+                        "name": prompt["name"],
+                        "type": "private",
+                    }
+                )
+            for prompt in shared_prompts:
+                list_prompts.append(
+                    {
+                        "id": str(prompt["id"]),
+                        "name": prompt["name"],
+                        "type": "team",
+                        "team_access": team_shared.get(str(prompt["id"])),
+                    }
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving prompts: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_prompts), 200)
+
+
+@prompts_ns.route("/get_single_prompt")
+class GetSinglePrompt(Resource):
+    @api.doc(params={"id": "ID of the prompt"}, description="Get a single prompt by ID")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        prompt_id = request.args.get("id")
+        if not prompt_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            if prompt_id == "default":
+                with open(
+                    os.path.join(current_dir, "prompts", "chat_combine_default.txt"),
+                    "r",
+                ) as f:
+                    chat_combine_template = f.read()
+                return make_response(jsonify({"content": chat_combine_template}), 200)
+            elif prompt_id == "creative":
+                with open(
+                    os.path.join(current_dir, "prompts", "chat_combine_creative.txt"),
+                    "r",
+                ) as f:
+                    chat_reduce_creative = f.read()
+                return make_response(jsonify({"content": chat_reduce_creative}), 200)
+            elif prompt_id == "strict":
+                with open(
+                    os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r"
+                ) as f:
+                    chat_reduce_strict = f.read()
+                return make_response(jsonify({"content": chat_reduce_strict}), 200)
+            with db_readonly() as conn:
+                repo = PromptsRepository(conn)
+                prompt = repo.get_any(prompt_id, user)
+                if not prompt and team_access_for(conn, user, "prompt", prompt_id):
+                    # Team fallback: ownerless fetch only after a grant check.
+                    prompt = repo.get_for_rendering(prompt_id)
+            if not prompt:
+                return make_response(
+                    jsonify({"success": False, "message": "Prompt not found"}), 404
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"content": prompt["content"]}), 200)
+
+
+@prompts_ns.route("/delete_prompt")
+class DeletePrompt(Resource):
+    delete_prompt_model = api.model(
+        "DeletePromptModel",
+        {"id": fields.String(required=True, description="Prompt ID to delete")},
+    )
+
+    @api.expect(delete_prompt_model)
+    @api.doc(description="Delete a prompt by ID")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            with db_session() as conn:
+                repo = PromptsRepository(conn)
+                prompt = repo.get_any(data["id"], user)
+                if not prompt:
+                    return make_response(
+                        jsonify({"success": False, "message": "Prompt not found"}),
+                        404,
+                    )
+                repo.delete(str(prompt["id"]), user)
+        except Exception as err:
+            current_app.logger.error(f"Error deleting prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@prompts_ns.route("/update_prompt")
+class UpdatePrompt(Resource):
+    update_prompt_model = api.model(
+        "UpdatePromptModel",
+        {
+            "id": fields.String(required=True, description="Prompt ID to update"),
+            "name": fields.String(required=True, description="New name of the prompt"),
+            "content": fields.String(
+                required=True, description="New content of the prompt"
+            ),
+        },
+    )
+
+    @api.expect(update_prompt_model)
+    @api.doc(description="Update an existing prompt")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "name", "content"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            with db_session() as conn:
+                repo = PromptsRepository(conn)
+                prompt = repo.get_any(data["id"], user)
+                if prompt:
+                    repo.update(str(prompt["id"]), user, data["name"], data["content"])
+                else:
+                    # Team editor write path (viewer is read-only).
+                    access = team_access_for(conn, user, "prompt", data["id"])
+                    if access == "editor":
+                        result = repo.update_by_id(
+                            data["id"],
+                            data["name"],
+                            data["content"],
+                            expected_updated_at=data.get("expected_updated_at"),
+                        )
+                        if result is None:
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": "Prompt was modified by someone else",
+                                        "code": "stale_write",
+                                    }
+                                ),
+                                409,
+                            )
+                    elif access == "viewer":
+                        return make_response(
+                            jsonify(
+                                {"success": False, "message": "Read-only: editor access required"}
+                            ),
+                            403,
+                        )
+                    else:
+                        return make_response(
+                            jsonify({"success": False, "message": "Prompt not found"}),
+                            404,
+                        )
+        except Exception as err:
+            current_app.logger.error(f"Error updating prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)

application/api/user/reconciliation.py

@@ -0,0 +1,435 @@
+"""Reconciler tick: sweep stuck rows and escalate to terminal status + alert."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional, TYPE_CHECKING
+
+from sqlalchemy import Connection
+
+from application.api.user.idempotency import MAX_TASK_ATTEMPTS
+from application.core.settings import settings
+from application.storage.db.engine import get_engine
+from application.storage.db.repositories.pending_tool_state import (
+    PendingToolStateRepository,
+)
+from application.storage.db.repositories.reconciliation import (
+    ReconciliationRepository,
+)
+from application.storage.db.repositories.stack_logs import StackLogsRepository
+
+if TYPE_CHECKING:
+    from application.storage.db.repositories.schedules import SchedulesRepository
+
+logger = logging.getLogger(__name__)
+
+
+MAX_MESSAGE_RECONCILE_ATTEMPTS = 3
+
+
+def run_reconciliation() -> Dict[str, Any]:
+    """Single tick of the reconciler. Five sweeps, FOR UPDATE SKIP LOCKED.
+
+    Stuck ``executed`` tool calls always flip to ``failed`` — operators
+    handle cleanup manually via the structured alert. The side effect is
+    assumed to have committed; no automated rollback is attempted.
+
+    Stuck ``task_dedup`` rows (lease expired AND attempts >= max)
+    promote to ``failed`` so a same-key retry can re-claim instead of
+    sitting in ``pending`` until 24 h TTL.
+    """
+    if not settings.POSTGRES_URI:
+        return {
+            "messages_failed": 0,
+            "tool_calls_failed": 0,
+            "skipped": "POSTGRES_URI not set",
+        }
+
+    engine = get_engine()
+    summary = {
+        "messages_failed": 0,
+        "tool_calls_failed": 0,
+        "ingests_stalled": 0,
+        "idempotency_pending_failed": 0,
+        "schedule_runs_failed": 0,
+    }
+    # User-facing events to fan out once their DB writes have committed
+    # (publish-after-commit). Each item is
+    # ``(user_id, event_type, payload, scope)``.
+    events: list[tuple] = []
+
+    with engine.begin() as conn:
+        repo = ReconciliationRepository(conn)
+        pt_repo = PendingToolStateRepository(conn)
+        for msg in repo.find_and_lock_stuck_messages():
+            new_count = repo.increment_message_reconcile_attempts(msg["id"])
+            if new_count >= MAX_MESSAGE_RECONCILE_ATTEMPTS:
+                repo.mark_message_failed(
+                    msg["id"],
+                    error=(
+                        "reconciler: stuck in pending/streaming for >5 min "
+                        f"after {new_count} attempts"
+                    ),
+                )
+                summary["messages_failed"] += 1
+                # Revoke any awaiting-approval prompt: the resumable state
+                # dies with the message, so the durable
+                # ``tool.approval.required`` envelope must be cleared or the
+                # UI toast lingers on reconnect. Only emit when a row was
+                # actually deleted so non-approval failures stay quiet.
+                user_id = msg.get("user_id")
+                conversation_id = msg.get("conversation_id")
+                if (
+                    user_id
+                    and conversation_id
+                    and pt_repo.delete_state(str(conversation_id), str(user_id))
+                ):
+                    events.append(
+                        (
+                            str(user_id),
+                            "tool.approval.cleared",
+                            {
+                                "conversation_id": str(conversation_id),
+                                "message_id": str(msg["id"]),
+                                "reason": "failed",
+                            },
+                            {"kind": "conversation", "id": str(conversation_id)},
+                        )
+                    )
+                _emit_alert(
+                    conn,
+                    name="reconciler_message_failed",
+                    user_id=msg.get("user_id"),
+                    detail={
+                        "message_id": str(msg["id"]),
+                        "attempts": new_count,
+                    },
+                )
+
+    with engine.begin() as conn:
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_and_lock_proposed_tool_calls():
+            repo.mark_tool_call_failed(
+                row["call_id"],
+                error=(
+                    "reconciler: stuck in 'proposed' for >5 min; "
+                    "side effect status unknown"
+                ),
+            )
+            summary["tool_calls_failed"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_tool_call_failed_proposed",
+                user_id=None,
+                detail={
+                    "call_id": row["call_id"],
+                    "tool_name": row.get("tool_name"),
+                },
+            )
+
+    with engine.begin() as conn:
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_and_lock_executed_tool_calls():
+            repo.mark_tool_call_failed(
+                row["call_id"],
+                error=(
+                    "reconciler: executed-not-confirmed; side effect "
+                    "assumed committed, manual cleanup required"
+                ),
+            )
+            summary["tool_calls_failed"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_tool_call_failed_executed",
+                user_id=None,
+                detail={
+                    "call_id": row["call_id"],
+                    "tool_name": row.get("tool_name"),
+                    "action_name": row.get("action_name"),
+                },
+            )
+
+    # Q4: ingest checkpoints whose heartbeat has gone silent. Each is
+    # escalated to terminal ``status='stalled'`` and alerted once — no
+    # worker kill, no rollback of the partial embed. The 'stalled' flag
+    # ends the re-alert loop and drives the "indexing failed" badge the
+    # sources list derives from this row.
+    with engine.begin() as conn:
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_and_lock_stalled_ingests():
+            summary["ingests_stalled"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_ingest_stalled",
+                user_id=row.get("user_id"),
+                detail={
+                    "source_id": str(row.get("source_id")),
+                    "embedded_chunks": row.get("embedded_chunks"),
+                    "total_chunks": row.get("total_chunks"),
+                    "last_updated": str(row.get("last_updated")),
+                },
+            )
+            repo.mark_ingest_stalled(str(row["source_id"]))
+            # Tell the upload toast the ingest is done-for. Without it the
+            # toast spins on "Training…" for the whole live session; only
+            # ``source.ingest.failed`` flips it to a terminal error.
+            user_id = row.get("user_id")
+            source_id = row.get("source_id")
+            if user_id and source_id:
+                events.append(
+                    (
+                        str(user_id),
+                        "source.ingest.failed",
+                        {
+                            "source_id": str(source_id),
+                            "filename": row.get("source_name") or "",
+                            "operation": "upload",
+                            "error": "Indexing stopped after a stall.",
+                        },
+                        {"kind": "source", "id": str(source_id)},
+                    )
+                )
+
+    # Q5: idempotency rows whose lease expired with attempts exhausted.
+    # The wrapper's poison-loop guard normally finalises these, but if
+    # the wrapper itself died mid-task (worker SIGKILL, OOM during
+    # heartbeat) the row sits in ``pending`` blocking same-key retries
+    # via ``_lookup_completed`` returning None for the whole 24 h TTL.
+    # Promote to ``failed`` so a retry can re-claim and either resume
+    # or fail loudly.
+    with engine.begin() as conn:
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_stuck_idempotency_pending(
+            max_attempts=MAX_TASK_ATTEMPTS,
+        ):
+            error_msg = (
+                "reconciler: idempotency lease expired with attempts "
+                f"({row['attempt_count']}) >= {MAX_TASK_ATTEMPTS}; "
+                "task abandoned"
+            )
+            repo.mark_idempotency_pending_failed(
+                row["idempotency_key"], error=error_msg,
+            )
+            summary["idempotency_pending_failed"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_idempotency_pending_failed",
+                user_id=None,
+                detail={
+                    "idempotency_key": row["idempotency_key"],
+                    "task_name": row.get("task_name"),
+                    "task_id": row.get("task_id"),
+                    "attempts": row.get("attempt_count"),
+                },
+            )
+
+    # Q6: scheduler runs stuck in 'running' past the soft-time-limit window.
+    from application.storage.db.repositories.schedule_runs import (
+        ScheduleRunsRepository,
+    )
+    from application.storage.db.repositories.schedules import SchedulesRepository
+    from application.core.settings import settings as _settings
+
+    stuck_age = max(
+        15, int(_settings.SCHEDULE_RUN_TIMEOUT // 60) + 5,
+    )
+    with engine.begin() as conn:
+        runs_repo = ScheduleRunsRepository(conn)
+        schedules_repo = SchedulesRepository(conn)
+        for run in runs_repo.list_stuck_running(age_minutes=stuck_age):
+            runs_repo.update(
+                run["id"],
+                {
+                    "status": "timeout",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "timeout",
+                    "error": (
+                        "reconciler: schedule_run stuck in 'running' past "
+                        f"{stuck_age} min"
+                    ),
+                },
+            )
+            schedules_repo.bump_failure_count(str(run["schedule_id"]))
+            flipped = _terminal_flip_once_schedule(
+                schedules_repo, str(run["schedule_id"]),
+            )
+            summary["schedule_runs_failed"] += 1
+            events.extend(
+                _schedule_terminal_events(
+                    run,
+                    error_type="timeout",
+                    error=(
+                        "reconciler: schedule_run stuck in 'running' past "
+                        f"{stuck_age} min"
+                    ),
+                    once_completed=flipped,
+                )
+            )
+            _emit_alert(
+                conn,
+                name="reconciler_schedule_run_timeout",
+                user_id=run.get("user_id"),
+                detail={
+                    "run_id": str(run["id"]),
+                    "schedule_id": str(run["schedule_id"]),
+                },
+            )
+
+    # Q7: scheduler runs orphaned in 'pending' — dispatcher committed but
+    # apply_async failed (broker outage / crash mid-dispatch).
+    with engine.begin() as conn:
+        runs_repo = ScheduleRunsRepository(conn)
+        schedules_repo = SchedulesRepository(conn)
+        for run in runs_repo.list_stuck_pending(age_minutes=stuck_age):
+            runs_repo.update(
+                run["id"],
+                {
+                    "status": "failed",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "internal",
+                    "error": (
+                        "reconciler: schedule_run stuck in 'pending' past "
+                        f"{stuck_age} min (worker_never_started)"
+                    ),
+                },
+            )
+            schedules_repo.bump_failure_count(str(run["schedule_id"]))
+            flipped = _terminal_flip_once_schedule(
+                schedules_repo, str(run["schedule_id"]),
+            )
+            summary["schedule_runs_failed"] += 1
+            events.extend(
+                _schedule_terminal_events(
+                    run,
+                    error_type="internal",
+                    error=(
+                        "reconciler: schedule_run stuck in 'pending' past "
+                        f"{stuck_age} min (worker_never_started)"
+                    ),
+                    once_completed=flipped,
+                )
+            )
+            _emit_alert(
+                conn,
+                name="reconciler_schedule_run_pending",
+                user_id=run.get("user_id"),
+                detail={
+                    "run_id": str(run["id"]),
+                    "schedule_id": str(run["schedule_id"]),
+                },
+            )
+
+    _publish_events(events)
+    return summary
+
+
+def _terminal_flip_once_schedule(
+    schedules_repo: "SchedulesRepository", schedule_id: str,
+) -> bool:
+    """Flip a once-schedule to 'completed' after its run terminates.
+
+    Recurring schedules keep firing; once-schedules would otherwise read
+    'active forever' since next_run_at is already NULL. Returns ``True``
+    when the flip happened so the caller can publish a status event.
+    """
+    schedule = schedules_repo.get_internal(schedule_id)
+    if schedule is None or schedule.get("trigger_type") != "once":
+        return False
+    if schedule.get("status") in {"completed", "cancelled"}:
+        return False
+    schedules_repo.update_internal(
+        schedule_id, {"status": "completed", "next_run_at": None},
+    )
+    return True
+
+
+def _schedule_terminal_events(
+    run: Dict[str, Any],
+    *,
+    error_type: str,
+    error: str,
+    once_completed: bool,
+) -> list:
+    """Build the user-facing events for a reconciler-failed schedule run.
+
+    Always a ``schedule.run.failed`` (so a watching run-log updates live),
+    plus ``schedule.completed`` when a once-schedule flipped terminal — the
+    UI otherwise shows it 'active' with stale Edit/Cancel actions until a
+    manual refresh.
+    """
+    user_id = run.get("user_id")
+    schedule_id = run.get("schedule_id")
+    if not user_id or not schedule_id:
+        return []
+    scope = {"kind": "schedule", "id": str(schedule_id)}
+    out: list = [
+        (
+            str(user_id),
+            "schedule.run.failed",
+            {
+                "run_id": str(run["id"]),
+                "schedule_id": str(schedule_id),
+                "status": "failed",
+                "error_type": error_type,
+                "error": error,
+            },
+            scope,
+        )
+    ]
+    if once_completed:
+        out.append(
+            (
+                str(user_id),
+                "schedule.completed",
+                {"schedule_id": str(schedule_id), "status": "completed"},
+                scope,
+            )
+        )
+    return out
+
+
+def _publish_events(events: list) -> None:
+    """Fan out user-facing events after their DB writes have committed.
+
+    Each item is ``(user_id, event_type, payload, scope)``. Best-effort:
+    a publish miss is swallowed per event so one failure can't strand the
+    rest, and notifications never surface as a reconciler-task failure.
+    """
+    if not events:
+        return
+    from application.events.publisher import publish_user_event
+
+    for user_id, event_type, payload, scope in events:
+        try:
+            publish_user_event(user_id, event_type, payload, scope=scope)
+        except Exception:
+            logger.exception(
+                "reconciler: failed to publish %s for user=%s",
+                event_type,
+                user_id,
+            )
+
+
+def _emit_alert(
+    conn: Connection,
+    *,
+    name: str,
+    user_id: Optional[str],
+    detail: Dict[str, Any],
+) -> None:
+    """Structured ``logger.error`` plus a ``stack_logs`` row for operators."""
+    extra = {"alert": name, **detail}
+    logger.error("reconciler alert: %s", name, extra=extra)
+    try:
+        StackLogsRepository(conn).insert(
+            activity_id=str(uuid.uuid4()),
+            endpoint="reconciliation_worker",
+            level="ERROR",
+            user_id=user_id,
+            query=name,
+            stacks=[extra],
+        )
+    except Exception:
+        logger.exception("reconciler: failed to write stack_logs row for %s", name)

application/api/user/routes.py

@@ -0,0 +1,79 @@
+"""
+Main user API routes - registers all namespace modules.
+"""
+
+from flask import Blueprint
+
+from application.api import api
+from .agents import (
+    agents_folders_ns,
+    agents_ns,
+    agents_portability_ns,
+    agents_sharing_ns,
+    agents_webhooks_ns,
+)
+from .analytics import analytics_ns
+from .artifacts import artifacts_ns
+from .attachments import attachments_ns
+from .conversations import conversations_ns
+from .me import me_ns
+from .models import models_ns
+from .prompts import prompts_ns
+from .schedules import schedules_ns
+from .sharing import sharing_ns
+from .sources import sources_chunks_ns, sources_ns, sources_upload_ns
+from .teams import teams_ns
+from .tools import tools_mcp_ns, tools_ns
+from .workflows import workflows_ns
+
+
+user = Blueprint("user", __name__)
+
+# Analytics
+api.add_namespace(analytics_ns)
+
+# Artifacts
+api.add_namespace(artifacts_ns)
+
+# Attachments
+api.add_namespace(attachments_ns)
+
+# Conversations
+api.add_namespace(conversations_ns)
+
+# Current user (identity + roles)
+api.add_namespace(me_ns)
+
+# Models
+api.add_namespace(models_ns)
+
+# Agents (main, sharing, webhooks, folders, import/export)
+api.add_namespace(agents_ns)
+api.add_namespace(agents_sharing_ns)
+api.add_namespace(agents_webhooks_ns)
+api.add_namespace(agents_folders_ns)
+api.add_namespace(agents_portability_ns)
+
+# Prompts
+api.add_namespace(prompts_ns)
+
+# Schedules
+api.add_namespace(schedules_ns)
+
+# Sharing
+api.add_namespace(sharing_ns)
+
+# Sources (main, chunks, upload)
+api.add_namespace(sources_ns)
+api.add_namespace(sources_chunks_ns)
+api.add_namespace(sources_upload_ns)
+
+# Teams (CRUD, membership, resource-sharing grants)
+api.add_namespace(teams_ns)
+
+# Tools (main, MCP)
+api.add_namespace(tools_ns)
+api.add_namespace(tools_mcp_ns)
+
+# Workflows
+api.add_namespace(workflows_ns)

application/api/user/scheduler_dispatcher.py

@@ -0,0 +1,186 @@
+"""Schedule dispatcher: poll Postgres, claim due rows under FOR UPDATE SKIP LOCKED,
+advance next_run_at atomically with the run claim, then enqueue.
+
+Per-schedule IANA tz semantics (croniter+zoneinfo) outside Celery's app-wide tz,
+plus Postgres-native dedup avoid Redis visibility_timeout double-fires.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+from application.agents.scheduler_utils import next_cron_run
+from application.core.settings import settings
+from application.storage.db.engine import get_engine
+from application.storage.db.repositories.schedule_runs import (
+    ScheduleRunsRepository,
+)
+from application.storage.db.repositories.schedules import SchedulesRepository
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_dt(value: Any) -> Optional[datetime]:
+    """Accept a datetime / ISO string / None and return a tz-aware UTC dt."""
+    if value is None:
+        return None
+    if isinstance(value, datetime):
+        return value.astimezone(timezone.utc) if value.tzinfo else (
+            value.replace(tzinfo=timezone.utc)
+        )
+    if isinstance(value, str):
+        try:
+            parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return None
+        return parsed.astimezone(timezone.utc) if parsed.tzinfo else (
+            parsed.replace(tzinfo=timezone.utc)
+        )
+    return None
+
+
+def _compute_next(
+    schedule: Dict[str, Any],
+    *,
+    after: datetime,
+) -> Optional[datetime]:
+    """Next next_run_at for a recurring schedule, or None when past end_at."""
+    cron = schedule.get("cron")
+    if not cron:
+        return None
+    end_at = _normalize_dt(schedule.get("end_at"))
+    candidate = next_cron_run(cron, schedule.get("timezone"), after=after)
+    if end_at is not None and candidate > end_at:
+        return None
+    return candidate
+
+
+def dispatch_due_runs() -> Dict[str, int]:
+    """One dispatcher tick; returns counts for schedule_syncs-style logging."""
+    if not settings.POSTGRES_URI:
+        return {"enqueued": 0, "skipped": 0, "advanced": 0}
+
+    from application.api.user.tasks import execute_scheduled_run
+
+    now = datetime.now(timezone.utc)
+    grace = timedelta(seconds=max(0, settings.SCHEDULE_MISFIRE_GRACE))
+    engine = get_engine()
+    counts = {"enqueued": 0, "skipped": 0, "advanced": 0}
+    enqueue_args: List[str] = []
+
+    with engine.begin() as conn:
+        schedules_repo = SchedulesRepository(conn)
+        runs_repo = ScheduleRunsRepository(conn)
+        for schedule in schedules_repo.list_due():
+            scheduled_for = _normalize_dt(schedule.get("next_run_at"))
+            if scheduled_for is None:
+                continue
+
+            trigger_type = schedule.get("trigger_type")
+            agent_id_raw = schedule.get("agent_id")
+            agent_id = str(agent_id_raw) if agent_id_raw else None
+
+            # Misfire grace applies to recurring only — once-tasks fire late, not vanish.
+            if (
+                trigger_type == "recurring"
+                and grace > timedelta(0)
+                and (now - scheduled_for) > grace
+            ):
+                runs_repo.record_skipped(
+                    str(schedule["id"]),
+                    schedule["user_id"],
+                    agent_id,
+                    scheduled_for,
+                    error_type="missed",
+                    error="misfire grace exceeded",
+                )
+                counts["skipped"] += 1
+                nxt = _compute_next(schedule, after=now)
+                if nxt is None:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"status": "completed", "next_run_at": None,
+                         "last_run_at": now},
+                    )
+                else:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": nxt, "last_run_at": now},
+                    )
+                counts["advanced"] += 1
+                continue
+
+            # Overlap guard: never enqueue while a previous run is active.
+            if runs_repo.has_active_run(str(schedule["id"])):
+                runs_repo.record_skipped(
+                    str(schedule["id"]),
+                    schedule["user_id"],
+                    agent_id,
+                    scheduled_for,
+                    error_type="overlap",
+                    error="previous run still active",
+                )
+                counts["skipped"] += 1
+                if trigger_type == "recurring":
+                    nxt = _compute_next(schedule, after=scheduled_for)
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": nxt, "last_run_at": now},
+                    )
+                else:
+                    # Once: null next_run_at so we don't re-pick; the in-flight
+                    # run will terminal-flip the schedule when it finishes.
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": None, "last_run_at": now},
+                    )
+                continue
+
+            # Dedup primitive: two racing dispatchers see exactly one row.
+            run = runs_repo.record_pending(
+                str(schedule["id"]),
+                schedule["user_id"],
+                agent_id,
+                scheduled_for,
+                trigger_source="cron",
+            )
+            if run is None:
+                counts["skipped"] += 1
+            else:
+                enqueue_args.append(str(run["id"]))
+                counts["enqueued"] += 1
+
+            # Advance: recurring picks next tick, once nulls next_run_at
+            # (worker terminal-flips status on completion).
+            if trigger_type == "recurring":
+                nxt = _compute_next(schedule, after=scheduled_for)
+                if nxt is None:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"status": "completed", "next_run_at": None,
+                         "last_run_at": now},
+                    )
+                else:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": nxt, "last_run_at": now},
+                    )
+            else:
+                schedules_repo.update_internal(
+                    str(schedule["id"]),
+                    {"next_run_at": None, "last_run_at": now},
+                )
+            counts["advanced"] += 1
+
+    # Enqueue after commit so the worker sees the schedule_runs row on pick-up.
+    for run_id in enqueue_args:
+        try:
+            execute_scheduled_run.apply_async(args=[run_id], queue="docsgpt")
+        except Exception:
+            logger.exception(
+                "dispatcher: failed to enqueue execute_scheduled_run for %s",
+                run_id,
+            )
+    return counts

application/api/user/scheduler_worker.py

@@ -0,0 +1,434 @@
+"""Body of ``execute_scheduled_run`` — runs a single agent execution.
+
+Not a DURABLE_TASK: agent runs have side effects (messages, CRM writes)
+and blind auto-retry would double them. Failures after agent.gen starts
+are terminal and recorded; only the pre-start load is retry-safe.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from celery.exceptions import SoftTimeLimitExceeded
+from sqlalchemy import text as sql_text
+
+from application.agents.headless_runner import run_agent_headless
+from application.core.settings import settings
+from application.events.publisher import publish_user_event
+from application.storage.db.base_repository import row_to_dict
+from application.storage.db.engine import get_engine
+from application.storage.db.repositories.conversations import (
+    ConversationsRepository,
+)
+from application.storage.db.repositories.schedule_runs import (
+    ScheduleRunsRepository,
+)
+from application.storage.db.repositories.schedules import SchedulesRepository
+from application.storage.db.repositories.token_usage import TokenUsageRepository
+
+logger = logging.getLogger(__name__)
+
+
+# Cap output verbatim in the run log; beyond the cap we keep the head and stamp output_truncated.
+_OUTPUT_CAP_CHARS = 24_000
+
+
+def _agent_config_for_schedule(schedule: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Resolve the agent row (agent-bound) or build an ephemeral classic config.
+
+    For agentless schedules (``agent_id IS NULL``), the worker constructs an
+    in-memory agent shape carrying just enough fields for ``run_agent_headless``:
+    classic agent type, system-default retriever/chunks/prompt, no source, and
+    the optional ``model_id`` override. The runtime toolset is rebuilt by
+    ``ToolExecutor`` at fire time (current ``user_tools`` + non-disabled,
+    non-headless-excluded defaults), so a snapshot here would be dead code.
+    """
+    if schedule.get("agent_id"):
+        engine = get_engine()
+        with engine.connect() as conn:
+            row = conn.execute(
+                sql_text("SELECT * FROM agents WHERE id = CAST(:id AS uuid)"),
+                {"id": str(schedule["agent_id"])},
+            ).fetchone()
+        return row_to_dict(row) if row is not None else None
+    return _ephemeral_agent_for_agentless(schedule)
+
+
+def _ephemeral_agent_for_agentless(
+    schedule: Dict[str, Any],
+) -> Optional[Dict[str, Any]]:
+    """Build an agent-shaped config for a schedule with no parent agent."""
+    # ``agent_config["tools"]`` is intentionally omitted: ``run_agent_headless``
+    # never reads it. The runtime toolset is rebuilt by
+    # ``ToolExecutor._get_user_tools(owner)`` at fire time — same dereference
+    # the agent-bound path uses, so a tool added/disabled after creation is
+    # reflected. Headless mode there filters chat-only tools (``scheduler``).
+    user_id = schedule.get("user_id")
+    if not user_id:
+        return None
+    return {
+        "id": None,
+        "user_id": user_id,
+        "agent_type": "classic",
+        "retriever": "classic",
+        "chunks": 2,
+        "prompt_id": "default",
+        "source_id": None,
+        "default_model_id": schedule.get("model_id") or "",
+    }
+
+
+def _load_chat_history(schedule: Dict[str, Any]) -> list:
+    """Originating conversation history (one-time only; recurring has none)."""
+    origin = schedule.get("origin_conversation_id")
+    if not origin or schedule.get("trigger_type") != "once":
+        return []
+    user_id = schedule.get("user_id")
+    if not user_id:
+        return []
+    try:
+        engine = get_engine()
+        with engine.connect() as conn:
+            conv = ConversationsRepository(conn).get_any(str(origin), user_id)
+            if conv is None:
+                return []
+            messages = ConversationsRepository(conn).get_messages(str(conv["id"]))
+    except Exception:
+        logger.exception("scheduler: failed loading chat history")
+        return []
+    history: list = []
+    for msg in messages:
+        if msg.get("prompt") and msg.get("response"):
+            history.append({
+                "prompt": msg["prompt"],
+                "response": msg["response"],
+            })
+    return history
+
+
+def _publish_run_event(
+    event_type: str, run: Dict[str, Any], schedule: Dict[str, Any], **extra: Any
+) -> None:
+    """Best-effort SSE publish for a scheduler run state transition."""
+    user_id = run.get("user_id") or schedule.get("user_id")
+    if not user_id:
+        return
+    agent_id_raw = schedule.get("agent_id")
+    payload = {
+        "run_id": str(run["id"]),
+        "schedule_id": str(schedule["id"]),
+        "agent_id": str(agent_id_raw) if agent_id_raw else None,
+        "trigger_type": schedule.get("trigger_type"),
+        "status": run.get("status"),
+        **extra,
+    }
+    try:
+        publish_user_event(
+            user_id,
+            event_type,
+            payload,
+            scope={"kind": "schedule", "id": str(schedule["id"])},
+        )
+    except Exception:
+        logger.exception(
+            "scheduler: SSE publish failed event=%s run=%s",
+            event_type, run.get("id"),
+        )
+
+
+def _publish_message_appended(
+    user_id: str,
+    conversation_id: str,
+    message: Dict[str, Any],
+    schedule_id: str,
+    run_id: str,
+) -> None:
+    """SSE message-appended event for a one-time run's chat turn."""
+    try:
+        publish_user_event(
+            user_id,
+            "schedule.message.appended",
+            {
+                "conversation_id": str(conversation_id),
+                "message_id": str(message["id"]),
+                "schedule_id": str(schedule_id),
+                "run_id": str(run_id),
+                "position": int(message.get("position", 0)),
+            },
+            scope={"kind": "conversation", "id": str(conversation_id)},
+        )
+    except Exception:
+        logger.exception(
+            "scheduler: message.appended publish failed run=%s", run_id,
+        )
+
+
+def _append_one_time_turn(
+    schedule: Dict[str, Any],
+    run: Dict[str, Any],
+    outcome: Dict[str, Any],
+) -> Optional[Dict[str, Any]]:
+    """Insert an assistant turn in the originating conversation (once only)."""
+    origin = schedule.get("origin_conversation_id")
+    if not origin:
+        return None
+    engine = get_engine()
+    user_id = schedule.get("user_id")
+    metadata = {
+        "scheduled": True,
+        "schedule_id": str(schedule["id"]),
+        "run_id": str(run["id"]),
+        "scheduled_run_at": (
+            run.get("scheduled_for")
+            if isinstance(run.get("scheduled_for"), str)
+            else None
+        ),
+    }
+    with engine.begin() as conn:
+        conv = ConversationsRepository(conn).get_any(str(origin), user_id)
+        if conv is None:
+            return None
+        message = ConversationsRepository(conn).append_message(
+            str(conv["id"]),
+            {
+                "prompt": schedule.get("instruction") or "",
+                "response": outcome.get("answer") or "",
+                "thought": outcome.get("thought") or "",
+                "sources": outcome.get("sources") or [],
+                "tool_calls": outcome.get("tool_calls") or [],
+                "model_id": outcome.get("model_id"),
+                "metadata": metadata,
+            },
+        )
+    return message
+
+
+def execute_scheduled_run_body(run_id: str, celery_task_id: Optional[str]) -> Dict[str, Any]:
+    """Execute one scheduled run by id; returns a result dict for tracing."""
+    if not settings.POSTGRES_URI:
+        return {"status": "skipped", "reason": "POSTGRES_URI not set"}
+
+    engine = get_engine()
+
+    with engine.connect() as conn:
+        run = ScheduleRunsRepository(conn).get_internal(run_id)
+        if run is None:
+            return {"status": "skipped", "reason": "run not found"}
+        schedule = SchedulesRepository(conn).get_internal(str(run["schedule_id"]))
+    if schedule is None:
+        return {"status": "skipped", "reason": "schedule not found"}
+
+    # Refuse non-runnable terminal states; manual run-now bypasses.
+    if run.get("status") != "pending":
+        return {"status": "skipped", "reason": f"run status={run.get('status')}"}
+    if schedule.get("status") in {"cancelled", "completed"} and run.get(
+        "trigger_source"
+    ) != "manual":
+        with engine.begin() as conn:
+            ScheduleRunsRepository(conn).update(
+                run_id,
+                {
+                    "status": "skipped",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "internal",
+                    "error": "schedule no longer active",
+                },
+            )
+        return {"status": "skipped", "reason": "schedule terminal"}
+
+    agent_config = _agent_config_for_schedule(schedule)
+    if agent_config is None:
+        with engine.begin() as conn:
+            updated = ScheduleRunsRepository(conn).update(
+                run_id,
+                {
+                    "status": "failed",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "internal",
+                    "error": "agent missing",
+                },
+            )
+            SchedulesRepository(conn).bump_failure_count(str(schedule["id"]))
+        _publish_run_event("schedule.run.failed", updated or run, schedule,
+                           error="agent missing")
+        return {"status": "failed", "reason": "agent missing"}
+
+    with engine.begin() as conn:
+        if not ScheduleRunsRepository(conn).mark_running(run_id, celery_task_id):
+            return {"status": "skipped", "reason": "lost race to mark_running"}
+
+    started = datetime.now(timezone.utc)
+    instruction = schedule.get("instruction") or ""
+    allowlist = schedule.get("tool_allowlist") or []
+    chat_history = _load_chat_history(schedule)
+    outcome: Dict[str, Any]
+    error_type: Optional[str] = None
+    error_text: Optional[str] = None
+    timed_out = False
+    try:
+        outcome = run_agent_headless(
+            agent_config,
+            instruction,
+            tool_allowlist=allowlist,
+            model_id_override=schedule.get("model_id"),
+            endpoint="schedule",
+            conversation_id=schedule.get("origin_conversation_id"),
+            chat_history=chat_history,
+        )
+    except SoftTimeLimitExceeded:
+        timed_out = True
+        outcome = {"answer": "", "tool_calls": [], "sources": [], "thought": ""}
+        error_type = "timeout"
+        error_text = "run exceeded soft time limit"
+    except Exception as exc:
+        outcome = {"answer": "", "tool_calls": [], "sources": [], "thought": ""}
+        error_type = "agent_error"
+        error_text = str(exc)
+        logger.exception("scheduler: agent run failed run=%s", run_id)
+
+    finished = datetime.now(timezone.utc)
+
+    # Headless denial with no usable output → tool_not_allowed.
+    if (
+        error_type is None
+        and (outcome.get("denied") or [])
+        and not (outcome.get("answer") or "").strip()
+    ):
+        error_type = "tool_not_allowed"
+        error_text = "headless allowlist blocked required tool"
+
+    prompt_tokens = int(outcome.get("prompt_tokens", 0) or 0)
+    generated_tokens = int(outcome.get("generated_tokens", 0) or 0)
+    used_tokens = prompt_tokens + generated_tokens
+    if (
+        schedule.get("token_budget") is not None
+        and int(schedule["token_budget"]) > 0
+        and used_tokens > int(schedule["token_budget"])
+    ):
+        error_type = "budget_exceeded"
+        error_text = (
+            f"used {used_tokens} tokens exceeds budget "
+            f"{schedule['token_budget']}"
+        )
+
+    answer = outcome.get("answer") or ""
+    truncated = False
+    if len(answer) > _OUTPUT_CAP_CHARS:
+        answer = answer[:_OUTPUT_CAP_CHARS]
+        truncated = True
+
+    new_status = (
+        "timeout" if timed_out else ("failed" if error_type else "success")
+    )
+
+    with engine.begin() as conn:
+        update_fields: Dict[str, Any] = {
+            "status": new_status,
+            "started_at": started,
+            "finished_at": finished,
+            "output": answer or None,
+            "output_truncated": truncated,
+            "prompt_tokens": prompt_tokens,
+            "generated_tokens": generated_tokens,
+        }
+        if error_type:
+            update_fields["error_type"] = error_type
+            update_fields["error"] = error_text
+        updated_run = ScheduleRunsRepository(conn).update(run_id, update_fields)
+        if used_tokens > 0:
+            agent_id_raw = schedule.get("agent_id")
+            try:
+                TokenUsageRepository(conn).insert(
+                    user_id=schedule.get("user_id"),
+                    api_key=None,
+                    prompt_tokens=prompt_tokens,
+                    generated_tokens=generated_tokens,
+                    timestamp=finished,
+                    agent_id=str(agent_id_raw) if agent_id_raw else None,
+                    source="schedule",
+                    request_id=str(run_id),
+                    model_id=outcome.get("model_id"),
+                )
+            except Exception:
+                logger.exception(
+                    "scheduler: token_usage insert failed run=%s", run_id,
+                )
+        schedules_repo = SchedulesRepository(conn)
+        autopaused = False
+        if new_status == "success":
+            schedules_repo.reset_failure_count(str(schedule["id"]))
+        elif new_status in ("failed", "timeout"):
+            count = schedules_repo.bump_failure_count(str(schedule["id"]))
+            if (
+                settings.SCHEDULE_AUTOPAUSE_FAILURES > 0
+                and count >= settings.SCHEDULE_AUTOPAUSE_FAILURES
+                and schedule.get("trigger_type") == "recurring"
+            ):
+                autopaused = schedules_repo.autopause(str(schedule["id"]))
+        # Once: terminal-flip on cron-fired runs only; manual runs on a
+        # still-active once-schedule leave the future cadence intact.
+        if (
+            schedule.get("trigger_type") == "once"
+            and run.get("trigger_source") != "manual"
+            and schedule.get("status") == "active"
+        ):
+            schedules_repo.update_internal(
+                str(schedule["id"]),
+                {"status": "completed", "next_run_at": None},
+            )
+
+    appended: Optional[Dict[str, Any]] = None
+    if (
+        schedule.get("trigger_type") == "once"
+        and new_status == "success"
+        and schedule.get("origin_conversation_id")
+    ):
+        try:
+            appended = _append_one_time_turn(schedule, updated_run or run, outcome)
+        except Exception:
+            logger.exception(
+                "scheduler: append turn failed run=%s", run_id,
+            )
+        if appended is not None:
+            with engine.begin() as conn:
+                ScheduleRunsRepository(conn).update(
+                    run_id,
+                    {
+                        "conversation_id": str(appended["conversation_id"]),
+                        "message_id": str(appended["id"]),
+                    },
+                )
+            _publish_message_appended(
+                schedule.get("user_id"),
+                str(appended["conversation_id"]),
+                appended,
+                str(schedule["id"]),
+                run_id,
+            )
+
+    if new_status == "success":
+        _publish_run_event("schedule.run.completed", updated_run or run, schedule)
+    else:
+        _publish_run_event(
+            "schedule.run.failed",
+            updated_run or run,
+            schedule,
+            error_type=error_type,
+            error=error_text,
+        )
+
+    if autopaused:
+        _publish_run_event(
+            "schedule.autopaused",
+            updated_run or run,
+            schedule,
+            consecutive_failure_count=settings.SCHEDULE_AUTOPAUSE_FAILURES,
+        )
+
+    return {
+        "status": new_status,
+        "run_id": run_id,
+        "error_type": error_type,
+    }

application/api/user/schedules/__init__.py

@@ -0,0 +1,5 @@
+"""Schedules module."""
+
+from .routes import schedules_ns
+
+__all__ = ["schedules_ns"]

application/api/user/schedules/routes.py

@@ -0,0 +1,621 @@
+"""Schedules REST API (owner-scoped via request.decoded_token)."""
+
+from __future__ import annotations
+
+import functools
+import logging
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, Optional
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource, fields
+
+from application.agents.scheduler_utils import (
+    ScheduleValidationError,
+    clamp_once_horizon,
+    cron_interval_seconds,
+    next_cron_run,
+    parse_cron,
+    parse_run_at,
+    resolve_timezone,
+)
+from application.api import api
+from application.core.settings import settings
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.schedule_runs import (
+    ScheduleRunsRepository,
+)
+from application.storage.db.repositories.schedules import SchedulesRepository
+from application.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+schedules_ns = Namespace(
+    "schedules", description="Agent schedule management", path="/api",
+)
+
+
+def _ok(data: Any, status: int = 200):
+    return make_response(jsonify(data), status)
+
+
+def _err(message: str, status: int = 400):
+    return make_response(jsonify({"success": False, "message": message}), status)
+
+
+def _safe_route(func: Callable) -> Callable:
+    """Decorator: log + mask exceptions that escape a route body as 500.
+
+    ``ScheduleValidationError`` messages are explicitly surfaced via
+    ``_err(str(exc))`` at each call site (deliberate, user-safe). This
+    decorator only fires for *unexpected* exceptions (DB driver errors,
+    NPEs, etc.) that escape the route body. The full trace is logged
+    server-side; the response body carries no internal detail.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as exc:  # noqa: BLE001
+            try:
+                current_app.logger.exception(
+                    "unhandled exception in schedules route %s: %s",
+                    func.__qualname__, exc,
+                )
+            except RuntimeError:
+                # Out of Flask app context (rare in tests); use module logger.
+                logger.exception(
+                    "unhandled exception in schedules route %s: %s",
+                    func.__qualname__, exc,
+                )
+            return _err("internal error", 500)
+    return wrapper
+
+
+def _format_schedule(row: Dict[str, Any]) -> Dict[str, Any]:
+    """Render a schedule row for the API (id-as-string + ISO timestamps)."""
+    if not row:
+        return {}
+    out = dict(row)
+    for key in (
+        "id", "agent_id", "origin_conversation_id",
+    ):
+        if out.get(key) is not None:
+            out[key] = str(out[key])
+    out.pop("_id", None)  # drop dual-id legacy mirror
+    return out
+
+
+def _format_run(row: Dict[str, Any]) -> Dict[str, Any]:
+    """Render a schedule_run row for the API."""
+    if not row:
+        return {}
+    out = dict(row)
+    for key in (
+        "id", "schedule_id", "agent_id", "conversation_id", "message_id",
+    ):
+        if out.get(key) is not None:
+            out[key] = str(out[key])
+    out.pop("_id", None)
+    return out
+
+
+def _agent_owned(agent_id: str, user_id: str) -> Optional[Dict[str, Any]]:
+    if not looks_like_uuid(str(agent_id)):
+        return None
+    with db_readonly() as conn:
+        return AgentsRepository(conn).get_any(agent_id, user_id)
+
+
+def _user_id() -> Optional[str]:
+    decoded = getattr(request, "decoded_token", None)
+    if not decoded:
+        return None
+    return decoded.get("sub")
+
+
+def _publish_schedule_event(
+    user_id: str, event_type: str, schedule_id: str, *, status: str,
+) -> None:
+    """Best-effort SSE so other clients revoke a stale schedule state.
+
+    A resume/cancel must override an earlier ``schedule.autopaused``; that
+    envelope is durable and replays from the backlog on reconnect, so
+    without a newer event the schedule reads 'paused' even though it will
+    fire on the next tick.
+    """
+    try:
+        from application.events.publisher import publish_user_event
+
+        publish_user_event(
+            user_id,
+            event_type,
+            {"schedule_id": str(schedule_id), "status": status},
+            scope={"kind": "schedule", "id": str(schedule_id)},
+        )
+    except Exception:
+        logger.exception(
+            "schedules: publish %s failed for %s", event_type, schedule_id,
+        )
+
+
+@schedules_ns.route("/agents/<string:agent_id>/schedules")
+class AgentSchedules(Resource):
+    @api.doc(description="List schedules for an agent (recurring + one-time).")
+    @_safe_route
+    def get(self, agent_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        agent = _agent_owned(agent_id, user_id)
+        if agent is None:
+            return _err("agent not found", 404)
+        try:
+            with db_readonly() as conn:
+                rows = SchedulesRepository(conn).list_for_agent(
+                    str(agent["id"]), user_id,
+                )
+        except Exception as exc:
+            current_app.logger.error("list schedules failed: %s", exc, exc_info=True)
+            return _err("internal error", 500)
+        return _ok({"schedules": [_format_schedule(r) for r in rows]})
+
+    create_model = api.model(
+        "ScheduleCreate",
+        {
+            "instruction": fields.String(required=True),
+            "trigger_type": fields.String(
+                required=False,
+                description="'recurring' (default) or 'once'",
+            ),
+            "cron": fields.String(
+                required=False,
+                description="Required when trigger_type == 'recurring'",
+            ),
+            "run_at": fields.String(
+                required=False,
+                description="ISO 8601 — required when trigger_type == 'once'",
+            ),
+            "timezone": fields.String(required=False),
+            "name": fields.String(required=False),
+            "end_at": fields.String(required=False, description="ISO 8601"),
+            "tool_allowlist": fields.List(fields.String, required=False),
+            "model_id": fields.String(required=False),
+            "token_budget": fields.Integer(required=False),
+        },
+    )
+
+    @api.expect(create_model)
+    @api.doc(description="Create a schedule (recurring or one-time) for an agent.")
+    @_safe_route
+    def post(self, agent_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        agent = _agent_owned(agent_id, user_id)
+        if agent is None:
+            return _err("agent not found", 404)
+        data = request.get_json(silent=True) or {}
+        instruction = (data.get("instruction") or "").strip()
+        tz_name = (data.get("timezone") or "UTC").strip() or "UTC"
+        trigger_type = (data.get("trigger_type") or "recurring").strip().lower()
+        if trigger_type not in ("recurring", "once"):
+            return _err("trigger_type must be 'recurring' or 'once'")
+        if not instruction:
+            return _err("instruction is required")
+        try:
+            resolve_timezone(tz_name)
+        except ScheduleValidationError as exc:
+            return _err(str(exc))
+        token_budget = data.get("token_budget")
+        if token_budget is not None:
+            try:
+                token_budget = int(token_budget)
+                if token_budget < 0:
+                    raise ValueError
+            except (TypeError, ValueError):
+                return _err("token_budget must be a non-negative integer")
+        with db_readonly() as conn:
+            count = SchedulesRepository(conn).count_active_for_user(user_id)
+        if (
+            settings.SCHEDULE_MAX_PER_USER > 0
+            and count >= settings.SCHEDULE_MAX_PER_USER
+        ):
+            return _err("max schedules per user reached", 429)
+
+        if trigger_type == "once":
+            run_at_raw = (data.get("run_at") or "").strip()
+            if not run_at_raw:
+                return _err("run_at is required for trigger_type 'once'")
+            try:
+                fire = parse_run_at(run_at_raw, tz_name)
+                clamp_once_horizon(
+                    fire, settings.SCHEDULE_ONCE_MAX_HORIZON,
+                )
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            try:
+                with db_session() as conn:
+                    created = SchedulesRepository(conn).create(
+                        user_id=user_id,
+                        agent_id=str(agent["id"]),
+                        trigger_type="once",
+                        instruction=instruction,
+                        run_at=fire,
+                        next_run_at=fire,
+                        timezone=tz_name,
+                        name=(data.get("name") or "").strip() or None,
+                        tool_allowlist=data.get("tool_allowlist") or [],
+                        model_id=(data.get("model_id") or None),
+                        token_budget=token_budget,
+                        created_via="ui",
+                    )
+            except Exception as exc:
+                current_app.logger.error(
+                    "create one-time schedule failed: %s", exc, exc_info=True,
+                )
+                return _err("internal error", 500)
+            return _ok({"schedule": _format_schedule(created)}, status=201)
+
+        cron = (data.get("cron") or "").strip()
+        if not cron:
+            return _err("cron is required")
+        try:
+            parse_cron(cron)
+        except ScheduleValidationError as exc:
+            return _err(str(exc))
+        min_interval = max(0, int(settings.SCHEDULE_MIN_INTERVAL))
+        if min_interval > 0:
+            try:
+                cadence = cron_interval_seconds(cron, tz_name)
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            if cadence < min_interval:
+                return _err(
+                    "cadence below minimum interval "
+                    f"({cadence}s < {min_interval}s)",
+                )
+        end_at = None
+        if data.get("end_at"):
+            try:
+                end_at = datetime.fromisoformat(
+                    str(data["end_at"]).replace("Z", "+00:00"),
+                )
+            except ValueError:
+                return _err("invalid end_at")
+        try:
+            next_run = next_cron_run(cron, tz_name, after=datetime.now(timezone.utc))
+        except ScheduleValidationError as exc:
+            return _err(str(exc))
+        if end_at is not None and next_run > end_at:
+            return _err("end_at is before the first cron tick")
+        try:
+            with db_session() as conn:
+                created = SchedulesRepository(conn).create(
+                    user_id=user_id,
+                    agent_id=str(agent["id"]),
+                    trigger_type="recurring",
+                    instruction=instruction,
+                    cron=cron,
+                    timezone=tz_name,
+                    next_run_at=next_run,
+                    end_at=end_at,
+                    name=(data.get("name") or "").strip() or None,
+                    tool_allowlist=data.get("tool_allowlist") or [],
+                    model_id=(data.get("model_id") or None),
+                    token_budget=token_budget,
+                    created_via="ui",
+                )
+        except Exception as exc:
+            current_app.logger.error(
+                "create schedule failed: %s", exc, exc_info=True,
+            )
+            return _err("internal error", 500)
+        return _ok({"schedule": _format_schedule(created)}, status=201)
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>")
+class ScheduleResource(Resource):
+    @api.doc(description="Get schedule by id.")
+    @_safe_route
+    def get(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        with db_readonly() as conn:
+            row = SchedulesRepository(conn).get(schedule_id, user_id)
+        if row is None:
+            return _err("schedule not found", 404)
+        return _ok({"schedule": _format_schedule(row)})
+
+    @api.doc(description="Edit a schedule's editable fields.")
+    @_safe_route
+    def put(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        data = request.get_json(silent=True) or {}
+        fields_in: Dict[str, Any] = {}
+        if "instruction" in data:
+            inst = (data["instruction"] or "").strip()
+            if not inst:
+                return _err("instruction must not be empty")
+            fields_in["instruction"] = inst
+        if "cron" in data:
+            cron = (data["cron"] or "").strip()
+            try:
+                parse_cron(cron)
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            fields_in["cron"] = cron
+        if "timezone" in data:
+            tz_name = (data["timezone"] or "UTC").strip() or "UTC"
+            try:
+                resolve_timezone(tz_name)
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            fields_in["timezone"] = tz_name
+        if "tool_allowlist" in data:
+            fields_in["tool_allowlist"] = data["tool_allowlist"] or []
+        if "name" in data:
+            fields_in["name"] = (data["name"] or "").strip() or None
+        if "model_id" in data:
+            fields_in["model_id"] = (data["model_id"] or None)
+        if "token_budget" in data:
+            tb = data["token_budget"]
+            if tb is not None:
+                try:
+                    tb = int(tb)
+                    if tb < 0:
+                        raise ValueError
+                except (TypeError, ValueError):
+                    return _err("token_budget must be a non-negative integer")
+            fields_in["token_budget"] = tb
+        if "end_at" in data:
+            if data["end_at"]:
+                try:
+                    fields_in["end_at"] = datetime.fromisoformat(
+                        str(data["end_at"]).replace("Z", "+00:00"),
+                    )
+                except ValueError:
+                    return _err("invalid end_at")
+            else:
+                fields_in["end_at"] = None
+        # Recompute next_run_at when cron/tz changes.
+        with db_session() as conn:
+            existing = SchedulesRepository(conn).get(schedule_id, user_id)
+            if existing is None:
+                return _err("schedule not found", 404)
+            if (
+                ("cron" in fields_in or "timezone" in fields_in)
+                and existing.get("trigger_type") == "recurring"
+            ):
+                cron_eff = fields_in.get("cron") or existing.get("cron")
+                tz_eff = fields_in.get("timezone") or existing.get("timezone")
+                if cron_eff:
+                    min_interval = max(0, int(settings.SCHEDULE_MIN_INTERVAL))
+                    if min_interval > 0:
+                        try:
+                            cadence = cron_interval_seconds(cron_eff, tz_eff)
+                        except ScheduleValidationError as exc:
+                            return _err(str(exc))
+                        if cadence < min_interval:
+                            return _err(
+                                "cadence below minimum interval "
+                                f"({cadence}s < {min_interval}s)",
+                            )
+                    try:
+                        fields_in["next_run_at"] = next_cron_run(
+                            cron_eff, tz_eff, after=datetime.now(timezone.utc),
+                        )
+                    except ScheduleValidationError as exc:
+                        return _err(str(exc))
+            updated = SchedulesRepository(conn).update(
+                schedule_id, user_id, fields_in,
+            )
+        return _ok({"schedule": _format_schedule(updated or {})})
+
+    @api.doc(description="Pause / resume a schedule.")
+    @_safe_route
+    def patch(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        data = request.get_json(silent=True) or {}
+        action = (data.get("action") or "").lower().strip()
+        if action not in {"pause", "resume"}:
+            return _err("action must be 'pause' or 'resume'")
+        with db_session() as conn:
+            existing = SchedulesRepository(conn).get(schedule_id, user_id)
+            if existing is None:
+                return _err("schedule not found", 404)
+            if existing.get("status") in ("cancelled", "completed"):
+                return _err("schedule is terminal", 409)
+            if action == "pause":
+                fields_in: Dict[str, Any] = {"status": "paused", "next_run_at": None}
+            else:
+                # Resume: recurring recomputes from now; once honours run_at if still future.
+                fields_in = {"status": "active"}
+                if existing.get("trigger_type") == "recurring":
+                    try:
+                        fields_in["next_run_at"] = next_cron_run(
+                            existing["cron"],
+                            existing["timezone"],
+                            after=datetime.now(timezone.utc),
+                        )
+                    except ScheduleValidationError as exc:
+                        return _err(str(exc))
+                else:
+                    new_run_at = data.get("run_at")
+                    if new_run_at:
+                        try:
+                            run_at_dt = datetime.fromisoformat(
+                                str(new_run_at).replace("Z", "+00:00"),
+                            )
+                        except ValueError:
+                            return _err("invalid run_at")
+                        if run_at_dt <= datetime.now(timezone.utc):
+                            return _err(
+                                "run_at must be in the future to resume", 409,
+                            )
+                        fields_in["next_run_at"] = run_at_dt
+                        fields_in["run_at"] = run_at_dt
+                    else:
+                        run_at = existing.get("run_at")
+                        if run_at:
+                            if isinstance(run_at, str):
+                                try:
+                                    run_at_dt = datetime.fromisoformat(
+                                        run_at.replace("Z", "+00:00"),
+                                    )
+                                except ValueError:
+                                    return _err("schedule run_at is invalid")
+                            else:
+                                run_at_dt = run_at
+                            if run_at_dt <= datetime.now(timezone.utc):
+                                return _err(
+                                    "the once schedule has elapsed; recreate "
+                                    "it or supply a new run_at",
+                                    409,
+                                )
+                            fields_in["next_run_at"] = run_at_dt
+            updated = SchedulesRepository(conn).update(
+                schedule_id, user_id, fields_in,
+            )
+            if action == "resume":
+                SchedulesRepository(conn).reset_failure_count(schedule_id)
+        if action == "resume" and updated:
+            _publish_schedule_event(
+                user_id, "schedule.resumed", schedule_id, status="active",
+            )
+        return _ok({"schedule": _format_schedule(updated or {})})
+
+    @api.doc(description="Cancel / delete a schedule.")
+    @_safe_route
+    def delete(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        with db_session() as conn:
+            ok = SchedulesRepository(conn).delete(schedule_id, user_id)
+        if not ok:
+            return _err("schedule not found", 404)
+        _publish_schedule_event(
+            user_id, "schedule.cancelled", schedule_id, status="cancelled",
+        )
+        return _ok({"success": True})
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>/run")
+class ScheduleRunNow(Resource):
+    @api.doc(description="Run a schedule immediately (trigger_source='manual').")
+    @_safe_route
+    def post(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        # FOR UPDATE serializes concurrent Run-Now POSTs (timestamp-unique
+        # scheduled_for values would otherwise sneak past the unique index).
+        with db_session() as conn:
+            schedule = SchedulesRepository(conn).get_for_update(
+                schedule_id, user_id,
+            )
+            if schedule is None:
+                return _err("schedule not found", 404)
+            if schedule.get("status") == "cancelled":
+                return _err("schedule is cancelled", 409)
+            if ScheduleRunsRepository(conn).has_active_run(schedule_id):
+                return _err("a run is already in flight", 409)
+            scheduled_for = datetime.now(timezone.utc)
+            agent_id_raw = schedule.get("agent_id")
+            run = ScheduleRunsRepository(conn).record_pending(
+                schedule_id,
+                user_id,
+                str(agent_id_raw) if agent_id_raw else None,
+                scheduled_for,
+                trigger_source="manual",
+            )
+        if run is None:
+            return _err("could not claim run (concurrent dispatch)", 409)
+        # Import inside the handler to avoid a circular tasks <-> routes import.
+        try:
+            from application.api.user.tasks import execute_scheduled_run
+            execute_scheduled_run.apply_async(args=[str(run["id"])], queue="docsgpt")
+        except Exception as exc:
+            current_app.logger.error(
+                "run-now enqueue failed: %s", exc, exc_info=True,
+            )
+            return _err("enqueue failed", 500)
+        return _ok({"run": _format_run(run)}, status=202)
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>/runs")
+class ScheduleRunList(Resource):
+    @api.doc(
+        description="Paginated run log for a schedule.",
+        params={"limit": "Page size (default 50)", "offset": "Page offset"},
+    )
+    @_safe_route
+    def get(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        try:
+            limit = max(1, min(int(request.args.get("limit", 50)), 200))
+        except (TypeError, ValueError):
+            limit = 50
+        try:
+            offset = max(0, int(request.args.get("offset", 0)))
+        except (TypeError, ValueError):
+            offset = 0
+        with db_readonly() as conn:
+            schedule = SchedulesRepository(conn).get(schedule_id, user_id)
+            if schedule is None:
+                return _err("schedule not found", 404)
+            rows = ScheduleRunsRepository(conn).list_runs(
+                schedule_id, user_id, limit=limit, offset=offset,
+            )
+        return _ok(
+            {
+                "runs": [_format_run(r) for r in rows],
+                "limit": limit,
+                "offset": offset,
+            }
+        )
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>/runs/<string:run_id>")
+class ScheduleRunDetail(Resource):
+    @api.doc(description="Full output / error for a single run.")
+    @_safe_route
+    def get(self, schedule_id, run_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id) or not looks_like_uuid(run_id):
+            return _err("invalid id", 400)
+        with db_readonly() as conn:
+            schedule = SchedulesRepository(conn).get(schedule_id, user_id)
+            if schedule is None:
+                return _err("schedule not found", 404)
+            run = ScheduleRunsRepository(conn).get(run_id, user_id)
+            if run is None or str(run.get("schedule_id")) != str(
+                schedule["id"]
+            ):
+                return _err("run not found", 404)
+        return _ok({"run": _format_run(run)})

application/api/user/sharing/__init__.py

@@ -0,0 +1,5 @@
+"""Sharing module."""
+
+from .routes import sharing_ns
+
+__all__ = ["sharing_ns"]

application/api/user/sharing/routes.py

@@ -0,0 +1,363 @@
+"""Conversation sharing routes."""
+
+import uuid
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, inputs, Namespace, Resource
+from sqlalchemy import text as _sql_text
+
+from application.api import api
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.attachments import AttachmentsRepository
+from application.storage.db.repositories.conversations import ConversationsRepository
+from application.storage.db.repositories.shared_conversations import (
+    SharedConversationsRepository,
+)
+from application.storage.db.session import db_readonly, db_session
+from application.utils import check_required_fields
+
+
+sharing_ns = Namespace(
+    "sharing", description="Conversation sharing operations", path="/api"
+)
+
+
+def _resolve_prompt_pg_id(conn, prompt_id_raw, user_id):
+    """Translate an incoming prompt id (UUID or legacy Mongo ObjectId) to a PG UUID.
+
+    Scoped by ``user_id`` so a caller can't link another user's prompt
+    into their share record. Returns ``None`` for sentinel values
+    (``"default"``) or unresolved ids.
+    """
+    if not prompt_id_raw or prompt_id_raw == "default":
+        return None
+    value = str(prompt_id_raw)
+    # Already UUID — trust it but still require ownership. A shape-gate
+    # (rather than a loose ``len == 36 and '-' in value`` check) keeps
+    # non-UUID input out of ``CAST(:pid AS uuid)``; the cast would raise
+    # and poison the readonly transaction otherwise.
+    if looks_like_uuid(value):
+        row = conn.execute(
+            _sql_text(
+                "SELECT id FROM prompts WHERE id = CAST(:pid AS uuid) "
+                "AND user_id = :uid"
+            ),
+            {"pid": value, "uid": user_id},
+        ).fetchone()
+        return str(row[0]) if row else None
+    # Legacy Mongo ObjectId fallback.
+    row = conn.execute(
+        _sql_text(
+            "SELECT id FROM prompts WHERE legacy_mongo_id = :pid "
+            "AND user_id = :uid"
+        ),
+        {"pid": value, "uid": user_id},
+    ).fetchone()
+    return str(row[0]) if row else None
+
+
+def _resolve_source_pg_id(conn, source_raw):
+    """Translate a source id (UUID or legacy Mongo ObjectId) to a PG UUID."""
+    if not source_raw:
+        return None
+    value = str(source_raw)
+    # See ``_resolve_prompt_pg_id`` for the shape-gate rationale.
+    if looks_like_uuid(value):
+        row = conn.execute(
+            _sql_text(
+                "SELECT id FROM sources WHERE id = CAST(:sid AS uuid)"
+            ),
+            {"sid": value},
+        ).fetchone()
+        return str(row[0]) if row else None
+    row = conn.execute(
+        _sql_text("SELECT id FROM sources WHERE legacy_mongo_id = :sid"),
+        {"sid": value},
+    ).fetchone()
+    return str(row[0]) if row else None
+
+
+def _find_reusable_share_agent(
+    conn, user_id, *, prompt_pg_id, chunks, source_pg_id, retriever,
+):
+    """Find an existing share-as-agent key row matching these parameters.
+
+    Mirrors the legacy Mongo ``agents_collection.find_one`` pre-existence
+    check. Used to reuse an api key across repeated shares of the same
+    conversation with the same prompt/chunks/source/retriever.
+    """
+    clauses = ["user_id = :uid", "key IS NOT NULL"]
+    params: dict = {"uid": user_id}
+    if prompt_pg_id is None:
+        clauses.append("prompt_id IS NULL")
+    else:
+        clauses.append("prompt_id = CAST(:pid AS uuid)")
+        params["pid"] = prompt_pg_id
+    if chunks is None:
+        clauses.append("chunks IS NULL")
+    else:
+        clauses.append("chunks = :chunks")
+        params["chunks"] = int(chunks)
+    if source_pg_id is None:
+        clauses.append("source_id IS NULL")
+    else:
+        clauses.append("source_id = CAST(:sid AS uuid)")
+        params["sid"] = source_pg_id
+    if retriever is None:
+        clauses.append("retriever IS NULL")
+    else:
+        clauses.append("retriever = :retr")
+        params["retr"] = retriever
+    sql = (
+        "SELECT * FROM agents WHERE "
+        + " AND ".join(clauses)
+        + " LIMIT 1"
+    )
+    row = conn.execute(_sql_text(sql), params).fetchone()
+    if row is None:
+        return None
+    mapping = dict(row._mapping)
+    mapping["id"] = str(mapping["id"]) if mapping.get("id") else None
+    return mapping
+
+
+@sharing_ns.route("/share")
+class ShareConversation(Resource):
+    share_conversation_model = api.model(
+        "ShareConversationModel",
+        {
+            "conversation_id": fields.String(
+                required=True, description="Conversation ID"
+            ),
+            "user": fields.String(description="User ID (optional)"),
+            "prompt_id": fields.String(description="Prompt ID (optional)"),
+            "chunks": fields.Integer(description="Chunks count (optional)"),
+        },
+    )
+
+    @api.expect(share_conversation_model)
+    @api.doc(description="Share a conversation")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["conversation_id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        is_promptable = request.args.get("isPromptable", type=inputs.boolean)
+        if is_promptable is None:
+            return make_response(
+                jsonify({"success": False, "message": "isPromptable is required"}), 400
+            )
+        conversation_id = data["conversation_id"]
+
+        try:
+            with db_session() as conn:
+                conv_repo = ConversationsRepository(conn)
+                shared_repo = SharedConversationsRepository(conn)
+                agents_repo = AgentsRepository(conn)
+
+                conversation = conv_repo.get_any(conversation_id, user)
+                if conversation is None:
+                    return make_response(
+                        jsonify(
+                            {
+                                "status": "error",
+                                "message": "Conversation does not exist",
+                            }
+                        ),
+                        404,
+                    )
+                conv_pg_id = str(conversation["id"])
+                current_n_queries = conv_repo.message_count(conv_pg_id)
+
+                if is_promptable:
+                    prompt_id_raw = data.get("prompt_id", "default")
+                    chunks_raw = data.get("chunks", "2")
+                    try:
+                        chunks_int = int(chunks_raw) if chunks_raw not in (None, "") else None
+                    except (TypeError, ValueError):
+                        chunks_int = None
+
+                    prompt_pg_id = _resolve_prompt_pg_id(conn, prompt_id_raw, user)
+                    source_pg_id = _resolve_source_pg_id(conn, data.get("source"))
+                    retriever = data.get("retriever")
+
+                    reusable = _find_reusable_share_agent(
+                        conn, user,
+                        prompt_pg_id=prompt_pg_id,
+                        chunks=chunks_int,
+                        source_pg_id=source_pg_id,
+                        retriever=retriever,
+                    )
+                    if reusable:
+                        api_uuid = reusable.get("key")
+                    else:
+                        api_uuid = str(uuid.uuid4())
+                        name = (conversation.get("name") or "") + "(shared)"
+                        agents_repo.create(
+                            user,
+                            name,
+                            "published",
+                            key=api_uuid,
+                            retriever=retriever,
+                            chunks=chunks_int,
+                            prompt_id=prompt_pg_id,
+                            source_id=source_pg_id,
+                        )
+
+                    share = shared_repo.get_or_create(
+                        conv_pg_id,
+                        user,
+                        is_promptable=True,
+                        first_n_queries=current_n_queries,
+                        api_key=api_uuid,
+                        prompt_id=prompt_pg_id,
+                        chunks=chunks_int,
+                    )
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": True,
+                                "identifier": str(share["uuid"]),
+                            }
+                        ),
+                        201 if reusable is None else 200,
+                    )
+
+                # Non-promptable share path.
+                share = shared_repo.get_or_create(
+                    conv_pg_id,
+                    user,
+                    is_promptable=False,
+                    first_n_queries=current_n_queries,
+                    api_key=None,
+                )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": True,
+                            "identifier": str(share["uuid"]),
+                        }
+                    ),
+                    201,
+                )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error sharing conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+
+
+@sharing_ns.route("/shared_conversation/<string:identifier>")
+class GetPubliclySharedConversations(Resource):
+    @api.doc(description="Get publicly shared conversations by identifier")
+    def get(self, identifier: str):
+        try:
+            with db_readonly() as conn:
+                shared_repo = SharedConversationsRepository(conn)
+                conv_repo = ConversationsRepository(conn)
+                attach_repo = AttachmentsRepository(conn)
+
+                shared = shared_repo.find_by_uuid(identifier)
+                if not shared or not shared.get("conversation_id"):
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "might have broken url or the conversation does not exist",
+                            }
+                        ),
+                        404,
+                    )
+                conv_pg_id = str(shared["conversation_id"])
+                owner_user = shared.get("user_id")
+
+                conversation = conv_repo.get_owned(conv_pg_id, owner_user) if owner_user else None
+                if conversation is None:
+                    # Fall back to any-user lookup in case shared row's
+                    # user_id is missing — still keyed by PG UUID.
+                    row = conn.execute(
+                        _sql_text(
+                            "SELECT * FROM conversations WHERE id = CAST(:id AS uuid)"
+                        ),
+                        {"id": conv_pg_id},
+                    ).fetchone()
+                    if row is None:
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": False,
+                                    "error": "might have broken url or the conversation does not exist",
+                                }
+                            ),
+                            404,
+                        )
+                    conversation = dict(row._mapping)
+
+                messages = conv_repo.get_messages(conv_pg_id)
+                first_n = shared.get("first_n_queries") or 0
+                conversation_queries = []
+                for msg in messages[:first_n]:
+                    query = {
+                        "prompt": msg.get("prompt"),
+                        "response": msg.get("response"),
+                        "thought": msg.get("thought"),
+                        "sources": msg.get("sources") or [],
+                        "tool_calls": msg.get("tool_calls") or [],
+                        "timestamp": (
+                            msg["timestamp"].isoformat()
+                            if hasattr(msg.get("timestamp"), "isoformat")
+                            else msg.get("timestamp")
+                        ),
+                        "feedback": msg.get("feedback"),
+                    }
+                    attachments = msg.get("attachments") or []
+                    if attachments:
+                        attachment_details = []
+                        for attachment_id in attachments:
+                            try:
+                                attachment = attach_repo.get_any(
+                                    str(attachment_id), owner_user,
+                                ) if owner_user else None
+                                if attachment:
+                                    attachment_details.append(
+                                        {
+                                            "id": str(attachment["id"]),
+                                            "fileName": attachment.get(
+                                                "filename", "Unknown file"
+                                            ),
+                                        }
+                                    )
+                            except Exception as e:
+                                current_app.logger.error(
+                                    f"Error retrieving attachment {attachment_id}: {e}",
+                                    exc_info=True,
+                                )
+                        query["attachments"] = attachment_details
+                    conversation_queries.append(query)
+
+                created = conversation.get("created_at") or conversation.get("date")
+                date_iso = (
+                    created.isoformat()
+                    if hasattr(created, "isoformat")
+                    else (str(created) if created is not None else None)
+                )
+                res = {
+                    "success": True,
+                    "queries": conversation_queries,
+                    "title": conversation.get("name"),
+                    "timestamp": date_iso,
+                }
+                if shared.get("is_promptable") and shared.get("api_key"):
+                    res["api_key"] = shared["api_key"]
+                return make_response(jsonify(res), 200)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting shared conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)

application/api/user/sources/__init__.py

@@ -0,0 +1,7 @@
+"""Sources module."""
+
+from .chunks import sources_chunks_ns
+from .routes import sources_ns
+from .upload import sources_upload_ns
+
+__all__ = ["sources_ns", "sources_chunks_ns", "sources_upload_ns"]

application/api/user/sources/chunks.py

@@ -0,0 +1,311 @@
+"""Source document management chunk management."""
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import get_vector_store
+from application.api.user.team_sharing import effective_write_owner
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.session import db_readonly
+from application.utils import check_required_fields, num_tokens_from_string
+
+sources_chunks_ns = Namespace(
+    "sources", description="Source document management operations", path="/api"
+)
+
+
+def _resolve_source(doc_id: str, user: str):
+    """Resolve a source (UUID or legacy ObjectId) for the caller.
+
+    Returns the row dict (with PG UUID in ``id``) or ``None`` if missing.
+    """
+    with db_readonly() as conn:
+        return SourcesRepository(conn).get_any(doc_id, user)
+
+
+def _resolve_source_for_write(doc_id: str, user: str):
+    """Resolve a source the caller may WRITE chunks on.
+
+    Returns the row dict when ``user`` owns the source, or when they hold a
+    team ``editor`` grant (adding/removing/editing documents is editor-allowed
+    — the vector partition is keyed by source_id, owner-agnostic). Returns
+    ``None`` for viewer-only / no access. Source deletion stays owner-only and
+    is handled elsewhere.
+    """
+    with db_readonly() as conn:
+        doc = SourcesRepository(conn).get_any(doc_id, user)
+        if doc is not None:
+            return doc
+        owner = effective_write_owner(conn, "source", doc_id, user)
+        if not owner:
+            return None
+        return SourcesRepository(conn).get_any(doc_id, owner)
+
+
+@sources_chunks_ns.route("/get_chunks")
+class GetChunks(Resource):
+    @api.doc(
+        description="Retrieves chunks from a document, optionally filtered by file path and search term",
+        params={
+            "id": "The document ID",
+            "page": "Page number for pagination",
+            "per_page": "Number of chunks per page",
+            "path": "Optional: Filter chunks by relative file path",
+            "search": "Optional: Search term to filter chunks by title or content",
+        },
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        doc_id = request.args.get("id")
+        page = int(request.args.get("page", 1))
+        per_page = int(request.args.get("per_page", 10))
+        path = request.args.get("path")
+        search_term = request.args.get("search", "").strip().lower()
+
+        if not doc_id:
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        try:
+            doc = _resolve_source(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(
+                jsonify({"error": "Document not found or access denied"}), 404
+            )
+        resolved_id = str(doc["id"])
+        try:
+            store = get_vector_store(resolved_id)
+            chunks = store.get_chunks()
+
+            filtered_chunks = []
+            for chunk in chunks:
+                metadata = chunk.get("metadata", {})
+
+                if path:
+                    chunk_source = metadata.get("source", "")
+                    chunk_file_path = metadata.get("file_path", "")
+                    source_match = chunk_source and chunk_source.endswith(path)
+                    file_path_match = chunk_file_path and chunk_file_path.endswith(path)
+
+                    if not (source_match or file_path_match):
+                        continue
+                if search_term:
+                    text_match = search_term in chunk.get("text", "").lower()
+                    title_match = search_term in metadata.get("title", "").lower()
+
+                    if not (text_match or title_match):
+                        continue
+                filtered_chunks.append(chunk)
+            chunks = filtered_chunks
+
+            total_chunks = len(chunks)
+            start = (page - 1) * per_page
+            end = start + per_page
+            paginated_chunks = chunks[start:end]
+
+            return make_response(
+                jsonify(
+                    {
+                        "page": page,
+                        "per_page": per_page,
+                        "total": total_chunks,
+                        "chunks": paginated_chunks,
+                        "path": path if path else None,
+                        "search": search_term if search_term else None,
+                    }
+                ),
+                200,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error getting chunks: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+@sources_chunks_ns.route("/add_chunk")
+class AddChunk(Resource):
+    @api.expect(
+        api.model(
+            "AddChunkModel",
+            {
+                "id": fields.String(required=True, description="Document ID"),
+                "text": fields.String(required=True, description="Text of the chunk"),
+                "metadata": fields.Raw(
+                    required=False,
+                    description="Metadata associated with the chunk",
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Adds a new chunk to the document",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "text"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        doc_id = data.get("id")
+        text = data.get("text")
+        metadata = data.get("metadata", {})
+        token_count = num_tokens_from_string(text)
+        metadata["token_count"] = token_count
+
+        try:
+            doc = _resolve_source_for_write(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(jsonify({"error": "Source not accessible"}), 403)
+        try:
+            store = get_vector_store(str(doc["id"]))
+            chunk_id = store.add_chunk(text, metadata)
+            return make_response(
+                jsonify({"message": "Chunk added successfully", "chunk_id": chunk_id}),
+                201,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error adding chunk: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+@sources_chunks_ns.route("/delete_chunk")
+class DeleteChunk(Resource):
+    @api.doc(
+        description="Deletes a specific chunk from the document.",
+        params={"id": "The document ID", "chunk_id": "The ID of the chunk to delete"},
+    )
+    def delete(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        doc_id = request.args.get("id")
+        chunk_id = request.args.get("chunk_id")
+
+        try:
+            doc = _resolve_source_for_write(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(jsonify({"error": "Source not accessible"}), 403)
+        try:
+            store = get_vector_store(str(doc["id"]))
+            deleted = store.delete_chunk(chunk_id)
+            if deleted:
+                return make_response(
+                    jsonify({"message": "Chunk deleted successfully"}), 200
+                )
+            else:
+                return make_response(
+                    jsonify({"message": "Chunk not found or could not be deleted"}),
+                    404,
+                )
+        except Exception as e:
+            current_app.logger.error(f"Error deleting chunk: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+@sources_chunks_ns.route("/update_chunk")
+class UpdateChunk(Resource):
+    @api.expect(
+        api.model(
+            "UpdateChunkModel",
+            {
+                "id": fields.String(required=True, description="Document ID"),
+                "chunk_id": fields.String(
+                    required=True, description="Chunk ID to update"
+                ),
+                "text": fields.String(
+                    required=False, description="New text of the chunk"
+                ),
+                "metadata": fields.Raw(
+                    required=False,
+                    description="Updated metadata associated with the chunk",
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Updates an existing chunk in the document.",
+    )
+    def put(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "chunk_id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        doc_id = data.get("id")
+        chunk_id = data.get("chunk_id")
+        text = data.get("text")
+        metadata = data.get("metadata")
+
+        if text is not None:
+            token_count = num_tokens_from_string(text)
+            if metadata is None:
+                metadata = {}
+            metadata["token_count"] = token_count
+        try:
+            doc = _resolve_source_for_write(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(jsonify({"error": "Source not accessible"}), 403)
+        try:
+            store = get_vector_store(str(doc["id"]))
+
+            chunks = store.get_chunks()
+            existing_chunk = next((c for c in chunks if c["doc_id"] == chunk_id), None)
+            if not existing_chunk:
+                return make_response(jsonify({"error": "Chunk not found"}), 404)
+            new_text = text if text is not None else existing_chunk["text"]
+
+            if metadata is not None:
+                new_metadata = existing_chunk["metadata"].copy()
+                new_metadata.update(metadata)
+            else:
+                new_metadata = existing_chunk["metadata"].copy()
+            if text is not None:
+                new_metadata["token_count"] = num_tokens_from_string(new_text)
+            try:
+                new_chunk_id = store.add_chunk(new_text, new_metadata)
+
+                deleted = store.delete_chunk(chunk_id)
+                if not deleted:
+                    current_app.logger.warning(
+                        f"Failed to delete old chunk {chunk_id}, but new chunk {new_chunk_id} was created"
+                    )
+                return make_response(
+                    jsonify(
+                        {
+                            "message": "Chunk updated successfully",
+                            "chunk_id": new_chunk_id,
+                            "original_chunk_id": chunk_id,
+                        }
+                    ),
+                    200,
+                )
+            except Exception as add_error:
+                current_app.logger.error(f"Failed to add updated chunk: {add_error}")
+                return make_response(
+                    jsonify({"error": "Failed to update chunk - addition failed"}), 500
+                )
+        except Exception as e:
+            current_app.logger.error(f"Error updating chunk: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)

application/api/user/tasks.py

@@ -0,0 +1,650 @@
+import logging
+from datetime import timedelta
+
+from application.api.user.idempotency import with_idempotency
+from application.celery_init import celery
+from application.worker import (
+    agent_webhook_worker,
+    attachment_worker,
+    ingest_worker,
+    mcp_oauth,
+    parse_document_worker,
+    reembed_wiki_page_worker,
+    remote_worker,
+    sync,
+    sync_worker,
+)
+
+
+# Shared decorator config for long-running, side-effecting tasks. ``acks_late``
+# is also the celeryconfig default but stays explicit here so each task's
+# durability story is grep-able next to the body. Combined with
+# ``autoretry_for=(Exception,)`` and a bounded ``max_retries`` so a poison
+# message can't loop forever.
+DURABLE_TASK = dict(
+    bind=True,
+    acks_late=True,
+    autoretry_for=(Exception,),
+    retry_kwargs={"max_retries": 3, "countdown": 60},
+    retry_backoff=True,
+)
+
+
+# operation tag for the poison-path source.ingest.failed event, per task.
+_INGEST_POISON_OPERATION = {
+    "ingest": "upload",
+    "ingest_remote": "upload",
+    "ingest_connector_task": "upload",
+    "reingest_source_task": "reingest",
+}
+
+
+def _emit_ingest_poison_event(task_name, bound):
+    """Publish a terminal ``source.ingest.failed`` when the poison-guard trips.
+
+    The guard returns before the worker runs, so the worker's own failed
+    event never fires — without this the upload toast spins on "training".
+    """
+    user = bound.get("user")
+    source_id = bound.get("source_id")
+    if not user or not source_id:
+        return
+    from application.events.publisher import publish_user_event
+
+    publish_user_event(
+        user,
+        "source.ingest.failed",
+        {
+            "source_id": str(source_id),
+            "filename": bound.get("filename") or "",
+            "operation": _INGEST_POISON_OPERATION.get(task_name, "upload"),
+            "error": "Ingestion stopped after repeated failures.",
+        },
+        scope={"kind": "source", "id": str(source_id)},
+    )
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="ingest", on_poison=_emit_ingest_poison_event)
+def ingest(
+    self,
+    directory,
+    formats,
+    job_name,
+    user,
+    file_path,
+    filename,
+    file_name_map=None,
+    config=None,
+    idempotency_key=None,
+    source_id=None,
+):
+    resp = ingest_worker(
+        self,
+        directory,
+        formats,
+        job_name,
+        file_path,
+        filename,
+        user,
+        file_name_map=file_name_map,
+        config=config,
+        idempotency_key=idempotency_key,
+        source_id=source_id,
+    )
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="ingest_remote", on_poison=_emit_ingest_poison_event)
+def ingest_remote(
+    self, source_data, job_name, user, loader,
+    config=None, idempotency_key=None, source_id=None,
+):
+    resp = remote_worker(
+        self, source_data, job_name, user, loader,
+        config=config,
+        idempotency_key=idempotency_key,
+        source_id=source_id,
+    )
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(
+    task_name="reingest_source_task", on_poison=_emit_ingest_poison_event,
+)
+def reingest_source_task(self, source_id, user, idempotency_key=None):
+    from application.worker import reingest_source_worker
+
+    resp = reingest_source_worker(self, source_id, user)
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="reembed_wiki_page")
+def reembed_wiki_page(
+    self, source_id, path, content_hash, user, idempotency_key=None,
+):
+    resp = reembed_wiki_page_worker(self, source_id, path, content_hash, user)
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="convert_source_to_wiki")
+def convert_source_to_wiki(self, source_id, user, idempotency_key=None):
+    from application.worker import convert_source_to_wiki_worker
+
+    resp = convert_source_to_wiki_worker(self, source_id, user)
+    return resp
+
+
+def _emit_graph_poison_event(task_name, bound):
+    """Publish a terminal ``graph.extract.failed`` when the poison-guard trips.
+
+    The guard returns before the worker runs, so the worker's own failed event
+    never fires — without this the build UI spins forever.
+    """
+    user = bound.get("user")
+    source_id = bound.get("source_id")
+    if not user or not source_id:
+        return
+    from application.events.publisher import publish_user_event
+
+    publish_user_event(
+        user,
+        "graph.extract.failed",
+        {
+            "source_id": str(source_id),
+            "error": "Graph extraction stopped after repeated failures.",
+        },
+        scope={"kind": "source", "id": str(source_id)},
+    )
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="extract_graph", on_poison=_emit_graph_poison_event)
+def extract_graph(self, source_id, user, idempotency_key=None):
+    from application.worker import extract_graph_worker
+
+    resp = extract_graph_worker(self, source_id, user)
+    return resp
+
+
+# Beat-driven dispatch tasks default to ``acks_late=False``: a SIGKILL
+# of a beat tick is harmless to redeliver only if the dispatch itself is
+# idempotent. We keep these early-ACK so the broker doesn't replay a
+# dispatch that already enqueued downstream work.
+@celery.task(bind=True, acks_late=False)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.task(bind=True)
+def sync_source(
+    self,
+    source_data,
+    job_name,
+    user,
+    loader,
+    sync_frequency,
+    retriever,
+    doc_id,
+):
+    resp = sync(
+        self,
+        source_data,
+        job_name,
+        user,
+        loader,
+        sync_frequency,
+        retriever,
+        doc_id,
+    )
+    return resp
+
+
+def _emit_attachment_poison_event(task_name, bound):
+    """Publish a terminal ``attachment.failed`` when the poison-guard trips.
+
+    Mirrors ``_emit_ingest_poison_event``: the guard returns before the
+    worker runs, so ``attachment_worker``'s own events never fire and the
+    upload toast would otherwise spin on "processing" forever.
+    """
+    user = bound.get("user")
+    file_info = bound.get("file_info") or {}
+    attachment_id = file_info.get("attachment_id")
+    if not user or not attachment_id:
+        return
+    from application.events.publisher import publish_user_event
+
+    publish_user_event(
+        user,
+        "attachment.failed",
+        {
+            "attachment_id": str(attachment_id),
+            "filename": file_info.get("filename") or "",
+            "error": "Attachment processing stopped after repeated failures.",
+        },
+        scope={"kind": "attachment", "id": str(attachment_id)},
+    )
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(
+    task_name="store_attachment", on_poison=_emit_attachment_poison_event,
+)
+def store_attachment(self, file_info, user, idempotency_key=None):
+    resp = attachment_worker(self, file_info, user)
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="process_agent_webhook")
+def process_agent_webhook(self, agent_id, payload, idempotency_key=None):
+    resp = agent_webhook_worker(self, agent_id, payload)
+    return resp
+
+
+# Not DURABLE: the read_document tool awaits this synchronously with a timeout, so a
+# blind autoretry would double-parse and the caller would already have degraded. The
+# task is routed to the dedicated ``parsing`` queue (celeryconfig task_routes) so a
+# parse enqueued from inside a Celery worker (headless/scheduled agent) is served by a
+# separate parsing worker and never self-deadlocks the awaiting worker.
+@celery.task(bind=True, acks_late=False, autoretry_for=())
+def parse_document(self, artifact_id, parent, user_id, options=None):
+    """Parse an input artifact on the parsing queue; self-terminate at the soft time limit."""
+    from celery.exceptions import SoftTimeLimitExceeded
+
+    try:
+        return parse_document_worker(self, artifact_id, parent, user_id, options or {})
+    except SoftTimeLimitExceeded:
+        # A pathological/malicious document must not pin a parsing-worker slot past the
+        # window the caller already abandoned. Return the worker's clean error shape so
+        # the slot frees and the Redis result backend still gets a terminal result.
+        limit = getattr(self, "soft_time_limit", None)
+        suffix = f" after {int(limit)}s" if limit else ""
+        return {"status": "error", "error": f"document parsing timed out{suffix}."}
+
+
+# Bind the soft limit to DOCUMENT_PARSE_TIMEOUT (the same window read_document awaits) so
+# the prefork worker self-terminates a runaway parse instead of pinning the slot; the hard
+# limit is the SIGKILL backstop if the soft handler can't unwind in time.
+try:
+    from application.core.settings import settings as _parse_settings
+    parse_document.soft_time_limit = int(_parse_settings.DOCUMENT_PARSE_TIMEOUT)
+    parse_document.time_limit = parse_document.soft_time_limit + 30
+except Exception:
+    pass
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(
+    task_name="ingest_connector_task", on_poison=_emit_ingest_poison_event,
+)
+def ingest_connector_task(
+    self,
+    job_name,
+    user,
+    source_type,
+    session_token=None,
+    file_ids=None,
+    folder_ids=None,
+    recursive=True,
+    retriever="classic",
+    operation_mode="upload",
+    doc_id=None,
+    sync_frequency="never",
+    config=None,
+    idempotency_key=None,
+    source_id=None,
+):
+    from application.worker import ingest_connector
+
+    resp = ingest_connector(
+        self,
+        job_name,
+        user,
+        source_type,
+        session_token=session_token,
+        file_ids=file_ids,
+        folder_ids=folder_ids,
+        recursive=recursive,
+        retriever=retriever,
+        operation_mode=operation_mode,
+        doc_id=doc_id,
+        sync_frequency=sync_frequency,
+        config=config,
+        idempotency_key=idempotency_key,
+        source_id=source_id,
+    )
+    return resp
+
+
+@celery.task(bind=True, acks_late=False)
+def dispatch_scheduled_runs(self):
+    """Beat-driven scheduler poller (body in scheduler_dispatcher)."""
+    from application.api.user.scheduler_dispatcher import dispatch_due_runs
+
+    return dispatch_due_runs()
+
+
+@celery.task(
+    bind=True,
+    acks_late=True,
+    # Not DURABLE_TASK: agent runs have side effects; blind retry would double them.
+    autoretry_for=(),
+    max_retries=0,
+)
+def execute_scheduled_run(self, run_id):
+    """Execute one scheduled run; soft-time-limit honors SCHEDULE_RUN_TIMEOUT."""
+    from application.api.user.scheduler_worker import execute_scheduled_run_body
+
+    return execute_scheduled_run_body(run_id, getattr(self.request, "id", None))
+
+
+# Bind runtime soft-time-limit so the prefork worker can raise mid-agent.
+try:
+    from application.core.settings import settings as _scheduler_settings
+    execute_scheduled_run.soft_time_limit = max(
+        30, int(_scheduler_settings.SCHEDULE_RUN_TIMEOUT),
+    )
+    execute_scheduled_run.time_limit = (
+        execute_scheduled_run.soft_time_limit + 60
+    )
+except Exception:
+    pass
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_schedule_runs(self):
+    """Trim ``schedule_runs`` per ``SCHEDULE_RUN_OUTPUT_RETENTION_DAYS``."""
+    from application.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from application.storage.db.engine import get_engine
+    from application.storage.db.repositories.schedule_runs import (
+        ScheduleRunsRepository,
+    )
+
+    ttl_days = settings.SCHEDULE_RUN_OUTPUT_RETENTION_DAYS
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = ScheduleRunsRepository(conn).cleanup_older_than(ttl_days)
+    return {"deleted": deleted, "ttl_days": ttl_days}
+
+
+@celery.task(bind=True, acks_late=False)
+def reap_sandbox_sessions(self):
+    """Close sandbox sessions idle past their TTL in this worker process.
+
+    The SandboxManager registry is per-process, so this reaps only sessions
+    bound in THIS worker; the API processes reap their own opportunistically on
+    ``open``. Artifacts are persisted eagerly, so reaping only closes idle
+    kernels and never loses a user-facing artifact.
+    """
+    try:
+        from application.sandbox.sandbox_creator import SandboxCreator
+
+        reaped = SandboxCreator.get_manager().reap_expired()
+    except Exception:  # noqa: BLE001 - housekeeping must never crash the beat loop
+        logging.getLogger(__name__).exception("reap_sandbox_sessions failed")
+        return {"reaped": 0, "error": True}
+    return {"reaped": len(reaped)}
+
+
+@celery.task(bind=True, acks_late=False)
+def reap_stale_workflow_runs(self):
+    """Fail workflow runs stranded in ``running`` past the stale deadline.
+
+    A run row is pre-created as ``running`` and finalized when its generator
+    finishes; a client disconnect or worker crash can leave it ``running``
+    forever. This closes those rows out so the UI/API stop showing a run that
+    will never complete.
+    """
+    from datetime import datetime, timezone
+
+    from application.core.settings import settings
+    from application.storage.db.engine import get_engine
+    from application.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+
+    try:
+        stale_seconds = max(60, int(settings.WORKFLOW_RUN_STALE_SECONDS))
+        cutoff = datetime.now(timezone.utc) - timedelta(seconds=stale_seconds)
+        engine = get_engine()
+        with engine.begin() as conn:
+            reaped = WorkflowRunsRepository(conn).mark_stale_running_failed(cutoff)
+    except Exception:  # noqa: BLE001 - housekeeping must never crash the beat loop
+        logging.getLogger(__name__).exception("reap_stale_workflow_runs failed")
+        return {"reaped": 0, "error": True}
+    return {"reaped": reaped}
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    from application.core.settings import settings
+
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )
+    # Replaces Mongo's TTL index on pending_tool_state.expires_at.
+    sender.add_periodic_task(
+        timedelta(seconds=60),
+        cleanup_pending_tool_state.s(),
+        name="cleanup-pending-tool-state",
+    )
+    # Pure housekeeping for ``task_dedup`` / ``webhook_dedup`` — the
+    # upsert paths already handle stale rows, so cadence only bounds
+    # table size. Hourly is plenty for typical traffic.
+    sender.add_periodic_task(
+        timedelta(hours=1),
+        cleanup_idempotency_dedup.s(),
+        name="cleanup-idempotency-dedup",
+    )
+    sender.add_periodic_task(
+        timedelta(seconds=30),
+        reconciliation_task.s(),
+        name="reconciliation",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=7),
+        version_check_task.s(),
+        name="version-check",
+    )
+    # Bound ``message_events`` growth — every streamed SSE chunk writes
+    # one row, so retained chats accumulate hundreds of rows per
+    # message. Reconnect-replay is only meaningful for streams the user
+    # could plausibly still be waiting on, so 14 days is generous.
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_message_events.s(),
+        name="cleanup-message-events",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_orphan_memories.s(),
+        name="cleanup-orphan-memories",
+    )
+    # Scheduler dispatcher and run-log trim.
+    sender.add_periodic_task(
+        timedelta(seconds=max(15, settings.SCHEDULE_DISPATCHER_INTERVAL)),
+        dispatch_scheduled_runs.s(),
+        name="dispatch-scheduled-runs",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_schedule_runs.s(),
+        name="cleanup-schedule-runs",
+    )
+    # Close idle-past-TTL sandbox sessions roughly every minute. The on-open
+    # opportunistic reap still runs in the API processes; this covers worker
+    # processes (and quiet periods where no new session is opened).
+    sender.add_periodic_task(
+        timedelta(seconds=60),
+        reap_sandbox_sessions.s(),
+        name="reap-sandbox-sessions",
+    )
+    # Fail workflow runs stranded in ``running`` (client disconnect / crash) so
+    # they don't linger forever. Every few minutes is plenty; the cutoff is hours.
+    sender.add_periodic_task(
+        timedelta(seconds=300),
+        reap_stale_workflow_runs.s(),
+        name="reap-stale-workflow-runs",
+    )
+
+
+# Bound time limits so a hung OAuth discovery (user never finishes the
+# consent flow, upstream never redirects) self-terminates instead of
+# stranding the ``mcp.oauth.awaiting_redirect`` envelope forever. The
+# soft limit raises inside ``mcp_oauth``'s ``try`` so it publishes a
+# terminal ``mcp.oauth.failed``; the hard limit is the prefork backstop.
+# Generous so a human actively clicking through OAuth isn't cut off.
+@celery.task(bind=True, soft_time_limit=600, time_limit=660)
+def mcp_oauth_task(self, config, user):
+    resp = mcp_oauth(self, config, user)
+    return resp
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_pending_tool_state(self):
+    """Revert stale ``resuming`` rows, then delete TTL-expired rows."""
+    from application.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "reverted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from application.storage.db.engine import get_engine
+    from application.storage.db.repositories.pending_tool_state import (
+        PendingToolStateRepository,
+    )
+
+    engine = get_engine()
+    with engine.begin() as conn:
+        repo = PendingToolStateRepository(conn)
+        reverted = repo.revert_stale_resuming(grace_seconds=600)
+        cleared = repo.cleanup_expired()
+
+    # Reaping the resumable state retires any awaiting-approval prompt
+    # tied to it. Without a clearing event the durable
+    # ``tool.approval.required`` envelope replays on reconnect and the UI
+    # toast lingers for a conversation that can no longer be resumed.
+    from application.events.publisher import publish_user_event
+
+    for row in cleared:
+        user_id = row.get("user_id")
+        conversation_id = row.get("conversation_id")
+        if not user_id or not conversation_id:
+            continue
+        publish_user_event(
+            str(user_id),
+            "tool.approval.cleared",
+            {"conversation_id": str(conversation_id), "reason": "expired"},
+            scope={"kind": "conversation", "id": str(conversation_id)},
+        )
+    return {"deleted": len(cleared), "reverted": reverted}
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_idempotency_dedup(self):
+    """Delete TTL-expired rows from ``task_dedup`` and ``webhook_dedup``.
+
+    Pure housekeeping — the upsert paths already ignore stale rows
+    (TTL-aware ``ON CONFLICT DO UPDATE``), so this only bounds table
+    growth and keeps SELECT planning tight on large deployments.
+    """
+    from application.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {
+            "task_dedup_deleted": 0,
+            "webhook_dedup_deleted": 0,
+            "skipped": "POSTGRES_URI not set",
+        }
+
+    from application.storage.db.engine import get_engine
+    from application.storage.db.repositories.idempotency import (
+        IdempotencyRepository,
+    )
+
+    engine = get_engine()
+    with engine.begin() as conn:
+        return IdempotencyRepository(conn).cleanup_expired()
+
+
+@celery.task(bind=True, acks_late=False)
+def reconciliation_task(self):
+    """Sweep stuck durability rows and escalate them to terminal status + alert."""
+    from application.api.user.reconciliation import run_reconciliation
+
+    return run_reconciliation()
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_message_events(self):
+    """Delete ``message_events`` rows older than the retention window.
+
+    Streamed answer responses write one journal row per SSE yield,
+    so unbounded growth would dominate Postgres for any retained-
+    conversations deployment. The reconnect-replay path only needs
+    rows for in-flight streams; 14 days covers paused/tool-action
+    flows comfortably.
+    """
+    from application.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from application.storage.db.engine import get_engine
+    from application.storage.db.repositories.message_events import (
+        MessageEventsRepository,
+    )
+
+    ttl_days = settings.MESSAGE_EVENTS_RETENTION_DAYS
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = MessageEventsRepository(conn).cleanup_older_than(ttl_days)
+    return {"deleted": deleted, "ttl_days": ttl_days}
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_orphan_memories(self):
+    """Sweep orphan memories left by the 0009 FK-to-trigger orphan window.
+
+    A ``memories`` INSERT for a real ``tool_id`` racing a ``user_tools``
+    DELETE leaves a permanent orphan the dropped FK would have rejected.
+    Default-tool synthetic ids are preserved (legitimate built-in data).
+    """
+    from application.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from application.agents.default_tools import default_tool_ids
+    from application.storage.db.engine import get_engine
+    from application.storage.db.repositories.memories import MemoriesRepository
+
+    keep_tool_ids = list(default_tool_ids().values())
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = MemoriesRepository(conn).delete_orphans(keep_tool_ids)
+    return {"deleted": deleted}
+
+
+@celery.task(bind=True, acks_late=False)
+def version_check_task(self):
+    """Periodic anonymous version check.
+
+    Complements the ``worker_ready`` boot trigger so long-running
+    deployments (>6h cache TTL) still refresh advisories. ``run_check``
+    is fail-silent and coordinates across replicas via Redis lock +
+    cache (see ``application.updates.version_check``).
+    """
+    from application.updates.version_check import run_check
+    run_check()

application/api/user/team_authz.py

@@ -0,0 +1,109 @@
+"""Team-scoped authorization — the SECOND authz plane beside global RBAC.
+
+Mirrors ``application/api/user/authz.py`` (the global admin/user plane) but for
+per-team roles. The two planes never mix: a global ``admin`` is a superuser over
+all teams (short-circuit below), but a ``team_admin`` is NOT a global admin.
+
+Key differences from the global resolver, all deliberate:
+- Team roles are resolved PER ROUTE (lazy), not eagerly in the app.py chokepoint
+  — most requests never touch a team route, so we don't pay a team query on the
+  universal path.
+- Team roles are NEVER read from the JWT (same reason as global RBAC: simple_jwt
+  / session_jwt are self-mintable).
+- ``team_id`` is read ONLY from ``request.view_args`` (the URL path), NEVER the
+  request body — a body-supplied team_id would be a trivial escalation vector.
+- Resolution FAILS CLOSED: a DB error while reading membership denies access
+  (the inverse of the global resolver, which fails open — here, failing open
+  would grant team access during an outage).
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import wraps
+
+from flask import jsonify, make_response, request
+
+from application.api.user.authz import ROLE_ADMIN, has_role
+from application.storage.db.repositories.team_members import (
+    ROLE_TEAM_ADMIN,
+    ROLE_TEAM_MEMBER,
+    TeamMembersRepository,
+)
+from application.storage.db.session import db_readonly
+
+logger = logging.getLogger(__name__)
+
+
+def team_role_for(token: dict | None, team_id: str | None) -> str | None:
+    """Strongest team role the principal holds in ``team_id``, or None.
+
+    Fails closed to None on a DB error so a membership-read outage denies team
+    access rather than granting it.
+    """
+    if not token or not team_id:
+        return None
+    sub = token.get("sub")
+    if not sub:
+        return None
+    try:
+        with db_readonly() as conn:
+            return TeamMembersRepository(conn).role_for(sub, team_id)
+    except Exception:
+        logger.error(
+            "team_role_for: team_members read failed for sub=%s team=%s",
+            sub,
+            team_id,
+            exc_info=True,
+        )
+        return None
+
+
+def has_team_role(token: dict | None, team_id: str | None, name: str) -> bool:
+    """True if the principal satisfies ``name`` for ``team_id``.
+
+    A global ``admin`` satisfies any team role (superuser over all teams).
+    Otherwise ``team_admin`` implies ``team_member`` (admin ⊇ member).
+    """
+    if has_role(token, ROLE_ADMIN):
+        return True
+    role = team_role_for(token, team_id)
+    if role is None:
+        return False
+    if name == ROLE_TEAM_MEMBER:
+        return True
+    return role == name
+
+
+def require_team_role(name: str):
+    """Decorator factory: 401 when unauthenticated, 403 when lacking ``name``.
+
+    Reads ``team_id`` from the route kwargs (URL path) ONLY — never the body.
+    Fails closed: a missing token, missing team_id, or membership-read error
+    never passes through.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            token = getattr(request, "decoded_token", None)
+            if not token:
+                return make_response(
+                    jsonify({"success": False, "message": "Authentication required"}), 401
+                )
+            team_id = kwargs.get("team_id")
+            if not team_id:
+                return make_response(
+                    jsonify({"success": False, "message": "Team not specified"}), 400
+                )
+            if not has_team_role(token, team_id, name):
+                return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+team_admin_required = require_team_role(ROLE_TEAM_ADMIN)
+team_member_required = require_team_role(ROLE_TEAM_MEMBER)

application/api/user/team_sharing.py

@@ -0,0 +1,124 @@
+"""Cross-cutting helpers for team resource sharing.
+
+Centralises the polymorphic dispatch over the four shareable resource types so
+the grant endpoint and the per-entity list/get/update paths share one source of
+truth for "does this user own resource R?" and "what can this user see/edit via
+their teams?". Every shareable repo exposes the same ``get_any(id, user_id)``
+ownership accessor, which is what makes the dispatch uniform.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from sqlalchemy import Connection
+
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.prompts import PromptsRepository
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.repositories.team_resource_grants import (
+    TeamResourceGrantsRepository,
+)
+from application.storage.db.repositories.team_scope import TeamScopeRepository
+from application.storage.db.repositories.user_tools import UserToolsRepository
+
+RESOURCE_TYPES = ("agent", "source", "prompt", "tool")
+
+_REPO_FOR_TYPE = {
+    "agent": AgentsRepository,
+    "source": SourcesRepository,
+    "prompt": PromptsRepository,
+    "tool": UserToolsRepository,
+}
+
+
+def is_valid_resource_type(resource_type: str) -> bool:
+    return resource_type in _REPO_FOR_TYPE
+
+
+def owns_resource(
+    conn: Connection, resource_type: str, resource_id: str, user_id: str
+) -> bool:
+    """True if ``user_id`` is the OWNER of the resource.
+
+    Dispatches to the correct repo by ``resource_type`` — this is the guard that
+    stops a caller registering a grant with a mismatched ``resource_type`` /
+    ``resource_id`` (the polymorphic grant table has no FK to catch it). Owners
+    only — team-granted ``editor`` access does NOT make you an owner.
+    """
+    repo_cls = _REPO_FOR_TYPE.get(resource_type)
+    if repo_cls is None:
+        return False
+    return repo_cls(conn).get_any(resource_id, user_id) is not None
+
+
+def team_access_for(
+    conn: Connection, user_id: str, resource_type: str, resource_id: str
+) -> Optional[str]:
+    """Strongest team access (``editor``/``viewer``) ``user_id`` has on a resource.
+
+    None when no team grant reaches the user (they may still be the owner — that
+    is a separate dual-key check at the repo).
+    """
+    return TeamScopeRepository(conn).effective_access(user_id, resource_type, resource_id)
+
+
+def visible_ids_via_teams(
+    conn: Connection, user_id: str, resource_type: str
+) -> set[str]:
+    """The id set of ``resource_type`` shared to any team ``user_id`` belongs to."""
+    return TeamScopeRepository(conn).visible_resource_ids(user_id, resource_type)
+
+
+def visible_with_access(
+    conn: Connection, user_id: str, resource_type: str
+) -> dict[str, str]:
+    """Map ``resource_id -> strongest access`` shared to the user's teams."""
+    return TeamScopeRepository(conn).visible_with_access(user_id, resource_type)
+
+
+def effective_write_owner(
+    conn: Connection, resource_type: str, resource_id: str, user_id: str
+) -> Optional[str]:
+    """The owner id to write a resource AS, or None if the caller can't write.
+
+    Returns ``user_id`` when the caller owns the resource, or the resource's
+    real ``owner_id`` when the caller holds a team ``editor`` grant — so callers
+    can pass the result straight to the existing owner-scoped ``update(id, owner,
+    ...)`` repo methods (which match on ``WHERE id AND user_id = :owner``) without
+    a separate ownerless write path. None means viewer-only or no access → the
+    route should answer 403/404. Delete is never authorized here — owner-only.
+    """
+    if owns_resource(conn, resource_type, resource_id, user_id):
+        return user_id
+    # Past the ownership check, only canonical-UUID resources can carry a team
+    # grant; a legacy/non-UUID id can't, and casting it would poison the txn.
+    if not looks_like_uuid(resource_id):
+        return None
+    grants = TeamResourceGrantsRepository(conn).list_for_resource(
+        resource_type, resource_id
+    )
+    if not grants:
+        return None
+    if TeamScopeRepository(conn).can_write(user_id, resource_type, resource_id):
+        # All grant rows carry the same denormalised owner_id.
+        return grants[0].get("owner_id")
+    return None
+
+
+def can_access(
+    conn: Connection, resource_type: str, resource_id: str, user_id: str
+) -> bool:
+    """True if ``user_id`` owns the resource OR has any team grant on it (read).
+
+    This is the write-path gate for *referencing* a resource (e.g. attaching a
+    ``source_id`` to an agent): you may reference what you own or what a team has
+    shared with you directly. Transitive access *through* a shared agent is a
+    separate, run-time concept and is intentionally NOT gated here.
+    """
+    if not resource_id:
+        return True
+    if owns_resource(conn, resource_type, resource_id, user_id):
+        return True
+    return TeamScopeRepository(conn).can_read(user_id, resource_type, resource_id)

application/api/user/teams/__init__.py

@@ -0,0 +1,5 @@
+"""Teams API namespace: team CRUD, membership, and resource-sharing grants."""
+
+from .routes import teams_ns
+
+__all__ = ["teams_ns"]

application/api/user/teams/routes.py

@@ -0,0 +1,636 @@
+"""Team management API: CRUD, membership, and resource-sharing grants.
+
+Authorization model (two planes, see ``team_authz.py``):
+- Any authenticated user may create a team (self-serve); the creator becomes a
+  ``team_admin`` in the same transaction.
+- Team detail / member list / grant list require team membership.
+- Member management and team edit require ``team_admin``.
+- Team deletion and owner transfer are owner-only (a global ``admin`` overrides).
+- Sharing a resource requires the caller to OWN it (dispatched by resource_type)
+  and be a member of the target team. Sharing is additive visibility — the
+  resource's owner is never changed.
+
+``team_id`` always comes from the URL path (never the body) — enforced by
+``require_team_role`` and by reading ``team_id`` as a route kwarg here.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from application.api.user.authz import ROLE_ADMIN, has_role
+from application.api.user.team_authz import (
+    has_team_role,
+    team_admin_required,
+    team_member_required,
+)
+from application.api.user.team_sharing import is_valid_resource_type, owns_resource
+from application.events.publisher import publish_user_event
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.repositories.auth_events import AuthEventsRepository
+from application.storage.db.repositories.prompts import PromptsRepository
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.repositories.team_members import (
+    ROLE_TEAM_ADMIN,
+    ROLE_TEAM_MEMBER,
+    TeamMembersRepository,
+)
+from application.storage.db.repositories.team_resource_grants import (
+    TeamResourceGrantsRepository,
+)
+from application.storage.db.repositories.teams import TeamsRepository
+from application.storage.db.repositories.user_tools import UserToolsRepository
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+teams_ns = Namespace("teams", description="Team management and resource sharing", path="/api")
+
+_VALID_TEAM_ROLES = (ROLE_TEAM_ADMIN, ROLE_TEAM_MEMBER)
+_VALID_ACCESS_LEVELS = ("viewer", "editor")
+
+
+def _current_user() -> str | None:
+    token = getattr(request, "decoded_token", None)
+    return token.get("sub") if isinstance(token, dict) else None
+
+
+def _audit(conn, actor: str | None, event: str, **metadata) -> None:
+    """Append a team management event to the audit trail (best-effort).
+
+    Runs inside the action's transaction so the audit row commits atomically
+    with the change. Never raises into the request path — an audit failure must
+    not fail the operation.
+    """
+    try:
+        # SAVEPOINT: a failed audit insert poisons the surrounding txn, so
+        # nest it — on failure only the audit rolls back, not the action.
+        with conn.begin_nested():
+            AuthEventsRepository(conn).insert(
+                user_id=actor or "unknown",
+                event=event,
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata={k: v for k, v in metadata.items() if v is not None},
+            )
+    except Exception:
+        logger.warning("team audit insert failed for event=%s", event, exc_info=True)
+
+
+def _slugify(name: str) -> str:
+    base = re.sub(r"[^a-z0-9]+", "-", (name or "").lower()).strip("-")
+    return base or "team"
+
+
+def _unique_slug(repo: TeamsRepository, base: str) -> str:
+    """A slug not yet taken: ``base`` if free, else ``base-<4hex>`` until unique."""
+    if not repo.slug_exists(base):
+        return base
+    for _ in range(8):
+        candidate = f"{base}-{uuid.uuid4().hex[:4]}"
+        if not repo.slug_exists(candidate):
+            return candidate
+    return f"{base}-{uuid.uuid4().hex}"
+
+
+# --- Notifications (best-effort, fire-and-forget; never fail the request) ----
+# Emitted AFTER the mutation commits, over their own read connection, so a
+# name-lookup or Redis hiccup can never roll back the share/membership change.
+
+
+def _resource_display_name(conn, resource_type: str, resource_id: str) -> str | None:
+    """Best-effort human label for a shared resource (ownerless fetch)."""
+    try:
+        if resource_type == "agent":
+            row = AgentsRepository(conn).get_by_id(resource_id)
+        elif resource_type == "source":
+            row = SourcesRepository(conn).get_by_id(resource_id)
+        elif resource_type == "prompt":
+            row = PromptsRepository(conn).get_for_rendering(resource_id)
+        elif resource_type == "tool":
+            row = UserToolsRepository(conn).get_by_id(resource_id)
+        else:
+            return None
+        if not row:
+            return None
+        return row.get("custom_name") or row.get("display_name") or row.get("name")
+    except Exception:
+        logger.warning("resource name resolve failed (%s)", resource_type, exc_info=True)
+        return None
+
+
+def _notify_member_added(
+    team_id: str, new_user: str | None, role: str, actor: str | None
+) -> None:
+    """Toast the new member that they were added to a team."""
+    if not new_user or new_user == actor:
+        return
+    try:
+        with db_readonly() as conn:
+            team = TeamsRepository(conn).get(team_id)
+        publish_user_event(
+            new_user,
+            "team.member_added",
+            {
+                "team_id": str(team_id),
+                "team_name": team.get("name") if team else None,
+                "role": role,
+                "added_by": actor,
+            },
+            scope={"kind": "team", "id": str(team_id)},
+        )
+    except Exception:
+        logger.warning("member_added notify failed", exc_info=True)
+
+
+def _notify_resource_shared(
+    actor: str | None,
+    team_id: str,
+    resource_type: str,
+    resource_id: str,
+    access_level: str,
+    target_user_id: str | None,
+) -> None:
+    """Toast the recipient(s) of a share.
+
+    Per-member share notifies just that member; whole-team share notifies every
+    current member except the sharer.
+    """
+    try:
+        with db_readonly() as conn:
+            resource_name = _resource_display_name(conn, resource_type, resource_id)
+            team = TeamsRepository(conn).get(team_id)
+            if target_user_id:
+                recipients = [target_user_id]
+            else:
+                members = TeamMembersRepository(conn).list_members(team_id)
+                recipients = list({m["user_id"] for m in members})
+        payload = {
+            "resource_type": resource_type,
+            "resource_id": str(resource_id),
+            "resource_name": resource_name,
+            "access_level": access_level,
+            "team_id": str(team_id),
+            "team_name": team.get("name") if team else None,
+            "shared_by": actor,
+        }
+        for recipient in recipients:
+            if recipient and recipient != actor:
+                publish_user_event(
+                    recipient,
+                    "resource.shared",
+                    payload,
+                    scope={"kind": "resource", "id": str(resource_id)},
+                )
+    except Exception:
+        logger.warning("resource_shared notify failed", exc_info=True)
+
+
+@teams_ns.route("/teams")
+class Teams(Resource):
+    def get(self):
+        """List the teams the caller belongs to, each annotated with their role."""
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        try:
+            with db_readonly() as conn:
+                teams = TeamsRepository(conn).list_for_user(user)
+            return make_response(jsonify({"success": True, "teams": teams}), 200)
+        except Exception as err:
+            logger.error("List teams failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    def post(self):
+        """Create a team (self-serve). The creator becomes its first team_admin."""
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        data = request.get_json(silent=True) or {}
+        name = (data.get("name") or "").strip()
+        if not name:
+            return {"success": False, "message": "name required"}, 400
+        description = data.get("description")
+        try:
+            with db_session() as conn:
+                teams = TeamsRepository(conn)
+                slug = _unique_slug(teams, _slugify(name))
+                team = teams.create(name, slug, owner_id=user, description=description)
+                TeamMembersRepository(conn).add_member(
+                    team["id"], user, role=ROLE_TEAM_ADMIN, source="manual", granted_by=user
+                )
+                _audit(conn, user, "team.create", team_id=team["id"], name=name)
+            team["member_role"] = ROLE_TEAM_ADMIN
+            return make_response(jsonify({"success": True, "team": team}), 201)
+        except Exception as err:
+            logger.error("Create team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>")
+class Team(Resource):
+    @team_member_required
+    def get(self, team_id):
+        """Team detail with members and the caller's role. Requires membership."""
+        user = _current_user()
+        try:
+            with db_readonly() as conn:
+                team = TeamsRepository(conn).get(team_id)
+                if not team:
+                    return {"success": False, "message": "Not found"}, 404
+                members = TeamMembersRepository(conn)
+                team["members"] = members.list_members(team_id)
+                team["member_role"] = members.role_for(user, team_id)
+            return make_response(jsonify({"success": True, "team": team}), 200)
+        except Exception as err:
+            logger.error("Get team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_admin_required
+    def put(self, team_id):
+        """Update team name/description. Requires team_admin."""
+        data = request.get_json(silent=True) or {}
+        fields = {k: data[k] for k in ("name", "description") if k in data}
+        if not fields:
+            return {"success": False, "message": "nothing to update"}, 400
+        try:
+            with db_session() as conn:
+                updated = TeamsRepository(conn).update(team_id, fields)
+            return make_response(jsonify({"success": bool(updated)}), 200)
+        except Exception as err:
+            logger.error("Update team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    def delete(self, team_id):
+        """Delete the team. Owner-only (a global admin overrides)."""
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        try:
+            with db_session() as conn:
+                team = TeamsRepository(conn).get(team_id)
+                if not team:
+                    return {"success": False, "message": "Not found"}, 404
+                token = getattr(request, "decoded_token", None)
+                if team["owner_id"] != user and not has_role(token, ROLE_ADMIN):
+                    return {"success": False, "message": "Only the team owner can delete"}, 403
+                TeamsRepository(conn).delete(team_id)
+                _audit(conn, user, "team.delete", team_id=team_id)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Delete team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>/members")
+class TeamMembers(Resource):
+    @team_member_required
+    def get(self, team_id):
+        """List members. Requires membership."""
+        try:
+            with db_readonly() as conn:
+                members = TeamMembersRepository(conn).list_members(team_id)
+            return make_response(jsonify({"success": True, "members": members}), 200)
+        except Exception as err:
+            logger.error("List members failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_admin_required
+    def post(self, team_id):
+        """Add a member by email (preferred) or raw user_id. Requires team_admin."""
+        data = request.get_json(silent=True) or {}
+        new_user = (data.get("user_id") or "").strip()
+        email = (data.get("email") or "").strip()
+        role = data.get("role", ROLE_TEAM_MEMBER)
+        if role not in _VALID_TEAM_ROLES:
+            return {"success": False, "message": "invalid role"}, 400
+        if not new_user and not email:
+            return {"success": False, "message": "email or user_id required"}, 400
+        try:
+            with db_session() as conn:
+                # Resolve an email to its sub (the user must have logged in at
+                # least once for their email to be on file).
+                if not new_user and email:
+                    user_row = UsersRepository(conn).find_by_email(email)
+                    if not user_row:
+                        return {
+                            "success": False,
+                            "message": "No user found with that email (they must sign in once first)",
+                        }, 404
+                    new_user = user_row["user_id"]
+                TeamMembersRepository(conn).set_manual_role(
+                    team_id, new_user, role, granted_by=_current_user()
+                )
+                _audit(
+                    conn,
+                    _current_user(),
+                    "team.member_add",
+                    team_id=team_id,
+                    target_user=new_user,
+                    role=role,
+                )
+            # Post-commit, best-effort: tell the new member they were added.
+            _notify_member_added(team_id, new_user, role, _current_user())
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Add member failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>/members/<string:member_id>")
+class TeamMember(Resource):
+    @team_admin_required
+    def put(self, team_id, member_id):
+        """Change a member's role. Requires team_admin. Guards the last admin."""
+        data = request.get_json(silent=True) or {}
+        role = data.get("role")
+        if role not in _VALID_TEAM_ROLES:
+            return {"success": False, "message": "invalid role"}, 400
+        try:
+            with db_session() as conn:
+                members = TeamMembersRepository(conn)
+                if role == ROLE_TEAM_MEMBER and self._would_orphan_admins(
+                    members, team_id, member_id
+                ):
+                    return {
+                        "success": False,
+                        "message": "Cannot demote the last team admin",
+                    }, 409
+                members.set_manual_role(team_id, member_id, role, granted_by=_current_user())
+                _audit(
+                    conn,
+                    _current_user(),
+                    "team.member_role",
+                    team_id=team_id,
+                    target_user=member_id,
+                    role=role,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Update member role failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_member_required
+    def delete(self, team_id, member_id):
+        """Remove a member. team_admin removes anyone; a member may remove self
+        (leave). Guards the last admin."""
+        user = _current_user()
+        token = getattr(request, "decoded_token", None)
+        is_self = member_id == user
+        if not is_self and not has_team_role(token, team_id, ROLE_TEAM_ADMIN):
+            return {"success": False, "message": "Forbidden"}, 403
+        try:
+            with db_session() as conn:
+                members = TeamMembersRepository(conn)
+                if self._would_orphan_admins(members, team_id, member_id):
+                    return {
+                        "success": False,
+                        "message": "Cannot remove the last team admin",
+                    }, 409
+                members.remove_member(team_id, member_id)
+                _audit(
+                    conn,
+                    user,
+                    "team.member_remove",
+                    team_id=team_id,
+                    target_user=member_id,
+                    self_leave=is_self,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Remove member failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @staticmethod
+    def _would_orphan_admins(
+        members: TeamMembersRepository, team_id: str, member_id: str
+    ) -> bool:
+        """True if removing/demoting ``member_id`` leaves the team with no admin.
+
+        Locks the admin rows (FOR UPDATE) so concurrent demote/remove calls
+        serialize — preventing two simultaneous removals of distinct admins from
+        both passing the guard and orphaning the team.
+        """
+        admins = members.lock_admins(team_id)
+        if member_id not in admins:
+            return False
+        return len(admins) <= 1
+
+
+@teams_ns.route("/teams/<string:team_id>/grants")
+class TeamGrants(Resource):
+    @team_member_required
+    def get(self, team_id):
+        """List resources shared with this team. Requires membership."""
+        resource_type = request.args.get("resource_type")
+        try:
+            with db_readonly() as conn:
+                grants = TeamResourceGrantsRepository(conn).list_for_team(
+                    team_id, resource_type
+                )
+            return make_response(jsonify({"success": True, "grants": grants}), 200)
+        except Exception as err:
+            logger.error("List grants failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_member_required
+    def post(self, team_id):
+        """Share a resource the caller OWNS with this team (additive visibility)."""
+        user = _current_user()
+        data = request.get_json(silent=True) or {}
+        resource_type = data.get("resource_type")
+        resource_id = data.get("resource_id")
+        access_level = data.get("access_level", "viewer")
+        # None → share with the whole team; a sub → share with that one member.
+        target_user_id = (data.get("target_user_id") or "").strip() or None
+        if (
+            not is_valid_resource_type(resource_type)
+            or not resource_id
+            or not looks_like_uuid(resource_id)
+        ):
+            return {"success": False, "message": "invalid resource"}, 400
+        if access_level not in _VALID_ACCESS_LEVELS:
+            return {"success": False, "message": "invalid access_level"}, 400
+        try:
+            with db_session() as conn:
+                # Ownership is the security boundary: dispatch by resource_type so
+                # a mismatched type/id can't register a bogus grant.
+                if not owns_resource(conn, resource_type, resource_id, user):
+                    return {"success": False, "message": "Not the resource owner"}, 403
+                # A per-member share target must actually be a member of the team.
+                if target_user_id and not TeamMembersRepository(conn).is_member(
+                    target_user_id, team_id
+                ):
+                    return {"success": False, "message": "Target is not a team member"}, 400
+                grant = TeamResourceGrantsRepository(conn).grant(
+                    team_id,
+                    resource_type,
+                    resource_id,
+                    owner_id=user,
+                    granted_by=user,
+                    access_level=access_level,
+                    target_user_id=target_user_id,
+                )
+                _audit(
+                    conn,
+                    user,
+                    "team.share",
+                    team_id=team_id,
+                    resource_type=resource_type,
+                    resource_id=resource_id,
+                    access_level=access_level,
+                    target_user_id=target_user_id,
+                )
+            # Post-commit, best-effort: tell the recipient(s) it was shared.
+            _notify_resource_shared(
+                user, team_id, resource_type, resource_id, access_level, target_user_id
+            )
+            return make_response(jsonify({"success": True, "grant": grant}), 201)
+        except Exception as err:
+            logger.error("Share resource failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_member_required
+    def delete(self, team_id):
+        """Unshare a resource. Allowed for the resource owner or a team_admin.
+
+        Identifiers come from query params (some proxies strip DELETE bodies),
+        with a JSON-body fallback for older clients.
+        """
+        user = _current_user()
+        token = getattr(request, "decoded_token", None)
+        data = request.get_json(silent=True) or {}
+        resource_type = request.args.get("resource_type") or data.get("resource_type")
+        resource_id = request.args.get("resource_id") or data.get("resource_id")
+        # Which grant to remove: whole-team (None) or a specific member's.
+        target_user_id = (
+            request.args.get("target_user_id") or data.get("target_user_id") or ""
+        ).strip() or None
+        if (
+            not is_valid_resource_type(resource_type)
+            or not resource_id
+            or not looks_like_uuid(resource_id)
+        ):
+            return {"success": False, "message": "invalid resource"}, 400
+        try:
+            with db_session() as conn:
+                is_owner = owns_resource(conn, resource_type, resource_id, user)
+                if not is_owner and not has_team_role(token, team_id, ROLE_TEAM_ADMIN):
+                    return {"success": False, "message": "Forbidden"}, 403
+                revoked = TeamResourceGrantsRepository(conn).revoke(
+                    team_id, resource_type, resource_id, target_user_id=target_user_id
+                )
+                if revoked:
+                    _audit(
+                        conn,
+                        user,
+                        "team.unshare",
+                        team_id=team_id,
+                        resource_type=resource_type,
+                        resource_id=resource_id,
+                        target_user_id=target_user_id,
+                    )
+            return make_response(jsonify({"success": bool(revoked)}), 200)
+        except Exception as err:
+            logger.error("Unshare resource failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>/transfer_owner")
+class TeamOwnerTransfer(Resource):
+    def post(self, team_id):
+        """Transfer team ownership. Owner-only (global admin overrides).
+
+        The new owner must already be a member; they are promoted to team_admin.
+        """
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        data = request.get_json(silent=True) or {}
+        new_owner = (data.get("user_id") or "").strip()
+        if not new_owner:
+            return {"success": False, "message": "user_id required"}, 400
+        token = getattr(request, "decoded_token", None)
+        try:
+            with db_session() as conn:
+                teams = TeamsRepository(conn)
+                team = teams.get(team_id)
+                if not team:
+                    return {"success": False, "message": "Not found"}, 404
+                if team["owner_id"] != user and not has_role(token, ROLE_ADMIN):
+                    return {"success": False, "message": "Only the team owner can transfer"}, 403
+                members = TeamMembersRepository(conn)
+                if not members.is_member(new_owner, team_id):
+                    return {"success": False, "message": "New owner must be a member"}, 400
+                members.set_manual_role(team_id, new_owner, ROLE_TEAM_ADMIN, granted_by=user)
+                teams.reassign_owner(team_id, new_owner)
+                _audit(
+                    conn,
+                    user,
+                    "team.transfer_owner",
+                    team_id=team_id,
+                    new_owner=new_owner,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Transfer owner failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/resource_shares")
+class ResourceShares(Resource):
+    def get(self):
+        """List the teams a resource the caller OWNS is shared with.
+
+        Powers the share dialog (show current shares + unshare). Owner-only so a
+        non-owner can't enumerate a resource's sharing graph.
+        """
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        resource_type = request.args.get("resource_type")
+        resource_id = request.args.get("resource_id")
+        if (
+            not is_valid_resource_type(resource_type)
+            or not resource_id
+            or not looks_like_uuid(resource_id)
+        ):
+            return {"success": False, "message": "invalid resource"}, 400
+        try:
+            with db_readonly() as conn:
+                if not owns_resource(conn, resource_type, resource_id, user):
+                    return {"success": False, "message": "Not the resource owner"}, 403
+                shares = TeamResourceGrantsRepository(conn).list_for_resource(
+                    resource_type, resource_id
+                )
+            return make_response(jsonify({"success": True, "shares": shares}), 200)
+        except Exception as err:
+            logger.error("List resource shares failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/admin/teams")
+class AllTeams(Resource):
+    method_decorators = []
+
+    def get(self):
+        """Global-admin oversight: every team with member counts."""
+        token = getattr(request, "decoded_token", None)
+        if not token:
+            return {"success": False}, 401
+        if not has_role(token, ROLE_ADMIN):
+            return {"success": False, "message": "Forbidden"}, 403
+        try:
+            with db_readonly() as conn:
+                teams = TeamsRepository(conn).list_all()
+            return make_response(jsonify({"success": True, "teams": teams}), 200)
+        except Exception as err:
+            logger.error("List all teams failed: %s", err, exc_info=True)
+            return {"success": False}, 400

application/api/user/tools/__init__.py

@@ -0,0 +1,6 @@
+"""Tools module."""
+
+from .mcp import tools_mcp_ns
+from .routes import tools_ns
+
+__all__ = ["tools_ns", "tools_mcp_ns"]

application/api/user/tools/mcp.py

@@ -0,0 +1,516 @@
+"""Tool management MCP server integration."""
+
+from urllib.parse import urlencode, urlparse
+
+from flask import current_app, jsonify, make_response, redirect, request
+from flask_restx import Namespace, Resource, fields
+
+from application.agents.tools.mcp_tool import MCPOAuthManager, MCPTool
+from application.api import api
+from application.api.user.tools.routes import transform_actions
+from application.cache import get_redis_instance
+from application.core.url_validation import SSRFError, validate_url
+from application.security.encryption import decrypt_credentials, encrypt_credentials
+from application.storage.db.repositories.connector_sessions import (
+    ConnectorSessionsRepository,
+)
+from application.storage.db.repositories.user_tools import UserToolsRepository
+from application.storage.db.session import db_readonly, db_session
+from application.utils import check_required_fields
+
+tools_mcp_ns = Namespace("tools", description="Tool management operations", path="/api")
+
+_ALLOWED_TRANSPORTS = {"auto", "sse", "http"}
+
+
+def _sanitize_mcp_transport(config):
+    """Normalise and validate the transport_type field.
+
+    Strips ``command`` / ``args`` keys that are only valid for local STDIO
+    transports and returns the cleaned transport type string.
+    """
+    transport_type = (config.get("transport_type") or "auto").lower()
+    if transport_type not in _ALLOWED_TRANSPORTS:
+        raise ValueError(f"Unsupported transport_type: {transport_type}")
+    config.pop("command", None)
+    config.pop("args", None)
+    config["transport_type"] = transport_type
+    return transport_type
+
+
+def _extract_auth_credentials(config):
+    """Build an ``auth_credentials`` dict from the raw MCP config."""
+    auth_credentials = {}
+    auth_type = config.get("auth_type", "none")
+
+    if auth_type == "api_key":
+        if config.get("api_key"):
+            auth_credentials["api_key"] = config["api_key"]
+        if config.get("api_key_header"):
+            auth_credentials["api_key_header"] = config["api_key_header"]
+    elif auth_type == "bearer":
+        if config.get("bearer_token"):
+            auth_credentials["bearer_token"] = config["bearer_token"]
+    elif auth_type == "basic":
+        if config.get("username"):
+            auth_credentials["username"] = config["username"]
+        if config.get("password"):
+            auth_credentials["password"] = config["password"]
+
+    return auth_credentials
+
+
+def _validate_mcp_server_url(config: dict) -> None:
+    """Validate the server_url in an MCP config to prevent SSRF.
+
+    Raises:
+        ValueError: If the URL is missing or points to a blocked address.
+    """
+    server_url = (config.get("server_url") or "").strip()
+    if not server_url:
+        raise ValueError("server_url is required")
+    try:
+        validate_url(server_url)
+    except SSRFError as exc:
+        raise ValueError(f"Invalid server URL: {exc}") from exc
+
+
+@tools_mcp_ns.route("/mcp_server/test")
+class TestMCPServerConfig(Resource):
+    @api.expect(
+        api.model(
+            "MCPServerTestModel",
+            {
+                "config": fields.Raw(
+                    required=True, description="MCP server configuration to test"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Test MCP server connection with provided configuration")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+
+        required_fields = ["config"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            config = data["config"]
+            try:
+                _sanitize_mcp_transport(config)
+            except ValueError:
+                return make_response(
+                    jsonify({"success": False, "error": "Unsupported transport_type"}),
+                    400,
+                )
+
+            _validate_mcp_server_url(config)
+
+            auth_credentials = _extract_auth_credentials(config)
+            test_config = config.copy()
+            test_config["auth_credentials"] = auth_credentials
+
+            mcp_tool = MCPTool(config=test_config, user_id=user)
+            result = mcp_tool.test_connection()
+
+            if result.get("requires_oauth"):
+                safe_result = {
+                    k: v
+                    for k, v in result.items()
+                    if k in ("success", "requires_oauth", "auth_url")
+                }
+                return make_response(jsonify(safe_result), 200)
+
+            if not result.get("success"):
+                current_app.logger.error(
+                    f"MCP connection test failed: {result.get('message')}"
+                )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Connection test failed",
+                            "tools_count": 0,
+                        }
+                    ),
+                    200,
+                )
+
+            safe_result = {
+                "success": True,
+                "message": result.get("message", "Connection successful"),
+                "tools_count": result.get("tools_count", 0),
+                "tools": result.get("tools", []),
+            }
+            return make_response(jsonify(safe_result), 200)
+        except ValueError as e:
+            current_app.logger.warning(f"Invalid MCP server test request: {e}")
+            return make_response(
+                jsonify({"success": False, "error": "Invalid MCP server configuration"}),
+                400,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error testing MCP server: {e}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "error": "Connection test failed"}),
+                500,
+            )
+
+
+@tools_mcp_ns.route("/mcp_server/save")
+class MCPServerSave(Resource):
+    @api.expect(
+        api.model(
+            "MCPServerSaveModel",
+            {
+                "id": fields.String(
+                    required=False, description="Tool ID for updates (optional)"
+                ),
+                "displayName": fields.String(
+                    required=True, description="Display name for the MCP server"
+                ),
+                "config": fields.Raw(
+                    required=True, description="MCP server configuration"
+                ),
+                "status": fields.Boolean(
+                    required=False, default=True, description="Tool status"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Create or update MCP server with automatic tool discovery")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+
+        required_fields = ["displayName", "config"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            config = data["config"]
+            try:
+                _sanitize_mcp_transport(config)
+            except ValueError:
+                return make_response(
+                    jsonify({"success": False, "error": "Unsupported transport_type"}),
+                    400,
+                )
+
+            _validate_mcp_server_url(config)
+
+            auth_credentials = _extract_auth_credentials(config)
+            auth_type = config.get("auth_type", "none")
+            mcp_config = config.copy()
+            mcp_config["auth_credentials"] = auth_credentials
+
+            if auth_type == "oauth":
+                if not config.get("oauth_task_id"):
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "Connection not authorized. Please complete the OAuth authorization first.",
+                            }
+                        ),
+                        400,
+                    )
+                redis_client = get_redis_instance()
+                manager = MCPOAuthManager(redis_client)
+                result = manager.get_oauth_status(
+                    config["oauth_task_id"], user
+                )
+                if not result.get("status") == "completed":
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "OAuth failed or not completed. Please try authorizing again.",
+                            }
+                        ),
+                        400,
+                    )
+                actions_metadata = result.get("tools", [])
+            elif auth_type == "none" or auth_credentials:
+                mcp_tool = MCPTool(config=mcp_config, user_id=user)
+                mcp_tool.discover_tools()
+                actions_metadata = mcp_tool.get_actions_metadata()
+            else:
+                raise Exception(
+                    "No valid credentials provided for the selected authentication type"
+                )
+            storage_config = config.copy()
+
+            tool_id = data.get("id")
+            existing_doc = None
+            existing_encrypted = None
+            if tool_id:
+                with db_readonly() as conn:
+                    repo = UserToolsRepository(conn)
+                    existing_doc = repo.get_any(tool_id, user)
+                if existing_doc and existing_doc.get("name") == "mcp_tool":
+                    existing_encrypted = (existing_doc.get("config") or {}).get(
+                        "encrypted_credentials"
+                    )
+                else:
+                    existing_doc = None
+
+            if auth_credentials:
+                if existing_encrypted:
+                    existing_secrets = decrypt_credentials(existing_encrypted, user)
+                    existing_secrets.update(auth_credentials)
+                    auth_credentials = existing_secrets
+                storage_config["encrypted_credentials"] = encrypt_credentials(
+                    auth_credentials, user
+                )
+            elif existing_encrypted:
+                storage_config["encrypted_credentials"] = existing_encrypted
+
+            for field in [
+                "api_key",
+                "bearer_token",
+                "username",
+                "password",
+                "api_key_header",
+                "redirect_uri",
+            ]:
+                storage_config.pop(field, None)
+            transformed_actions = transform_actions(actions_metadata)
+
+            display_name = data["displayName"]
+            description = f"MCP Server: {storage_config.get('server_url', 'Unknown')}"
+            status_bool = bool(data.get("status", True))
+
+            with db_session() as conn:
+                repo = UserToolsRepository(conn)
+                if existing_doc:
+                    repo.update(
+                        str(existing_doc["id"]), user,
+                        {
+                            "display_name": display_name,
+                            "custom_name": display_name,
+                            "description": description,
+                            "config": storage_config,
+                            "actions": transformed_actions,
+                            "status": status_bool,
+                        },
+                    )
+                    saved_id = str(existing_doc["id"])
+                    response_data = {
+                        "success": True,
+                        "id": saved_id,
+                        "message": f"MCP server updated successfully! Discovered {len(transformed_actions)} tools.",
+                        "tools_count": len(transformed_actions),
+                    }
+                else:
+                    # Fall back to find_by_user_and_name — the original
+                    # dual-write path also ran an existence check before
+                    # deciding between insert and update.
+                    existing_by_name = repo.find_by_user_and_name(user, "mcp_tool")
+                    if tool_id is None and existing_by_name and (
+                        (existing_by_name.get("config") or {}).get("server_url")
+                        == storage_config.get("server_url")
+                    ):
+                        repo.update(
+                            str(existing_by_name["id"]), user,
+                            {
+                                "display_name": display_name,
+                                "custom_name": display_name,
+                                "description": description,
+                                "config": storage_config,
+                                "actions": transformed_actions,
+                                "status": status_bool,
+                            },
+                        )
+                        saved_id = str(existing_by_name["id"])
+                        response_data = {
+                            "success": True,
+                            "id": saved_id,
+                            "message": f"MCP server updated successfully! Discovered {len(transformed_actions)} tools.",
+                            "tools_count": len(transformed_actions),
+                        }
+                    else:
+                        created = repo.create(
+                            user, "mcp_tool",
+                            config=storage_config,
+                            custom_name=display_name,
+                            display_name=display_name,
+                            description=description,
+                            config_requirements={},
+                            actions=transformed_actions,
+                            status=status_bool,
+                        )
+                        saved_id = str(created["id"])
+                        response_data = {
+                            "success": True,
+                            "id": saved_id,
+                            "message": f"MCP server created successfully! Discovered {len(transformed_actions)} tools.",
+                            "tools_count": len(transformed_actions),
+                        }
+                    if tool_id and existing_doc is None:
+                        # Client requested update on a non-existent tool id.
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": False,
+                                    "error": "Tool not found or access denied",
+                                }
+                            ),
+                            404,
+                        )
+            return make_response(jsonify(response_data), 200)
+        except ValueError as e:
+            current_app.logger.warning(f"Invalid MCP server save request: {e}")
+            return make_response(
+                jsonify({"success": False, "error": "Invalid MCP server configuration"}),
+                400,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error saving MCP server: {e}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "error": "Failed to save MCP server"}),
+                500,
+            )
+
+
+@tools_mcp_ns.route("/mcp_server/callback")
+class MCPOAuthCallback(Resource):
+    @api.expect(
+        api.model(
+            "MCPServerCallbackModel",
+            {
+                "code": fields.String(required=True, description="Authorization code"),
+                "state": fields.String(required=True, description="State parameter"),
+                "error": fields.String(
+                    required=False, description="Error message (if any)"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Handle OAuth callback by providing the authorization code and state"
+    )
+    def get(self):
+        code = request.args.get("code")
+        state = request.args.get("state")
+        error = request.args.get("error")
+
+        if error:
+            params = {
+                "status": "error",
+                "message": f"OAuth error: {error}. Please try again and make sure to grant all requested permissions, including offline access.",
+                "provider": "mcp_tool",
+            }
+            return redirect(f"/api/connectors/callback-status?{urlencode(params)}")
+        if not code or not state:
+            return redirect(
+                "/api/connectors/callback-status?status=error&message=Authorization+code+or+state+not+provided.+Please+complete+the+authorization+process+and+make+sure+to+grant+offline+access.&provider=mcp_tool"
+            )
+        try:
+            redis_client = get_redis_instance()
+            if not redis_client:
+                return redirect(
+                    "/api/connectors/callback-status?status=error&message=Internal+server+error:+Redis+not+available.&provider=mcp_tool"
+                )
+            manager = MCPOAuthManager(redis_client)
+            success = manager.handle_oauth_callback(state, code, error)
+            if success:
+                return redirect(
+                    "/api/connectors/callback-status?status=success&message=Authorization+code+received+successfully.+You+can+close+this+window.&provider=mcp_tool"
+                )
+            else:
+                return redirect(
+                    "/api/connectors/callback-status?status=error&message=OAuth+callback+failed.&provider=mcp_tool"
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error handling MCP OAuth callback: {str(e)}", exc_info=True
+            )
+            return redirect(
+                "/api/connectors/callback-status?status=error&message=Internal+server+error.&provider=mcp_tool"
+            )
+
+
+@tools_mcp_ns.route("/mcp_server/auth_status")
+class MCPAuthStatus(Resource):
+    @api.doc(
+        description="Batch check auth status for all MCP tools. "
+        "Lightweight DB-only check — no network calls to MCP servers."
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                tools_repo = UserToolsRepository(conn)
+                sessions_repo = ConnectorSessionsRepository(conn)
+                all_tools = tools_repo.list_for_user(user)
+                mcp_tools = [t for t in all_tools if t.get("name") == "mcp_tool"]
+                if not mcp_tools:
+                    return make_response(
+                        jsonify({"success": True, "statuses": {}}), 200
+                    )
+
+                oauth_server_urls: dict = {}
+                statuses: dict = {}
+                for tool in mcp_tools:
+                    tool_id = str(tool["id"])
+                    config = tool.get("config") or {}
+                    auth_type = config.get("auth_type", "none")
+                    if auth_type == "oauth":
+                        server_url = config.get("server_url", "")
+                        if server_url:
+                            parsed = urlparse(server_url)
+                            base_url = f"{parsed.scheme}://{parsed.netloc}"
+                            oauth_server_urls[tool_id] = base_url
+                        else:
+                            statuses[tool_id] = "needs_auth"
+                    else:
+                        statuses[tool_id] = "configured"
+
+                if oauth_server_urls:
+                    # Look up a session per distinct base URL. MCP sessions
+                    # are stored with ``provider = "mcp:<server_url>"``
+                    # and the URL in ``server_url``; reuse the repo's
+                    # per-URL accessor rather than an ad-hoc $in query.
+                    url_has_tokens: dict = {}
+                    for base_url in set(oauth_server_urls.values()):
+                        session = sessions_repo.get_by_user_and_server_url(
+                            user, base_url,
+                        )
+                        tokens = (
+                            (session or {}).get("session_data", {}) or {}
+                        ).get("tokens", {}) or {}
+                        # MCP code also stashes tokens into token_info on
+                        # the row; consider either present as "connected".
+                        token_info = (session or {}).get("token_info") or {}
+                        url_has_tokens[base_url] = bool(
+                            tokens.get("access_token")
+                            or token_info.get("access_token")
+                        )
+
+                    for tool_id, base_url in oauth_server_urls.items():
+                        if url_has_tokens.get(base_url):
+                            statuses[tool_id] = "connected"
+                        else:
+                            statuses[tool_id] = "needs_auth"
+
+            return make_response(jsonify({"success": True, "statuses": statuses}), 200)
+        except Exception as e:
+            current_app.logger.error(
+                "Error checking MCP auth status: %s", e, exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "error": "Failed to check auth status"}),
+                500,
+            )

application/api/user/tools/routes.py

@@ -0,0 +1,995 @@
+"""Tool management routes."""
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.agents.default_tools import (
+    builtin_agent_tools_for_management,
+    BUILTIN_AGENT_TOOLS,
+    default_tool_name_for_id,
+    default_tools_for_management,
+    is_builtin_agent_tool_id,
+    is_default_tool_id,
+    is_synthesized_tool_id,
+    WORKFLOW_ONLY_BUILTINS,
+)
+from application.agents.tools.spec_parser import parse_spec
+from application.agents.tools.tool_manager import ToolManager
+from application.api import api
+from application.api.user.artifacts.authz import Principal, authorize_artifact
+from application.api.user.team_sharing import effective_write_owner, visible_with_access
+from application.core.url_validation import SSRFError, validate_url
+from application.security.encryption import decrypt_credentials, encrypt_credentials
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.artifacts import ArtifactsRepository
+from application.storage.db.repositories.notes import NotesRepository
+from application.storage.db.repositories.todos import TodosRepository
+from application.storage.db.repositories.user_tools import UserToolsRepository
+from application.storage.db.repositories.users import UsersRepository
+from application.storage.db.session import db_readonly, db_session
+from application.utils import check_required_fields, validate_function_name
+
+tool_config = {}
+tool_manager = ToolManager(config=tool_config)
+
+
+# ---------------------------------------------------------------------------
+# Shape translation helpers
+# ---------------------------------------------------------------------------
+# The frontend speaks camelCase (``displayName`` / ``customName`` /
+# ``configRequirements``). The PG ``user_tools`` table stores snake_case
+# (``display_name`` / ``custom_name`` / ``config_requirements``). Keep the
+# translation localized to this module so repositories stay pure.
+
+_CAMEL_TO_SNAKE = {
+    "displayName": "display_name",
+    "customName": "custom_name",
+    "configRequirements": "config_requirements",
+}
+_SNAKE_TO_CAMEL = {v: k for k, v in _CAMEL_TO_SNAKE.items()}
+
+
+def _row_to_api(row: dict) -> dict:
+    """Rename DB-native snake_case keys to the camelCase shape the frontend expects."""
+    out = dict(row)
+    for snake, camel in _SNAKE_TO_CAMEL.items():
+        if snake in out:
+            out[camel] = out.pop(snake)
+    # ``user_id`` is exposed as ``user`` in the legacy API shape.
+    if "user_id" in out:
+        out["user"] = out.pop("user_id")
+    return out
+
+
+def _api_to_update_fields(data: dict) -> dict:
+    """Rename incoming camelCase update keys to the repo's snake_case columns."""
+    fields_out: dict = {}
+    for key, value in data.items():
+        fields_out[_CAMEL_TO_SNAKE.get(key, key)] = value
+    return fields_out
+
+
+def _encrypt_secret_fields(config, config_requirements, user_id):
+    secret_keys = [
+        key for key, spec in config_requirements.items()
+        if spec.get("secret") and key in config and config[key]
+    ]
+    if not secret_keys:
+        return config
+
+    storage_config = config.copy()
+    secret_values = {k: config[k] for k in secret_keys}
+    storage_config["encrypted_credentials"] = encrypt_credentials(secret_values, user_id)
+    for key in secret_keys:
+        storage_config.pop(key, None)
+    return storage_config
+
+
+def _validate_config(config, config_requirements, has_existing_secrets=False):
+    errors = {}
+    for key, spec in config_requirements.items():
+        depends_on = spec.get("depends_on")
+        if depends_on:
+            if not all(config.get(dk) == dv for dk, dv in depends_on.items()):
+                continue
+        if spec.get("required") and not config.get(key):
+            if has_existing_secrets and spec.get("secret"):
+                continue
+            errors[key] = f"{spec.get('label', key)} is required"
+        value = config.get(key)
+        if value is not None and value != "":
+            if spec.get("type") == "number":
+                try:
+                    num = float(value)
+                    if key == "timeout" and (num < 1 or num > 300):
+                        errors[key] = "Timeout must be between 1 and 300"
+                except (ValueError, TypeError):
+                    errors[key] = f"{spec.get('label', key)} must be a number"
+            if spec.get("enum") and value not in spec["enum"]:
+                errors[key] = f"Invalid value for {spec.get('label', key)}"
+    return errors
+
+
+def _merge_secrets_on_update(new_config, existing_config, config_requirements, user_id):
+    """Merge incoming config with existing encrypted secrets and re-encrypt.
+
+    For updates, the client may omit unchanged secret values.  This helper
+    decrypts any previously stored secrets, overlays whatever the client *did*
+    send, strips plain-text secrets from the stored config, and re-encrypts
+    the merged result.
+
+    Returns the final ``config`` dict ready for persistence.
+    """
+    secret_keys = [
+        key for key, spec in config_requirements.items()
+        if spec.get("secret")
+    ]
+
+    if not secret_keys:
+        return new_config
+
+    existing_secrets = {}
+    if "encrypted_credentials" in existing_config:
+        existing_secrets = decrypt_credentials(
+            existing_config["encrypted_credentials"], user_id
+        )
+
+    merged_secrets = existing_secrets.copy()
+    for key in secret_keys:
+        if key in new_config and new_config[key]:
+            merged_secrets[key] = new_config[key]
+
+    # Start from existing non-secret values, then overlay incoming non-secrets
+    storage_config = {
+        k: v for k, v in existing_config.items()
+        if k not in secret_keys and k != "encrypted_credentials"
+    }
+    storage_config.update(
+        {k: v for k, v in new_config.items() if k not in secret_keys}
+    )
+
+    if merged_secrets:
+        storage_config["encrypted_credentials"] = encrypt_credentials(
+            merged_secrets, user_id
+        )
+    else:
+        storage_config.pop("encrypted_credentials", None)
+
+    storage_config.pop("has_encrypted_credentials", None)
+    return storage_config
+
+
+def transform_actions(actions_metadata):
+    """Set default flags on action metadata for storage.
+
+    Marks each action as active, sets ``filled_by_llm`` and ``value`` on every
+    parameter property. Used by both the generic create_tool and MCP save routes.
+    """
+    transformed = []
+    for action in actions_metadata:
+        action["active"] = True
+        if "parameters" in action:
+            props = action["parameters"].get("properties", {})
+            for param_details in props.values():
+                param_details["filled_by_llm"] = True
+                param_details["value"] = ""
+        transformed.append(action)
+    return transformed
+
+
+tools_ns = Namespace("tools", description="Tool management operations", path="/api")
+
+
+@tools_ns.route("/available_tools")
+class AvailableTools(Resource):
+    @api.doc(description="Get available tools for a user")
+    def get(self):
+        if not request.decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        try:
+            tools_metadata = []
+            for tool_name, tool_instance in tool_manager.tools.items():
+                doc = tool_instance.__doc__.strip()
+                lines = doc.split("\n", 1)
+                name = lines[0].strip()
+                description = lines[1].strip() if len(lines) > 1 else ""
+                config_req = tool_instance.get_config_requirements()
+                actions = tool_instance.get_actions_metadata()
+                tools_metadata.append(
+                    {
+                        "name": tool_name,
+                        "displayName": name,
+                        "description": description,
+                        "configRequirements": config_req,
+                        "actions": actions,
+                    }
+                )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting available tools: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True, "data": tools_metadata}), 200)
+
+
+@tools_ns.route("/get_tools")
+class GetTools(Resource):
+    @api.doc(description="Get tools created by a user")
+    def get(self):
+        try:
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False}), 401)
+            user = decoded_token.get("sub")
+            with db_readonly() as conn:
+                tools_repo = UserToolsRepository(conn)
+                rows = tools_repo.list_for_user(user)
+                user_doc = UsersRepository(conn).get(user)
+                owned_ids = {str(r["id"]) for r in rows}
+                # Tools shared with the caller's teams. Secrets are stripped
+                # unconditionally below — a grantee never sees the owner's
+                # credentials (they only run with the owner's creds server-side).
+                team_shared = visible_with_access(conn, user, "tool")
+                shared_ids = [tid for tid in team_shared if tid not in owned_ids]
+                shared_rows = tools_repo.list_by_ids(shared_ids)
+            user_tools = []
+
+            def _shape_tool(row, *, ownership="user", force_strip_secret=False):
+                tool_copy = _row_to_api(row)
+                config_req = tool_copy.get("configRequirements", {})
+                if not config_req:
+                    tool_instance = tool_manager.tools.get(tool_copy.get("name"))
+                    if tool_instance:
+                        config_req = tool_instance.get_config_requirements()
+                        tool_copy["configRequirements"] = config_req
+                has_secrets = any(
+                    spec.get("secret") for spec in config_req.values()
+                ) if config_req else False
+                if (has_secrets or force_strip_secret) and "encrypted_credentials" in tool_copy.get(
+                    "config", {}
+                ):
+                    tool_copy["config"]["has_encrypted_credentials"] = True
+                    tool_copy["config"].pop("encrypted_credentials", None)
+                tool_copy["ownership"] = ownership
+                return tool_copy
+
+            for row in rows:
+                user_tools.append(_shape_tool(row))
+            for row in shared_rows:
+                shaped = _shape_tool(row, ownership="team", force_strip_secret=True)
+                shaped["team_access"] = team_shared.get(str(row["id"]))
+                user_tools.append(shaped)
+
+            # ``scheduler`` is dual-registered (default chat tool + agent-
+            # selectable builtin) and resolves to the same synthetic uuid5 id.
+            # Surface a single row with both flags so the frontend can show it
+            # in the management page (toggle) and the agent picker.
+            seen_ids: set = set()
+            for default_row in default_tools_for_management(user_doc):
+                default_copy = _row_to_api(default_row)
+                default_copy["default"] = True
+                if default_copy.get("name") in BUILTIN_AGENT_TOOLS:
+                    default_copy["builtin"] = True
+                seen_ids.add(str(default_copy["id"]))
+                user_tools.append(default_copy)
+            # Builtins (e.g. scheduler) hidden from Add-Tool catalog, visible
+            # to the agent picker. Skip ones already added via the default
+            # path — both registries share ``_DEFAULT_TOOL_NAMESPACE``.
+            # ``workflow_only`` builtins (e.g. ``read_document``) carry that
+            # flag so the classic picker can hide them and the workflow node
+            # picker can keep them.
+            for builtin_row in builtin_agent_tools_for_management():
+                builtin_copy = _row_to_api(builtin_row)
+                if str(builtin_copy["id"]) in seen_ids:
+                    continue
+                builtin_copy["builtin"] = True
+                builtin_copy["default"] = False
+                builtin_copy["workflow_only"] = (
+                    builtin_copy.get("name") in WORKFLOW_ONLY_BUILTINS
+                )
+                user_tools.append(builtin_copy)
+        except Exception as err:
+            current_app.logger.error(f"Error getting user tools: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True, "tools": user_tools}), 200)
+
+
+@tools_ns.route("/create_tool")
+class CreateTool(Resource):
+    @api.expect(
+        api.model(
+            "CreateToolModel",
+            {
+                "name": fields.String(required=True, description="Name of the tool"),
+                "displayName": fields.String(
+                    required=True, description="Display name for the tool"
+                ),
+                "description": fields.String(
+                    required=True, description="Tool description"
+                ),
+                "config": fields.Raw(
+                    required=True, description="Configuration of the tool"
+                ),
+                "customName": fields.String(
+                    required=False, description="Custom name for the tool"
+                ),
+                "status": fields.Boolean(
+                    required=True, description="Status of the tool"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Create a new tool")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = [
+            "name",
+            "displayName",
+            "description",
+            "config",
+            "status",
+        ]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            if data["name"] == "mcp_tool":
+                server_url = (data.get("config", {}).get("server_url") or "").strip()
+                if server_url:
+                    try:
+                        validate_url(server_url)
+                    except SSRFError:
+                        return make_response(
+                            jsonify({"success": False, "message": "Invalid server URL"}),
+                            400,
+                        )
+            tool_instance = tool_manager.tools.get(data["name"])
+            if not tool_instance:
+                return make_response(
+                    jsonify({"success": False, "message": "Tool not found"}), 404
+                )
+            actions_metadata = tool_instance.get_actions_metadata()
+            transformed_actions = transform_actions(actions_metadata)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting tool actions: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        try:
+            config_requirements = tool_instance.get_config_requirements()
+            if config_requirements:
+                validation_errors = _validate_config(
+                    data["config"], config_requirements
+                )
+                if validation_errors:
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "message": "Validation failed",
+                                "errors": validation_errors,
+                            }
+                        ),
+                        400,
+                    )
+            storage_config = _encrypt_secret_fields(
+                data["config"], config_requirements, user
+            )
+            with db_session() as conn:
+                created = UserToolsRepository(conn).create(
+                    user,
+                    data["name"],
+                    config=storage_config,
+                    custom_name=data.get("customName", ""),
+                    display_name=data["displayName"],
+                    description=data["description"],
+                    config_requirements=config_requirements,
+                    actions=transformed_actions,
+                    status=bool(data.get("status", True)),
+                )
+            new_id = str(created["id"])
+        except Exception as err:
+            current_app.logger.error(f"Error creating tool: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"id": new_id}), 200)
+
+
+@tools_ns.route("/update_tool")
+class UpdateTool(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "name": fields.String(description="Name of the tool"),
+                "displayName": fields.String(description="Display name for the tool"),
+                "customName": fields.String(description="Custom name for the tool"),
+                "description": fields.String(description="Tool description"),
+                "config": fields.Raw(description="Configuration of the tool"),
+                "actions": fields.List(
+                    fields.Raw, description="Actions the tool can perform"
+                ),
+                "status": fields.Boolean(description="Status of the tool"),
+            },
+        )
+    )
+    @api.doc(description="Update a tool by ID")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        # Default-tool branch first: a dual-registered tool (e.g. ``scheduler``)
+        # matches BOTH ``is_default_tool_id`` and ``is_builtin_agent_tool_id``.
+        # The toggle in Tools settings is the per-user opt-out for the
+        # agentless default — it must reach the ``set_default_tool_enabled``
+        # path, not the builtin "not editable" reject.
+        if is_default_tool_id(data["id"]):
+            if "status" not in data:
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Default tools are not editable; "
+                            "only their on/off status can be changed.",
+                        }
+                    ),
+                    400,
+                )
+            tool_name = default_tool_name_for_id(data["id"])
+            try:
+                with db_session() as conn:
+                    UsersRepository(conn).set_default_tool_enabled(
+                        user, tool_name, bool(data["status"])
+                    )
+            except Exception as err:
+                current_app.logger.error(
+                    f"Error updating default tool: {err}", exc_info=True
+                )
+                return make_response(jsonify({"success": False}), 400)
+            return make_response(jsonify({"success": True}), 200)
+        if is_builtin_agent_tool_id(data["id"]):
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Built-in agent tools are not editable; "
+                        "add them to an agent via the agent picker.",
+                    }
+                ),
+                400,
+            )
+        try:
+            update_data: dict = {}
+            for key in ("name", "displayName", "customName", "description", "actions"):
+                if key in data:
+                    update_data[key] = data[key]
+            if "config" in data:
+                if "actions" in data["config"]:
+                    for action_name in list(data["config"]["actions"].keys()):
+                        if not validate_function_name(action_name):
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": f"Invalid function name '{action_name}'. Function names must match pattern '^[a-zA-Z0-9_-]+$'.",
+                                        "param": "tools[].function.name",
+                                    }
+                                ),
+                                400,
+                            )
+                with db_session() as conn:
+                    repo = UserToolsRepository(conn)
+                    tool_doc = repo.get_any(data["id"], user)
+                    if not tool_doc:
+                        return make_response(
+                            jsonify({"success": False, "message": "Tool not found"}),
+                            404,
+                        )
+                    tool_name = tool_doc.get("name", data.get("name"))
+                    tool_instance = tool_manager.tools.get(tool_name)
+                    config_requirements = (
+                        tool_instance.get_config_requirements()
+                        if tool_instance
+                        else {}
+                    )
+                    existing_config = tool_doc.get("config", {}) or {}
+                    has_existing_secrets = "encrypted_credentials" in existing_config
+
+                    if config_requirements:
+                        validation_errors = _validate_config(
+                            data["config"], config_requirements,
+                            has_existing_secrets=has_existing_secrets,
+                        )
+                        if validation_errors:
+                            return make_response(
+                                jsonify({
+                                    "success": False,
+                                    "message": "Validation failed",
+                                    "errors": validation_errors,
+                                }),
+                                400,
+                            )
+
+                    update_data["config"] = _merge_secrets_on_update(
+                        data["config"], existing_config, config_requirements, user
+                    )
+                    if "status" in data:
+                        update_data["status"] = bool(data["status"])
+                    repo.update(
+                        str(tool_doc["id"]), user, _api_to_update_fields(update_data),
+                    )
+            else:
+                if "status" in data:
+                    update_data["status"] = bool(data["status"])
+                with db_session() as conn:
+                    repo = UserToolsRepository(conn)
+                    tool_doc = repo.get_any(data["id"], user)
+                    if not tool_doc:
+                        return make_response(
+                            jsonify({"success": False, "message": "Tool not found"}),
+                            404,
+                        )
+                    repo.update(
+                        str(tool_doc["id"]), user, _api_to_update_fields(update_data),
+                    )
+        except Exception as err:
+            current_app.logger.error(f"Error updating tool: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@tools_ns.route("/update_tool_config")
+class UpdateToolConfig(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolConfigModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "config": fields.Raw(
+                    required=True, description="Configuration of the tool"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Update the configuration of a tool")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "config"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        if is_synthesized_tool_id(data["id"]):
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Default and built-in tools are config-free "
+                        "and cannot be configured.",
+                    }
+                ),
+                400,
+            )
+        try:
+            with db_session() as conn:
+                repo = UserToolsRepository(conn)
+                tool_doc = repo.get_any(data["id"], user)
+                if not tool_doc:
+                    return make_response(jsonify({"success": False}), 404)
+
+                tool_name = tool_doc.get("name")
+                if tool_name == "mcp_tool":
+                    server_url = (data["config"].get("server_url") or "").strip()
+                    if server_url:
+                        try:
+                            validate_url(server_url)
+                        except SSRFError:
+                            return make_response(
+                                jsonify({"success": False, "message": "Invalid server URL"}),
+                                400,
+                            )
+                tool_instance = tool_manager.tools.get(tool_name)
+                config_requirements = (
+                    tool_instance.get_config_requirements() if tool_instance else {}
+                )
+                existing_config = tool_doc.get("config", {}) or {}
+                has_existing_secrets = "encrypted_credentials" in existing_config
+
+                if config_requirements:
+                    validation_errors = _validate_config(
+                        data["config"], config_requirements,
+                        has_existing_secrets=has_existing_secrets,
+                    )
+                    if validation_errors:
+                        return make_response(
+                            jsonify({
+                                "success": False,
+                                "message": "Validation failed",
+                                "errors": validation_errors,
+                            }),
+                            400,
+                        )
+
+                final_config = _merge_secrets_on_update(
+                    data["config"], existing_config, config_requirements, user
+                )
+
+                repo.update(str(tool_doc["id"]), user, {"config": final_config})
+        except Exception as err:
+            current_app.logger.error(
+                f"Error updating tool config: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@tools_ns.route("/update_tool_actions")
+class UpdateToolActions(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolActionsModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "actions": fields.List(
+                    fields.Raw,
+                    required=True,
+                    description="Actions the tool can perform",
+                ),
+            },
+        )
+    )
+    @api.doc(description="Update the actions of a tool")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "actions"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        if is_synthesized_tool_id(data["id"]):
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Default and built-in tools' actions are not editable.",
+                    }
+                ),
+                400,
+            )
+        try:
+            with db_session() as conn:
+                repo = UserToolsRepository(conn)
+                tool_doc = repo.get_any(data["id"], user)
+                if tool_doc:
+                    repo.update(str(tool_doc["id"]), user, {"actions": data["actions"]})
+                else:
+                    # Team editor write path (secrets stay owner-only — actions
+                    # carry no credentials, so editing them is safe).
+                    owner = effective_write_owner(conn, "tool", data["id"], user)
+                    if not owner:
+                        return make_response(
+                            jsonify({"success": False, "message": "Tool not found"}),
+                            404,
+                        )
+                    repo.update(data["id"], owner, {"actions": data["actions"]})
+        except Exception as err:
+            current_app.logger.error(
+                f"Error updating tool actions: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@tools_ns.route("/update_tool_status")
+class UpdateToolStatus(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolStatusModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "status": fields.Boolean(
+                    required=True, description="Status of the tool"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Update the status of a tool")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "status"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            # Default branch first so a dual-registered id (e.g. ``scheduler``)
+            # writes the per-user opt-out instead of being rejected as a
+            # not-editable builtin (both predicates match the same uuid5).
+            if is_default_tool_id(data["id"]):
+                tool_name = default_tool_name_for_id(data["id"])
+                with db_session() as conn:
+                    UsersRepository(conn).set_default_tool_enabled(
+                        user, tool_name, bool(data["status"])
+                    )
+                return make_response(jsonify({"success": True}), 200)
+            if is_builtin_agent_tool_id(data["id"]):
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Built-in agent tools have no per-user "
+                            "toggle; add them to an agent via the agent picker.",
+                        }
+                    ),
+                    400,
+                )
+            with db_session() as conn:
+                repo = UserToolsRepository(conn)
+                tool_doc = repo.get_any(data["id"], user)
+                if not tool_doc:
+                    return make_response(
+                        jsonify({"success": False, "message": "Tool not found"}),
+                        404,
+                    )
+                repo.update(
+                    str(tool_doc["id"]), user, {"status": bool(data["status"])},
+                )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error updating tool status: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@tools_ns.route("/delete_tool")
+class DeleteTool(Resource):
+    @api.expect(
+        api.model(
+            "DeleteToolModel",
+            {"id": fields.String(required=True, description="Tool ID")},
+        )
+    )
+    @api.doc(description="Delete a tool by ID")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        if is_synthesized_tool_id(data["id"]):
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Built-in tools cannot be deleted; disable them instead.",
+                    }
+                ),
+                400,
+            )
+        try:
+            with db_session() as conn:
+                repo = UserToolsRepository(conn)
+                tool_doc = repo.get_any(data["id"], user)
+                if not tool_doc:
+                    return make_response(
+                        jsonify({"success": False, "message": "Tool not found"}), 404
+                    )
+                repo.delete(str(tool_doc["id"]), user)
+        except Exception as err:
+            current_app.logger.error(f"Error deleting tool: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@tools_ns.route("/parse_spec")
+class ParseSpec(Resource):
+    @api.doc(
+        description="Parse an API specification (OpenAPI 3.x or Swagger 2.0) and return actions"
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        if "file" in request.files:
+            file = request.files["file"]
+            if not file.filename:
+                return make_response(
+                    jsonify({"success": False, "message": "No file selected"}), 400
+                )
+            try:
+                spec_content = file.read().decode("utf-8")
+            except UnicodeDecodeError:
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid file encoding"}), 400
+                )
+        elif request.is_json:
+            data = request.get_json()
+            spec_content = data.get("spec_content", "")
+        else:
+            return make_response(
+                jsonify({"success": False, "message": "No spec provided"}), 400
+            )
+        if not spec_content or not spec_content.strip():
+            return make_response(
+                jsonify({"success": False, "message": "Empty spec content"}), 400
+            )
+        try:
+            metadata, actions = parse_spec(spec_content)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "metadata": metadata,
+                        "actions": actions,
+                    }
+                ),
+                200,
+            )
+        except ValueError as e:
+            current_app.logger.error(f"Spec validation error: {e}")
+            return make_response(jsonify({"success": False, "error": "Invalid specification format"}), 400)
+        except Exception as err:
+            current_app.logger.error(f"Error parsing spec: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to parse specification"}), 500)
+
+
+@tools_ns.route("/artifact/<artifact_id>")
+class GetArtifact(Resource):
+    @api.doc(description="Get artifact data by artifact ID. Returns all todos for the tool when fetching a todo artifact.")
+    def get(self, artifact_id: str):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user_id = decoded_token.get("sub")
+
+        try:
+            with db_readonly() as conn:
+                notes_repo = NotesRepository(conn)
+                todos_repo = TodosRepository(conn)
+
+                # Artifact IDs may be PG UUIDs (post-cutover) or legacy
+                # Mongo ObjectIds embedded in older conversation history.
+                # Both repos' ``get_any`` handles the id-shape branching
+                # internally so a non-UUID input never reaches
+                # ``CAST(:id AS uuid)`` (which would poison the readonly
+                # transaction and break the fallback below).
+                note_doc = notes_repo.get_any(artifact_id, user_id)
+
+                if note_doc:
+                    content = note_doc.get("note", "") or note_doc.get("content", "")
+                    line_count = len(content.split("\n")) if content else 0
+                    updated = note_doc.get("updated_at")
+                    artifact = {
+                        "artifact_type": "note",
+                        "data": {
+                            "content": content,
+                            "line_count": line_count,
+                            "updated_at": (
+                                updated.isoformat()
+                                if hasattr(updated, "isoformat")
+                                else updated
+                            ),
+                        },
+                    }
+                    return make_response(
+                        jsonify({"success": True, "artifact": artifact}), 200
+                    )
+
+                todo_doc = todos_repo.get_any(artifact_id, user_id)
+                if todo_doc:
+                    tool_id = todo_doc.get("tool_id")
+                    all_todos = todos_repo.list_for_tool(user_id, tool_id) if tool_id else []
+                    items = []
+                    open_count = 0
+                    completed_count = 0
+                    for t in all_todos:
+                        # PG ``todos`` stores a ``completed BOOLEAN`` column;
+                        # the legacy Mongo shape used a ``status`` string.
+                        # Keep the response shape stable by translating here.
+                        status = "completed" if t.get("completed") else "open"
+                        if status == "open":
+                            open_count += 1
+                        else:
+                            completed_count += 1
+                        created = t.get("created_at")
+                        updated = t.get("updated_at")
+                        items.append({
+                            "todo_id": t.get("todo_id"),
+                            "title": t.get("title", ""),
+                            "status": status,
+                            "created_at": (
+                                created.isoformat()
+                                if hasattr(created, "isoformat")
+                                else created
+                            ),
+                            "updated_at": (
+                                updated.isoformat()
+                                if hasattr(updated, "isoformat")
+                                else updated
+                            ),
+                        })
+                    artifact = {
+                        "artifact_type": "todo_list",
+                        "data": {
+                            "items": items,
+                            "total_count": len(items),
+                            "open_count": open_count,
+                            "completed_count": completed_count,
+                        },
+                    }
+                    return make_response(
+                        jsonify({"success": True, "artifact": artifact}), 200
+                    )
+
+                # Generalized document/file artifacts live in the
+                # ``artifacts`` store, authorized by their parent (not user_id).
+                # Shape-gate the UUID lookup so a non-UUID id (legacy note/todo
+                # ObjectId already handled above) never reaches the CAST that
+                # would poison this read-only transaction.
+                artifact_doc = None
+                if looks_like_uuid(artifact_id):
+                    artifacts_repo = ArtifactsRepository(conn)
+                    artifact_doc = artifacts_repo.get_artifact(artifact_id)
+                if artifact_doc and authorize_artifact(
+                    conn, artifact_doc, Principal(user_id=user_id)
+                ):
+                    current = artifacts_repo.get_version(
+                        artifact_id, artifact_doc.get("current_version")
+                    )
+                    artifact = {
+                        "artifact_type": (
+                            "document"
+                            if artifact_doc.get("kind") != "file"
+                            else "file"
+                        ),
+                        "data": {
+                            "id": str(artifact_doc.get("id")),
+                            "kind": artifact_doc.get("kind"),
+                            "title": artifact_doc.get("title"),
+                            "current_version": artifact_doc.get("current_version"),
+                            "mime_type": current.get("mime_type") if current else None,
+                            "filename": current.get("filename") if current else None,
+                            "size": current.get("size") if current else None,
+                            "download_url": (
+                                f"/api/artifacts/{artifact_id}/download"
+                                if current and current.get("storage_path")
+                                else None
+                            ),
+                        },
+                    }
+                    return make_response(
+                        jsonify({"success": True, "artifact": artifact}), 200
+                    )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving artifact: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+
+        return make_response(
+            jsonify({"success": False, "message": "Artifact not found"}), 404
+        )

application/api/user/utils.py

@@ -0,0 +1,75 @@
+"""Centralized utilities for API routes.
+
+Post-Mongo-cutover slim: the old Mongo-shaped helpers (``validate_object_id``,
+``check_resource_ownership``, ``paginated_response``, ``serialize_object_id``,
+``safe_db_operation``, ``validate_enum``, ``extract_sort_params``) have been
+removed — they carried ``bson`` / ``pymongo`` imports and had zero callers.
+"""
+
+from functools import wraps
+from typing import Callable, Optional
+
+from flask import (
+    Response,
+    jsonify,
+    make_response,
+    request,
+)
+
+
+def get_user_id() -> Optional[str]:
+    """Extract user ID from decoded JWT token, or None if unauthenticated."""
+    decoded_token = getattr(request, "decoded_token", None)
+    return decoded_token.get("sub") if decoded_token else None
+
+
+def require_auth(func: Callable) -> Callable:
+    """Decorator to require authentication. Returns 401 when absent."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        user_id = get_user_id()
+        if not user_id:
+            return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def success_response(
+    data=None, message: Optional[str] = None, status: int = 200
+) -> Response:
+    """Shape a successful JSON response."""
+    body = {"success": True}
+    if data is not None:
+        body["data"] = data
+    if message is not None:
+        body["message"] = message
+    return make_response(jsonify(body), status)
+
+
+def error_response(message: str, status: int = 400, **kwargs) -> Response:
+    """Shape an error JSON response; any kwargs are merged into the body."""
+    body = {"success": False, "error": message, **kwargs}
+    return make_response(jsonify(body), status)
+
+
+def require_fields(required: list) -> Callable:
+    """Decorator: return 400 if any listed field is missing/falsy in the JSON body."""
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            data = request.get_json()
+            if not data:
+                return error_response("Request body required")
+            missing = [field for field in required if not data.get(field)]
+            if missing:
+                return error_response(
+                    f"Missing required fields: {', '.join(missing)}"
+                )
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

application/api/user/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .routes import workflows_ns
+
+__all__ = ["workflows_ns"]

application/api/user/workflows/routes.py

@@ -0,0 +1,532 @@
+"""Workflow management routes."""
+
+from typing import Any, Dict, List, Optional, Set
+
+from flask import current_app, request
+from flask_restx import Namespace, Resource
+
+from application.storage.db.base_repository import looks_like_uuid
+from application.storage.db.repositories.workflow_edges import WorkflowEdgesRepository
+from application.storage.db.repositories.workflow_nodes import WorkflowNodesRepository
+from application.storage.db.repositories.workflows import WorkflowsRepository
+from application.storage.db.session import db_readonly, db_session
+from application.core.json_schema_utils import (
+    JsonSchemaValidationError,
+    normalize_json_schema_payload,
+)
+from application.core.model_utils import get_model_capabilities
+from application.api.user.utils import (
+    error_response,
+    get_user_id,
+    require_auth,
+    require_fields,
+    success_response,
+)
+
+workflows_ns = Namespace("workflows", path="/api")
+
+
+def _workflow_error_response(message: str, err: Exception):
+    current_app.logger.error(f"{message}: {err}", exc_info=True)
+    return error_response(message)
+
+
+def _resolve_workflow(repo: WorkflowsRepository, workflow_id: str, user_id: str):
+    """Resolve a workflow by UUID or legacy Mongo id, scoped to user."""
+    if not workflow_id:
+        return None
+    if looks_like_uuid(workflow_id):
+        row = repo.get(workflow_id, user_id)
+        if row is not None:
+            return row
+    return repo.get_by_legacy_id(workflow_id, user_id)
+
+
+def _write_graph(
+    conn,
+    pg_workflow_id: str,
+    graph_version: int,
+    nodes_data: List[Dict],
+    edges_data: List[Dict],
+) -> List[Dict]:
+    """Bulk-create nodes + edges for one graph version. Uses ON CONFLICT upsert.
+
+    Edges arrive with source/target as user-provided node-id strings. We
+    insert nodes first, capture their ``node_id → UUID`` map, then
+    translate edges before insertion. Edges referencing missing nodes are
+    dropped with a warning.
+    """
+    nodes_repo = WorkflowNodesRepository(conn)
+    edges_repo = WorkflowEdgesRepository(conn)
+
+    if nodes_data:
+        created_nodes = nodes_repo.bulk_create(
+            pg_workflow_id, graph_version,
+            [
+                {
+                    "node_id": n["id"],
+                    "node_type": n["type"],
+                    "title": n.get("title", ""),
+                    "description": n.get("description", ""),
+                    "position": n.get("position", {"x": 0, "y": 0}),
+                    "config": n.get("data", {}),
+                }
+                for n in nodes_data
+            ],
+        )
+        node_uuid_by_str = {n["node_id"]: n["id"] for n in created_nodes}
+    else:
+        created_nodes = []
+        node_uuid_by_str = {}
+
+    if edges_data:
+        translated_edges: List[Dict] = []
+        for e in edges_data:
+            src = e.get("source")
+            tgt = e.get("target")
+            from_uuid = node_uuid_by_str.get(src)
+            to_uuid = node_uuid_by_str.get(tgt)
+            if not from_uuid or not to_uuid:
+                current_app.logger.warning(
+                    "Workflow graph write: dropping edge %s; node refs unresolved "
+                    "(source=%s, target=%s)",
+                    e.get("id"), src, tgt,
+                )
+                continue
+            translated_edges.append({
+                "edge_id": e["id"],
+                "from_node_id": from_uuid,
+                "to_node_id": to_uuid,
+                "source_handle": e.get("sourceHandle"),
+                "target_handle": e.get("targetHandle"),
+            })
+        if translated_edges:
+            edges_repo.bulk_create(
+                pg_workflow_id, graph_version, translated_edges,
+            )
+
+    return created_nodes
+
+
+def serialize_workflow(w: Dict) -> Dict:
+    """Serialize workflow row to API response format."""
+    created_at = w.get("created_at")
+    updated_at = w.get("updated_at")
+    return {
+        "id": str(w["id"]),
+        "name": w.get("name"),
+        "description": w.get("description"),
+        "created_at": created_at.isoformat() if hasattr(created_at, "isoformat") else created_at,
+        "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at,
+    }
+
+
+def serialize_node(n: Dict) -> Dict:
+    """Serialize workflow node row to API response format."""
+    return {
+        "id": n["node_id"],
+        "type": n["node_type"],
+        "title": n.get("title"),
+        "description": n.get("description"),
+        "position": n.get("position"),
+        "data": n.get("config", {}) or {},
+    }
+
+
+def serialize_edge(e: Dict) -> Dict:
+    """Serialize workflow edge row to API response format."""
+    return {
+        "id": e["edge_id"],
+        "source": e.get("source_id"),
+        "target": e.get("target_id"),
+        "sourceHandle": e.get("source_handle"),
+        "targetHandle": e.get("target_handle"),
+    }
+
+
+def get_workflow_graph_version(workflow: Dict) -> int:
+    """Get current graph version with fallback."""
+    raw_version = workflow.get("current_graph_version", 1)
+    try:
+        version = int(raw_version)
+        return version if version > 0 else 1
+    except (ValueError, TypeError):
+        return 1
+
+
+def validate_json_schema_payload(
+    json_schema: Any,
+) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
+    """Validate and normalize optional JSON schema payload for structured output."""
+    if json_schema is None:
+        return None, None
+    try:
+        return normalize_json_schema_payload(json_schema), None
+    except JsonSchemaValidationError as exc:
+        return None, str(exc)
+
+
+def normalize_agent_node_json_schemas(nodes: List[Dict]) -> List[Dict]:
+    """Normalize agent-node JSON schema payloads before persistence."""
+    normalized_nodes: List[Dict] = []
+    for node in nodes:
+        if not isinstance(node, dict):
+            normalized_nodes.append(node)
+            continue
+
+        normalized_node = dict(node)
+        if normalized_node.get("type") != "agent":
+            normalized_nodes.append(normalized_node)
+            continue
+
+        raw_config = normalized_node.get("data")
+        if not isinstance(raw_config, dict) or "json_schema" not in raw_config:
+            normalized_nodes.append(normalized_node)
+            continue
+
+        normalized_config = dict(raw_config)
+        try:
+            normalized_config["json_schema"] = normalize_json_schema_payload(
+                raw_config.get("json_schema")
+            )
+        except JsonSchemaValidationError:
+            # Validation runs before normalization; keep original on unexpected shape.
+            normalized_config["json_schema"] = raw_config.get("json_schema")
+        normalized_node["data"] = normalized_config
+        normalized_nodes.append(normalized_node)
+
+    return normalized_nodes
+
+
+def validate_workflow_structure(
+    nodes: List[Dict], edges: List[Dict], user_id: str | None = None
+) -> List[str]:
+    """Validate workflow graph structure.
+
+    ``user_id`` is required so per-user BYOM custom-model UUIDs resolve
+    when checking each agent node's structured-output capability.
+    """
+    errors = []
+
+    if not nodes:
+        errors.append("Workflow must have at least one node")
+        return errors
+
+    start_nodes = [n for n in nodes if n.get("type") == "start"]
+    if len(start_nodes) != 1:
+        errors.append("Workflow must have exactly one start node")
+
+    end_nodes = [n for n in nodes if n.get("type") == "end"]
+    if not end_nodes:
+        errors.append("Workflow must have at least one end node")
+
+    node_ids = {n.get("id") for n in nodes}
+    node_map = {n.get("id"): n for n in nodes}
+    end_ids = {n.get("id") for n in end_nodes}
+
+    for edge in edges:
+        source_id = edge.get("source")
+        target_id = edge.get("target")
+        if source_id not in node_ids:
+            errors.append(f"Edge references non-existent source: {source_id}")
+        if target_id not in node_ids:
+            errors.append(f"Edge references non-existent target: {target_id}")
+
+    if start_nodes:
+        start_id = start_nodes[0].get("id")
+        if not any(e.get("source") == start_id for e in edges):
+            errors.append("Start node must have at least one outgoing edge")
+
+    condition_nodes = [n for n in nodes if n.get("type") == "condition"]
+    for cnode in condition_nodes:
+        cnode_id = cnode.get("id")
+        cnode_title = cnode.get("title", cnode_id)
+        outgoing = [e for e in edges if e.get("source") == cnode_id]
+        if len(outgoing) < 2:
+            errors.append(
+                f"Condition node '{cnode_title}' must have at least 2 outgoing edges"
+            )
+        node_data = cnode.get("data", {}) or {}
+        cases = node_data.get("cases", [])
+        if not isinstance(cases, list):
+            cases = []
+        if not cases or not any(
+            isinstance(c, dict) and str(c.get("expression", "")).strip() for c in cases
+        ):
+            errors.append(
+                f"Condition node '{cnode_title}' must have at least one case with an expression"
+            )
+
+        case_handles: Set[str] = set()
+        duplicate_case_handles: Set[str] = set()
+        for case in cases:
+            if not isinstance(case, dict):
+                continue
+            raw_handle = case.get("sourceHandle", "")
+            handle = raw_handle.strip() if isinstance(raw_handle, str) else ""
+            if not handle:
+                errors.append(
+                    f"Condition node '{cnode_title}' has a case without a branch handle"
+                )
+                continue
+            if handle in case_handles:
+                duplicate_case_handles.add(handle)
+            case_handles.add(handle)
+
+        for handle in duplicate_case_handles:
+            errors.append(
+                f"Condition node '{cnode_title}' has duplicate case handle '{handle}'"
+            )
+
+        outgoing_by_handle: Dict[str, List[Dict]] = {}
+        for out_edge in outgoing:
+            raw_handle = out_edge.get("sourceHandle", "")
+            handle = raw_handle.strip() if isinstance(raw_handle, str) else ""
+            outgoing_by_handle.setdefault(handle, []).append(out_edge)
+
+        for handle, handle_edges in outgoing_by_handle.items():
+            if not handle:
+                errors.append(
+                    f"Condition node '{cnode_title}' has an outgoing edge without sourceHandle"
+                )
+                continue
+            if handle != "else" and handle not in case_handles:
+                errors.append(
+                    f"Condition node '{cnode_title}' has a connection from unknown branch '{handle}'"
+                )
+            if len(handle_edges) > 1:
+                errors.append(
+                    f"Condition node '{cnode_title}' has multiple outgoing edges from branch '{handle}'"
+                )
+
+        if "else" not in outgoing_by_handle:
+            errors.append(f"Condition node '{cnode_title}' must have an 'else' branch")
+
+        for case in cases:
+            if not isinstance(case, dict):
+                continue
+            raw_handle = case.get("sourceHandle", "")
+            handle = raw_handle.strip() if isinstance(raw_handle, str) else ""
+            if not handle:
+                continue
+
+            raw_expression = case.get("expression", "")
+            has_expression = isinstance(raw_expression, str) and bool(
+                raw_expression.strip()
+            )
+            has_outgoing = bool(outgoing_by_handle.get(handle))
+            if has_expression and not has_outgoing:
+                errors.append(
+                    f"Condition node '{cnode_title}' case '{handle}' has an expression but no outgoing edge"
+                )
+            if not has_expression and has_outgoing:
+                errors.append(
+                    f"Condition node '{cnode_title}' case '{handle}' has an outgoing edge but no expression"
+                )
+
+        for handle, handle_edges in outgoing_by_handle.items():
+            if not handle:
+                continue
+            for out_edge in handle_edges:
+                target = out_edge.get("target")
+                if target and not _can_reach_end(target, edges, node_map, end_ids):
+                    errors.append(
+                        f"Branch '{handle}' of condition '{cnode_title}' "
+                        f"must eventually reach an end node"
+                    )
+
+    agent_nodes = [n for n in nodes if n.get("type") == "agent"]
+    for agent_node in agent_nodes:
+        agent_title = agent_node.get("title", agent_node.get("id", "unknown"))
+        raw_config = agent_node.get("data", {}) or {}
+        if not isinstance(raw_config, dict):
+            errors.append(f"Agent node '{agent_title}' has invalid configuration")
+            continue
+        normalized_schema, schema_error = validate_json_schema_payload(
+            raw_config.get("json_schema")
+        )
+        has_json_schema = normalized_schema is not None
+
+        model_id = raw_config.get("model_id")
+        if has_json_schema and isinstance(model_id, str) and model_id.strip():
+            capabilities = get_model_capabilities(model_id.strip(), user_id=user_id)
+            if capabilities and not capabilities.get("supports_structured_output", False):
+                errors.append(
+                    f"Agent node '{agent_title}' selected model does not support structured output"
+                )
+        if schema_error:
+            errors.append(f"Agent node '{agent_title}' JSON schema {schema_error}")
+
+    code_nodes = [n for n in nodes if n.get("type") == "code"]
+    for code_node in code_nodes:
+        code_title = code_node.get("title", code_node.get("id", "unknown"))
+        raw_config = code_node.get("data", {}) or {}
+        if not isinstance(raw_config, dict):
+            errors.append(f"Code node '{code_title}' has invalid configuration")
+            continue
+        if not str(raw_config.get("code", "")).strip():
+            errors.append(f"Code node '{code_title}' must have code to execute")
+        _, schema_error = validate_json_schema_payload(raw_config.get("json_schema"))
+        if schema_error:
+            errors.append(f"Code node '{code_title}' JSON schema {schema_error}")
+
+    for node in nodes:
+        if not node.get("id"):
+            errors.append("All nodes must have an id")
+        if not node.get("type"):
+            errors.append(f"Node {node.get('id', 'unknown')} must have a type")
+
+    return errors
+
+
+def _can_reach_end(
+    node_id: str, edges: List[Dict], node_map: Dict, end_ids: set, visited: set = None
+) -> bool:
+    if visited is None:
+        visited = set()
+    if node_id in end_ids:
+        return True
+    if node_id in visited or node_id not in node_map:
+        return False
+    visited.add(node_id)
+    outgoing = [e.get("target") for e in edges if e.get("source") == node_id]
+    return any(_can_reach_end(t, edges, node_map, end_ids, visited) for t in outgoing if t)
+
+
+@workflows_ns.route("/workflows")
+class WorkflowList(Resource):
+
+    @require_auth
+    @require_fields(["name"])
+    def post(self):
+        """Create a new workflow with nodes and edges."""
+        user_id = get_user_id()
+        data = request.get_json()
+
+        name = data.get("name", "").strip()
+        description = data.get("description", "")
+        nodes_data = data.get("nodes", [])
+        edges_data = data.get("edges", [])
+
+        validation_errors = validate_workflow_structure(
+            nodes_data, edges_data, user_id=user_id
+        )
+        if validation_errors:
+            return error_response(
+                "Workflow validation failed", errors=validation_errors
+            )
+        nodes_data = normalize_agent_node_json_schemas(nodes_data)
+
+        try:
+            with db_session() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = repo.create(user_id, name, description=description)
+                pg_workflow_id = str(workflow["id"])
+                _write_graph(conn, pg_workflow_id, 1, nodes_data, edges_data)
+        except Exception as err:
+            return _workflow_error_response("Failed to create workflow", err)
+
+        return success_response({"id": pg_workflow_id}, 201)
+
+
+@workflows_ns.route("/workflows/<string:workflow_id>")
+class WorkflowDetail(Resource):
+
+    @require_auth
+    def get(self, workflow_id: str):
+        """Get workflow details with nodes and edges."""
+        user_id = get_user_id()
+        try:
+            with db_readonly() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = _resolve_workflow(repo, workflow_id, user_id)
+                if workflow is None:
+                    return error_response("Workflow not found", 404)
+                pg_workflow_id = str(workflow["id"])
+                graph_version = get_workflow_graph_version(workflow)
+                nodes = WorkflowNodesRepository(conn).find_by_version(
+                    pg_workflow_id, graph_version,
+                )
+                edges = WorkflowEdgesRepository(conn).find_by_version(
+                    pg_workflow_id, graph_version,
+                )
+        except Exception as err:
+            return _workflow_error_response("Failed to fetch workflow", err)
+
+        return success_response(
+            {
+                "workflow": serialize_workflow(workflow),
+                "nodes": [serialize_node(n) for n in nodes],
+                "edges": [serialize_edge(e) for e in edges],
+            }
+        )
+
+    @require_auth
+    @require_fields(["name"])
+    def put(self, workflow_id: str):
+        """Update workflow and replace nodes/edges."""
+        user_id = get_user_id()
+        data = request.get_json()
+        name = data.get("name", "").strip()
+        description = data.get("description", "")
+        nodes_data = data.get("nodes", [])
+        edges_data = data.get("edges", [])
+
+        validation_errors = validate_workflow_structure(
+            nodes_data, edges_data, user_id=user_id
+        )
+        if validation_errors:
+            return error_response(
+                "Workflow validation failed", errors=validation_errors
+            )
+        nodes_data = normalize_agent_node_json_schemas(nodes_data)
+
+        try:
+            with db_session() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = _resolve_workflow(repo, workflow_id, user_id)
+                if workflow is None:
+                    return error_response("Workflow not found", 404)
+                pg_workflow_id = str(workflow["id"])
+                current_graph_version = get_workflow_graph_version(workflow)
+                next_graph_version = current_graph_version + 1
+
+                _write_graph(
+                    conn, pg_workflow_id, next_graph_version,
+                    nodes_data, edges_data,
+                )
+                repo.update(
+                    pg_workflow_id, user_id,
+                    {
+                        "name": name,
+                        "description": description,
+                        "current_graph_version": next_graph_version,
+                    },
+                )
+                WorkflowNodesRepository(conn).delete_other_versions(
+                    pg_workflow_id, next_graph_version,
+                )
+                WorkflowEdgesRepository(conn).delete_other_versions(
+                    pg_workflow_id, next_graph_version,
+                )
+        except Exception as err:
+            return _workflow_error_response("Failed to update workflow", err)
+
+        return success_response()
+
+    @require_auth
+    def delete(self, workflow_id: str):
+        """Delete workflow and its graph."""
+        user_id = get_user_id()
+        try:
+            with db_session() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = _resolve_workflow(repo, workflow_id, user_id)
+                if workflow is None:
+                    return error_response("Workflow not found", 404)
+                # ON DELETE CASCADE on workflow_nodes/edges cleans children.
+                repo.delete(str(workflow["id"]), user_id)
+        except Exception as err:
+            return _workflow_error_response("Failed to delete workflow", err)
+
+        return success_response()

application/api/v1/__init__.py

@@ -0,0 +1,3 @@
+from application.api.v1.routes import v1_bp
+
+__all__ = ["v1_bp"]

application/api/v1/idempotency.py

@@ -0,0 +1,260 @@
+"""Layer-1 idempotency for the OpenAI-compatible ``/v1/chat/completions`` route.
+
+The ``/v1`` tool round-trip is fully stateless (the pause finalizes the prior
+turn as ``complete`` and resumes via ``build_continuation_from_messages`` with
+no ``pending_tool_state``). Dropping the native ``resume_from_tool_actions``
+path also dropped its ``mark_resuming`` guard, so a duplicated/retried POST
+could re-run the agent → a duplicate answer row + double token billing.
+
+This module restores protection the OpenAI-compatible way: a client-supplied
+``Idempotency-Key`` header makes retries return the *stored first response*
+instead of re-running the agent. It is opt-in (no header → today's behavior,
+byte-for-byte) and scoped to **non-streaming** requests only (the b2b client
+and the actual regression); streaming replay is intentionally unsupported.
+
+Storage reuses the existing ``task_dedup`` table via
+:class:`~application.storage.db.repositories.idempotency.IdempotencyRepository`
+— no new table or migration. The contract maps onto its claim/finalize
+semantics:
+
+- **No record** → ``claim_task`` inserts a ``pending`` row (we run + finalize).
+- **``completed`` within 24h TTL** → return the cached body + status code.
+- **Fresh ``pending``** (in-flight) → HTTP 409 idempotency conflict.
+- **Stale ``pending``** (older than :data:`STALE_PENDING_SECONDS` — the
+  original request likely died) → release and re-claim.
+- **``failed`` / past-TTL** → ``claim_task`` re-claims automatically.
+
+Only successful (2xx) responses are cached; a 4xx/5xx releases the claim so a
+genuine retry can still succeed (matches OpenAI).
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any, Dict, Optional, Tuple
+
+from flask import jsonify, make_response, request, Response
+from sqlalchemy import text as sql_text
+
+from application.storage.db.repositories.idempotency import IdempotencyRepository
+from application.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+# Distinct ``task_name`` so v1 chat dedup rows never collide with ingest /
+# webhook rows that share the ``task_dedup`` table.
+TASK_NAME = "v1_chat_completion"
+
+_IDEMPOTENCY_KEY_MAX_LEN = 256
+
+# A ``pending`` claim older than this is treated as a dead in-flight request
+# (the process crashed before finalize), so a genuine retry may re-claim it
+# rather than waiting out the full 24h TTL or getting a permanent 409. Kept
+# short enough to unblock retries quickly, long enough that a normal
+# non-streaming completion (which finalizes on the same request) never trips
+# it while still running.
+STALE_PENDING_SECONDS = 300
+
+
+def read_idempotency_key() -> Tuple[Optional[str], Optional[Response]]:
+    """Read and validate the ``Idempotency-Key`` request header.
+
+    Returns:
+        ``(key, error_response)``. An absent/empty header yields
+        ``(None, None)`` (idempotency is opt-in). An oversized header yields
+        ``(None, <400 response>)`` so the caller can short-circuit.
+    """
+    key = request.headers.get("Idempotency-Key")
+    if not key:
+        return None, None
+    if len(key) > _IDEMPOTENCY_KEY_MAX_LEN:
+        return None, make_response(
+            jsonify(
+                {
+                    "error": {
+                        "message": (
+                            f"Idempotency-Key exceeds maximum length of "
+                            f"{_IDEMPOTENCY_KEY_MAX_LEN} characters"
+                        ),
+                        "type": "invalid_request",
+                    }
+                }
+            ),
+            400,
+        )
+    return key, None
+
+
+def scoped_key(idempotency_key: Optional[str], agent_id: Optional[str]) -> Optional[str]:
+    """Compose ``{agent_id}:{idempotency_key}`` so tenants never collide.
+
+    Two agents replaying the same key value resolve to distinct stored rows.
+    Falls back to ``api_key`` scoping at the call site when no agent id is
+    available; returns ``None`` when either component is missing (idempotency
+    is then skipped, preserving today's behavior).
+    """
+    if not idempotency_key or not agent_id:
+        return None
+    return f"{agent_id}:{idempotency_key}"
+
+
+def _release_stale_pending(key: str) -> None:
+    """Delete a stale ``pending`` claim so the caller can re-claim it.
+
+    Scoped to ``status = 'pending'`` and the staleness window so we never
+    clobber a live in-flight claim or a ``completed`` cache row.
+    """
+    try:
+        with db_session() as conn:
+            conn.execute(
+                sql_text(
+                    "DELETE FROM task_dedup "
+                    "WHERE idempotency_key = :k "
+                    "AND status = 'pending' "
+                    "AND created_at <= now() - make_interval(secs => :secs)"
+                ),
+                {"k": key, "secs": STALE_PENDING_SECONDS},
+            )
+    except Exception:
+        logger.exception("Failed to release stale v1 idempotency claim for key=%s", key)
+
+
+def claim_or_replay(key: str) -> Tuple[bool, Optional[Response]]:
+    """Claim ``key`` for this request, or return the prior outcome.
+
+    Claim-before-process: atomically insert a ``pending`` row. The three
+    outcomes map onto the existing ``task_dedup`` contract:
+
+    - **claimed** → ``(True, None)``: this caller runs the agent and must call
+      :func:`finalize` (success) or :func:`release` (error) afterwards.
+    - **``completed`` within TTL** → ``(False, <cached response>)``: replay the
+      stored body + status code without re-running.
+    - **fresh ``pending``** → ``(False, <409 response>)``: a same-key request is
+      already in progress.
+
+    A ``pending`` row older than :data:`STALE_PENDING_SECONDS` is released and
+    re-claimed (the original request likely died). ``failed`` / past-TTL rows
+    are re-claimed by ``claim_task`` itself.
+
+    Args:
+        key: The tenant-scoped idempotency key.
+
+    Returns:
+        ``(claimed, response)``. When ``claimed`` is True the caller owns the
+        run; otherwise ``response`` is the replay/409 to return immediately.
+    """
+    predetermined_id = str(uuid.uuid4())
+    with db_session() as conn:
+        claimed = IdempotencyRepository(conn).claim_task(
+            key=key, task_name=TASK_NAME, task_id=predetermined_id,
+        )
+    if claimed is not None:
+        return True, None
+
+    # Lost the claim — resolve why against the within-TTL row.
+    with db_readonly() as conn:
+        existing = IdempotencyRepository(conn).get_task(key)
+
+    if existing is not None and existing.get("status") == "completed":
+        return False, _replay_response(existing.get("result_json"))
+
+    if existing is not None and existing.get("status") == "pending":
+        # In-flight? Re-claim only if the prior claim is stale (dead request).
+        _release_stale_pending(key)
+        with db_session() as conn:
+            reclaimed = IdempotencyRepository(conn).claim_task(
+                key=key, task_name=TASK_NAME, task_id=predetermined_id,
+            )
+        if reclaimed is not None:
+            return True, None
+        return False, _conflict_response()
+
+    # Row vanished between claim and read (TTL cleanup / release race) — one
+    # more claim attempt; treat a persistent loss as a conflict.
+    with db_session() as conn:
+        reclaimed = IdempotencyRepository(conn).claim_task(
+            key=key, task_name=TASK_NAME, task_id=predetermined_id,
+        )
+    if reclaimed is not None:
+        return True, None
+    with db_readonly() as conn:
+        existing = IdempotencyRepository(conn).get_task(key)
+    if existing is not None and existing.get("status") == "completed":
+        return False, _replay_response(existing.get("result_json"))
+    return False, _conflict_response()
+
+
+def finalize(key: str, response: Response) -> None:
+    """Cache a successful (2xx) response under ``key``; release otherwise.
+
+    Stores ``{"status_code", "body"}`` in ``task_dedup.result_json`` so a
+    retry replays byte-for-byte. Non-2xx responses are not cached — the claim
+    is released so a genuine retry can still succeed (matches OpenAI).
+
+    Args:
+        key: The tenant-scoped idempotency key claimed by :func:`claim_or_replay`.
+        response: The Flask response produced by running the request.
+    """
+    status_code = response.status_code
+    if not (200 <= status_code < 300):
+        release(key)
+        return
+    try:
+        body = response.get_json(silent=True)
+    except Exception:
+        body = None
+    result_json = {"status_code": status_code, "body": body}
+    try:
+        with db_session() as conn:
+            IdempotencyRepository(conn).finalize_task(
+                key=key, result_json=result_json, status="completed",
+            )
+    except Exception:
+        logger.exception("Failed to finalize v1 idempotency record for key=%s", key)
+
+
+def release(key: str) -> None:
+    """Drop this request's ``pending`` claim so a retry can re-claim it.
+
+    Used on the error path (non-2xx or an exception before finalize) so a
+    failed first attempt never blocks a legitimate retry for the full TTL.
+    """
+    try:
+        with db_session() as conn:
+            conn.execute(
+                sql_text(
+                    "DELETE FROM task_dedup "
+                    "WHERE idempotency_key = :k AND status = 'pending'"
+                ),
+                {"k": key},
+            )
+    except Exception:
+        logger.exception("Failed to release v1 idempotency claim for key=%s", key)
+
+
+def _replay_response(result_json: Optional[Dict[str, Any]]) -> Response:
+    """Rebuild a Flask response from a cached ``result_json`` row."""
+    status_code = 200
+    body: Any = None
+    if isinstance(result_json, dict):
+        status_code = int(result_json.get("status_code", 200))
+        body = result_json.get("body")
+    return make_response(jsonify(body), status_code)
+
+
+def _conflict_response() -> Response:
+    """OpenAI-shaped 409 for a same-key request already in progress."""
+    return make_response(
+        jsonify(
+            {
+                "error": {
+                    "message": (
+                        "A request with this Idempotency-Key is already in progress"
+                    ),
+                    "type": "idempotency_conflict",
+                }
+            }
+        ),
+        409,
+    )

application/api/v1/routes.py

@@ -0,0 +1,458 @@
+"""Standard chat completions API routes.
+
+Exposes ``/v1/chat/completions`` and ``/v1/models`` endpoints that
+follow the widely-adopted chat completions protocol so external tools
+(opencode, continue, etc.) can connect to DocsGPT agents.
+"""
+
+import json
+import logging
+import time
+import traceback
+from datetime import datetime
+from typing import Any, Dict, Generator, Optional
+
+from flask import Blueprint, jsonify, make_response, request, Response
+
+from application.api.answer.routes.base import BaseAnswerResource
+from application.api.answer.services.persistence_policy import resolve_persistence
+from application.api.answer.services.stream_processor import StreamProcessor
+from application.api.v1 import idempotency as v1_idempotency
+from application.api.v1.translator import (
+    translate_request,
+    translate_response,
+    translate_stream_event,
+)
+from application.storage.db.repositories.agents import AgentsRepository
+from application.storage.db.session import db_readonly
+
+logger = logging.getLogger(__name__)
+
+v1_bp = Blueprint("v1", __name__, url_prefix="/v1")
+
+
+def _extract_bearer_token() -> Optional[str]:
+    """Extract API key from Authorization: Bearer header."""
+    auth = request.headers.get("Authorization", "")
+    if auth.startswith("Bearer "):
+        return auth[7:].strip()
+    return None
+
+
+def _lookup_agent(api_key: str) -> Optional[Dict]:
+    """Look up the agent document for this API key."""
+    try:
+        with db_readonly() as conn:
+            return AgentsRepository(conn).find_by_key(api_key)
+    except Exception:
+        logger.warning("Failed to look up agent for API key", exc_info=True)
+        return None
+
+
+def _get_model_name(agent: Optional[Dict], api_key: str) -> str:
+    """Return agent name for display as model name."""
+    if agent:
+        return agent.get("name", api_key)
+    return api_key
+
+
+class _V1AnswerHelper(BaseAnswerResource):
+    """Thin wrapper to access complete_stream / process_response_stream."""
+    pass
+
+
+@v1_bp.route("/chat/completions", methods=["POST"])
+def chat_completions():
+    """Handle POST /v1/chat/completions."""
+    api_key = _extract_bearer_token()
+    if not api_key:
+        return make_response(
+            jsonify({"error": {"message": "Missing Authorization header", "type": "auth_error"}}),
+            401,
+        )
+
+    data = request.get_json()
+    if not data or not data.get("messages"):
+        return make_response(
+            jsonify({"error": {"message": "messages field is required", "type": "invalid_request"}}),
+            400,
+        )
+
+    is_stream = data.get("stream", False)
+    agent_doc = _lookup_agent(api_key)
+    model_name = _get_model_name(agent_doc, api_key)
+
+    # ---- Layer-1 idempotency (opt-in, non-streaming only) ----
+    # An ``Idempotency-Key`` header makes a retried non-streaming request
+    # return the stored first response instead of re-running the agent
+    # (restoring the guard lost when the v1 tool round dropped the native
+    # ``resume_from_tool_actions`` / ``mark_resuming`` path → would otherwise
+    # duplicate the answer row and double-bill tokens). Streaming replay is
+    # intentionally NOT supported (see the ``is_stream`` branch below), so we
+    # only resolve a key for non-streaming requests. No header → byte-for-byte
+    # today's behavior.
+    idem_key: Optional[str] = None
+    if not is_stream:
+        raw_key, key_error = v1_idempotency.read_idempotency_key()
+        if key_error is not None:
+            return key_error
+        # Scope per tenant: ``{agent_id}:{key}`` so two agents using the same
+        # key value never collide. Fall back to api_key scoping when the agent
+        # has no resolvable id (idempotency still keyed, just per api_key).
+        agent_scope = None
+        if agent_doc is not None:
+            agent_scope = str(agent_doc.get("id") or agent_doc.get("_id") or "") or None
+        idem_key = v1_idempotency.scoped_key(raw_key, agent_scope or api_key)
+
+    try:
+        internal_data = translate_request(data, api_key)
+    except Exception as e:
+        logger.error(f"/v1/chat/completions translate error: {e}", exc_info=True)
+        return make_response(
+            jsonify({"error": {"message": "Failed to process request", "type": "invalid_request"}}),
+            400,
+        )
+
+    # Link decoded_token to the agent's owner so continuation state,
+    # logs, and tool execution use the correct user identity. The PG
+    # ``agents`` row exposes the owner via ``user_id`` (``user`` is the
+    # legacy Mongo field name kept in ``row_to_dict`` only for the
+    # mapping ``id``/``_id``).
+    agent_user = (
+        (agent_doc.get("user_id") or agent_doc.get("user"))
+        if agent_doc else None
+    )
+    decoded_token = {"sub": agent_user or "api_key_user"}
+
+    try:
+        processor = StreamProcessor(internal_data, decoded_token)
+
+        if internal_data.get("tool_actions"):
+            # Continuation mode — coherent Option B: the v1 tool round-trip is
+            # fully stateless. The pause finalized the prior turn's row as
+            # ``complete`` and wrote NO ``pending_tool_state`` (see
+            # ``complete_stream(finalize_tool_pause_as_complete=True)``), so we
+            # ALWAYS rebuild the agent + pending calls from the re-POSTed
+            # message history — even when the client threads back the
+            # ``conversation_id`` it got from the first response.
+            #
+            # We deliberately do NOT call ``resume_from_tool_actions`` here:
+            # its ``load_state`` would find no pending state and raise (→ HTTP
+            # 400), since OpenAI clients resume statelessly rather than via a
+            # native resume. ``resume_from_tool_actions`` stays in place for
+            # the native ``/stream`` + ``/api/answer`` routes, which are
+            # unchanged.
+            conversation_id = internal_data.get("conversation_id")
+            (
+                agent,
+                messages,
+                tools_dict,
+                pending_tool_calls,
+                tool_actions,
+                reasoning_content,
+            ) = processor.build_continuation_from_messages(
+                internal_data.get("messages", []),
+                internal_data["tool_actions"],
+            )
+            # When a conversation_id is carried, target it for persistence so
+            # the final answer appends as a NEW terminal turn in that
+            # conversation (``save_conversation`` keys off ``conversation_id``)
+            # rather than creating an orphan sibling. ``build_continuation_from_messages``
+            # leaves the processor's ``conversation_id`` (set from the request
+            # in ``__init__``) intact; pin it explicitly for clarity.
+            if conversation_id:
+                processor.conversation_id = conversation_id
+            continuation = {
+                "messages": messages,
+                "tools_dict": tools_dict,
+                "pending_tool_calls": pending_tool_calls,
+                "tool_actions": tool_actions,
+                "reasoning_content": reasoning_content,
+            }
+            question = ""
+        else:
+            # Normal mode
+            question = internal_data.get("question", "")
+            agent = processor.build_agent(question)
+            continuation = None
+
+        if not processor.decoded_token:
+            return make_response(
+                jsonify({"error": {"message": "Unauthorized", "type": "auth_error"}}),
+                401,
+            )
+
+        helper = _V1AnswerHelper()
+        usage_error = helper.check_usage(processor.agent_config)
+        if usage_error:
+            return usage_error
+
+        # v1 always persists (unless the translator opted out for a stateless
+        # tool round) and never lists in the agent owner's sidebar — only the
+        # first-party UI opts a conversation into ``visibility: "listed"``.
+        should_persist, visibility = resolve_persistence(
+            persist_flag=internal_data.get("persist"),
+        )
+        # Only strip leaked reasoning from content for structured requests -- the
+        # only path where models echo reasoning into content -- so legitimate
+        # answers that mention the marker text are never corrupted.
+        strip_reasoning_leak = bool(
+            internal_data.get("json_schema") or internal_data.get("json_object")
+        )
+
+        if is_stream:
+            # Idempotency replay is NOT supported for streaming: there is no
+            # safe way to re-emit a recorded SSE stream (and the regression /
+            # b2b client is non-streaming), so a streaming request never
+            # claims a key. This is a known, accepted limitation.
+            return Response(
+                _stream_response(
+                    helper,
+                    question,
+                    agent,
+                    processor,
+                    model_name,
+                    continuation,
+                    should_persist,
+                    visibility,
+                    strip_reasoning_leak,
+                ),
+                mimetype="text/event-stream",
+                headers={
+                    "Cache-Control": "no-cache",
+                    "X-Accel-Buffering": "no",
+                },
+            )
+
+        # ---- Non-streaming: claim-before-process, then finalize/release ----
+        # Claim happens here (after auth + agent resolution + continuation
+        # build, immediately before running the agent) so a duplicate retry
+        # short-circuits to the cached body / 409 instead of re-running.
+        if idem_key:
+            claimed, replay = v1_idempotency.claim_or_replay(idem_key)
+            if not claimed:
+                # ``completed`` cache hit, or a 409 for an in-flight same-key
+                # request — either way return without re-running the agent.
+                return replay
+
+        # An exception from the agent run propagates to the ``except`` handlers
+        # below, which release the claim so a genuine retry can re-claim.
+        response = _non_stream_response(
+            helper,
+            question,
+            agent,
+            processor,
+            model_name,
+            continuation,
+            should_persist,
+            visibility,
+            strip_reasoning_leak,
+        )
+
+        # Cache only successful (2xx) responses; ``finalize`` releases the
+        # claim on a non-2xx so a real retry can still succeed (matches OpenAI).
+        if idem_key:
+            v1_idempotency.finalize(idem_key, response)
+        return response
+
+    except ValueError as e:
+        if idem_key:
+            v1_idempotency.release(idem_key)
+        logger.error(
+            f"/v1/chat/completions error: {e} - {traceback.format_exc()}",
+            extra={"error": str(e)},
+        )
+        return make_response(
+            jsonify({"error": {"message": "Failed to process request", "type": "invalid_request"}}),
+            400,
+        )
+    except Exception as e:
+        if idem_key:
+            v1_idempotency.release(idem_key)
+        logger.error(
+            f"/v1/chat/completions error: {e} - {traceback.format_exc()}",
+            extra={"error": str(e)},
+        )
+        return make_response(
+            jsonify({"error": {"message": "Internal server error", "type": "server_error"}}),
+            500,
+        )
+
+
+def _stream_response(
+    helper: _V1AnswerHelper,
+    question: str,
+    agent: Any,
+    processor: StreamProcessor,
+    model_name: str,
+    continuation: Optional[Dict],
+    should_persist: bool,
+    visibility: str,
+    strip_reasoning_leak: bool = False,
+) -> Generator[str, None, None]:
+    """Generate translated SSE chunks for streaming response."""
+    completion_id = f"chatcmpl-{int(time.time())}"
+
+    internal_stream = helper.complete_stream(
+        question=question,
+        agent=agent,
+        conversation_id=processor.conversation_id,
+        user_api_key=processor.agent_config.get("user_api_key"),
+        decoded_token=processor.decoded_token,
+        agent_id=processor.agent_id,
+        model_id=processor.model_id,
+        model_user_id=processor.model_user_id,
+        should_persist=should_persist,
+        visibility=visibility,
+        _continuation=continuation,
+        # OpenAI clients resume tool calls statelessly (no slot for our
+        # reserved_message_id), so a tool pause must finalize the row as
+        # ``complete`` here rather than stranding it for a native resume.
+        finalize_tool_pause_as_complete=True,
+    )
+
+    for line in internal_stream:
+        if not line.strip():
+            continue
+        # ``complete_stream`` prefixes each frame with ``id: <seq>\n``
+        # before the ``data:`` line. Extract just the data line so JSON
+        # decode doesn't choke on the SSE framing.
+        event_str = ""
+        for raw in line.split("\n"):
+            if raw.startswith("data:"):
+                event_str = raw[len("data:") :].lstrip()
+                break
+        if not event_str:
+            continue
+        try:
+            event_data = json.loads(event_str)
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+        # Skip the informational ``message_id`` event — it has no v1 /
+        # OpenAI-compatible analog.
+        if event_data.get("type") == "message_id":
+            continue
+
+        # Update completion_id when we get the conversation id
+        if event_data.get("type") == "id":
+            conv_id = event_data.get("id", "")
+            if conv_id and conv_id != "None":
+                completion_id = f"chatcmpl-{conv_id}"
+
+        # Translate to standard format
+        translated = translate_stream_event(
+            event_data, completion_id, model_name, strip_reasoning_leak
+        )
+        for chunk in translated:
+            yield chunk
+
+
+def _non_stream_response(
+    helper: _V1AnswerHelper,
+    question: str,
+    agent: Any,
+    processor: StreamProcessor,
+    model_name: str,
+    continuation: Optional[Dict],
+    should_persist: bool,
+    visibility: str,
+    strip_reasoning_leak: bool = False,
+) -> Response:
+    """Collect full response and return as single JSON."""
+    stream = helper.complete_stream(
+        question=question,
+        agent=agent,
+        conversation_id=processor.conversation_id,
+        user_api_key=processor.agent_config.get("user_api_key"),
+        decoded_token=processor.decoded_token,
+        agent_id=processor.agent_id,
+        model_id=processor.model_id,
+        model_user_id=processor.model_user_id,
+        should_persist=should_persist,
+        visibility=visibility,
+        _continuation=continuation,
+        # OpenAI clients resume tool calls statelessly (no slot for our
+        # reserved_message_id), so a tool pause must finalize the row as
+        # ``complete`` here rather than stranding it for a native resume.
+        finalize_tool_pause_as_complete=True,
+    )
+
+    result = helper.process_response_stream(stream)
+
+    if result["error"]:
+        return make_response(
+            jsonify({"error": {"message": result["error"], "type": "server_error"}}),
+            500,
+        )
+
+    extra = result.get("extra")
+    pending = extra.get("pending_tool_calls") if isinstance(extra, dict) else None
+
+    response = translate_response(
+        conversation_id=result["conversation_id"],
+        answer=result["answer"] or "",
+        sources=result["sources"],
+        tool_calls=result["tool_calls"],
+        thought=result["thought"] or "",
+        model_name=model_name,
+        pending_tool_calls=pending,
+        strip_reasoning_leak=strip_reasoning_leak,
+    )
+    return make_response(jsonify(response), 200)
+
+
+@v1_bp.route("/models", methods=["GET"])
+def list_models():
+    """Handle GET /v1/models — return agents as models."""
+    api_key = _extract_bearer_token()
+    if not api_key:
+        return make_response(
+            jsonify({"error": {"message": "Missing Authorization header", "type": "auth_error"}}),
+            401,
+        )
+
+    try:
+        with db_readonly() as conn:
+            agents_repo = AgentsRepository(conn)
+            agent = agents_repo.find_by_key(api_key)
+            if not agent:
+                return make_response(
+                    jsonify({"error": {"message": "Invalid API key", "type": "auth_error"}}),
+                    401,
+                )
+
+        # Repository rows now go through ``coerce_pg_native`` at SELECT
+        # time, so timestamps arrive as ISO 8601 strings. Parse before
+        # taking ``.timestamp()``; fall back to ``time.time()`` only when
+        # the value is genuinely missing or unparseable.
+        created = agent.get("created_at") or agent.get("createdAt")
+        if isinstance(created, str):
+            try:
+                created = datetime.fromisoformat(created)
+            except (ValueError, TypeError):
+                created = None
+        created_ts = (
+            int(created.timestamp()) if hasattr(created, "timestamp")
+            else int(time.time())
+        )
+        model_id = str(agent.get("id") or agent.get("_id") or "")
+        model = {
+            "id": model_id,
+            "object": "model",
+            "created": created_ts,
+            "owned_by": "docsgpt",
+            "name": agent.get("name", ""),
+            "description": agent.get("description", ""),
+        }
+
+        return make_response(
+            jsonify({"object": "list", "data": [model]}),
+            200,
+        )
+    except Exception as e:
+        logger.error(f"/v1/models error: {e}", exc_info=True)
+        return make_response(
+            jsonify({"error": {"message": "Internal server error", "type": "server_error"}}),
+            500,
+        )

application/api/v1/translator.py

@@ -0,0 +1,623 @@
+"""Translate between standard chat completions format and DocsGPT internals.
+
+This module handles:
+- Request translation (chat completions -> DocsGPT internal format)
+- Response translation (DocsGPT response -> chat completions format)
+- Streaming event translation (DocsGPT SSE -> standard SSE chunks)
+"""
+
+import json
+import re
+import time
+from typing import Any, Dict, List, Optional
+
+# Some upstream models/proxies echo their reasoning into ``content`` as
+# stringified ``{'type': 'thought', 'thought': '...'}`` event reprs (instead of
+# using the separate reasoning channel) — most visibly when ``response_format``
+# is set. OpenAI's API never puts reasoning in ``content``, so for the
+# OpenAI-compatible endpoint we strip these and reroute them to
+# ``reasoning_content`` to keep ``content`` clean and compatible.
+# The thought value is a Python string repr: single-quoted, or double-quoted when
+# the token contains an apostrophe (e.g. "'ll"). Match the full quoted value
+# (honoring escapes) so tokens containing ``}`` or newlines don't truncate the
+# match and leave stray ``'}`` tails in the content.
+_LEAKED_THOUGHT_RE = re.compile(
+    r"""\{'type': 'thought', 'thought': ('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")\}""",
+    re.DOTALL,
+)
+
+
+def _strip_repr_quotes(value: str) -> str:
+    value = value.strip()
+    if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0]:
+        return value[1:-1]
+    return value
+
+
+def _split_leaked_reasoning(content: Optional[str]) -> tuple:
+    """Return ``(clean_content, leaked_reasoning)``.
+
+    ``clean_content`` has any stringified thought-event reprs removed;
+    ``leaked_reasoning`` is the concatenated reasoning text that was extracted.
+    A no-op (returns the input unchanged) when no leak markers are present.
+    """
+    if not content or "'type': 'thought'" not in content:
+        return content, ""
+    extracted: List[str] = []
+    cleaned = _LEAKED_THOUGHT_RE.sub(
+        lambda m: (extracted.append(_strip_repr_quotes(m.group(1))) or ""), content
+    )
+    return cleaned, "".join(extracted)
+
+
+def _get_client_tool_name(tc: Dict) -> str:
+    """Return the original tool name for client-facing responses.
+
+    For client-side tools the ``tool_name`` field carries the name the
+    client originally registered.  Fall back to ``action_name`` (which
+    is now the clean LLM-visible name) or ``name``.
+    """
+    return tc.get("tool_name", tc.get("action_name", tc.get("name", "")))
+
+
+# ---------------------------------------------------------------------------
+# Request translation
+# ---------------------------------------------------------------------------
+
+
+def is_continuation(messages: List[Dict]) -> bool:
+    """Check if messages represent a tool-call continuation.
+
+    A continuation is detected when the last message(s) have ``role: "tool"``
+    immediately after an assistant message with ``tool_calls``.
+    """
+    if not messages:
+        return False
+    # Walk backwards: if we see tool messages before hitting a non-tool, non-assistant message
+    # and there's an assistant message with tool_calls, it's a continuation.
+    i = len(messages) - 1
+    while i >= 0 and messages[i].get("role") == "tool":
+        i -= 1
+    if i < 0:
+        return False
+    return (
+        messages[i].get("role") == "assistant"
+        and bool(messages[i].get("tool_calls"))
+    )
+
+
+def extract_tool_results(messages: List[Dict]) -> List[Dict]:
+    """Extract tool results from trailing tool messages for continuation.
+
+    Returns a list of ``tool_actions`` dicts with ``call_id`` and ``result``.
+    """
+    results = []
+    for msg in reversed(messages):
+        if msg.get("role") != "tool":
+            break
+        call_id = msg.get("tool_call_id", "")
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            try:
+                content = json.loads(content)
+            except (json.JSONDecodeError, TypeError):
+                pass
+        results.append({"call_id": call_id, "result": content})
+    results.reverse()
+    return results
+
+
+def extract_conversation_id(messages: List[Dict]) -> Optional[str]:
+    """Try to extract conversation_id from the assistant message before tool results.
+
+    The conversation_id may be stored in a custom field on the assistant message
+    from a previous response cycle.
+    """
+    for msg in reversed(messages):
+        if msg.get("role") == "assistant":
+            # Check docsgpt extension
+            return msg.get("docsgpt", {}).get("conversation_id")
+    return None
+
+
+def content_to_text(content: Any) -> str:
+    """Flatten an OpenAI message ``content`` to plain text.
+
+    ``content`` may be a string or a list of typed parts
+    (``{"type":"text",...}`` / ``{"type":"image_url",...}`` / ...). Only text
+    parts contribute; image/other parts are dropped here. The full content
+    array is preserved separately (see ``multimodal_content``) so images still
+    reach the model in the final user message.
+    """
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        out = []
+        for part in content:
+            if isinstance(part, dict) and part.get("type") == "text":
+                out.append(part.get("text", "") or "")
+            elif isinstance(part, str):
+                out.append(part)
+        return "\n".join(out)
+    return "" if content is None else str(content)
+
+
+def extract_system_prompt(messages: List[Dict]) -> Optional[str]:
+    """Extract the first system message content from the messages array.
+
+    Returns None if no system message is present.
+    """
+    for msg in messages:
+        if msg.get("role") == "system":
+            return content_to_text(msg.get("content", ""))
+    return None
+
+
+def convert_history(messages: List[Dict]) -> List[Dict]:
+    """Convert chat completions messages array to DocsGPT history format.
+
+    DocsGPT history is a list of ``{prompt, response}`` dicts.
+    Excludes the last user message (that becomes the ``question``).
+    """
+    history = []
+    i = 0
+    while i < len(messages):
+        msg = messages[i]
+        if msg.get("role") == "system":
+            i += 1
+            continue
+        if msg.get("role") == "user":
+            # Look ahead for assistant response
+            if i + 1 < len(messages) and messages[i + 1].get("role") == "assistant":
+                content = content_to_text(messages[i + 1].get("content") or "")
+                history.append({
+                    "prompt": content_to_text(msg.get("content", "")),
+                    "response": content,
+                })
+                i += 2
+                continue
+            # Last user message without response — skip (it's the question)
+            i += 1
+            continue
+        i += 1
+    return history
+
+
+def extract_response_schema(data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Extract a JSON schema for structured output from a chat-completions request.
+
+    Supports two request shapes:
+    - OpenAI ``response_format``:
+      ``{"type": "json_schema", "json_schema": {"name": ..., "schema": {...}}}``
+      (a bare schema under ``json_schema`` is also tolerated).
+    - ``response_schema`` convenience field: a raw JSON Schema object, or a
+      ``{"schema": {...}}`` wrapper.
+
+    Returns a raw JSON Schema object, or None. ``response_format``
+    ``{"type": "json_object"}`` carries no schema to enforce and yields None
+    (the model is still steered by the system prompt).
+    """
+    response_schema = data.get("response_schema")
+    if isinstance(response_schema, dict) and response_schema:
+        inner = response_schema.get("schema")
+        return inner if isinstance(inner, dict) else response_schema
+
+    response_format = data.get("response_format")
+    if isinstance(response_format, dict) and response_format.get("type") == "json_schema":
+        json_schema = response_format.get("json_schema")
+        if isinstance(json_schema, dict):
+            schema = json_schema.get("schema")
+            if isinstance(schema, dict):
+                return schema
+            if "type" in json_schema:
+                return json_schema
+    return None
+
+
+def translate_request(
+    data: Dict[str, Any], api_key: str
+) -> Dict[str, Any]:
+    """Translate a chat completions request to DocsGPT internal format.
+
+    Args:
+        data: The incoming request body.
+        api_key: Agent API key from the Authorization header.
+
+    Returns:
+        Dict suitable for passing to ``StreamProcessor``.
+    """
+    messages = data.get("messages", [])
+    response_schema = extract_response_schema(data)
+    _rf = data.get("response_format")
+    _rf = _rf if isinstance(_rf, dict) else {}
+    # OpenAI Structured Outputs default to strict; honor an explicit strict:false.
+    json_schema_strict = bool((_rf.get("json_schema") or {}).get("strict", True))
+    json_object_mode = _rf.get("type") == "json_object"
+
+    # OpenAI sampling params, forwarded to the LLM gen call (the agent otherwise
+    # uses its configured defaults).
+    sampling_params = {}
+    for _k in (
+        "temperature", "max_tokens", "max_completion_tokens",
+        "top_p", "frequency_penalty", "presence_penalty", "stop", "seed",
+    ):
+        if data.get(_k) is not None:
+            sampling_params[_k] = data[_k]
+    # OpenAI rejects sending both; the provider maps max_tokens ->
+    # max_completion_tokens, so drop the alias when the canonical key is present.
+    if "max_completion_tokens" in sampling_params:
+        sampling_params.pop("max_tokens", None)
+
+    # Check for continuation (tool results after assistant tool_calls)
+    if is_continuation(messages):
+        tool_actions = extract_tool_results(messages)
+        conversation_id = extract_conversation_id(messages)
+        if not conversation_id:
+            conversation_id = data.get("conversation_id")
+        result = {
+            "conversation_id": conversation_id,
+            "tool_actions": tool_actions,
+            "api_key": api_key,
+            # Full messages array for STATELESS continuation: OpenAI clients
+            # (opencode, etc.) don't carry a conversation_id, so the agent is
+            # rebuilt from the resent messages instead of server-side state.
+            "messages": messages,
+        }
+        # Persistence: stateful continuations (carrying a conversation_id)
+        # persist the final turn; stateless ones (no conversation_id, e.g.
+        # opencode) skip it, else every tool round writes an orphan conversation
+        # with an empty question. ``docsgpt.persist`` overrides. Visibility is
+        # not request-controllable on v1 — rows always persist hidden, so the
+        # legacy ``docsgpt.save_conversation`` flag is ignored.
+        docsgpt_ext = data.get("docsgpt", {})
+        result["persist"] = bool(docsgpt_ext.get("persist", bool(conversation_id)))
+        # Carry tools forward for next iteration
+        if data.get("tools"):
+            result["client_tools"] = data["tools"]
+        if response_schema is not None:
+            result["json_schema"] = response_schema
+            result["json_schema_strict"] = json_schema_strict
+        if json_object_mode:
+            result["json_object"] = True
+        if sampling_params:
+            result["llm_params"] = sampling_params
+        return result
+
+    # Normal request — extract the question (text) from the last user message,
+    # and keep its full content array (text + image_url parts) when multimodal so
+    # images still reach the model in the final user message.
+    last_user_content = None
+    for msg in reversed(messages):
+        if msg.get("role") == "user":
+            last_user_content = msg.get("content")
+            break
+    question = content_to_text(last_user_content)
+    multimodal_content = last_user_content if isinstance(last_user_content, list) else None
+
+    history = convert_history(messages)
+    system_prompt_override = extract_system_prompt(messages)
+
+    docsgpt = data.get("docsgpt", {})
+
+    result = {
+        "question": question,
+        "api_key": api_key,
+        "history": json.dumps(history),
+        # v1 conversations always persist and stay hidden from the agent
+        # owner's sidebar; the legacy ``docsgpt.save_conversation`` flag
+        # (old meaning: "persist this conversation") is ignored.
+    }
+
+    if system_prompt_override is not None:
+        result["system_prompt_override"] = system_prompt_override
+
+    # Client tools
+    if data.get("tools"):
+        result["client_tools"] = data["tools"]
+
+    # DocsGPT extensions
+    if docsgpt.get("attachments"):
+        result["attachments"] = docsgpt["attachments"]
+
+    if response_schema is not None:
+        result["json_schema"] = response_schema
+        result["json_schema_strict"] = json_schema_strict
+    if json_object_mode:
+        result["json_object"] = True
+    if sampling_params:
+        result["llm_params"] = sampling_params
+    if multimodal_content is not None:
+        result["multimodal_content"] = multimodal_content
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Response translation (non-streaming)
+# ---------------------------------------------------------------------------
+
+
+def translate_response(
+    conversation_id: str,
+    answer: str,
+    sources: Optional[List[Dict]],
+    tool_calls: Optional[List[Dict]],
+    thought: str,
+    model_name: str,
+    pending_tool_calls: Optional[List[Dict]] = None,
+    strip_reasoning_leak: bool = False,
+) -> Dict[str, Any]:
+    """Translate DocsGPT response to chat completions format.
+
+    Args:
+        conversation_id: The DocsGPT conversation ID.
+        answer: The assistant's text response.
+        sources: RAG retrieval sources.
+        tool_calls: Completed tool call results.
+        thought: Reasoning/thinking tokens.
+        model_name: Model/agent identifier.
+        pending_tool_calls: Pending client-side tool calls (if paused).
+
+    Returns:
+        Dict in the standard chat completions response format.
+    """
+    created = int(time.time())
+    completion_id = f"chatcmpl-{conversation_id}" if conversation_id else f"chatcmpl-{created}"
+
+    # Build message
+    message: Dict[str, Any] = {"role": "assistant"}
+
+    if pending_tool_calls:
+        # Tool calls pending — return them for client execution
+        message["content"] = None
+        message["tool_calls"] = [
+            {
+                "id": tc.get("call_id", ""),
+                "type": "function",
+                "function": {
+                    "name": _get_client_tool_name(tc),
+                    "arguments": (
+                        json.dumps(tc["arguments"])
+                        if isinstance(tc.get("arguments"), dict)
+                        else tc.get("arguments", "{}")
+                    ),
+                },
+            }
+            for tc in pending_tool_calls
+        ]
+        finish_reason = "tool_calls"
+    else:
+        if strip_reasoning_leak:
+            clean_answer, leaked_reasoning = _split_leaked_reasoning(answer)
+        else:
+            clean_answer, leaked_reasoning = answer, ""
+        message["content"] = clean_answer
+        combined_reasoning = (thought or "") + leaked_reasoning
+        if combined_reasoning:
+            message["reasoning_content"] = combined_reasoning
+        finish_reason = "stop"
+
+    result: Dict[str, Any] = {
+        "id": completion_id,
+        "object": "chat.completion",
+        "created": created,
+        "model": model_name,
+        "choices": [
+            {
+                "index": 0,
+                "message": message,
+                "finish_reason": finish_reason,
+            }
+        ],
+        "usage": {
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "total_tokens": 0,
+        },
+    }
+
+    # DocsGPT extensions
+    docsgpt: Dict[str, Any] = {}
+    if conversation_id:
+        docsgpt["conversation_id"] = conversation_id
+    if sources:
+        docsgpt["sources"] = sources
+    if tool_calls:
+        docsgpt["tool_calls"] = tool_calls
+    if docsgpt:
+        result["docsgpt"] = docsgpt
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Streaming event translation
+# ---------------------------------------------------------------------------
+
+
+def _make_chunk(
+    completion_id: str,
+    model_name: str,
+    delta: Dict[str, Any],
+    finish_reason: Optional[str] = None,
+) -> str:
+    """Build a single SSE chunk in the standard streaming format."""
+    chunk = {
+        "id": completion_id,
+        "object": "chat.completion.chunk",
+        "created": int(time.time()),
+        "model": model_name,
+        "choices": [
+            {
+                "index": 0,
+                "delta": delta,
+                "finish_reason": finish_reason,
+            }
+        ],
+    }
+    return f"data: {json.dumps(chunk)}\n\n"
+
+
+def _make_docsgpt_chunk(data: Dict[str, Any], completion_id: str, model_name: str) -> str:
+    """Build a DocsGPT extension chunk that is ALSO a valid ``chat.completion.chunk``.
+
+    Strict OpenAI clients (e.g. the Vercel AI SDK used by opencode) validate every
+    SSE ``data:`` frame as a chat.completion.chunk, so the DocsGPT extension is
+    attached to an otherwise-empty (no-op) chunk rather than sent as a bare
+    ``{"docsgpt": ...}`` object — which has no ``choices`` and fails validation.
+    OpenAI clients ignore the extra top-level ``docsgpt`` field.
+    """
+    chunk = {
+        "id": completion_id,
+        "object": "chat.completion.chunk",
+        "created": int(time.time()),
+        "model": model_name,
+        "choices": [{"index": 0, "delta": {}, "finish_reason": None}],
+        "docsgpt": data,
+    }
+    return f"data: {json.dumps(chunk)}\n\n"
+
+
+def translate_stream_event(
+    event_data: Dict[str, Any],
+    completion_id: str,
+    model_name: str,
+    strip_reasoning_leak: bool = False,
+) -> List[str]:
+    """Translate a DocsGPT SSE event dict to standard streaming chunks.
+
+    May return 0, 1, or 2 chunks per input event. For example, a completed
+    tool call produces both a docsgpt extension chunk and nothing on the
+    standard side (since server-side tool calls aren't surfaced in standard
+    format).
+
+    Args:
+        event_data: Parsed DocsGPT event dict.
+        completion_id: The completion ID for this response.
+        model_name: Model/agent identifier.
+
+    Returns:
+        List of SSE-formatted strings to send to the client.
+    """
+    event_type = event_data.get("type")
+    chunks: List[str] = []
+
+    if event_type == "answer":
+        raw = event_data.get("answer", "")
+        clean, leaked = (
+            _split_leaked_reasoning(raw) if strip_reasoning_leak else (raw, "")
+        )
+        if leaked:
+            chunks.append(
+                _make_chunk(completion_id, model_name, {"reasoning_content": leaked})
+            )
+        if clean:
+            chunks.append(
+                _make_chunk(completion_id, model_name, {"content": clean})
+            )
+
+    elif event_type == "thought":
+        chunks.append(
+            _make_chunk(
+                completion_id, model_name,
+                {"reasoning_content": event_data.get("thought", "")},
+            )
+        )
+
+    elif event_type == "source":
+        chunks.append(
+            _make_docsgpt_chunk(
+                {"type": "source", "sources": event_data.get("source", [])},
+                completion_id, model_name,
+            )
+        )
+
+    elif event_type == "tool_call":
+        tc_data = event_data.get("data", {})
+        status = tc_data.get("status")
+
+        if status == "requires_client_execution":
+            # Standard: stream as tool_calls delta
+            args = tc_data.get("arguments", {})
+            args_str = json.dumps(args) if isinstance(args, dict) else str(args)
+            chunks.append(
+                _make_chunk(completion_id, model_name, {
+                    "tool_calls": [{
+                        "index": 0,
+                        "id": tc_data.get("call_id", ""),
+                        "type": "function",
+                        "function": {
+                            "name": _get_client_tool_name(tc_data),
+                            "arguments": args_str,
+                        },
+                    }],
+                })
+            )
+        elif status == "awaiting_approval":
+            # Extension: approval needed
+            chunks.append(_make_docsgpt_chunk({"type": "tool_call", "data": tc_data}, completion_id, model_name))
+        elif status in ("completed", "pending", "error", "denied", "skipped"):
+            # Extension: tool call progress
+            chunks.append(_make_docsgpt_chunk({"type": "tool_call", "data": tc_data}, completion_id, model_name))
+
+    elif event_type == "tool_calls_pending":
+        # Standard: finish_reason = tool_calls
+        chunks.append(
+            _make_chunk(completion_id, model_name, {}, finish_reason="tool_calls")
+        )
+        # Also emit as docsgpt extension
+        chunks.append(
+            _make_docsgpt_chunk(
+                {
+                    "type": "tool_calls_pending",
+                    "pending_tool_calls": event_data.get("data", {}).get("pending_tool_calls", []),
+                },
+                completion_id, model_name,
+            )
+        )
+
+    elif event_type == "end":
+        chunks.append(
+            _make_chunk(completion_id, model_name, {}, finish_reason="stop")
+        )
+        chunks.append("data: [DONE]\n\n")
+
+    elif event_type == "id":
+        # Skip the "None" placeholder conversation_id emitted when the call is
+        # not persisted (persist=false tool rounds) — nothing useful to surface.
+        conv_id = event_data.get("id", "")
+        if conv_id and conv_id != "None":
+            chunks.append(
+                _make_docsgpt_chunk(
+                    {"type": "id", "conversation_id": conv_id},
+                    completion_id, model_name,
+                )
+            )
+
+    elif event_type == "error":
+        # Emit as standard error (non-standard but widely supported)
+        error_data = {
+            "error": {
+                "message": event_data.get("error", "An error occurred"),
+                "type": "server_error",
+            }
+        }
+        chunks.append(f"data: {json.dumps(error_data)}\n\n")
+
+    elif event_type == "structured_answer":
+        raw = event_data.get("answer", "")
+        clean, leaked = (
+            _split_leaked_reasoning(raw) if strip_reasoning_leak else (raw, "")
+        )
+        if leaked:
+            chunks.append(
+                _make_chunk(completion_id, model_name, {"reasoning_content": leaked})
+            )
+        if clean:
+            chunks.append(
+                _make_chunk(completion_id, model_name, {"content": clean})
+            )
+
+    # Skip: tool_calls (redundant), research_plan, research_progress
+
+    return chunks