chore: import upstream snapshot with attribution

apps/sim/lib/copilot/async-runs/lifecycle.test.ts

@@ -0,0 +1,38 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import {
+  ASYNC_TOOL_CONFIRMATION_STATUS,
+  ASYNC_TOOL_STATUS,
+  isAsyncEphemeralConfirmationStatus,
+  isAsyncTerminalConfirmationStatus,
+  isDeliveredAsyncStatus,
+  isTerminalAsyncStatus,
+} from './lifecycle'
+
+describe('async tool lifecycle helpers', () => {
+  it('treats only completed, failed, and cancelled as terminal execution states', () => {
+    expect(isTerminalAsyncStatus(ASYNC_TOOL_STATUS.pending)).toBe(false)
+    expect(isTerminalAsyncStatus(ASYNC_TOOL_STATUS.running)).toBe(false)
+    expect(isTerminalAsyncStatus(ASYNC_TOOL_STATUS.completed)).toBe(true)
+    expect(isTerminalAsyncStatus(ASYNC_TOOL_STATUS.failed)).toBe(true)
+    expect(isTerminalAsyncStatus(ASYNC_TOOL_STATUS.cancelled)).toBe(true)
+    expect(isTerminalAsyncStatus(ASYNC_TOOL_STATUS.delivered)).toBe(false)
+  })
+
+  it('treats delivered rows as distinct from terminal execution states', () => {
+    expect(isDeliveredAsyncStatus(ASYNC_TOOL_STATUS.delivered)).toBe(true)
+  })
+
+  it('distinguishes background from terminal completion statuses', () => {
+    expect(isAsyncEphemeralConfirmationStatus(ASYNC_TOOL_CONFIRMATION_STATUS.background)).toBe(true)
+    expect(isAsyncEphemeralConfirmationStatus(ASYNC_TOOL_CONFIRMATION_STATUS.success)).toBe(false)
+
+    expect(isAsyncTerminalConfirmationStatus(ASYNC_TOOL_CONFIRMATION_STATUS.success)).toBe(true)
+    expect(isAsyncTerminalConfirmationStatus(ASYNC_TOOL_CONFIRMATION_STATUS.error)).toBe(true)
+    expect(isAsyncTerminalConfirmationStatus(ASYNC_TOOL_CONFIRMATION_STATUS.cancelled)).toBe(true)
+    expect(isAsyncTerminalConfirmationStatus(ASYNC_TOOL_CONFIRMATION_STATUS.background)).toBe(true)
+  })
+})

apps/sim/lib/copilot/async-runs/lifecycle.ts

@@ -0,0 +1,115 @@
+import type { CopilotAsyncToolStatus } from '@sim/db/schema'
+import {
+  MothershipStreamV1AsyncToolRecordStatus,
+  MothershipStreamV1ToolOutcome,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+
+export const ASYNC_TOOL_STATUS = MothershipStreamV1AsyncToolRecordStatus
+
+export type AsyncLifecycleStatus =
+  | typeof ASYNC_TOOL_STATUS.pending
+  | typeof ASYNC_TOOL_STATUS.running
+  | typeof ASYNC_TOOL_STATUS.completed
+  | typeof ASYNC_TOOL_STATUS.failed
+  | typeof ASYNC_TOOL_STATUS.cancelled
+
+export type AsyncTerminalStatus =
+  | typeof ASYNC_TOOL_STATUS.completed
+  | typeof ASYNC_TOOL_STATUS.failed
+  | typeof ASYNC_TOOL_STATUS.cancelled
+
+/**
+ * Confirmation statuses sent on the request-local async tool confirmation channel.
+ *
+ * `background` is still emitted by the current browser workflow runtime on `pagehide`.
+ */
+export const ASYNC_TOOL_CONFIRMATION_STATUS = {
+  background: 'background',
+  success: MothershipStreamV1ToolOutcome.success,
+  error: MothershipStreamV1ToolOutcome.error,
+  cancelled: MothershipStreamV1ToolOutcome.cancelled,
+} as const
+
+export type AsyncConfirmationStatus =
+  (typeof ASYNC_TOOL_CONFIRMATION_STATUS)[keyof typeof ASYNC_TOOL_CONFIRMATION_STATUS]
+
+export type AsyncTerminalConfirmationStatus = AsyncConfirmationStatus
+
+export type AsyncConfirmationProgressStatus =
+  | typeof ASYNC_TOOL_STATUS.pending
+  | typeof ASYNC_TOOL_STATUS.running
+
+export type AsyncEphemeralConfirmationStatus = typeof ASYNC_TOOL_CONFIRMATION_STATUS.background
+
+export type AsyncConfirmationStateStatus = AsyncConfirmationProgressStatus | AsyncConfirmationStatus
+
+export type AsyncPromiseStatus = typeof ASYNC_TOOL_STATUS.running | AsyncTerminalConfirmationStatus
+
+export type AsyncCompletionData = unknown
+
+export interface AsyncCompletionEnvelope {
+  toolCallId: string
+  status: AsyncConfirmationStatus
+  message?: string
+  data?: AsyncCompletionData
+  runId?: string
+  checkpointId?: string
+  executionId?: string
+  chatId?: string
+  timestamp?: string
+}
+
+export type AsyncCompletionSnapshot = Pick<
+  AsyncCompletionEnvelope,
+  'status' | 'message' | 'data' | 'timestamp'
+>
+
+export interface AsyncTerminalCompletionSnapshot extends AsyncCompletionSnapshot {
+  status: AsyncTerminalConfirmationStatus
+}
+
+export interface AsyncConfirmationState {
+  status: AsyncConfirmationStateStatus
+  message?: string
+  data?: AsyncCompletionData
+  timestamp?: string
+}
+
+export interface AsyncCompletionSignal {
+  status: AsyncPromiseStatus
+  message?: string
+  data?: AsyncCompletionData
+}
+
+export function isTerminalAsyncStatus(
+  status: CopilotAsyncToolStatus | AsyncLifecycleStatus | string | null | undefined
+): status is AsyncTerminalStatus {
+  return (
+    status === ASYNC_TOOL_STATUS.completed ||
+    status === ASYNC_TOOL_STATUS.failed ||
+    status === ASYNC_TOOL_STATUS.cancelled
+  )
+}
+
+export function isDeliveredAsyncStatus(
+  status: CopilotAsyncToolStatus | string | null | undefined
+): status is typeof ASYNC_TOOL_STATUS.delivered {
+  return status === ASYNC_TOOL_STATUS.delivered
+}
+
+export function isAsyncTerminalConfirmationStatus(
+  status: string | null | undefined
+): status is AsyncTerminalConfirmationStatus {
+  return (
+    status === ASYNC_TOOL_CONFIRMATION_STATUS.background ||
+    status === ASYNC_TOOL_CONFIRMATION_STATUS.success ||
+    status === ASYNC_TOOL_CONFIRMATION_STATUS.error ||
+    status === ASYNC_TOOL_CONFIRMATION_STATUS.cancelled
+  )
+}
+
+export function isAsyncEphemeralConfirmationStatus(
+  status: string | null | undefined
+): status is AsyncEphemeralConfirmationStatus {
+  return status === ASYNC_TOOL_CONFIRMATION_STATUS.background
+}

apps/sim/lib/copilot/async-runs/repository.test.ts

@@ -0,0 +1,83 @@
+/**
+ * @vitest-environment node
+ */
+
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+import {
+  claimCompletedAsyncToolCall,
+  completeAsyncToolCall,
+  markAsyncToolDelivered,
+} from './repository'
+
+describe('async tool repository single-row semantics', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('does not overwrite a delivered row on late completion', async () => {
+    const deliveredRow = {
+      toolCallId: 'tool-1',
+      status: 'delivered',
+      result: { ok: true },
+      error: null,
+    }
+    dbChainMockFns.limit.mockResolvedValueOnce([deliveredRow])
+
+    const result = await completeAsyncToolCall({
+      toolCallId: 'tool-1',
+      status: 'completed',
+      result: { ok: false },
+      error: null,
+    })
+
+    expect(result).toEqual(deliveredRow)
+    expect(dbChainMockFns.returning).not.toHaveBeenCalled()
+  })
+
+  it('marks a row delivered and clears the claim fields', async () => {
+    dbChainMockFns.returning.mockResolvedValueOnce([
+      {
+        toolCallId: 'tool-1',
+        status: 'delivered',
+      },
+    ])
+
+    await markAsyncToolDelivered('tool-1')
+
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({
+        status: 'delivered',
+        claimedBy: null,
+        claimedAt: null,
+      })
+    )
+  })
+
+  it('claims only completed rows for delivery handoff', async () => {
+    dbChainMockFns.returning.mockResolvedValueOnce([
+      {
+        toolCallId: 'tool-1',
+        status: 'completed',
+        claimedBy: 'worker-1',
+      },
+    ])
+
+    const result = await claimCompletedAsyncToolCall('tool-1', 'worker-1')
+
+    expect(result).toEqual({
+      toolCallId: 'tool-1',
+      status: 'completed',
+      claimedBy: 'worker-1',
+    })
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({
+        claimedBy: 'worker-1',
+      })
+    )
+  })
+})

apps/sim/lib/copilot/async-runs/repository.ts

@@ -0,0 +1,498 @@
+import { trace } from '@opentelemetry/api'
+import { db } from '@sim/db'
+import {
+  type CopilotAsyncToolStatus,
+  type CopilotRunStatus,
+  copilotAsyncToolCalls,
+  copilotRunCheckpoints,
+  copilotRuns,
+} from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { filterUndefined } from '@sim/utils/object'
+import { and, desc, eq, inArray, isNull } from 'drizzle-orm'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { markSpanForError } from '@/lib/copilot/request/otel'
+import {
+  ASYNC_TOOL_STATUS,
+  type AsyncCompletionData,
+  isDeliveredAsyncStatus,
+  isTerminalAsyncStatus,
+} from './lifecycle'
+
+const logger = createLogger('CopilotAsyncRunsRepo')
+// Resolve the tracer lazily per-call to avoid capturing the NoOp tracer
+// before NodeSDK installs the global TracerProvider (Next.js 16/Turbopack
+// can evaluate modules before instrumentation-node.ts finishes).
+const getAsyncRunsTracer = () => trace.getTracer('sim-copilot-async-runs', '1.0.0')
+
+// Wrap an async DB op in a client-kind span with canonical `db.*` attrs.
+// Cancellation is routed through `markSpanForError` so aborts record the
+// exception event but don't paint spans red.
+async function withDbSpan<T>(
+  name: string,
+  op: string,
+  table: string,
+  attrs: Record<string, string | number | boolean | undefined>,
+  fn: () => Promise<T>
+): Promise<T> {
+  const span = getAsyncRunsTracer().startSpan(name, {
+    attributes: {
+      [TraceAttr.DbSystem]: 'postgresql',
+      [TraceAttr.DbOperation]: op,
+      [TraceAttr.DbSqlTable]: table,
+      ...filterUndefined(attrs),
+    },
+  })
+  try {
+    return await fn()
+  } catch (error) {
+    markSpanForError(span, error)
+    throw error
+  } finally {
+    span.end()
+  }
+}
+
+export interface CreateRunSegmentInput {
+  id?: string
+  executionId: string
+  parentRunId?: string | null
+  chatId: string
+  userId: string
+  workflowId?: string | null
+  workspaceId?: string | null
+  streamId: string
+  agent?: string | null
+  model?: string | null
+  provider?: string | null
+  requestContext?: Record<string, unknown>
+  status?: CopilotRunStatus
+}
+
+export async function createRunSegment(input: CreateRunSegmentInput) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsCreateRunSegment,
+    'INSERT',
+    'copilot_runs',
+    {
+      [TraceAttr.CopilotExecutionId]: input.executionId,
+      [TraceAttr.ChatId]: input.chatId,
+      [TraceAttr.StreamId]: input.streamId,
+      [TraceAttr.UserId]: input.userId,
+      [TraceAttr.CopilotRunParentId]: input.parentRunId ?? undefined,
+      [TraceAttr.CopilotRunAgent]: input.agent ?? undefined,
+      [TraceAttr.CopilotRunModel]: input.model ?? undefined,
+      [TraceAttr.CopilotRunProvider]: input.provider ?? undefined,
+      [TraceAttr.CopilotRunStatus]: input.status ?? 'active',
+    },
+    async () => {
+      const [run] = await db
+        .insert(copilotRuns)
+        .values({
+          ...(input.id ? { id: input.id } : {}),
+          executionId: input.executionId,
+          parentRunId: input.parentRunId ?? null,
+          chatId: input.chatId,
+          userId: input.userId,
+          workflowId: input.workflowId ?? null,
+          workspaceId: input.workspaceId ?? null,
+          streamId: input.streamId,
+          agent: input.agent ?? null,
+          model: input.model ?? null,
+          provider: input.provider ?? null,
+          requestContext: input.requestContext ?? {},
+          status: input.status ?? 'active',
+        })
+        .returning()
+      return run
+    }
+  )
+}
+
+export async function updateRunStatus(
+  runId: string,
+  status: CopilotRunStatus,
+  updates: {
+    completedAt?: Date | null
+    error?: string | null
+    requestContext?: Record<string, unknown>
+  } = {}
+) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsUpdateRunStatus,
+    'UPDATE',
+    'copilot_runs',
+    {
+      [TraceAttr.RunId]: runId,
+      [TraceAttr.CopilotRunStatus]: status,
+      [TraceAttr.CopilotRunHasError]: !!updates.error,
+      [TraceAttr.CopilotRunHasCompletedAt]: !!updates.completedAt,
+    },
+    async () => {
+      const [run] = await db
+        .update(copilotRuns)
+        .set({
+          status,
+          completedAt: updates.completedAt,
+          error: updates.error,
+          requestContext: updates.requestContext,
+          updatedAt: new Date(),
+        })
+        .where(eq(copilotRuns.id, runId))
+        .returning()
+      return run ?? null
+    }
+  )
+}
+
+async function getLatestRunForExecution(executionId: string) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsGetLatestForExecution,
+    'SELECT',
+    'copilot_runs',
+    { [TraceAttr.CopilotExecutionId]: executionId },
+    async () => {
+      const [run] = await db
+        .select()
+        .from(copilotRuns)
+        .where(eq(copilotRuns.executionId, executionId))
+        .orderBy(desc(copilotRuns.startedAt))
+        .limit(1)
+      return run ?? null
+    }
+  )
+}
+
+// Un-instrumented: called from a 4 Hz resume poll; per-call spans
+// swamped traces. Use Prom histograms if latency visibility is needed.
+export async function getLatestRunForStream(streamId: string, userId?: string) {
+  const conditions = userId
+    ? and(eq(copilotRuns.streamId, streamId), eq(copilotRuns.userId, userId))
+    : eq(copilotRuns.streamId, streamId)
+  const [run] = await db
+    .select()
+    .from(copilotRuns)
+    .where(conditions)
+    .orderBy(desc(copilotRuns.startedAt))
+    .limit(1)
+  return run ?? null
+}
+
+export async function getRunSegment(runId: string) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsGetRunSegment,
+    'SELECT',
+    'copilot_runs',
+    { [TraceAttr.RunId]: runId },
+    async () => {
+      const [run] = await db
+        .select({ id: copilotRuns.id, userId: copilotRuns.userId })
+        .from(copilotRuns)
+        .where(eq(copilotRuns.id, runId))
+        .limit(1)
+      return run ?? null
+    }
+  )
+}
+
+async function createRunCheckpoint(input: {
+  runId: string
+  pendingToolCallId: string
+  conversationSnapshot: Record<string, unknown>
+  agentState: Record<string, unknown>
+  providerRequest: Record<string, unknown>
+}) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsCreateRunCheckpoint,
+    'INSERT',
+    'copilot_run_checkpoints',
+    {
+      [TraceAttr.RunId]: input.runId,
+      [TraceAttr.CopilotCheckpointPendingToolCallId]: input.pendingToolCallId,
+    },
+    async () => {
+      const [checkpoint] = await db
+        .insert(copilotRunCheckpoints)
+        .values({
+          runId: input.runId,
+          pendingToolCallId: input.pendingToolCallId,
+          conversationSnapshot: input.conversationSnapshot,
+          agentState: input.agentState,
+          providerRequest: input.providerRequest,
+        })
+        .returning()
+
+      return checkpoint
+    }
+  )
+}
+
+export async function upsertAsyncToolCall(input: {
+  runId?: string | null
+  checkpointId?: string | null
+  toolCallId: string
+  toolName: string
+  args?: Record<string, unknown>
+  status?: CopilotAsyncToolStatus
+}) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsUpsertAsyncToolCall,
+    'UPSERT',
+    'copilot_async_tool_calls',
+    {
+      [TraceAttr.ToolCallId]: input.toolCallId,
+      [TraceAttr.ToolName]: input.toolName,
+      [TraceAttr.CopilotAsyncToolStatus]: input.status ?? 'pending',
+      [TraceAttr.RunId]: input.runId ?? undefined,
+    },
+    async () => {
+      const existing = await getAsyncToolCall(input.toolCallId)
+      const incomingStatus = input.status ?? 'pending'
+      if (
+        existing &&
+        (isTerminalAsyncStatus(existing.status) || isDeliveredAsyncStatus(existing.status)) &&
+        !isTerminalAsyncStatus(incomingStatus) &&
+        !isDeliveredAsyncStatus(incomingStatus)
+      ) {
+        logger.info('Ignoring async tool upsert that would downgrade terminal state', {
+          toolCallId: input.toolCallId,
+          existingStatus: existing.status,
+          incomingStatus,
+        })
+        return existing
+      }
+      const effectiveRunId = input.runId ?? existing?.runId ?? null
+      if (!effectiveRunId) {
+        logger.warn('upsertAsyncToolCall missing runId and no existing row', {
+          toolCallId: input.toolCallId,
+          toolName: input.toolName,
+          status: input.status ?? 'pending',
+        })
+        return null
+      }
+
+      const now = new Date()
+      const [row] = await db
+        .insert(copilotAsyncToolCalls)
+        .values({
+          runId: effectiveRunId,
+          checkpointId: input.checkpointId ?? null,
+          toolCallId: input.toolCallId,
+          toolName: input.toolName,
+          args: input.args ?? {},
+          status: incomingStatus,
+          updatedAt: now,
+        })
+        .onConflictDoUpdate({
+          target: copilotAsyncToolCalls.toolCallId,
+          set: {
+            runId: effectiveRunId,
+            checkpointId: input.checkpointId ?? null,
+            toolName: input.toolName,
+            args: input.args ?? {},
+            status: incomingStatus,
+            updatedAt: now,
+          },
+        })
+        .returning()
+
+      return row
+    }
+  )
+}
+
+export async function getAsyncToolCall(toolCallId: string) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsGetAsyncToolCall,
+    'SELECT',
+    'copilot_async_tool_calls',
+    { [TraceAttr.ToolCallId]: toolCallId },
+    async () => {
+      const [row] = await db
+        .select()
+        .from(copilotAsyncToolCalls)
+        .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId))
+        .limit(1)
+      return row ?? null
+    }
+  )
+}
+
+async function markAsyncToolStatus(
+  toolCallId: string,
+  status: CopilotAsyncToolStatus,
+  updates: {
+    claimedBy?: string | null
+    claimedAt?: Date | null
+    result?: AsyncCompletionData | null
+    error?: string | null
+    completedAt?: Date | null
+  } = {}
+) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsMarkAsyncToolStatus,
+    'UPDATE',
+    'copilot_async_tool_calls',
+    {
+      [TraceAttr.ToolCallId]: toolCallId,
+      [TraceAttr.CopilotAsyncToolStatus]: status,
+      [TraceAttr.CopilotAsyncToolHasError]: !!updates.error,
+      [TraceAttr.CopilotAsyncToolClaimedBy]: updates.claimedBy ?? undefined,
+    },
+    async () => {
+      const claimedAt =
+        updates.claimedAt !== undefined
+          ? updates.claimedAt
+          : status === 'running' && updates.claimedBy
+            ? new Date()
+            : undefined
+
+      const [row] = await db
+        .update(copilotAsyncToolCalls)
+        .set({
+          status,
+          claimedBy: updates.claimedBy,
+          claimedAt,
+          result: updates.result,
+          error: updates.error,
+          completedAt: updates.completedAt,
+          updatedAt: new Date(),
+        })
+        .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId))
+        .returning()
+
+      return row ?? null
+    }
+  )
+}
+
+export async function markAsyncToolRunning(toolCallId: string, claimedBy: string) {
+  return markAsyncToolStatus(toolCallId, 'running', { claimedBy })
+}
+
+export async function completeAsyncToolCall(input: {
+  toolCallId: string
+  status: Extract<CopilotAsyncToolStatus, 'completed' | 'failed' | 'cancelled'>
+  result?: AsyncCompletionData | null
+  error?: string | null
+}) {
+  const existing = await getAsyncToolCall(input.toolCallId)
+
+  if (!existing) {
+    logger.warn('completeAsyncToolCall called before pending row existed', {
+      toolCallId: input.toolCallId,
+      status: input.status,
+    })
+    return null
+  }
+
+  if (isTerminalAsyncStatus(existing.status) || isDeliveredAsyncStatus(existing.status)) {
+    return existing
+  }
+
+  return markAsyncToolStatus(input.toolCallId, input.status, {
+    claimedBy: null,
+    claimedAt: null,
+    result: input.result ?? null,
+    error: input.error ?? null,
+    completedAt: new Date(),
+  })
+}
+
+export async function markAsyncToolDelivered(toolCallId: string) {
+  return markAsyncToolStatus(toolCallId, ASYNC_TOOL_STATUS.delivered, {
+    claimedBy: null,
+    claimedAt: null,
+  })
+}
+
+async function listAsyncToolCallsForRun(runId: string) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsListForRun,
+    'SELECT',
+    'copilot_async_tool_calls',
+    { [TraceAttr.RunId]: runId },
+    async () =>
+      db
+        .select()
+        .from(copilotAsyncToolCalls)
+        .where(eq(copilotAsyncToolCalls.runId, runId))
+        .orderBy(desc(copilotAsyncToolCalls.createdAt))
+  )
+}
+
+export async function getAsyncToolCalls(toolCallIds: string[]) {
+  if (toolCallIds.length === 0) return []
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsGetMany,
+    'SELECT',
+    'copilot_async_tool_calls',
+    { [TraceAttr.CopilotAsyncToolIdsCount]: toolCallIds.length },
+    async () =>
+      db
+        .select()
+        .from(copilotAsyncToolCalls)
+        .where(inArray(copilotAsyncToolCalls.toolCallId, toolCallIds))
+  )
+}
+
+export async function claimCompletedAsyncToolCall(toolCallId: string, workerId: string) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsClaimCompleted,
+    'UPDATE',
+    'copilot_async_tool_calls',
+    {
+      [TraceAttr.ToolCallId]: toolCallId,
+      [TraceAttr.CopilotAsyncToolWorkerId]: workerId,
+    },
+    async () => {
+      const [row] = await db
+        .update(copilotAsyncToolCalls)
+        .set({
+          claimedBy: workerId,
+          claimedAt: new Date(),
+          updatedAt: new Date(),
+        })
+        .where(
+          and(
+            eq(copilotAsyncToolCalls.toolCallId, toolCallId),
+            inArray(copilotAsyncToolCalls.status, ['completed', 'failed', 'cancelled']),
+            isNull(copilotAsyncToolCalls.claimedBy)
+          )
+        )
+        .returning()
+      return row ?? null
+    }
+  )
+}
+
+async function releaseCompletedAsyncToolClaim(toolCallId: string, workerId: string) {
+  return withDbSpan(
+    TraceSpan.CopilotAsyncRunsReleaseClaim,
+    'UPDATE',
+    'copilot_async_tool_calls',
+    {
+      [TraceAttr.ToolCallId]: toolCallId,
+      [TraceAttr.CopilotAsyncToolWorkerId]: workerId,
+    },
+    async () => {
+      const [row] = await db
+        .update(copilotAsyncToolCalls)
+        .set({
+          claimedBy: null,
+          claimedAt: null,
+          updatedAt: new Date(),
+        })
+        .where(
+          and(
+            eq(copilotAsyncToolCalls.toolCallId, toolCallId),
+            inArray(copilotAsyncToolCalls.status, ['completed', 'failed', 'cancelled']),
+            eq(copilotAsyncToolCalls.claimedBy, workerId)
+          )
+        )
+        .returning()
+      return row ?? null
+    }
+  )
+}

apps/sim/lib/copilot/auth/permissions.test.ts

@@ -0,0 +1,141 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockAuthorizeWorkflowByWorkspacePermission } = vi.hoisted(() => ({
+  mockAuthorizeWorkflowByWorkspacePermission: vi.fn(),
+}))
+
+vi.mock('@sim/platform-authz/workflow', () => ({
+  authorizeWorkflowByWorkspacePermission: mockAuthorizeWorkflowByWorkspacePermission,
+}))
+
+import { createPermissionError, verifyWorkflowAccess } from '@/lib/copilot/auth/permissions'
+
+describe('Copilot Auth Permissions', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('verifyWorkflowAccess', () => {
+    it('should return no access for non-existent workflow', async () => {
+      mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+        allowed: false,
+        status: 404,
+        workflow: null,
+        workspacePermission: null,
+      })
+
+      const result = await verifyWorkflowAccess('user-123', 'non-existent-workflow')
+
+      expect(result).toEqual({ hasAccess: false, userPermission: null })
+    })
+
+    it('should delegate to the shared workflow authorizer with a read action', async () => {
+      mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+        allowed: true,
+        status: 200,
+        workflow: { workspaceId: 'workspace-456' },
+        workspacePermission: 'write',
+      })
+
+      await verifyWorkflowAccess('user-123', 'workflow-789')
+
+      expect(mockAuthorizeWorkflowByWorkspacePermission).toHaveBeenCalledWith({
+        workflowId: 'workflow-789',
+        userId: 'user-123',
+        action: 'read',
+      })
+    })
+
+    it.each(['read', 'write', 'admin'] as const)(
+      'should grant access with %s permission through the workspace',
+      async (permission) => {
+        mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+          allowed: true,
+          status: 200,
+          workflow: { workspaceId: 'workspace-456' },
+          workspacePermission: permission,
+        })
+
+        const result = await verifyWorkflowAccess('user-123', 'workflow-789')
+
+        expect(result).toEqual({
+          hasAccess: true,
+          userPermission: permission,
+          workspaceId: 'workspace-456',
+        })
+      }
+    )
+
+    it('should report the workspaceId even when permission is denied for an existing workflow', async () => {
+      mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+        allowed: false,
+        status: 403,
+        workflow: { workspaceId: 'workspace-456' },
+        workspacePermission: null,
+      })
+
+      const result = await verifyWorkflowAccess('user-123', 'workflow-789')
+
+      expect(result).toEqual({
+        hasAccess: false,
+        userPermission: null,
+        workspaceId: 'workspace-456',
+      })
+    })
+
+    it('should return no access for a workflow without a workspace', async () => {
+      mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+        allowed: false,
+        status: 403,
+        workflow: { workspaceId: null },
+        workspacePermission: null,
+      })
+
+      const result = await verifyWorkflowAccess('user-123', 'workflow-789')
+
+      expect(result).toEqual({ hasAccess: false, userPermission: null })
+    })
+
+    it('should handle errors gracefully', async () => {
+      mockAuthorizeWorkflowByWorkspacePermission.mockRejectedValueOnce(
+        new Error('Database connection failed')
+      )
+
+      const result = await verifyWorkflowAccess('user-123', 'workflow-789')
+
+      expect(result).toEqual({ hasAccess: false, userPermission: null })
+    })
+  })
+
+  describe('createPermissionError', () => {
+    it('should create a permission error message for edit operation', () => {
+      const result = createPermissionError('edit')
+      expect(result).toBe('Access denied: You do not have permission to edit this workflow')
+    })
+
+    it('should create a permission error message for view operation', () => {
+      const result = createPermissionError('view')
+      expect(result).toBe('Access denied: You do not have permission to view this workflow')
+    })
+
+    it('should create a permission error message for delete operation', () => {
+      const result = createPermissionError('delete')
+      expect(result).toBe('Access denied: You do not have permission to delete this workflow')
+    })
+
+    it('should create a permission error message for deploy operation', () => {
+      const result = createPermissionError('deploy')
+      expect(result).toBe('Access denied: You do not have permission to deploy this workflow')
+    })
+
+    it('should create a permission error message for custom operation', () => {
+      const result = createPermissionError('modify settings of')
+      expect(result).toBe(
+        'Access denied: You do not have permission to modify settings of this workflow'
+      )
+    })
+  })
+})

apps/sim/lib/copilot/auth/permissions.ts

@@ -0,0 +1,44 @@
+import { createLogger } from '@sim/logger'
+import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow'
+import type { PermissionType } from '@/lib/workspaces/permissions/utils'
+
+const logger = createLogger('CopilotPermissions')
+
+/**
+ * Verifies if a user has access to a workflow for copilot operations
+ *
+ * @param userId - The authenticated user ID
+ * @param workflowId - The workflow ID to check access for
+ * @returns Promise<{ hasAccess: boolean; userPermission: PermissionType | null; workspaceId?: string }>
+ */
+export async function verifyWorkflowAccess(
+  userId: string,
+  workflowId: string
+): Promise<{
+  hasAccess: boolean
+  userPermission: PermissionType | null
+  workspaceId?: string
+}> {
+  try {
+    const result = await authorizeWorkflowByWorkspacePermission({
+      workflowId,
+      userId,
+      action: 'read',
+    })
+    return {
+      hasAccess: result.allowed,
+      userPermission: result.workspacePermission,
+      workspaceId: result.workflow?.workspaceId ?? undefined,
+    }
+  } catch (error) {
+    logger.error('Error verifying workflow access', { error, workflowId, userId })
+    return { hasAccess: false, userPermission: null }
+  }
+}
+
+/**
+ * Helper function to create consistent permission error messages
+ */
+export function createPermissionError(operation: string): string {
+  return `Access denied: You do not have permission to ${operation} this workflow`
+}

apps/sim/lib/copilot/block-visibility.ts

@@ -0,0 +1,58 @@
+import { LRUCache } from 'lru-cache'
+import { type BlockVisibilityState, getBlockVisibility } from '@/lib/core/config/block-visibility'
+import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils'
+
+/**
+ * Copilot-side resolver for the viewer's block-visibility projection.
+ *
+ * A single mothership turn fans out into many Go→Sim tool callbacks; resolving
+ * visibility per callback would repeat the workspace→org lookup (and, for
+ * admin-gated rules, a replica read) N times. This memoizes the resolved state
+ * per (userId, workspaceId) for a short TTL matching the AppConfig cache
+ * cadence, so a turn costs at most one resolution.
+ */
+const VISIBILITY_CACHE_TTL_MS = 30_000
+
+const visibilityCache = new LRUCache<string, Promise<BlockVisibilityState>>({
+  max: 1000,
+  ttl: VISIBILITY_CACHE_TTL_MS,
+})
+
+async function resolveVisibility(
+  userId: string,
+  workspaceId?: string
+): Promise<BlockVisibilityState> {
+  const orgId = workspaceId
+    ? (await getWorkspaceWithOwner(workspaceId, { includeArchived: true }))?.organizationId
+    : undefined
+  return getBlockVisibility({ userId, orgId })
+}
+
+/** The viewer's visibility state, memoized per (userId, workspaceId) for ~30s. */
+export function getBlockVisibilityForCopilot(
+  userId: string,
+  workspaceId?: string
+): Promise<BlockVisibilityState> {
+  const key = `${userId}:${workspaceId ?? ''}`
+  let promise = visibilityCache.get(key)
+  if (!promise) {
+    promise = resolveVisibility(userId, workspaceId).catch((error) => {
+      visibilityCache.delete(key)
+      throw error
+    })
+    visibilityCache.set(key, promise)
+  }
+  return promise
+}
+
+/**
+ * Stable signature of a visibility state, for keying caches whose contents
+ * depend on the gated projection (e.g. the integration tool-schema LRU).
+ */
+export function visibilitySignature(vis: BlockVisibilityState): string {
+  return JSON.stringify([
+    [...vis.revealed].sort(),
+    [...vis.disabled].sort(),
+    [...vis.previewTagged].sort(),
+  ])
+}

apps/sim/lib/copilot/chat-status.ts

@@ -0,0 +1,42 @@
+/**
+ * Chat Status Pub/Sub Adapter
+ *
+ * Broadcasts chat status events across processes using Redis Pub/Sub.
+ * Gracefully falls back to process-local EventEmitter when Redis is unavailable.
+ *
+ * The Redis channel and SSE label retain the legacy `task:status_changed`
+ * identifier so live status updates keep flowing across pods during a rolling
+ * deploy (old and new pods must publish/subscribe on the same channel).
+ */
+
+import { createPubSubChannel, type PubSubChannel } from '@/lib/events/pubsub'
+
+interface ChatStatusEvent {
+  workspaceId: string
+  chatId: string
+  type: 'started' | 'completed' | 'created' | 'deleted' | 'renamed'
+  streamId?: string
+}
+
+type ChatPubSubGlobal = typeof globalThis & {
+  _chatStatusChannel?: PubSubChannel<ChatStatusEvent> | null
+}
+
+const g = globalThis as ChatPubSubGlobal
+
+if (!('_chatStatusChannel' in g)) {
+  g._chatStatusChannel =
+    typeof window !== 'undefined'
+      ? null
+      : createPubSubChannel<ChatStatusEvent>({ channel: 'task:status_changed', label: 'task' })
+}
+
+const channel = g._chatStatusChannel
+
+export const chatPubSub = channel
+  ? {
+      publishStatusChanged: (event: ChatStatusEvent) => channel.publish(event),
+      onStatusChanged: (handler: (event: ChatStatusEvent) => void) => channel.subscribe(handler),
+      dispose: () => channel.dispose(),
+    }
+  : null

apps/sim/lib/copilot/chat/attachment-preview.ts

@@ -0,0 +1,9 @@
+export function getMothershipAttachmentPreviewUrl(file: {
+  key: string
+  media_type: string
+}): string | undefined {
+  if (!file.media_type.startsWith('image/') && !file.media_type.startsWith('video/')) {
+    return undefined
+  }
+  return `/api/files/serve/${encodeURIComponent(file.key)}?context=mothership`
+}

apps/sim/lib/copilot/chat/display-message.test.ts

@@ -0,0 +1,148 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import { toDisplayMessage } from './display-message'
+
+describe('display-message', () => {
+  it('maps canonical tool, subagent text, and cancelled complete blocks to display blocks', () => {
+    const display = toDisplayMessage({
+      id: 'msg-1',
+      role: 'assistant',
+      content: 'done',
+      timestamp: '2024-01-01T00:00:00.000Z',
+      requestId: 'req-1',
+      contentBlocks: [
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'tool-1',
+            name: 'read',
+            state: 'cancelled',
+            display: { title: 'Stopped by user' },
+          },
+        },
+        {
+          type: 'text',
+          lane: 'subagent',
+          channel: 'assistant',
+          content: 'subagent output',
+        },
+        {
+          type: 'complete',
+          status: 'cancelled',
+        },
+      ],
+    })
+
+    expect(display.contentBlocks).toEqual([
+      {
+        type: 'tool_call',
+        toolCall: {
+          id: 'tool-1',
+          name: 'read',
+          status: 'cancelled',
+          displayTitle: 'Stopped by user',
+          params: undefined,
+          calledBy: undefined,
+          result: undefined,
+        },
+      },
+      {
+        type: 'subagent_text',
+        content: 'subagent output',
+      },
+      {
+        type: 'stopped',
+      },
+    ])
+  })
+
+  it('hides load_agent_skill blocks from display output', () => {
+    const display = toDisplayMessage({
+      id: 'msg-2',
+      role: 'assistant',
+      content: '',
+      timestamp: '2024-01-01T00:00:00.000Z',
+      contentBlocks: [
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'tool-hidden',
+            name: 'load_agent_skill',
+            state: 'success',
+            display: { title: 'Loading skill' },
+          },
+        },
+        {
+          type: 'text',
+          channel: 'assistant',
+          content: 'visible text',
+        },
+      ],
+    })
+
+    expect(display.contentBlocks).toEqual([{ type: 'text', content: 'visible text' }])
+  })
+
+  it('preserves skipped and rejected tool outcomes', () => {
+    const display = toDisplayMessage({
+      id: 'msg-3',
+      role: 'assistant',
+      content: '',
+      timestamp: '2024-01-01T00:00:00.000Z',
+      contentBlocks: [
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'tool-skipped',
+            name: 'read',
+            state: 'skipped',
+            display: { title: 'Reading workflow' },
+          },
+        },
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'tool-rejected',
+            name: 'run_workflow',
+            state: 'rejected',
+            display: { title: 'Running workflow' },
+          },
+        },
+      ],
+    })
+
+    expect(display.contentBlocks).toEqual([
+      {
+        type: 'tool_call',
+        toolCall: {
+          id: 'tool-skipped',
+          name: 'read',
+          status: 'skipped',
+          displayTitle: 'Reading workflow',
+          params: undefined,
+          calledBy: undefined,
+          result: undefined,
+        },
+      },
+      {
+        type: 'tool_call',
+        toolCall: {
+          id: 'tool-rejected',
+          name: 'run_workflow',
+          status: 'rejected',
+          displayTitle: 'Running workflow',
+          params: undefined,
+          calledBy: undefined,
+          result: undefined,
+        },
+      },
+    ])
+  })
+})

apps/sim/lib/copilot/chat/display-message.ts

@@ -0,0 +1,202 @@
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+  MothershipStreamV1SpanLifecycleEvent,
+  MothershipStreamV1ToolOutcome,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { isToolHiddenInUi } from '@/lib/copilot/tools/client/hidden-tools'
+import {
+  type ChatContextKind,
+  type ChatMessage,
+  type ChatMessageAttachment,
+  type ChatMessageContext,
+  type ContentBlock,
+  ContentBlockType,
+  type ToolCallInfo,
+  ToolCallStatus,
+} from '@/app/workspace/[workspaceId]/home/types'
+import { getMothershipAttachmentPreviewUrl } from './attachment-preview'
+import type { PersistedContentBlock, PersistedMessage } from './persisted-message'
+import { withBlockTiming } from './persisted-message'
+
+const STATE_TO_STATUS: Record<string, ToolCallStatus> = {
+  [MothershipStreamV1ToolOutcome.success]: ToolCallStatus.success,
+  [MothershipStreamV1ToolOutcome.error]: ToolCallStatus.error,
+  [MothershipStreamV1ToolOutcome.cancelled]: ToolCallStatus.cancelled,
+  [MothershipStreamV1ToolOutcome.rejected]: ToolCallStatus.rejected,
+  [MothershipStreamV1ToolOutcome.skipped]: ToolCallStatus.skipped,
+  aborted: ToolCallStatus.cancelled,
+  failed: ToolCallStatus.error,
+  interrupted: ToolCallStatus.interrupted,
+  pending: ToolCallStatus.executing,
+  executing: ToolCallStatus.executing,
+}
+
+function toToolCallInfo(block: PersistedContentBlock): ToolCallInfo | undefined {
+  const tc = block.toolCall
+  if (!tc) return undefined
+  if (isToolHiddenInUi(tc.name)) return undefined
+  const status: ToolCallStatus = STATE_TO_STATUS[tc.state] ?? ToolCallStatus.error
+  return {
+    id: tc.id,
+    name: tc.name,
+    status,
+    displayTitle: status === ToolCallStatus.cancelled ? 'Stopped by user' : tc.display?.title,
+    params: tc.params,
+    calledBy: tc.calledBy,
+    result: tc.result,
+  }
+}
+
+function toDisplayBlock(block: PersistedContentBlock): ContentBlock | undefined {
+  const displayed = toDisplayBlockBody(block)
+  if (!displayed) return undefined
+  if (block.parentToolCallId && displayed.parentToolCallId === undefined) {
+    displayed.parentToolCallId = block.parentToolCallId
+  }
+  if (block.spanId && displayed.spanId === undefined) {
+    displayed.spanId = block.spanId
+  }
+  if (block.parentSpanId && displayed.parentSpanId === undefined) {
+    displayed.parentSpanId = block.parentSpanId
+  }
+  return withBlockTiming(displayed, block)
+}
+
+function toDisplayBlockBody(block: PersistedContentBlock): ContentBlock | undefined {
+  switch (block.type) {
+    case MothershipStreamV1EventType.text:
+      if (block.lane === 'subagent') {
+        if (block.channel === 'thinking') {
+          return { type: ContentBlockType.subagent_thinking, content: block.content }
+        }
+        return { type: ContentBlockType.subagent_text, content: block.content }
+      }
+      if (block.channel === 'thinking') {
+        return { type: ContentBlockType.thinking, content: block.content }
+      }
+      return { type: ContentBlockType.text, content: block.content }
+    case MothershipStreamV1EventType.tool:
+      if (!toToolCallInfo(block)) return undefined
+      return { type: ContentBlockType.tool_call, toolCall: toToolCallInfo(block) }
+    case MothershipStreamV1EventType.span:
+      if (block.lifecycle === MothershipStreamV1SpanLifecycleEvent.end) {
+        return { type: ContentBlockType.subagent_end }
+      }
+      return { type: ContentBlockType.subagent, content: block.content }
+    case MothershipStreamV1EventType.complete:
+      if (block.status === MothershipStreamV1CompletionStatus.cancelled) {
+        return { type: ContentBlockType.stopped }
+      }
+      return { type: ContentBlockType.text, content: block.content }
+    default:
+      return { type: ContentBlockType.text, content: block.content }
+  }
+}
+
+function toDisplayAttachment(f: PersistedMessage['fileAttachments']): ChatMessageAttachment[] {
+  if (!f || f.length === 0) return []
+  return f.map((a) => ({
+    id: a.id,
+    filename: a.filename,
+    media_type: a.media_type,
+    size: a.size,
+    previewUrl: getMothershipAttachmentPreviewUrl(a),
+  }))
+}
+
+function toDisplayContexts(
+  contexts: PersistedMessage['contexts']
+): ChatMessageContext[] | undefined {
+  if (!contexts || contexts.length === 0) return undefined
+  return contexts.map((c) => ({
+    kind: c.kind as ChatContextKind,
+    label: c.label,
+    ...(c.workflowId ? { workflowId: c.workflowId } : {}),
+    ...(c.knowledgeId ? { knowledgeId: c.knowledgeId } : {}),
+    ...(c.tableId ? { tableId: c.tableId } : {}),
+    ...(c.fileId ? { fileId: c.fileId } : {}),
+    ...(c.folderId ? { folderId: c.folderId } : {}),
+    ...(c.chatId ? { chatId: c.chatId } : {}),
+  }))
+}
+
+const WORKSPACE_FILE_TOOL = 'workspace_file'
+const EDIT_CONTENT_TOOL = 'edit_content'
+const MAIN_SPAN = 'main'
+
+/**
+ * Collapses an `edit_content` write into the most-recent `workspace_file` row in
+ * the same subagent span, mirroring the live turn-model fold. The live view
+ * folds these in `reduceEvent`, but the persisted transcript stores them as two
+ * separate tool blocks; without this a reloaded chat splits the file write into
+ * "workspace_file" + "edit_content" rows (and a refresh mid-write leaves the
+ * second row spinning). The reopened row inherits the edit_content's final
+ * status/result, exactly as the live single "writing" row resolves. Every other
+ * block is passed through untouched, so this only affects file writes.
+ */
+function foldFileWriteBlocks(blocks: ContentBlock[]): ContentBlock[] {
+  const folded: ContentBlock[] = []
+  const workspaceFileIndexBySpan = new Map<string, number>()
+  for (const block of blocks) {
+    const tc = block.type === ContentBlockType.tool_call ? block.toolCall : undefined
+    if (tc) {
+      const span = block.spanId ?? MAIN_SPAN
+      if (tc.name === EDIT_CONTENT_TOOL) {
+        const parentIndex = workspaceFileIndexBySpan.get(span)
+        const parent = parentIndex !== undefined ? folded[parentIndex] : undefined
+        if (parent?.type === ContentBlockType.tool_call && parent.toolCall) {
+          folded[parentIndex!] = {
+            ...parent,
+            toolCall: { ...parent.toolCall, status: tc.status, result: tc.result },
+          }
+          continue
+        }
+      } else if (tc.name === WORKSPACE_FILE_TOOL) {
+        workspaceFileIndexBySpan.set(span, folded.length)
+      }
+    }
+    folded.push(block)
+  }
+  return folded
+}
+
+const displayMessageCache = new WeakMap<PersistedMessage, ChatMessage>()
+
+/**
+ * Maps a `PersistedMessage` (server wire shape) to a `ChatMessage` (UI shape).
+ * Reference-stable: returns the same object for a given `PersistedMessage`
+ * instance so `React.memo` boundaries downstream of React Query's structural
+ * sharing can short-circuit on identity.
+ */
+export function toDisplayMessage(msg: PersistedMessage): ChatMessage {
+  const cached = displayMessageCache.get(msg)
+  if (cached) return cached
+
+  const display: ChatMessage = {
+    id: msg.id,
+    role: msg.role,
+    content: msg.content,
+  }
+
+  if (msg.requestId) {
+    display.requestId = msg.requestId
+  }
+
+  if (msg.contentBlocks && msg.contentBlocks.length > 0) {
+    const displayBlocks = msg.contentBlocks
+      .map(toDisplayBlock)
+      .filter((block): block is ContentBlock => !!block)
+    display.contentBlocks = foldFileWriteBlocks(displayBlocks)
+  }
+
+  const attachments = toDisplayAttachment(msg.fileAttachments)
+  if (attachments.length > 0) {
+    display.attachments = attachments
+  }
+
+  display.contexts = toDisplayContexts(msg.contexts)
+
+  displayMessageCache.set(msg, display)
+  return display
+}

apps/sim/lib/copilot/chat/effective-transcript.test.ts

@@ -0,0 +1,263 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import {
+  buildEffectiveChatTranscript,
+  getLiveAssistantMessageId,
+} from '@/lib/copilot/chat/effective-transcript'
+import { normalizeMessage } from '@/lib/copilot/chat/persisted-message'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+  MothershipStreamV1SessionKind,
+  MothershipStreamV1TextChannel,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import type { StreamBatchEvent } from '@/lib/copilot/request/session/types'
+
+function toBatchEvent(eventId: number, event: StreamBatchEvent['event']): StreamBatchEvent {
+  return {
+    eventId,
+    streamId: event.stream.streamId,
+    event,
+  }
+}
+
+function buildUserMessage(id: string, content: string) {
+  return normalizeMessage({
+    id,
+    role: 'user',
+    content,
+    timestamp: '2026-04-15T12:00:00.000Z',
+  })
+}
+
+describe('buildEffectiveChatTranscript', () => {
+  it('returns the existing transcript when the stream owner is no longer the trailing user', () => {
+    const messages = [
+      buildUserMessage('stream-1', 'Hello'),
+      normalizeMessage({
+        id: 'assistant-1',
+        role: 'assistant',
+        content: 'Persisted response',
+        timestamp: '2026-04-15T12:00:01.000Z',
+      }),
+    ]
+
+    const result = buildEffectiveChatTranscript({
+      messages,
+      activeStreamId: 'stream-1',
+      streamSnapshot: {
+        events: [
+          toBatchEvent(1, {
+            v: 1,
+            seq: 1,
+            ts: '2026-04-15T12:00:01.000Z',
+            type: MothershipStreamV1EventType.text,
+            stream: { streamId: 'stream-1' },
+            payload: {
+              channel: MothershipStreamV1TextChannel.assistant,
+              text: 'Live response',
+            },
+          }),
+        ],
+        previewSessions: [],
+        status: 'active',
+      },
+    })
+
+    expect(result).toEqual(messages)
+  })
+
+  it('appends a placeholder assistant while an active stream has not produced text yet', () => {
+    const result = buildEffectiveChatTranscript({
+      messages: [buildUserMessage('stream-1', 'Hello')],
+      activeStreamId: 'stream-1',
+      streamSnapshot: {
+        events: [
+          toBatchEvent(1, {
+            v: 1,
+            seq: 1,
+            ts: '2026-04-15T12:00:01.000Z',
+            type: MothershipStreamV1EventType.session,
+            stream: { streamId: 'stream-1' },
+            payload: {
+              kind: MothershipStreamV1SessionKind.start,
+            },
+          }),
+        ],
+        previewSessions: [],
+        status: 'active',
+      },
+    })
+
+    expect(result).toHaveLength(2)
+    expect(result[1]).toEqual(
+      expect.objectContaining({
+        id: getLiveAssistantMessageId('stream-1'),
+        role: 'assistant',
+        content: '',
+      })
+    )
+  })
+
+  it('materializes a live assistant response from redis-backed stream events', () => {
+    const result = buildEffectiveChatTranscript({
+      messages: [buildUserMessage('stream-1', 'Hello')],
+      activeStreamId: 'stream-1',
+      streamSnapshot: {
+        events: [
+          toBatchEvent(1, {
+            v: 1,
+            seq: 1,
+            ts: '2026-04-15T12:00:01.000Z',
+            type: MothershipStreamV1EventType.session,
+            stream: { streamId: 'stream-1' },
+            trace: { requestId: 'req-1' },
+            payload: {
+              kind: MothershipStreamV1SessionKind.trace,
+              requestId: 'req-1',
+            },
+          }),
+          toBatchEvent(2, {
+            v: 1,
+            seq: 2,
+            ts: '2026-04-15T12:00:02.000Z',
+            type: MothershipStreamV1EventType.text,
+            stream: { streamId: 'stream-1' },
+            trace: { requestId: 'req-1' },
+            payload: {
+              channel: MothershipStreamV1TextChannel.assistant,
+              text: 'Live response',
+            },
+          }),
+        ],
+        previewSessions: [],
+        status: 'active',
+      },
+    })
+
+    expect(result).toHaveLength(2)
+    expect(result[1]).toEqual(
+      expect.objectContaining({
+        id: getLiveAssistantMessageId('stream-1'),
+        role: 'assistant',
+        content: 'Live response',
+        requestId: 'req-1',
+      })
+    )
+  })
+
+  it('does not duplicate thinking-only text into a second assistant block', () => {
+    const result = buildEffectiveChatTranscript({
+      messages: [buildUserMessage('stream-1', 'Hello')],
+      activeStreamId: 'stream-1',
+      streamSnapshot: {
+        events: [
+          toBatchEvent(1, {
+            v: 1,
+            seq: 1,
+            ts: '2026-04-15T12:00:01.000Z',
+            type: MothershipStreamV1EventType.text,
+            stream: { streamId: 'stream-1' },
+            payload: {
+              channel: MothershipStreamV1TextChannel.thinking,
+              text: 'Internal reasoning',
+            },
+          }),
+        ],
+        previewSessions: [],
+        status: 'active',
+      },
+    })
+
+    expect(result).toHaveLength(2)
+    expect(result[1]).toEqual(
+      expect.objectContaining({
+        content: 'Internal reasoning',
+        contentBlocks: [
+          expect.objectContaining({
+            type: MothershipStreamV1EventType.text,
+            content: 'Internal reasoning',
+          }),
+        ],
+      })
+    )
+  })
+
+  it('treats user-cancelled tool results as cancelled', () => {
+    const result = buildEffectiveChatTranscript({
+      messages: [buildUserMessage('stream-1', 'Hello')],
+      activeStreamId: 'stream-1',
+      streamSnapshot: {
+        events: [
+          toBatchEvent(1, {
+            v: 1,
+            seq: 1,
+            ts: '2026-04-15T12:00:01.000Z',
+            type: MothershipStreamV1EventType.tool,
+            stream: { streamId: 'stream-1' },
+            payload: {
+              phase: 'result',
+              toolCallId: 'tool-1',
+              toolName: 'workspace_file',
+              executor: 'go',
+              mode: 'sync',
+              success: false,
+              output: {
+                reason: 'user_cancelled',
+              },
+            },
+          }),
+        ],
+        previewSessions: [],
+        status: 'active',
+      },
+    })
+
+    expect(result[1]?.contentBlocks).toEqual([
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.tool,
+        toolCall: expect.objectContaining({
+          id: 'tool-1',
+          name: 'workspace_file',
+          state: MothershipStreamV1CompletionStatus.cancelled,
+        }),
+      }),
+    ])
+  })
+
+  it('materializes a cancelled assistant tail when the stream ends before persistence', () => {
+    const result = buildEffectiveChatTranscript({
+      messages: [buildUserMessage('stream-1', 'Hello')],
+      activeStreamId: 'stream-1',
+      streamSnapshot: {
+        events: [
+          toBatchEvent(1, {
+            v: 1,
+            seq: 1,
+            ts: '2026-04-15T12:00:01.000Z',
+            type: MothershipStreamV1EventType.complete,
+            stream: { streamId: 'stream-1' },
+            payload: {
+              status: MothershipStreamV1CompletionStatus.cancelled,
+            },
+          }),
+        ],
+        previewSessions: [],
+        status: MothershipStreamV1CompletionStatus.cancelled,
+      },
+    })
+
+    expect(result).toHaveLength(2)
+    expect(result[1]?.contentBlocks).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          type: MothershipStreamV1EventType.complete,
+          status: MothershipStreamV1CompletionStatus.cancelled,
+        }),
+      ])
+    )
+  })
+})

apps/sim/lib/copilot/chat/effective-transcript.ts

@@ -0,0 +1,476 @@
+import { isRecordLike } from '@sim/utils/object'
+import { normalizeMessage, type PersistedMessage } from '@/lib/copilot/chat/persisted-message'
+import { resolveStreamToolOutcome } from '@/lib/copilot/chat/stream-tool-outcome'
+import {
+  MothershipStreamV1CompletionStatus,
+  type MothershipStreamV1ErrorPayload,
+  MothershipStreamV1EventType,
+  MothershipStreamV1RunKind,
+  MothershipStreamV1SessionKind,
+  MothershipStreamV1SpanLifecycleEvent,
+  MothershipStreamV1SpanPayloadKind,
+  MothershipStreamV1ToolOutcome,
+  MothershipStreamV1ToolPhase,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import type { FilePreviewSession } from '@/lib/copilot/request/session/file-preview-session-contract'
+import type { StreamBatchEvent } from '@/lib/copilot/request/session/types'
+import { getToolDisplayTitle } from '@/lib/copilot/tools/tool-display'
+
+interface StreamSnapshotLike {
+  events: StreamBatchEvent[]
+  previewSessions: FilePreviewSession[]
+  status: string
+}
+
+interface BuildEffectiveChatTranscriptParams {
+  messages: PersistedMessage[]
+  activeStreamId: string | null
+  streamSnapshot?: StreamSnapshotLike | null
+}
+
+type RawPersistedBlock = Record<string, unknown>
+
+export function getLiveAssistantMessageId(streamId: string): string {
+  return `live-assistant:${streamId}`
+}
+
+function asPayloadRecord(value: unknown): Record<string, unknown> | undefined {
+  return isRecordLike(value) ? value : undefined
+}
+
+function isTerminalStreamStatus(status: string | null | undefined): boolean {
+  return (
+    status === MothershipStreamV1CompletionStatus.complete ||
+    status === MothershipStreamV1CompletionStatus.error ||
+    status === MothershipStreamV1CompletionStatus.cancelled
+  )
+}
+
+function buildInlineErrorTag(payload: MothershipStreamV1ErrorPayload): string {
+  const message =
+    (typeof payload.displayMessage === 'string' ? payload.displayMessage : undefined) ||
+    (typeof payload.message === 'string' ? payload.message : undefined) ||
+    (typeof payload.error === 'string' ? payload.error : undefined) ||
+    'An unexpected error occurred'
+  const provider = typeof payload.provider === 'string' ? payload.provider : undefined
+  const code = typeof payload.code === 'string' ? payload.code : undefined
+  return `<mothership-error>${JSON.stringify({
+    message,
+    ...(code ? { code } : {}),
+    ...(provider ? { provider } : {}),
+  })}</mothership-error>`
+}
+
+function appendTextBlock(
+  blocks: RawPersistedBlock[],
+  content: string,
+  options: {
+    lane?: 'subagent'
+    parentToolCallId?: string
+    spanId?: string
+    parentSpanId?: string
+  }
+): void {
+  if (!content) return
+  const last = blocks[blocks.length - 1]
+  if (
+    last?.type === MothershipStreamV1EventType.text &&
+    last.lane === options.lane &&
+    last.parentToolCallId === options.parentToolCallId &&
+    last.spanId === options.spanId
+  ) {
+    last.content = `${typeof last.content === 'string' ? last.content : ''}${content}`
+    return
+  }
+
+  blocks.push({
+    type: MothershipStreamV1EventType.text,
+    ...(options.lane ? { lane: options.lane } : {}),
+    ...(options.parentToolCallId ? { parentToolCallId: options.parentToolCallId } : {}),
+    ...(options.spanId ? { spanId: options.spanId } : {}),
+    ...(options.parentSpanId ? { parentSpanId: options.parentSpanId } : {}),
+    content,
+  })
+}
+
+function buildLiveAssistantMessage(params: {
+  streamId: string
+  events: StreamBatchEvent[]
+  status: string | null | undefined
+}): PersistedMessage | null {
+  const { streamId, events, status } = params
+  const blocks: RawPersistedBlock[] = []
+  const toolIndexById = new Map<string, number>()
+  const subagentByParentToolCallId = new Map<string, string>()
+  const subagentBySpanId = new Map<string, string>()
+  let activeSubagent: string | undefined
+  let activeSubagentParentToolCallId: string | undefined
+  let activeCompactionId: string | undefined
+  let runningText = ''
+  let lastContentSource: 'main' | 'subagent' | null = null
+  let requestId: string | undefined
+  let lastTimestamp: string | undefined
+
+  // Scope-only resolution (mirrors the live browser stream loop): with
+  // concurrent subagents the legacy activeSubagent fallback / name-match scan
+  // would mis-attribute interleaved replayed events to the wrong lane.
+  const resolveScopedSubagent = (
+    agentId: string | undefined,
+    parentToolCallId: string | undefined,
+    spanId?: string
+  ): string | undefined => {
+    if (agentId) return agentId
+    if (spanId) {
+      const scoped = subagentBySpanId.get(spanId)
+      if (scoped) return scoped
+    }
+    if (parentToolCallId) {
+      const scoped = subagentByParentToolCallId.get(parentToolCallId)
+      if (scoped) return scoped
+    }
+    return undefined
+  }
+
+  const resolveParentForSubagentBlock = (
+    subagent: string | undefined,
+    scopedParent: string | undefined
+  ): string | undefined => {
+    if (!subagent) return undefined
+    return scopedParent
+  }
+
+  const ensureToolBlock = (input: {
+    toolCallId: string
+    toolName: string
+    calledBy?: string
+    parentToolCallId?: string
+    spanId?: string
+    parentSpanId?: string
+    displayTitle?: string
+    params?: Record<string, unknown>
+    result?: { success: boolean; output?: unknown; error?: string }
+    state?: string
+  }): RawPersistedBlock => {
+    const existingIndex = toolIndexById.get(input.toolCallId)
+    if (existingIndex !== undefined) {
+      const existing = blocks[existingIndex]
+      const existingToolCall = asPayloadRecord(existing.toolCall)
+      existing.toolCall = {
+        ...(existingToolCall ?? {}),
+        id: input.toolCallId,
+        name: input.toolName,
+        state:
+          input.state ??
+          (typeof existingToolCall?.state === 'string' ? existingToolCall.state : 'executing'),
+        ...(input.calledBy ? { calledBy: input.calledBy } : {}),
+        ...(input.params ? { params: input.params } : {}),
+        ...(input.result ? { result: input.result } : {}),
+        ...(input.displayTitle
+          ? {
+              display: {
+                title: input.displayTitle,
+              },
+            }
+          : existingToolCall?.display
+            ? { display: existingToolCall.display }
+            : {}),
+      }
+      if (input.parentToolCallId) existing.parentToolCallId = input.parentToolCallId
+      if (input.spanId) existing.spanId = input.spanId
+      if (input.parentSpanId) existing.parentSpanId = input.parentSpanId
+      return existing
+    }
+
+    const nextBlock: RawPersistedBlock = {
+      type: MothershipStreamV1EventType.tool,
+      phase: MothershipStreamV1ToolPhase.call,
+      toolCall: {
+        id: input.toolCallId,
+        name: input.toolName,
+        state: input.state ?? 'executing',
+        ...(input.calledBy ? { calledBy: input.calledBy } : {}),
+        ...(input.params ? { params: input.params } : {}),
+        ...(input.result ? { result: input.result } : {}),
+        ...(input.displayTitle
+          ? {
+              display: {
+                title: input.displayTitle,
+              },
+            }
+          : {}),
+      },
+      ...(input.parentToolCallId ? { parentToolCallId: input.parentToolCallId } : {}),
+      ...(input.spanId ? { spanId: input.spanId } : {}),
+      ...(input.parentSpanId ? { parentSpanId: input.parentSpanId } : {}),
+    }
+    toolIndexById.set(input.toolCallId, blocks.length)
+    blocks.push(nextBlock)
+    return nextBlock
+  }
+
+  for (const entry of events) {
+    const parsed = entry.event
+    lastTimestamp = parsed.ts
+    if (typeof parsed.trace?.requestId === 'string') {
+      requestId = parsed.trace.requestId
+    }
+    const scopedParentToolCallId =
+      typeof parsed.scope?.parentToolCallId === 'string' ? parsed.scope.parentToolCallId : undefined
+    const scopedAgentId =
+      typeof parsed.scope?.agentId === 'string' ? parsed.scope.agentId : undefined
+    const scopedSpanId = typeof parsed.scope?.spanId === 'string' ? parsed.scope.spanId : undefined
+    const scopedParentSpanId =
+      typeof parsed.scope?.parentSpanId === 'string' ? parsed.scope.parentSpanId : undefined
+    const scopedSubagent = resolveScopedSubagent(
+      scopedAgentId,
+      scopedParentToolCallId,
+      scopedSpanId
+    )
+    const spanIdentity: { spanId?: string; parentSpanId?: string } = {
+      ...(scopedSpanId ? { spanId: scopedSpanId } : {}),
+      ...(scopedParentSpanId ? { parentSpanId: scopedParentSpanId } : {}),
+    }
+
+    switch (parsed.type) {
+      case MothershipStreamV1EventType.session: {
+        if (parsed.payload.kind === MothershipStreamV1SessionKind.chat) {
+          continue
+        }
+        if (parsed.payload.kind === MothershipStreamV1SessionKind.start) {
+          continue
+        }
+        if (parsed.payload.kind === MothershipStreamV1SessionKind.trace) {
+          requestId = parsed.payload.requestId
+        }
+        continue
+      }
+      case MothershipStreamV1EventType.text: {
+        const chunk = parsed.payload.text
+        if (!chunk) {
+          continue
+        }
+        const contentSource: 'main' | 'subagent' = scopedSubagent ? 'subagent' : 'main'
+        const needsBoundaryNewline =
+          lastContentSource !== null &&
+          lastContentSource !== contentSource &&
+          runningText.length > 0 &&
+          !runningText.endsWith('\n')
+        const normalizedChunk = needsBoundaryNewline ? `\n${chunk}` : chunk
+        const parentForBlock = resolveParentForSubagentBlock(scopedSubagent, scopedParentToolCallId)
+        appendTextBlock(blocks, normalizedChunk, {
+          ...(scopedSubagent ? { lane: 'subagent' as const } : {}),
+          ...(parentForBlock ? { parentToolCallId: parentForBlock } : {}),
+          ...spanIdentity,
+        })
+        runningText += normalizedChunk
+        lastContentSource = contentSource
+        continue
+      }
+      case MothershipStreamV1EventType.tool: {
+        const payload = parsed.payload
+        const toolCallId = payload.toolCallId
+
+        if ('previewPhase' in payload) {
+          continue
+        }
+
+        if (payload.phase === MothershipStreamV1ToolPhase.args_delta) {
+          continue
+        }
+
+        const parentForBlock = resolveParentForSubagentBlock(scopedSubagent, scopedParentToolCallId)
+
+        if (payload.phase === MothershipStreamV1ToolPhase.result) {
+          ensureToolBlock({
+            toolCallId,
+            toolName: payload.toolName,
+            calledBy: scopedSubagent,
+            ...(parentForBlock ? { parentToolCallId: parentForBlock } : {}),
+            ...spanIdentity,
+            state: resolveStreamToolOutcome(payload),
+            result: {
+              success: payload.success,
+              ...(payload.output !== undefined ? { output: payload.output } : {}),
+              ...(typeof payload.error === 'string' ? { error: payload.error } : {}),
+            },
+          })
+          continue
+        }
+
+        ensureToolBlock({
+          toolCallId,
+          toolName: payload.toolName,
+          calledBy: scopedSubagent,
+          ...(parentForBlock ? { parentToolCallId: parentForBlock } : {}),
+          ...spanIdentity,
+          displayTitle: getToolDisplayTitle(
+            payload.toolName,
+            isRecordLike(payload.arguments) ? payload.arguments : undefined
+          ),
+          params: isRecordLike(payload.arguments) ? payload.arguments : undefined,
+          state: typeof payload.status === 'string' ? payload.status : 'executing',
+        })
+        continue
+      }
+      case MothershipStreamV1EventType.span: {
+        if (parsed.payload.kind !== MothershipStreamV1SpanPayloadKind.subagent) {
+          continue
+        }
+
+        const spanData = asPayloadRecord(parsed.payload.data)
+        const parentToolCallIdFromData =
+          typeof spanData?.tool_call_id === 'string'
+            ? spanData.tool_call_id
+            : typeof spanData?.toolCallId === 'string'
+              ? spanData.toolCallId
+              : undefined
+        const parentToolCallId = scopedParentToolCallId ?? parentToolCallIdFromData
+        const name = typeof parsed.payload.agent === 'string' ? parsed.payload.agent : scopedAgentId
+        if (parsed.payload.event === MothershipStreamV1SpanLifecycleEvent.start && name) {
+          if (scopedSpanId) {
+            subagentBySpanId.set(scopedSpanId, name)
+          }
+          if (parentToolCallId) {
+            subagentByParentToolCallId.set(parentToolCallId, name)
+          }
+          activeSubagent = name
+          activeSubagentParentToolCallId = parentToolCallId
+          blocks.push({
+            type: MothershipStreamV1EventType.span,
+            kind: MothershipStreamV1SpanPayloadKind.subagent,
+            lifecycle: MothershipStreamV1SpanLifecycleEvent.start,
+            content: name,
+            ...(parentToolCallId ? { parentToolCallId } : {}),
+            ...spanIdentity,
+          })
+          continue
+        }
+
+        if (parsed.payload.event === MothershipStreamV1SpanLifecycleEvent.end) {
+          if (spanData?.pending === true) {
+            continue
+          }
+          if (scopedSpanId) {
+            subagentBySpanId.delete(scopedSpanId)
+          }
+          if (parentToolCallId) {
+            subagentByParentToolCallId.delete(parentToolCallId)
+          }
+          // Clear the legacy pointer only for THIS lane (by parent tool call id)
+          // or an unscoped end — never by agent name, which would tear down a
+          // concurrent same-name sibling that is still open.
+          if (!parentToolCallId || parentToolCallId === activeSubagentParentToolCallId) {
+            activeSubagent = undefined
+            activeSubagentParentToolCallId = undefined
+          }
+          blocks.push({
+            type: MothershipStreamV1EventType.span,
+            kind: MothershipStreamV1SpanPayloadKind.subagent,
+            lifecycle: MothershipStreamV1SpanLifecycleEvent.end,
+            ...(parentToolCallId ? { parentToolCallId } : {}),
+            ...spanIdentity,
+          })
+        }
+        continue
+      }
+      case MothershipStreamV1EventType.run: {
+        if (parsed.payload.kind === MothershipStreamV1RunKind.compaction_start) {
+          activeCompactionId = `compaction_${entry.eventId}`
+          ensureToolBlock({
+            toolCallId: activeCompactionId,
+            toolName: 'context_compaction',
+            displayTitle: 'Compacting context...',
+            state: 'executing',
+          })
+          continue
+        }
+
+        if (parsed.payload.kind === MothershipStreamV1RunKind.compaction_done) {
+          const compactionId = activeCompactionId ?? `compaction_${entry.eventId}`
+          activeCompactionId = undefined
+          ensureToolBlock({
+            toolCallId: compactionId,
+            toolName: 'context_compaction',
+            displayTitle: 'Compacted context',
+            state: MothershipStreamV1ToolOutcome.success,
+          })
+        }
+        continue
+      }
+      case MothershipStreamV1EventType.error: {
+        const tag = buildInlineErrorTag(parsed.payload)
+        if (runningText.includes(tag)) {
+          continue
+        }
+        const prefix = runningText.length > 0 && !runningText.endsWith('\n') ? '\n' : ''
+        const content = `${prefix}${tag}`
+        const errorParent = resolveParentForSubagentBlock(scopedSubagent, scopedParentToolCallId)
+        appendTextBlock(blocks, content, {
+          ...(scopedSubagent ? { lane: 'subagent' as const } : {}),
+          ...(errorParent ? { parentToolCallId: errorParent } : {}),
+          ...spanIdentity,
+        })
+        runningText += content
+        continue
+      }
+      case MothershipStreamV1EventType.complete: {
+        if (parsed.payload.status === MothershipStreamV1CompletionStatus.cancelled) {
+          blocks.push({
+            type: MothershipStreamV1EventType.complete,
+            status: parsed.payload.status,
+          })
+        }
+        continue
+      }
+      case MothershipStreamV1EventType.resource: {
+        continue
+      }
+      default: {
+        continue
+      }
+    }
+  }
+
+  if (blocks.length === 0 && !runningText && isTerminalStreamStatus(status)) {
+    return null
+  }
+
+  return normalizeMessage({
+    id: getLiveAssistantMessageId(streamId),
+    role: 'assistant',
+    content: runningText,
+    timestamp: lastTimestamp ?? new Date().toISOString(),
+    ...(requestId ? { requestId } : {}),
+    ...(blocks.length > 0 ? { contentBlocks: blocks } : {}),
+  })
+}
+
+export function buildEffectiveChatTranscript({
+  messages,
+  activeStreamId,
+  streamSnapshot,
+}: BuildEffectiveChatTranscriptParams): PersistedMessage[] {
+  if (!activeStreamId || !streamSnapshot) {
+    return messages
+  }
+
+  const trailingMessage = messages[messages.length - 1]
+  if (
+    !trailingMessage ||
+    trailingMessage.role !== 'user' ||
+    trailingMessage.id !== activeStreamId
+  ) {
+    return messages
+  }
+
+  const liveAssistant = buildLiveAssistantMessage({
+    streamId: activeStreamId,
+    events: streamSnapshot.events,
+    status: streamSnapshot.status,
+  })
+  if (!liveAssistant) {
+    return messages
+  }
+
+  return [...messages, liveAssistant]
+}

apps/sim/lib/copilot/chat/lifecycle.test.ts

@@ -0,0 +1,165 @@
+/**
+ * @vitest-environment node
+ */
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+const { mockAuthorizeWorkflow, mockGetActiveWorkflow } = vi.hoisted(() => ({
+  mockAuthorizeWorkflow: vi.fn(),
+  mockGetActiveWorkflow: vi.fn(),
+}))
+
+vi.mock('@sim/platform-authz/workflow', () => ({
+  authorizeWorkflowByWorkspacePermission: mockAuthorizeWorkflow,
+  getActiveWorkflowRecord: mockGetActiveWorkflow,
+}))
+
+vi.mock('@/lib/workspaces/permissions/utils', () => ({
+  assertActiveWorkspaceAccess: vi.fn(),
+  checkWorkspaceAccess: vi.fn(),
+}))
+
+import {
+  getAccessibleCopilotChat,
+  getAccessibleCopilotChatWithMessages,
+  resolveOrCreateChat,
+} from '@/lib/copilot/chat/lifecycle'
+
+const CHAT_ID = 'chat-1'
+const USER_ID = 'user-1'
+
+// A chat with no workflow/workspace skips the authz lookups and authorizes directly.
+const chatRow = {
+  id: CHAT_ID,
+  userId: USER_ID,
+  workflowId: null,
+  workspaceId: null,
+  type: 'copilot',
+  title: 'Test',
+  conversationId: null,
+  resources: [],
+  createdAt: new Date('2026-01-01T00:00:00.000Z'),
+  updatedAt: new Date('2026-01-01T00:00:00.000Z'),
+}
+
+const userMsg = { id: 'm-user', role: 'user', content: 'Hi', timestamp: '2026-01-01T00:00:00.000Z' }
+const asstMsg = {
+  id: 'm-asst',
+  role: 'assistant',
+  content: 'Hello',
+  timestamp: '2026-01-01T00:00:01.000Z',
+}
+
+describe('lifecycle copilot chat reads (cutover to copilot_messages)', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('getAccessibleCopilotChatWithMessages sources messages from copilot_messages in seq order', async () => {
+    // 1st query: chat metadata (select().from().where().limit())
+    dbChainMockFns.limit.mockResolvedValueOnce([chatRow])
+    // 2nd query: messages (select().from().where().orderBy())
+    dbChainMockFns.orderBy.mockResolvedValueOnce([{ content: userMsg }, { content: asstMsg }])
+
+    const result = await getAccessibleCopilotChatWithMessages(CHAT_ID, USER_ID)
+
+    expect(result).not.toBeNull()
+    expect(result?.messages).toEqual([userMsg, asstMsg])
+    expect(dbChainMockFns.orderBy).toHaveBeenCalledTimes(1)
+  })
+
+  it('strips tool-result output on read, keeping success/error', async () => {
+    const toolMsg = {
+      id: 'm-tool',
+      role: 'assistant',
+      content: '',
+      timestamp: '2026-01-01T00:00:02.000Z',
+      contentBlocks: [
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'tc-1',
+            name: 'get_workflow_logs',
+            state: 'success',
+            result: { success: true, output: { huge: 'x'.repeat(5000) } },
+          },
+        },
+      ],
+    }
+    dbChainMockFns.limit.mockResolvedValueOnce([chatRow])
+    dbChainMockFns.orderBy.mockResolvedValueOnce([{ content: toolMsg }])
+
+    const result = await getAccessibleCopilotChatWithMessages(CHAT_ID, USER_ID)
+
+    expect(result?.messages?.[0].contentBlocks?.[0].toolCall?.result).toEqual({ success: true })
+    expect(JSON.stringify(result?.messages)).not.toContain('huge')
+  })
+
+  it('returns an empty transcript for a chat with no messages', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([chatRow])
+    dbChainMockFns.orderBy.mockResolvedValueOnce([])
+
+    const result = await getAccessibleCopilotChatWithMessages(CHAT_ID, USER_ID)
+
+    expect(result?.messages).toEqual([])
+  })
+
+  it('returns null and does NOT query messages when the chat is not found', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+
+    const result = await getAccessibleCopilotChatWithMessages(CHAT_ID, USER_ID)
+
+    expect(result).toBeNull()
+    expect(dbChainMockFns.orderBy).not.toHaveBeenCalled()
+  })
+
+  it('returns null and does NOT query messages when the row is found but authorization fails', async () => {
+    // Row exists but belongs to a workflow the user cannot read.
+    dbChainMockFns.limit.mockResolvedValueOnce([{ ...chatRow, workflowId: 'wf-1' }])
+    mockAuthorizeWorkflow.mockResolvedValueOnce({ allowed: false, workflow: null })
+
+    const result = await getAccessibleCopilotChatWithMessages(CHAT_ID, USER_ID)
+
+    expect(result).toBeNull()
+    expect(dbChainMockFns.orderBy).not.toHaveBeenCalled()
+  })
+
+  it('legacy getAccessibleCopilotChat also assembles messages from copilot_messages', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      { ...chatRow, model: 'm', planArtifact: null, config: null },
+    ])
+    dbChainMockFns.orderBy.mockResolvedValueOnce([{ content: userMsg }])
+
+    const result = await getAccessibleCopilotChat(CHAT_ID, USER_ID)
+
+    expect(result?.messages).toEqual([userMsg])
+  })
+
+  it('resolveOrCreateChat returns conversationHistory from the table for an existing chat', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([chatRow])
+    dbChainMockFns.orderBy.mockResolvedValueOnce([{ content: userMsg }, { content: asstMsg }])
+
+    const result = await resolveOrCreateChat({ chatId: CHAT_ID, userId: USER_ID, model: 'm' })
+
+    expect(result.isNew).toBe(false)
+    expect(result.conversationHistory).toEqual([userMsg, asstMsg])
+  })
+
+  it('resolveOrCreateChat creates a new chat with an empty transcript', async () => {
+    dbChainMockFns.returning.mockResolvedValueOnce([chatRow])
+
+    const result = await resolveOrCreateChat({ userId: USER_ID, model: 'm' })
+
+    expect(result.isNew).toBe(true)
+    expect(result.conversationHistory).toEqual([])
+    expect(result.chat?.messages).toEqual([])
+    const insertValues = dbChainMockFns.values.mock.calls[0]?.[0] as Record<string, unknown>
+    expect(Object.hasOwn(insertValues, 'messages')).toBe(false)
+    // a brand-new chat must not trigger a messages read
+    expect(dbChainMockFns.orderBy).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/copilot/chat/lifecycle.ts

@@ -0,0 +1,315 @@
+import { db } from '@sim/db'
+import { copilotChats, copilotMessages } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import {
+  authorizeWorkflowByWorkspacePermission,
+  getActiveWorkflowRecord,
+} from '@sim/platform-authz/workflow'
+import { and, asc, eq, isNull, sql } from 'drizzle-orm'
+import { type PersistedMessage, stripToolResultOutput } from '@/lib/copilot/chat/persisted-message'
+import {
+  assertActiveWorkspaceAccess,
+  checkWorkspaceAccess,
+} from '@/lib/workspaces/permissions/utils'
+
+const logger = createLogger('CopilotChatLifecycle')
+
+export interface ChatLoadResult {
+  chatId: string
+  chat: CopilotChatDetailRow | null
+  conversationHistory: unknown[]
+  isNew: boolean
+}
+
+/**
+ * Minimal column set needed to perform workflow/workspace authorization for a
+ * copilot chat. Heavy TOAST-able columns (messages, planArtifact, previewYaml,
+ * config, resources) are intentionally excluded — callers that only need to
+ * verify ownership should not pay the detoast cost for those fields.
+ */
+const copilotChatAuthColumns = {
+  id: copilotChats.id,
+  userId: copilotChats.userId,
+  workflowId: copilotChats.workflowId,
+  workspaceId: copilotChats.workspaceId,
+  type: copilotChats.type,
+} as const
+
+/**
+ * Column set for chat-detail callers that need chat metadata. The conversation
+ * transcript is no longer selected from `copilot_chats.messages` (JSONB) —
+ * reads now source it from the normalized `copilot_messages` table via
+ * `loadCopilotChatMessages`, which avoids detoasting the large messages blob on
+ * every load. The copilot-only TOAST-able fields (`previewYaml`,
+ * `planArtifact`, `config`) and unused metadata (`model`, `pinned`,
+ * `lastSeenAt`) remain excluded.
+ */
+const copilotChatDetailColumns = {
+  ...copilotChatAuthColumns,
+  title: copilotChats.title,
+  conversationId: copilotChats.conversationId,
+  resources: copilotChats.resources,
+  createdAt: copilotChats.createdAt,
+  updatedAt: copilotChats.updatedAt,
+} as const
+
+/**
+ * Column set for the legacy copilot chat detail endpoint. Extends
+ * `copilotChatDetailColumns` with `model`, `planArtifact`, and `config` — the
+ * fields the legacy `transformChat` response shape includes. Still drops
+ * `previewYaml` (JSONB), `pinned`, and `lastSeenAt`.
+ */
+const copilotChatLegacyDetailColumns = {
+  ...copilotChatDetailColumns,
+  model: copilotChats.model,
+  planArtifact: copilotChats.planArtifact,
+  config: copilotChats.config,
+} as const
+
+/**
+ * Load a chat's transcript from the normalized `copilot_messages` table in
+ * canonical order (`seq` first, then `created_at`/`id` as a deterministic
+ * tiebreak; `NULLS LAST` so any not-yet-sequenced row sorts after sequenced
+ * ones). Each row's `content` is the full message object — identical in shape
+ * to a legacy JSONB array element — so the downstream normalize/transcript
+ * pipeline is unchanged.
+ */
+export async function loadCopilotChatMessages(chatId: string): Promise<PersistedMessage[]> {
+  const rows = await db
+    .select({ content: copilotMessages.content })
+    .from(copilotMessages)
+    .where(and(eq(copilotMessages.chatId, chatId), isNull(copilotMessages.deletedAt)))
+    .orderBy(
+      sql`${copilotMessages.seq} asc nulls last`,
+      asc(copilotMessages.createdAt),
+      asc(copilotMessages.id)
+    )
+  // Also strip on read: rows written before the backfill still carry outputs.
+  return rows.map((row) => stripToolResultOutput(row.content as PersistedMessage))
+}
+
+type CopilotChatAuthRow = Pick<
+  typeof copilotChats.$inferSelect,
+  'id' | 'userId' | 'workflowId' | 'workspaceId' | 'type'
+>
+
+export type CopilotChatDetailRow = Pick<
+  typeof copilotChats.$inferSelect,
+  | 'id'
+  | 'userId'
+  | 'workflowId'
+  | 'workspaceId'
+  | 'type'
+  | 'title'
+  | 'conversationId'
+  | 'resources'
+  | 'createdAt'
+  | 'updatedAt'
+> & {
+  /** Transcript assembled from `copilot_messages` (no longer a chat-row column). */
+  messages: unknown[]
+}
+
+export type CopilotChatLegacyDetailRow = CopilotChatDetailRow &
+  Pick<typeof copilotChats.$inferSelect, 'model' | 'planArtifact' | 'config'>
+
+async function authorizeCopilotChatRow<T extends CopilotChatAuthRow>(
+  chat: T | undefined,
+  chatId: string,
+  userId: string
+): Promise<T | null> {
+  if (!chat) {
+    logger.warn('Copilot chat not found or not owned by user', { chatId, userId })
+    return null
+  }
+
+  if (chat.workflowId) {
+    const authorization = await authorizeWorkflowByWorkspacePermission({
+      workflowId: chat.workflowId,
+      userId,
+      action: 'read',
+    })
+    if (!authorization.allowed || !authorization.workflow) {
+      logger.warn('Copilot chat workflow not authorized for user', {
+        chatId,
+        userId,
+        workflowId: chat.workflowId,
+      })
+      return null
+    }
+  } else if (chat.workspaceId) {
+    const access = await checkWorkspaceAccess(chat.workspaceId, userId)
+    if (!access.exists || !access.hasAccess) {
+      logger.warn('Copilot chat workspace not accessible to user', {
+        chatId,
+        userId,
+        workspaceId: chat.workspaceId,
+      })
+      return null
+    }
+  }
+
+  return chat
+}
+
+/**
+ * Verify a copilot chat exists, is owned by the user, and the user has access
+ * to its workflow/workspace. Selects only the columns required for the
+ * authorization check — use this for routes that only need ownership
+ * verification before a mutation (rename, delete, update-messages).
+ */
+export async function getAccessibleCopilotChatAuth(
+  chatId: string,
+  userId: string
+): Promise<CopilotChatAuthRow | null> {
+  const [chat] = await db
+    .select(copilotChatAuthColumns)
+    .from(copilotChats)
+    .where(and(eq(copilotChats.id, chatId), eq(copilotChats.userId, userId)))
+    .limit(1)
+
+  return authorizeCopilotChatRow(chat, chatId, userId)
+}
+
+/**
+ * Load a copilot chat row for the legacy chat detail endpoint, including the
+ * transcript plus `model`, `planArtifact`, and `config`. Drops `previewYaml`
+ * (JSONB), `pinned`, and `lastSeenAt` — none of which the endpoint returns.
+ */
+export async function getAccessibleCopilotChat(
+  chatId: string,
+  userId: string
+): Promise<CopilotChatLegacyDetailRow | null> {
+  const [chat] = await db
+    .select(copilotChatLegacyDetailColumns)
+    .from(copilotChats)
+    .where(and(eq(copilotChats.id, chatId), eq(copilotChats.userId, userId)))
+    .limit(1)
+
+  const authorized = await authorizeCopilotChatRow(chat, chatId, userId)
+  if (!authorized) return null
+
+  const messages = await loadCopilotChatMessages(chatId)
+  return { ...authorized, messages }
+}
+
+/**
+ * Load a copilot chat with the conversation transcript and resources after
+ * authorization, omitting copilot-only TOAST-able fields (`previewYaml`,
+ * `planArtifact`, `config`) and unused metadata (`model`, `pinned`,
+ * `lastSeenAt`). Use this for the mothership chat detail endpoint and the
+ * shared `resolveOrCreateChat` path — every column read here is consumed
+ * downstream, and dropping the others avoids per-request detoast overhead.
+ */
+export async function getAccessibleCopilotChatWithMessages(
+  chatId: string,
+  userId: string
+): Promise<CopilotChatDetailRow | null> {
+  const [chat] = await db
+    .select(copilotChatDetailColumns)
+    .from(copilotChats)
+    .where(and(eq(copilotChats.id, chatId), eq(copilotChats.userId, userId)))
+    .limit(1)
+
+  const authorized = await authorizeCopilotChatRow(chat, chatId, userId)
+  if (!authorized) return null
+
+  const messages = await loadCopilotChatMessages(chatId)
+  return { ...authorized, messages }
+}
+
+/**
+ * Resolve or create a copilot chat session.
+ * If chatId is provided, loads the existing chat. Otherwise creates a new one.
+ * Supports both workflow-scoped and workspace-scoped chats.
+ */
+export async function resolveOrCreateChat(params: {
+  chatId?: string
+  userId: string
+  workflowId?: string
+  workspaceId?: string
+  model: string
+  type?: 'mothership' | 'copilot'
+}): Promise<ChatLoadResult> {
+  const { chatId, userId, workflowId, workspaceId, model, type } = params
+
+  if (workspaceId) {
+    await assertActiveWorkspaceAccess(workspaceId, userId)
+  }
+
+  if (chatId) {
+    const chat = await getAccessibleCopilotChatWithMessages(chatId, userId)
+
+    if (chat) {
+      if (workflowId && chat.workflowId !== workflowId) {
+        logger.warn('Copilot chat workflow mismatch', {
+          chatId,
+          userId,
+          requestWorkflowId: workflowId,
+          chatWorkflowId: chat.workflowId,
+        })
+        return { chatId, chat: null, conversationHistory: [], isNew: false }
+      }
+
+      if (workspaceId && chat.workspaceId !== workspaceId) {
+        logger.warn('Copilot chat workspace mismatch', {
+          chatId,
+          userId,
+          requestWorkspaceId: workspaceId,
+          chatWorkspaceId: chat.workspaceId,
+        })
+        return { chatId, chat: null, conversationHistory: [], isNew: false }
+      }
+
+      if (chat.workflowId) {
+        const activeWorkflow = await getActiveWorkflowRecord(chat.workflowId)
+        if (!activeWorkflow) {
+          logger.warn('Copilot chat workflow no longer active', {
+            chatId,
+            userId,
+            workflowId: chat.workflowId,
+          })
+          return { chatId, chat: null, conversationHistory: [], isNew: false }
+        }
+      }
+    }
+
+    return {
+      chatId,
+      chat: chat ?? null,
+      conversationHistory: chat && Array.isArray(chat.messages) ? chat.messages : [],
+      isNew: false,
+    }
+  }
+
+  const now = new Date()
+  const [newChat] = await db
+    .insert(copilotChats)
+    .values({
+      userId,
+      ...(workflowId ? { workflowId } : {}),
+      ...(workspaceId ? { workspaceId } : {}),
+      type: type ?? 'copilot',
+      title: null,
+      model,
+      lastSeenAt: now,
+    })
+    .returning(copilotChatDetailColumns)
+
+  if (!newChat) {
+    logger.warn('Failed to create new copilot chat row', { userId, workflowId, workspaceId })
+    return {
+      chatId: '',
+      chat: null,
+      conversationHistory: [],
+      isNew: true,
+    }
+  }
+
+  return {
+    chatId: newChat.id,
+    chat: { ...newChat, messages: [] },
+    conversationHistory: [],
+    isNew: true,
+  }
+}

apps/sim/lib/copilot/chat/list-mothership-chats.ts

@@ -0,0 +1,50 @@
+import { db } from '@sim/db'
+import { copilotChats } from '@sim/db/schema'
+import { and, desc, eq } from 'drizzle-orm'
+import type { MothershipChat } from '@/lib/api/contracts/mothership-chats'
+import { reconcileChatStreamMarkers } from '@/lib/copilot/chat/stream-liveness'
+
+/**
+ * Lists a user's mothership (home) chats for a workspace as the contract wire
+ * shape, shared by the `GET /api/mothership/chats` route and the workspace
+ * sidebar prefetch. Performs no auth or workspace-access checks — callers
+ * enforce access before invoking. Reconciles stale live-stream markers and
+ * normalizes timestamps to ISO strings to honor the wire contract.
+ */
+export async function listMothershipChats(
+  userId: string,
+  workspaceId: string
+): Promise<MothershipChat[]> {
+  const chats = await db
+    .select({
+      id: copilotChats.id,
+      title: copilotChats.title,
+      updatedAt: copilotChats.updatedAt,
+      activeStreamId: copilotChats.conversationId,
+      lastSeenAt: copilotChats.lastSeenAt,
+      pinned: copilotChats.pinned,
+    })
+    .from(copilotChats)
+    .where(
+      and(
+        eq(copilotChats.userId, userId),
+        eq(copilotChats.workspaceId, workspaceId),
+        eq(copilotChats.type, 'mothership')
+      )
+    )
+    .orderBy(desc(copilotChats.pinned), desc(copilotChats.updatedAt))
+
+  const streamMarkers = await reconcileChatStreamMarkers(
+    chats.map((c) => ({ chatId: c.id, streamId: c.activeStreamId })),
+    { repairVerifiedStaleMarkers: true }
+  )
+
+  return chats.map((c) => ({
+    id: c.id,
+    title: c.title,
+    updatedAt: c.updatedAt.toISOString(),
+    activeStreamId: streamMarkers.get(c.id)?.streamId ?? null,
+    lastSeenAt: c.lastSeenAt ? c.lastSeenAt.toISOString() : null,
+    pinned: c.pinned,
+  }))
+}

apps/sim/lib/copilot/chat/messages-store.test.ts

@@ -0,0 +1,237 @@
+/**
+ * @vitest-environment node
+ */
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+import {
+  appendCopilotChatMessages,
+  replaceCopilotChatMessages,
+} from '@/lib/copilot/chat/messages-store'
+import type { PersistedMessage } from '@/lib/copilot/chat/persisted-message'
+
+const userMsg: PersistedMessage = {
+  id: 'msg-user-1',
+  role: 'user',
+  content: 'Hello',
+  timestamp: '2026-01-01T00:00:00.000Z',
+}
+
+const assistantMsg: PersistedMessage = {
+  id: 'msg-asst-1',
+  role: 'assistant',
+  content: 'Hi back',
+  timestamp: '2026-01-01T00:00:01.000Z',
+}
+
+const toolMsg: PersistedMessage = {
+  id: 'msg-tool-1',
+  role: 'assistant',
+  content: '',
+  timestamp: '2026-01-01T00:00:02.000Z',
+  contentBlocks: [
+    {
+      type: 'tool',
+      phase: 'call',
+      toolCall: {
+        id: 'tc-1',
+        name: 'get_workflow_logs',
+        state: 'error',
+        params: { workflowId: 'wf-1' },
+        result: { success: false, output: { huge: 'x'.repeat(5000) }, error: 'too big' },
+      },
+    },
+  ],
+}
+
+/** The persisted `content` of the most recently inserted row at `index`. */
+function lastRowContent(index: number): PersistedMessage {
+  return lastValuesRows()[index].content as PersistedMessage
+}
+
+/** The first arg passed to the most recent `.values(...)` call. */
+function lastValuesRows() {
+  const calls = dbChainMockFns.values.mock.calls
+  return calls[calls.length - 1][0] as Array<Record<string, unknown>>
+}
+
+describe('messages-store', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  describe('appendCopilotChatMessages', () => {
+    it('is a no-op on empty array', async () => {
+      await appendCopilotChatMessages('chat-1', [])
+      expect(dbChainMockFns.insert).not.toHaveBeenCalled()
+    })
+
+    it('inserts rows built from PersistedMessage shape', async () => {
+      await appendCopilotChatMessages('chat-1', [userMsg, assistantMsg])
+
+      expect(dbChainMockFns.insert).toHaveBeenCalledTimes(1)
+      expect(dbChainMockFns.values).toHaveBeenCalledTimes(1)
+      const rows = lastValuesRows()
+      expect(rows).toHaveLength(2)
+
+      expect(rows[0]).toMatchObject({
+        chatId: 'chat-1',
+        messageId: 'msg-user-1',
+        role: 'user',
+        content: userMsg,
+        model: null,
+        streamId: null,
+      })
+      expect(rows[0].createdAt as Date).toEqual(new Date(userMsg.timestamp))
+      expect(rows[0].updatedAt as Date).toEqual(new Date(userMsg.timestamp))
+
+      expect(rows[1]).toMatchObject({
+        chatId: 'chat-1',
+        messageId: 'msg-asst-1',
+        role: 'assistant',
+        content: assistantMsg,
+      })
+      expect(rows[1].createdAt as Date).toEqual(new Date(assistantMsg.timestamp))
+    })
+
+    it('assigns seq as 0-based array index when the chat has no prior rows', async () => {
+      dbChainMockFns.where.mockResolvedValueOnce([{ maxSeq: null }])
+
+      await appendCopilotChatMessages('chat-1', [userMsg, assistantMsg])
+      const rows = lastValuesRows()
+      expect(rows[0].seq).toBe(0)
+      expect(rows[1].seq).toBe(1)
+    })
+
+    it('continues seq from MAX(seq)+1 when the chat already has rows', async () => {
+      dbChainMockFns.where.mockResolvedValueOnce([{ maxSeq: 4 }])
+
+      await appendCopilotChatMessages('chat-1', [userMsg, assistantMsg])
+      const rows = lastValuesRows()
+      expect(rows[0].seq).toBe(5)
+      expect(rows[1].seq).toBe(6)
+    })
+
+    it('passes chatModel and streamId options to every row', async () => {
+      await appendCopilotChatMessages('chat-1', [userMsg, assistantMsg], {
+        chatModel: 'claude-sonnet-4-5',
+        streamId: 'stream-xyz',
+      })
+
+      const rows = lastValuesRows()
+      expect(rows[0].model).toBe('claude-sonnet-4-5')
+      expect(rows[0].streamId).toBe('stream-xyz')
+      expect(rows[1].model).toBe('claude-sonnet-4-5')
+      expect(rows[1].streamId).toBe('stream-xyz')
+    })
+
+    it('uses ON CONFLICT DO UPDATE that PRESERVES existing seq', async () => {
+      await appendCopilotChatMessages('chat-1', [userMsg])
+
+      expect(dbChainMockFns.onConflictDoUpdate).toHaveBeenCalledTimes(1)
+      const conflictArg = dbChainMockFns.onConflictDoUpdate.mock.calls[0][0]
+      expect(conflictArg.target).toHaveLength(2)
+      expect(conflictArg.set).toHaveProperty('content')
+      expect(conflictArg.set).toHaveProperty('role')
+      expect(conflictArg.set).toHaveProperty('model')
+      expect(conflictArg.set).toHaveProperty('streamId')
+      expect(conflictArg.set).toHaveProperty('updatedAt')
+      expect(conflictArg.set.seq.strings.join('')).toContain('COALESCE(')
+    })
+
+    it('collapses duplicate message ids to a single row', async () => {
+      await appendCopilotChatMessages('chat-1', [userMsg, { ...userMsg, content: 'dupe' }])
+      const rows = lastValuesRows()
+      expect(rows).toHaveLength(1)
+      expect(rows[0].messageId).toBe('msg-user-1')
+    })
+
+    it('propagates DB errors — copilot_messages is the sole store', async () => {
+      dbChainMockFns.onConflictDoUpdate.mockRejectedValueOnce(new Error('connection lost'))
+
+      await expect(appendCopilotChatMessages('chat-1', [userMsg])).rejects.toThrow(
+        'connection lost'
+      )
+    })
+
+    it('strips tool-result output before persisting, keeping success/error', async () => {
+      await appendCopilotChatMessages('chat-1', [toolMsg])
+
+      const toolCall = lastRowContent(0).contentBlocks?.[0].toolCall
+      expect(toolCall?.result).toEqual({ success: false, error: 'too big' })
+      expect(JSON.stringify(lastValuesRows())).not.toContain('huge')
+    })
+  })
+
+  describe('replaceCopilotChatMessages', () => {
+    it('deletes all chat rows when given an empty snapshot', async () => {
+      await replaceCopilotChatMessages('chat-1', [])
+
+      expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1)
+      expect(dbChainMockFns.delete).toHaveBeenCalledTimes(1)
+      expect(dbChainMockFns.insert).not.toHaveBeenCalled()
+    })
+
+    it('deletes only rows whose message_id is not in the new snapshot, then upserts', async () => {
+      await replaceCopilotChatMessages('chat-1', [userMsg, assistantMsg])
+
+      expect(dbChainMockFns.delete).toHaveBeenCalledTimes(1)
+      expect(dbChainMockFns.insert).toHaveBeenCalledTimes(1)
+
+      const rows = lastValuesRows()
+      expect(rows).toHaveLength(2)
+      expect(rows.map((r) => r.messageId)).toEqual(['msg-user-1', 'msg-asst-1'])
+
+      expect(dbChainMockFns.onConflictDoUpdate).toHaveBeenCalledTimes(1)
+      const conflictArg = dbChainMockFns.onConflictDoUpdate.mock.calls[0][0]
+      expect(conflictArg.set).toHaveProperty('streamId')
+      expect(conflictArg.set).toHaveProperty('model')
+    })
+
+    it('assigns seq as the snapshot array index (0-based)', async () => {
+      await replaceCopilotChatMessages('chat-1', [userMsg, assistantMsg])
+      const rows = lastValuesRows()
+      expect(rows[0].seq).toBe(0)
+      expect(rows[1].seq).toBe(1)
+    })
+
+    it('OVERWRITES seq on conflict so positions re-densify after a delete', async () => {
+      await replaceCopilotChatMessages('chat-1', [userMsg])
+      const conflictArg = dbChainMockFns.onConflictDoUpdate.mock.calls[0][0]
+      expect(conflictArg.set.seq.strings.join('')).toBe('excluded.seq')
+    })
+
+    it('collapses duplicate message ids to a single row', async () => {
+      await replaceCopilotChatMessages('chat-1', [userMsg, { ...userMsg, content: 'dupe' }])
+      const rows = lastValuesRows()
+      expect(rows).toHaveLength(1)
+      expect(rows[0].seq).toBe(0)
+    })
+
+    it('passes chatModel to every row in the snapshot', async () => {
+      await replaceCopilotChatMessages('chat-1', [userMsg], {
+        chatModel: 'gpt-4o-mini',
+      })
+
+      const rows = lastValuesRows()
+      expect(rows[0].model).toBe('gpt-4o-mini')
+    })
+
+    it('propagates DB errors — the snapshot is authoritative', async () => {
+      dbChainMockFns.transaction.mockRejectedValueOnce(new Error('tx aborted'))
+
+      await expect(replaceCopilotChatMessages('chat-1', [userMsg])).rejects.toThrow('tx aborted')
+    })
+
+    it('strips tool-result output before persisting, keeping success/error', async () => {
+      await replaceCopilotChatMessages('chat-1', [toolMsg])
+
+      const toolCall = lastRowContent(0).contentBlocks?.[0].toolCall
+      expect(toolCall?.result).toEqual({ success: false, error: 'too big' })
+      expect(JSON.stringify(lastValuesRows())).not.toContain('huge')
+    })
+  })
+})

apps/sim/lib/copilot/chat/messages-store.ts

@@ -0,0 +1,122 @@
+import { db } from '@sim/db'
+import { copilotMessages } from '@sim/db/schema'
+import { and, eq, notInArray, sql } from 'drizzle-orm'
+import { type PersistedMessage, stripToolResultOutput } from '@/lib/copilot/chat/persisted-message'
+import type { DbOrTx } from '@/lib/db/types'
+
+/**
+ * Keep the first occurrence of each message id. A single `INSERT ... ON
+ * CONFLICT` cannot touch the same conflict target twice, so a repeated id
+ * would otherwise throw.
+ */
+function dedupeById(messages: PersistedMessage[]): PersistedMessage[] {
+  const seen = new Set<string>()
+  const out: PersistedMessage[] = []
+  for (const m of messages) {
+    if (seen.has(m.id)) continue
+    seen.add(m.id)
+    out.push(m)
+  }
+  return out
+}
+
+function toRow(
+  chatId: string,
+  message: PersistedMessage,
+  seq: number,
+  options?: { chatModel?: string | null; streamId?: string | null }
+): typeof copilotMessages.$inferInsert {
+  const ts = new Date(message.timestamp)
+  return {
+    chatId,
+    messageId: message.id,
+    role: message.role,
+    content: stripToolResultOutput(message),
+    seq,
+    model: options?.chatModel ?? null,
+    streamId: options?.streamId ?? null,
+    createdAt: ts,
+    updatedAt: ts,
+  }
+}
+
+/**
+ * Append messages to the `copilot_messages` table — the sole store for chat
+ * transcripts. Throws on failure (a swallowed write would lose messages).
+ * Pass `executor` to enlist the write in an existing transaction.
+ *
+ * `seq` is `MAX(seq) + index`, computed in JS. The read-then-insert is
+ * non-atomic, but per-chat appends are serialized by the pending-stream lock
+ * and the `seq, created_at, id` read order breaks any residual tie.
+ */
+export async function appendCopilotChatMessages(
+  chatId: string,
+  messages: PersistedMessage[],
+  options?: { chatModel?: string | null; streamId?: string | null },
+  executor: DbOrTx = db
+): Promise<void> {
+  if (messages.length === 0) return
+  const deduped = dedupeById(messages)
+  const [maxRow] = await executor
+    .select({ maxSeq: sql<number | null>`max(${copilotMessages.seq})` })
+    .from(copilotMessages)
+    .where(eq(copilotMessages.chatId, chatId))
+  const base = (maxRow?.maxSeq ?? -1) + 1
+  await executor
+    .insert(copilotMessages)
+    .values(deduped.map((m, i) => toRow(chatId, m, base + i, options)))
+    .onConflictDoUpdate({
+      target: [copilotMessages.chatId, copilotMessages.messageId],
+      set: {
+        content: sql`excluded.content`,
+        role: sql`excluded.role`,
+        model: sql`COALESCE(excluded.model, ${copilotMessages.model})`,
+        streamId: sql`COALESCE(excluded.stream_id, ${copilotMessages.streamId})`,
+        seq: sql`COALESCE(${copilotMessages.seq}, excluded.seq)`,
+        updatedAt: sql`now()`,
+      },
+    })
+}
+
+/**
+ * Replace all messages for a chat from a full snapshot (used by update-messages).
+ * Throws on failure. Pass `executor` to enlist the delete+insert in an existing
+ * transaction; otherwise it runs in its own.
+ */
+export async function replaceCopilotChatMessages(
+  chatId: string,
+  messages: PersistedMessage[],
+  options?: { chatModel?: string | null },
+  executor?: DbOrTx
+): Promise<void> {
+  const deduped = dedupeById(messages)
+  const newMessageIds = deduped.map((m) => m.id)
+  const run = async (tx: DbOrTx) => {
+    await tx
+      .delete(copilotMessages)
+      .where(
+        newMessageIds.length > 0
+          ? and(
+              eq(copilotMessages.chatId, chatId),
+              notInArray(copilotMessages.messageId, newMessageIds)
+            )
+          : eq(copilotMessages.chatId, chatId)
+      )
+    if (deduped.length === 0) return
+    await tx
+      .insert(copilotMessages)
+      .values(deduped.map((m, i) => toRow(chatId, m, i, options)))
+      .onConflictDoUpdate({
+        target: [copilotMessages.chatId, copilotMessages.messageId],
+        set: {
+          content: sql`excluded.content`,
+          role: sql`excluded.role`,
+          model: sql`COALESCE(excluded.model, ${copilotMessages.model})`,
+          streamId: sql`COALESCE(excluded.stream_id, ${copilotMessages.streamId})`,
+          seq: sql`excluded.seq`,
+          updatedAt: sql`now()`,
+        },
+      })
+  }
+  await (executor ? run(executor) : db.transaction(run))
+}

apps/sim/lib/copilot/chat/payload.test.ts

@@ -0,0 +1,270 @@
+/**
+ * @vitest-environment node
+ */
+import { envFlagsMock, workflowsUtilsMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockCreateUserToolSchema, mockGetHighestPrioritySubscription } = vi.hoisted(() => ({
+  mockCreateUserToolSchema: vi.fn(() => ({ type: 'object', properties: {} })),
+  mockGetHighestPrioritySubscription: vi.fn(),
+}))
+
+vi.mock('@/lib/billing/core/subscription', () => ({
+  getHighestPrioritySubscription: mockGetHighestPrioritySubscription,
+}))
+
+vi.mock('@/lib/billing/plan-helpers', () => ({
+  isPaid: vi.fn(
+    (plan: string | null) => plan === 'pro' || plan === 'team' || plan === 'enterprise'
+  ),
+}))
+
+vi.mock('@/lib/core/config/env-flags', () => envFlagsMock)
+
+vi.mock('@/lib/mcp/utils', () => ({
+  createMcpToolId: vi.fn(),
+}))
+
+vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock)
+
+vi.mock('@/tools/registry', () => ({
+  tools: {
+    gmail_send: {
+      id: 'gmail_send',
+      name: 'Gmail Send',
+      description: 'Send emails using Gmail',
+    },
+    brandfetch_search: {
+      id: 'brandfetch_search',
+      name: 'Brandfetch Search',
+      description: 'Search for brands by company name',
+    },
+    // Catalog marks run_workflow as client-routed / clientExecutable; registry ToolConfig has no routing fields.
+    run_workflow: {
+      id: 'run_workflow',
+      name: 'Run Workflow',
+      description: 'Run a workflow from the client',
+    },
+  },
+}))
+
+vi.mock('@/tools/utils', () => ({
+  getLatestVersionTools: vi.fn((input) => input),
+  stripVersionSuffix: vi.fn((toolId: string) => toolId),
+}))
+
+vi.mock('@/lib/copilot/block-visibility', () => ({
+  getBlockVisibilityForCopilot: vi.fn(async () => ({
+    revealed: new Set<string>(),
+    disabled: new Set<string>(),
+    previewTagged: new Set<string>(),
+  })),
+  visibilitySignature: vi.fn(() => 'vis:none'),
+}))
+
+vi.mock('@/lib/copilot/integration-tools', () => ({
+  filterExposedIntegrationTools: vi.fn((tools: unknown[]) => tools),
+  getExposedIntegrationTools: vi.fn(() => [
+    {
+      toolId: 'gmail_send',
+      config: { id: 'gmail_send', name: 'Gmail Send', description: 'Send emails using Gmail' },
+      service: 'gmail',
+      operation: 'send',
+    },
+    {
+      toolId: 'brandfetch_search',
+      config: {
+        id: 'brandfetch_search',
+        name: 'Brandfetch Search',
+        description: 'Search for brands by company name',
+      },
+      service: 'brandfetch',
+      operation: 'search',
+    },
+    {
+      toolId: 'run_workflow',
+      config: {
+        id: 'run_workflow',
+        name: 'Run Workflow',
+        description: 'Run a workflow from the client',
+      },
+      service: 'run',
+      operation: 'workflow',
+    },
+  ]),
+}))
+
+vi.mock('@/tools/params', () => ({
+  createUserToolSchema: mockCreateUserToolSchema,
+}))
+
+import {
+  buildCopilotRequestPayload,
+  buildIntegrationToolSchemas,
+  clearIntegrationToolSchemaCacheForTests,
+} from './payload'
+
+describe('buildIntegrationToolSchemas', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    clearIntegrationToolSchemaCacheForTests()
+    mockCreateUserToolSchema.mockReturnValue({ type: 'object', properties: {} })
+  })
+
+  it('appends the email footer prompt for free users', async () => {
+    mockGetHighestPrioritySubscription.mockResolvedValue(null)
+
+    const toolSchemas = await buildIntegrationToolSchemas('user-free')
+    const gmailTool = toolSchemas.find((tool) => tool.name === 'gmail_send')
+
+    expect(mockGetHighestPrioritySubscription).toHaveBeenCalledWith('user-free')
+    expect(gmailTool?.description).toContain('sent with sim ai')
+  })
+
+  it('does not append the email footer prompt for paid users', async () => {
+    mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' })
+
+    const toolSchemas = await buildIntegrationToolSchemas('user-paid')
+    const gmailTool = toolSchemas.find((tool) => tool.name === 'gmail_send')
+
+    expect(mockGetHighestPrioritySubscription).toHaveBeenCalledWith('user-paid')
+    expect(gmailTool?.description).toBe('Send emails using Gmail')
+  })
+
+  it('still builds integration tools when subscription lookup fails', async () => {
+    mockGetHighestPrioritySubscription.mockRejectedValue(new Error('db unavailable'))
+
+    const toolSchemas = await buildIntegrationToolSchemas('user-error')
+    const gmailTool = toolSchemas.find((tool) => tool.name === 'gmail_send')
+    const brandfetchTool = toolSchemas.find((tool) => tool.name === 'brandfetch_search')
+
+    expect(mockGetHighestPrioritySubscription).toHaveBeenCalledWith('user-error')
+    expect(gmailTool?.description).toBe('Send emails using Gmail')
+    expect(brandfetchTool?.description).toBe('Search for brands by company name')
+  })
+
+  it('emits executeLocally for dynamic client tools only', async () => {
+    mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' })
+
+    const toolSchemas = await buildIntegrationToolSchemas('user-client')
+    const gmailTool = toolSchemas.find((tool) => tool.name === 'gmail_send')
+    const runTool = toolSchemas.find((tool) => tool.name === 'run_workflow')
+
+    expect(gmailTool?.executeLocally).toBe(false)
+    expect(runTool?.executeLocally).toBe(true)
+  })
+
+  it('uses copilot-facing file schemas for integration tools', async () => {
+    mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' })
+
+    await buildIntegrationToolSchemas('user-copilot')
+
+    expect(mockCreateUserToolSchema).toHaveBeenCalledWith(
+      expect.objectContaining({ id: 'gmail_send' }),
+      { surface: 'copilot' }
+    )
+    expect(mockCreateUserToolSchema).toHaveBeenCalledWith(
+      expect.objectContaining({ id: 'brandfetch_search' }),
+      { surface: 'copilot' }
+    )
+  })
+
+  it('briefly reuses built schemas for the same user and surface', async () => {
+    mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' })
+
+    const first = await buildIntegrationToolSchemas('user-cache')
+    first[0].input_schema.mutated = true
+    const second = await buildIntegrationToolSchemas('user-cache')
+
+    expect(mockGetHighestPrioritySubscription).toHaveBeenCalledTimes(1)
+    expect(mockCreateUserToolSchema).toHaveBeenCalledTimes(3)
+    expect(second[0].input_schema).not.toHaveProperty('mutated')
+  })
+})
+
+describe('buildCopilotRequestPayload', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('passes workspaceContext through to the Go request payload', async () => {
+    const payload = await buildCopilotRequestPayload(
+      {
+        message: 'debug workspace',
+        userId: 'user-1',
+        userMessageId: 'msg-1',
+        mode: 'agent',
+        model: 'claude-opus-4-8',
+        workspaceId: 'ws-1',
+        workspaceContext: 'workspace inventory',
+      },
+      { selectedModel: 'claude-opus-4-8' }
+    )
+
+    expect(payload).toEqual(
+      expect.objectContaining({
+        workspaceId: 'ws-1',
+        workspaceContext: 'workspace inventory',
+      })
+    )
+  })
+
+  it('passes user metadata through to the Go request payload', async () => {
+    const payload = await buildCopilotRequestPayload(
+      {
+        message: 'what time is it',
+        userId: 'user-1',
+        userMessageId: 'msg-1',
+        mode: 'agent',
+        model: 'claude-opus-4-8',
+        workspaceId: 'ws-1',
+        userTimezone: 'America/Los_Angeles',
+        userMetadata: {
+          name: 'Sid',
+          timezone: 'America/Los_Angeles',
+        },
+      },
+      { selectedModel: 'claude-opus-4-8' }
+    )
+
+    expect(payload).toEqual(
+      expect.objectContaining({
+        userTimezone: 'America/Los_Angeles',
+        userMetadata: {
+          name: 'Sid',
+          timezone: 'America/Los_Angeles',
+        },
+      })
+    )
+  })
+
+  it('passes entitlements through and omits the field when empty', async () => {
+    const withEntitlements = await buildCopilotRequestPayload(
+      {
+        message: 'publish as a block',
+        userId: 'user-1',
+        userMessageId: 'msg-1',
+        mode: 'agent',
+        model: 'claude-opus-4-8',
+        workspaceId: 'ws-1',
+        entitlements: ['custom-blocks'],
+      },
+      { selectedModel: 'claude-opus-4-8' }
+    )
+    expect(withEntitlements).toEqual(expect.objectContaining({ entitlements: ['custom-blocks'] }))
+
+    const withoutEntitlements = await buildCopilotRequestPayload(
+      {
+        message: 'publish as a block',
+        userId: 'user-1',
+        userMessageId: 'msg-1',
+        mode: 'agent',
+        model: 'claude-opus-4-8',
+        workspaceId: 'ws-1',
+        entitlements: [],
+      },
+      { selectedModel: 'claude-opus-4-8' }
+    )
+    expect(withoutEntitlements).not.toHaveProperty('entitlements')
+  })
+})

apps/sim/lib/copilot/chat/payload.ts

@@ -0,0 +1,403 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { LRUCache } from 'lru-cache'
+import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
+import { isPaid } from '@/lib/billing/plan-helpers'
+import { getBlockVisibilityForCopilot, visibilitySignature } from '@/lib/copilot/block-visibility'
+import type { VfsSnapshotV1 } from '@/lib/copilot/generated/vfs-snapshot-v1'
+import {
+  filterExposedIntegrationTools,
+  getExposedIntegrationTools,
+} from '@/lib/copilot/integration-tools'
+import { getToolEntry } from '@/lib/copilot/tool-executor/router'
+import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions'
+import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils'
+import type { BlockVisibilityState } from '@/lib/core/config/block-visibility'
+import { isE2BDocEnabled, isHosted } from '@/lib/core/config/env-flags'
+import { buildUserSkillTool } from '@/lib/mothership/skills'
+import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+import { stripVersionSuffix } from '@/tools/utils'
+
+const logger = createLogger('CopilotChatPayload')
+const INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS = 5_000
+const INTEGRATION_TOOL_SCHEMA_CACHE_MAX_ENTRIES = 500
+
+interface BuildPayloadParams {
+  message: string
+  workflowId?: string
+  workflowName?: string
+  workspaceId?: string
+  userId: string
+  userMessageId: string
+  mode: string
+  model: string
+  provider?: string
+  contexts?: Array<{ type: string; content: string; tag?: string; path?: string }>
+  fileAttachments?: Array<{ id: string; key: string; size: number; [key: string]: unknown }>
+  commands?: string[]
+  chatId?: string
+  prefetch?: boolean
+  implicitFeedback?: string
+  workspaceContext?: string
+  vfs?: VfsSnapshotV1
+  userPermission?: string
+  /** Plan/flag-gated org capabilities (e.g. "custom-blocks") the mothership gates tools/prompts on. */
+  entitlements?: string[]
+  userTimezone?: string
+  userMetadata?: {
+    name?: string
+    email?: string
+    timezone?: string
+  }
+  includeMothershipTools?: boolean
+}
+
+export interface ToolSchema {
+  name: string
+  description: string
+  input_schema: Record<string, unknown>
+  defer_loading?: boolean
+  executeLocally?: boolean
+  params?: Record<string, unknown>
+  /** Canonical integration service/folder (e.g. "slack"), for server-side grouping. */
+  service?: string
+  oauth?: { required: boolean; provider: string }
+}
+
+interface BuildIntegrationToolSchemasOptions {
+  schemaSurface?: 'default' | 'copilot'
+}
+
+interface IntegrationToolSchemaCacheEntry {
+  promise: Promise<ToolSchema[]>
+}
+
+const integrationToolSchemaCache = new LRUCache<string, IntegrationToolSchemaCacheEntry>({
+  max: INTEGRATION_TOOL_SCHEMA_CACHE_MAX_ENTRIES,
+  ttl: INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS,
+})
+
+function getIntegrationToolSchemaCacheKey(
+  userId: string,
+  workspaceId: string | undefined,
+  schemaSurface: string,
+  visSignature: string
+): string {
+  // The visibility signature keys the entry to the viewer's gated projection —
+  // two users in one workspace with different preview reveals must not share.
+  return JSON.stringify([userId, workspaceId ?? null, schemaSurface, visSignature])
+}
+
+function cloneToolSchemas(toolSchemas: ToolSchema[]): ToolSchema[] {
+  return toolSchemas.map((tool) => {
+    const cloned: ToolSchema = {
+      ...tool,
+      input_schema: { ...tool.input_schema },
+    }
+    if (tool.params) cloned.params = { ...tool.params }
+    if (tool.oauth) cloned.oauth = { ...tool.oauth }
+    return cloned
+  })
+}
+
+export function clearIntegrationToolSchemaCacheForTests(): void {
+  integrationToolSchemaCache.clear()
+}
+
+/**
+ * Build deferred integration tool schemas from the Sim tool registry.
+ * Shared by the interactive chat payload builder and the non-interactive
+ * block execution route so both paths send the same tool definitions to Go.
+ *
+ * When `workspaceId` is provided the user's workspace permission config is
+ * loaded once and used to skip any tool whose owning block is not in the
+ * workspace's `allowedIntegrations` allowlist.
+ */
+export async function buildIntegrationToolSchemas(
+  userId: string,
+  messageId?: string,
+  options: BuildIntegrationToolSchemasOptions = { schemaSurface: 'copilot' },
+  workspaceId?: string
+): Promise<ToolSchema[]> {
+  const schemaSurface = options.schemaSurface ?? 'copilot'
+  const vis = await getBlockVisibilityForCopilot(userId, workspaceId)
+  const cacheKey = getIntegrationToolSchemaCacheKey(
+    userId,
+    workspaceId,
+    schemaSurface,
+    visibilitySignature(vis)
+  )
+  const cached = integrationToolSchemaCache.get(cacheKey)
+  if (cached) {
+    return cloneToolSchemas(await cached.promise)
+  }
+
+  const promise = buildIntegrationToolSchemasUncached(
+    userId,
+    messageId,
+    { schemaSurface },
+    workspaceId,
+    vis
+  ).catch((error) => {
+    integrationToolSchemaCache.delete(cacheKey)
+    throw error
+  })
+
+  integrationToolSchemaCache.set(cacheKey, {
+    promise,
+  })
+
+  return cloneToolSchemas(await promise)
+}
+
+async function buildIntegrationToolSchemasUncached(
+  userId: string,
+  messageId: string | undefined,
+  options: Required<BuildIntegrationToolSchemasOptions>,
+  workspaceId?: string,
+  vis: BlockVisibilityState | null = null
+): Promise<ToolSchema[]> {
+  const reqLogger = logger.withMetadata({ messageId })
+  const integrationTools: ToolSchema[] = []
+  try {
+    const { createUserToolSchema } = await import('@/tools/params')
+    let shouldAppendEmailTagline = false
+
+    try {
+      const subscription = await getHighestPrioritySubscription(userId)
+      shouldAppendEmailTagline = !subscription || !isPaid(subscription.plan)
+    } catch (error) {
+      reqLogger.warn('Failed to load subscription for copilot tool descriptions', {
+        userId,
+        error: toError(error).message,
+      })
+    }
+
+    let allowedIntegrations: Set<string> | null = null
+    let toolIdToBlockType: Map<string, string> | null = null
+    if (workspaceId) {
+      try {
+        const [{ getUserPermissionConfig }, { getAllBlocks }] = await Promise.all([
+          import('@/ee/access-control/utils/permission-check'),
+          import('@/blocks/registry'),
+        ])
+        const permissionConfig = await getUserPermissionConfig(userId, workspaceId)
+        if (permissionConfig?.allowedIntegrations) {
+          allowedIntegrations = new Set(
+            permissionConfig.allowedIntegrations.map((i) => i.toLowerCase())
+          )
+          toolIdToBlockType = new Map()
+          for (const blockConfig of getAllBlocks()) {
+            const access = blockConfig.tools?.access
+            if (!access) continue
+            for (const toolId of access) {
+              toolIdToBlockType.set(stripVersionSuffix(toolId), blockConfig.type.toLowerCase())
+            }
+          }
+        }
+      } catch (error) {
+        reqLogger.warn('Failed to load permission config for tool schema filter', {
+          userId,
+          workspaceId,
+          error: toError(error).message,
+        })
+      }
+    }
+
+    const exposedTools = filterExposedIntegrationTools(getExposedIntegrationTools(), vis)
+    for (const { toolId, config: toolConfig, service } of exposedTools) {
+      try {
+        if (allowedIntegrations && toolIdToBlockType) {
+          const owningBlock = toolIdToBlockType.get(stripVersionSuffix(toolId))
+          if (owningBlock && !allowedIntegrations.has(owningBlock)) {
+            continue
+          }
+        }
+        const userSchema = createUserToolSchema(toolConfig, {
+          surface: options.schemaSurface,
+        })
+        const catalogEntry = getToolEntry(toolId)
+        integrationTools.push({
+          name: toolId,
+          service,
+          description: getCopilotToolDescription(toolConfig, {
+            isHosted,
+            fallbackName: toolId,
+            appendEmailTagline: shouldAppendEmailTagline,
+          }),
+          input_schema: { ...userSchema },
+          defer_loading: true,
+          executeLocally:
+            catalogEntry?.clientExecutable === true || catalogEntry?.route === 'client',
+          ...(toolConfig.oauth?.required && {
+            oauth: {
+              required: true,
+              provider: toolConfig.oauth.provider,
+            },
+          }),
+        })
+      } catch (toolError) {
+        logger.warn(
+          messageId
+            ? `Failed to build schema for tool, skipping [messageId:${messageId}]`
+            : 'Failed to build schema for tool, skipping',
+          {
+            toolId,
+            error: toError(toolError).message,
+          }
+        )
+      }
+    }
+  } catch (error) {
+    logger.warn(
+      messageId
+        ? `Failed to build tool schemas [messageId:${messageId}]`
+        : 'Failed to build tool schemas',
+      {
+        error: toError(error).message,
+      }
+    )
+  }
+
+  return integrationTools
+}
+
+/**
+ * Build the request payload for the copilot backend.
+ */
+export async function buildCopilotRequestPayload(
+  params: BuildPayloadParams,
+  options: {
+    selectedModel: string
+  }
+): Promise<Record<string, unknown>> {
+  const {
+    message,
+    workflowId,
+    userId,
+    userMessageId,
+    mode,
+    provider,
+    contexts,
+    fileAttachments,
+    commands,
+    chatId,
+    prefetch,
+    implicitFeedback,
+  } = params
+
+  const selectedModel = options.selectedModel
+
+  const effectiveMode = mode === 'agent' ? 'build' : mode
+  const transportMode = effectiveMode === 'build' ? 'agent' : effectiveMode
+
+  // Track uploaded files in the DB and build context tags instead of base64 inlining
+  const uploadContexts: Array<{ type: string; content: string; tag?: string; path?: string }> = []
+  if (chatId && params.workspaceId && fileAttachments && fileAttachments.length > 0) {
+    for (const f of fileAttachments) {
+      const filename = (f.filename ?? f.name ?? 'file') as string
+      const mediaType = (f.media_type ?? f.mimeType ?? 'application/octet-stream') as string
+      try {
+        const { displayName } = await trackChatUpload(
+          params.workspaceId,
+          userId,
+          chatId,
+          f.key,
+          filename,
+          mediaType,
+          f.size
+        )
+        // Encode the read path per the percent-encoded VFS convention (matches
+        // files/ and the uploads glob output). The materialize_file `fileName`
+        // arg stays the raw display name — the upload resolver accepts both.
+        let encodedUploadName = displayName
+        try {
+          encodedUploadName = encodeVfsSegment(displayName)
+        } catch {
+          encodedUploadName = displayName
+        }
+        const lines = [
+          `File "${displayName}" (${mediaType}, ${f.size} bytes) uploaded.`,
+          `Read with: read("uploads/${encodedUploadName}")`,
+          `To save permanently: materialize_file(fileName: "${displayName}")`,
+        ]
+        if (displayName.endsWith('.json')) {
+          lines.push(
+            `To import as a workflow: materialize_file(fileName: "${displayName}", operation: "import")`
+          )
+        }
+        uploadContexts.push({
+          type: 'uploaded_file',
+          content: lines.join('\n'),
+        })
+      } catch (err) {
+        logger.warn('Failed to track chat upload', {
+          filename,
+          chatId,
+          error: toError(err).message,
+        })
+      }
+    }
+  }
+
+  const allContexts = [...(contexts ?? []), ...uploadContexts]
+
+  let integrationTools: ToolSchema[] = []
+  const mothershipTools: ToolSchema[] = []
+  const payloadLogger = logger.withMetadata({ messageId: userMessageId })
+
+  if (effectiveMode === 'build') {
+    integrationTools = await buildIntegrationToolSchemas(
+      userId,
+      userMessageId,
+      { schemaSurface: 'copilot' },
+      params.workspaceId
+    )
+
+    if (params.includeMothershipTools && params.workspaceId) {
+      // Expose all workspace user-created skills via the single load_user_skill
+      // tool. Available to every user; content is fetched sim-side when the
+      // model calls it.
+      try {
+        const userSkillTool = await buildUserSkillTool(params.workspaceId)
+        if (userSkillTool) mothershipTools.push(userSkillTool)
+      } catch (error) {
+        logger.warn('Failed to build load_user_skill tool', {
+          error: toError(error).message,
+        })
+      }
+    }
+  }
+
+  return {
+    message,
+    ...(workflowId ? { workflowId } : {}),
+    ...(params.workflowName ? { workflowName: params.workflowName } : {}),
+    ...(params.workspaceId ? { workspaceId: params.workspaceId } : {}),
+    userId,
+    ...(selectedModel ? { model: selectedModel } : {}),
+    ...(provider ? { provider } : {}),
+    mode: transportMode,
+    messageId: userMessageId,
+    ...(allContexts.length > 0 ? { context: allContexts } : {}),
+    ...(chatId ? { chatId } : {}),
+    ...(typeof prefetch === 'boolean' ? { prefetch } : {}),
+    ...(implicitFeedback ? { implicitFeedback } : {}),
+    ...(integrationTools.length > 0 ? { integrationTools } : {}),
+    ...(mothershipTools.length > 0 ? { mothershipTools } : {}),
+    ...(commands && commands.length > 0 ? { commands } : {}),
+    ...(params.workspaceContext ? { workspaceContext: params.workspaceContext } : {}),
+    ...(params.vfs ? { vfs: params.vfs } : {}),
+    ...(params.userPermission ? { userPermission: params.userPermission } : {}),
+    ...(params.entitlements?.length ? { entitlements: params.entitlements } : {}),
+    ...(params.userTimezone ? { userTimezone: params.userTimezone } : {}),
+    ...(params.userMetadata &&
+    (params.userMetadata.name || params.userMetadata.email || params.userMetadata.timezone)
+      ? { userMetadata: params.userMetadata }
+      : {}),
+    // Tell the copilot file subagent which document toolchain to write. Emitted
+    // only in Python mode so the JS path sends no new field (Go defaults to js).
+    ...(isE2BDocEnabled ? { docCompiler: 'python' } : {}),
+    isHosted,
+  }
+}

apps/sim/lib/copilot/chat/persisted-message.test.ts

@@ -0,0 +1,372 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+import {
+  buildPersistedAssistantMessage,
+  buildPersistedUserMessage,
+  normalizeMessage,
+  type PersistedMessage,
+  stripToolResultOutput,
+} from './persisted-message'
+
+describe('persisted-message', () => {
+  it('round-trips canonical tool blocks through normalizeMessage', () => {
+    const blockTimestamp = 1_700_000_000_000
+    const result: OrchestratorResult = {
+      success: true,
+      content: 'done',
+      requestId: 'req-1',
+      contentBlocks: [
+        {
+          type: 'tool_call',
+          timestamp: blockTimestamp,
+          calledBy: 'workflow',
+          toolCall: {
+            id: 'tool-1',
+            name: 'read',
+            status: 'success',
+            displayTitle: 'Reading foo.txt',
+            params: { path: 'foo.txt' },
+            result: { success: true, output: { ok: true } },
+          },
+        },
+      ],
+      toolCalls: [],
+    }
+
+    const persisted = buildPersistedAssistantMessage(result)
+    const normalized = normalizeMessage(persisted as unknown as Record<string, unknown>)
+
+    expect(normalized.contentBlocks).toEqual([
+      {
+        type: 'tool',
+        phase: 'call',
+        timestamp: blockTimestamp,
+        toolCall: {
+          id: 'tool-1',
+          name: 'read',
+          state: 'success',
+          display: { title: 'Reading foo.txt' },
+          params: { path: 'foo.txt' },
+          result: { success: true, output: { ok: true } },
+          calledBy: 'workflow',
+        },
+      },
+      {
+        type: 'text',
+        channel: 'assistant',
+        content: 'done',
+      },
+    ])
+  })
+
+  it('prefers an explicit persisted request ID override', () => {
+    const result: OrchestratorResult = {
+      success: true,
+      content: 'done',
+      requestId: 'go-trace-1',
+      contentBlocks: [],
+      toolCalls: [],
+    }
+
+    const persisted = buildPersistedAssistantMessage(result, 'sim-request-1')
+
+    expect(persisted.requestId).toBe('sim-request-1')
+  })
+
+  it('redacts sim_key credential tags so persisted assistant messages never re-expose the key', () => {
+    const live = `Here is your key: <credential>${JSON.stringify({ value: 'sk-sim-secret-123', type: 'sim_key' })}</credential> save it.`
+    const result: OrchestratorResult = {
+      success: true,
+      content: live,
+      requestId: 'req-1',
+      contentBlocks: [{ type: 'text', content: live }],
+      toolCalls: [],
+    }
+
+    const persisted = buildPersistedAssistantMessage(result)
+
+    expect(persisted.content).not.toContain('sk-sim-secret-123')
+    expect(persisted.content).toContain('"redacted":true')
+    const textBlock = persisted.contentBlocks?.find((b) => b.type === 'text')
+    expect(textBlock?.content).not.toContain('sk-sim-secret-123')
+    expect(textBlock?.content).toContain('"redacted":true')
+  })
+
+  it('redacts sim_key credential tags split across streamed text chunks', () => {
+    const chunks = [
+      'Here\'s your key:\n\n<credential>{"value": "sk-',
+      'sim-secret',
+      '-12345',
+      '", "type":',
+      ' "sim_key"}</credential>',
+      '\n\nDone.',
+    ]
+    const result: OrchestratorResult = {
+      success: true,
+      content: chunks.join(''),
+      requestId: 'req-1',
+      contentBlocks: chunks.map((c) => ({ type: 'text', content: c })),
+      toolCalls: [],
+    }
+
+    const persisted = buildPersistedAssistantMessage(result)
+
+    expect(persisted.content).not.toContain('sk-sim-secret-12345')
+    expect(persisted.contentBlocks).toBeDefined()
+    const joined = (persisted.contentBlocks ?? []).map((b) => b.content ?? '').join('')
+    expect(joined).not.toContain('sk-sim-secret-12345')
+    expect(joined).toContain('"redacted":true')
+  })
+
+  it('redacts the api key from a persisted generate_api_key tool result output', () => {
+    const result: OrchestratorResult = {
+      success: true,
+      content: '',
+      requestId: 'req-1',
+      contentBlocks: [
+        {
+          type: 'tool_call',
+          toolCall: {
+            id: 'tool-1',
+            name: 'generate_api_key',
+            status: 'success',
+            params: { name: 'workspace-key' },
+            result: {
+              success: true,
+              output: {
+                id: 'k1',
+                name: 'workspace-key',
+                key: 'sk-sim-tool-output-secret',
+              },
+            },
+          },
+        },
+      ],
+      toolCalls: [],
+    }
+
+    const persisted = buildPersistedAssistantMessage(result)
+    const toolBlock = persisted.contentBlocks?.find((b) => b.toolCall?.name === 'generate_api_key')
+    const output = toolBlock?.toolCall?.result?.output as Record<string, unknown> | undefined
+
+    expect(output?.key).toBe('[REDACTED]')
+    expect(output?.redacted).toBe(true)
+    expect(JSON.stringify(persisted)).not.toContain('sk-sim-tool-output-secret')
+  })
+
+  it('leaves non-sim_key credential tags untouched', () => {
+    const live = `<credential>${JSON.stringify({ value: 'https://oauth.example/connect', type: 'link', provider: 'slack' })}</credential>`
+    const result: OrchestratorResult = {
+      success: true,
+      content: live,
+      requestId: 'req-1',
+      contentBlocks: [{ type: 'text', content: live }],
+      toolCalls: [],
+    }
+
+    const persisted = buildPersistedAssistantMessage(result)
+
+    expect(persisted.content).toContain('https://oauth.example/connect')
+  })
+
+  it('normalizes legacy tool_call and top-level toolCalls shapes', () => {
+    const normalized = normalizeMessage({
+      id: 'msg-1',
+      role: 'assistant',
+      content: 'hello',
+      timestamp: '2024-01-01T00:00:00.000Z',
+      contentBlocks: [
+        {
+          type: 'tool_call',
+          toolCall: {
+            id: 'tool-1',
+            name: 'read',
+            state: 'cancelled',
+            display: { phaseLabel: 'Workspace' },
+          },
+        },
+      ],
+      toolCalls: [
+        {
+          id: 'tool-2',
+          name: 'glob',
+          status: 'success',
+          result: { matches: [] },
+        },
+      ],
+    })
+
+    expect(normalized.contentBlocks).toEqual([
+      {
+        type: 'tool',
+        phase: 'call',
+        toolCall: {
+          id: 'tool-1',
+          name: 'read',
+          state: 'cancelled',
+          display: { title: 'Workspace' },
+        },
+      },
+      {
+        type: 'text',
+        channel: 'assistant',
+        content: 'hello',
+      },
+    ])
+  })
+
+  it('builds normalized user messages with stripped optional empties', () => {
+    const msg = buildPersistedUserMessage({
+      id: 'user-1',
+      content: 'hello',
+      fileAttachments: [],
+      contexts: [],
+    })
+
+    expect(msg).toMatchObject({
+      id: 'user-1',
+      role: 'user',
+      content: 'hello',
+    })
+    expect(msg.fileAttachments).toBeUndefined()
+    expect(msg.contexts).toBeUndefined()
+  })
+})
+
+describe('stripToolResultOutput', () => {
+  it('drops result.output but keeps success and error', () => {
+    const message: PersistedMessage = {
+      id: 'msg-1',
+      role: 'assistant',
+      content: '',
+      timestamp: '2026-01-01T00:00:00.000Z',
+      contentBlocks: [
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'tool-1',
+            name: 'get_workflow_logs',
+            state: 'error',
+            params: { workflowId: 'wf-1' },
+            display: { title: 'Reading logs' },
+            result: { success: false, output: { huge: 'x'.repeat(1000) }, error: 'boom' },
+          },
+        },
+      ],
+    }
+
+    const stripped = stripToolResultOutput(message)
+
+    expect(stripped.contentBlocks?.[0].toolCall).toEqual({
+      id: 'tool-1',
+      name: 'get_workflow_logs',
+      state: 'error',
+      params: { workflowId: 'wf-1' },
+      display: { title: 'Reading logs' },
+      result: { success: false, error: 'boom' },
+    })
+    expect(message.contentBlocks?.[0].toolCall?.result).toHaveProperty('output')
+  })
+
+  it('omits error when the original result had none', () => {
+    const message: PersistedMessage = {
+      id: 'msg-1',
+      role: 'assistant',
+      content: '',
+      timestamp: '2026-01-01T00:00:00.000Z',
+      contentBlocks: [
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 't',
+            name: 'read',
+            state: 'success',
+            result: { success: true, output: [1, 2, 3] },
+          },
+        },
+      ],
+    }
+
+    expect(stripToolResultOutput(message).contentBlocks?.[0].toolCall?.result).toEqual({
+      success: true,
+    })
+  })
+
+  it('returns the same reference when there is nothing to strip', () => {
+    const noBlocks: PersistedMessage = {
+      id: 'u',
+      role: 'user',
+      content: 'hi',
+      timestamp: '2026-01-01T00:00:00.000Z',
+    }
+    expect(stripToolResultOutput(noBlocks)).toBe(noBlocks)
+
+    const noOutput: PersistedMessage = {
+      id: 'msg',
+      role: 'assistant',
+      content: 'done',
+      timestamp: '2026-01-01T00:00:00.000Z',
+      contentBlocks: [
+        { type: 'text', channel: 'assistant', content: 'done' },
+        { type: 'tool', phase: 'call', toolCall: { id: 't', name: 'read', state: 'pending' } },
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 't2',
+            name: 'read',
+            state: 'error',
+            result: { success: false, error: 'x' },
+          },
+        },
+      ],
+    }
+    expect(stripToolResultOutput(noOutput)).toBe(noOutput)
+  })
+
+  it('strips every tool block while leaving text/thinking blocks intact', () => {
+    const message: PersistedMessage = {
+      id: 'msg',
+      role: 'assistant',
+      content: '',
+      timestamp: '2026-01-01T00:00:00.000Z',
+      contentBlocks: [
+        { type: 'text', channel: 'thinking', content: 'hmm' },
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'a',
+            name: 'run_workflow',
+            state: 'success',
+            result: { success: true, output: { big: 1 } },
+          },
+        },
+        { type: 'text', channel: 'assistant', content: 'answer' },
+        {
+          type: 'tool',
+          phase: 'call',
+          toolCall: {
+            id: 'b',
+            name: 'read',
+            state: 'success',
+            result: { success: true, output: 'file contents' },
+          },
+        },
+      ],
+    }
+
+    const blocks = stripToolResultOutput(message).contentBlocks ?? []
+    expect(blocks[0]).toEqual({ type: 'text', channel: 'thinking', content: 'hmm' })
+    expect(blocks[1].toolCall?.result).toEqual({ success: true })
+    expect(blocks[2]).toEqual({ type: 'text', channel: 'assistant', content: 'answer' })
+    expect(blocks[3].toolCall?.result).toEqual({ success: true })
+    expect(JSON.stringify(blocks)).not.toContain('file contents')
+  })
+})

apps/sim/lib/copilot/chat/persisted-message.ts

@@ -0,0 +1,656 @@
+import { generateId } from '@sim/utils/id'
+import {
+  mergeAndRedactPersistedBlocks,
+  redactSensitiveContent,
+  redactToolCallResult,
+} from '@/lib/copilot/chat/sim-key-redaction'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+  MothershipStreamV1SpanLifecycleEvent,
+  MothershipStreamV1SpanPayloadKind,
+  type MothershipStreamV1StreamScope,
+  MothershipStreamV1TextChannel,
+  MothershipStreamV1ToolOutcome,
+  MothershipStreamV1ToolPhase,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import type {
+  ContentBlock,
+  LocalToolCallStatus,
+  OrchestratorResult,
+} from '@/lib/copilot/request/types'
+
+export type PersistedToolState = LocalToolCallStatus | MothershipStreamV1ToolOutcome | 'interrupted'
+
+interface PersistedToolCall {
+  id: string
+  name: string
+  state: PersistedToolState
+  params?: Record<string, unknown>
+  result?: { success: boolean; output?: unknown; error?: string }
+  error?: string
+  calledBy?: string
+  durationMs?: number
+  display?: { title?: string }
+}
+
+export interface PersistedContentBlock {
+  type: MothershipStreamV1EventType
+  lane?: MothershipStreamV1StreamScope['lane']
+  channel?: MothershipStreamV1TextChannel
+  phase?: MothershipStreamV1ToolPhase
+  kind?: MothershipStreamV1SpanPayloadKind
+  lifecycle?: MothershipStreamV1SpanLifecycleEvent
+  status?: MothershipStreamV1CompletionStatus
+  content?: string
+  toolCall?: PersistedToolCall
+  timestamp?: number
+  endedAt?: number
+  parentToolCallId?: string
+  spanId?: string
+  parentSpanId?: string
+}
+
+export interface PersistedFileAttachment {
+  id: string
+  key: string
+  filename: string
+  media_type: string
+  size: number
+}
+
+interface PersistedMessageContext {
+  kind: string
+  label: string
+  workflowId?: string
+  knowledgeId?: string
+  tableId?: string
+  fileId?: string
+  folderId?: string
+  chatId?: string
+}
+
+export interface PersistedMessage {
+  id: string
+  role: 'user' | 'assistant'
+  content: string
+  timestamp: string
+  requestId?: string
+  contentBlocks?: PersistedContentBlock[]
+  fileAttachments?: PersistedFileAttachment[]
+  contexts?: PersistedMessageContext[]
+}
+
+/**
+ * Drop the `output` of every persisted tool result, keeping `success` and
+ * `error`. Tool outputs are never rendered (the chat thread shows only the tool
+ * name/title/status) and never replayed to the model (the upstream copilot
+ * service owns conversation memory), so storing them only bloats
+ * `copilot_messages.content` — a single `get_workflow_logs`/`run_workflow`
+ * result can reach hundreds of MB and stall task loads.
+ *
+ * Applied on both the write path (so new rows never store outputs) and the read
+ * path (so already-bloated rows still load fast). Returns the original
+ * reference when there is nothing to strip, preserving memoized identity for
+ * read-side consumers.
+ */
+export function stripToolResultOutput(message: PersistedMessage): PersistedMessage {
+  if (!message.contentBlocks?.length) return message
+  let changed = false
+  const contentBlocks = message.contentBlocks.map((block) => {
+    const toolCall = block.toolCall
+    const result = toolCall?.result
+    if (!toolCall || !result || typeof result !== 'object' || !('output' in result)) return block
+    changed = true
+    const strippedResult: { success: boolean; error?: string } = { success: result.success }
+    if (result.error !== undefined) strippedResult.error = result.error
+    return { ...block, toolCall: { ...toolCall, result: strippedResult } }
+  })
+  return changed ? { ...message, contentBlocks } : message
+}
+
+// ---------------------------------------------------------------------------
+// Write: OrchestratorResult → PersistedMessage
+// ---------------------------------------------------------------------------
+
+function resolveToolState(block: ContentBlock): PersistedToolState {
+  const tc = block.toolCall
+  if (!tc) return 'pending'
+  if (tc.result?.success !== undefined) {
+    return tc.result.success
+      ? MothershipStreamV1ToolOutcome.success
+      : MothershipStreamV1ToolOutcome.error
+  }
+  return tc.status as PersistedToolState
+}
+
+/**
+ * Copy `timestamp` / `endedAt` from a source object onto a target object.
+ * Shared by every block mapper (persist, display, snapshot) so the timing
+ * metadata that drives the `Thought for Ns` chip survives the full
+ * persist → normalize → display round-trip — and one rule lives in one place.
+ */
+export function withBlockTiming<T>(target: T, src: { timestamp?: number; endedAt?: number }): T {
+  const writable = target as { timestamp?: number; endedAt?: number }
+  if (typeof src.timestamp === 'number') writable.timestamp = src.timestamp
+  if (typeof src.endedAt === 'number') writable.endedAt = src.endedAt
+  return target
+}
+
+function withBlockParent<T>(target: T, src: { parentToolCallId?: string }): T {
+  if (src.parentToolCallId) {
+    ;(target as { parentToolCallId?: string }).parentToolCallId = src.parentToolCallId
+  }
+  return target
+}
+
+/**
+ * Carry deterministic span identity (spanId / parentSpanId) across a block
+ * mapping so the nesting tree survives the persist → normalize → display round
+ * trip. Shared by both the write and read paths.
+ */
+function withBlockSpan<T>(target: T, src: { spanId?: string; parentSpanId?: string }): T {
+  const writable = target as { spanId?: string; parentSpanId?: string }
+  if (src.spanId) writable.spanId = src.spanId
+  if (src.parentSpanId) writable.parentSpanId = src.parentSpanId
+  return target
+}
+
+function mapContentBlock(block: ContentBlock): PersistedContentBlock {
+  const persisted = mapContentBlockBody(block)
+  return withBlockSpan(withBlockParent(withBlockTiming(persisted, block), block), block)
+}
+
+function mapContentBlockBody(block: ContentBlock): PersistedContentBlock {
+  switch (block.type) {
+    case 'text':
+      return {
+        type: MothershipStreamV1EventType.text,
+        channel: MothershipStreamV1TextChannel.assistant,
+        content: block.content,
+      }
+    case 'thinking':
+      return {
+        type: MothershipStreamV1EventType.text,
+        channel: MothershipStreamV1TextChannel.thinking,
+        content: block.content,
+      }
+    case 'subagent':
+      return {
+        type: MothershipStreamV1EventType.span,
+        kind: MothershipStreamV1SpanPayloadKind.subagent,
+        lifecycle: MothershipStreamV1SpanLifecycleEvent.start,
+        content: block.content,
+      }
+    case 'subagent_text':
+      return {
+        type: MothershipStreamV1EventType.text,
+        lane: 'subagent',
+        channel: MothershipStreamV1TextChannel.assistant,
+        content: block.content,
+      }
+    case 'subagent_thinking':
+      return {
+        type: MothershipStreamV1EventType.text,
+        lane: 'subagent',
+        channel: MothershipStreamV1TextChannel.thinking,
+        content: block.content,
+      }
+    case 'tool_call': {
+      if (!block.toolCall) {
+        return {
+          type: MothershipStreamV1EventType.tool,
+          phase: MothershipStreamV1ToolPhase.call,
+          content: block.content,
+        }
+      }
+      const state = resolveToolState(block)
+      const isSubagentTool = !!block.calledBy
+      const isNonTerminal =
+        state === MothershipStreamV1ToolOutcome.cancelled ||
+        state === 'pending' ||
+        state === 'executing'
+
+      const redactedResult = redactToolCallResult(block.toolCall.name, block.toolCall.result)
+
+      const toolCall: PersistedToolCall = {
+        id: block.toolCall.id,
+        name: block.toolCall.name,
+        state,
+        ...(isSubagentTool && isNonTerminal ? {} : { result: redactedResult }),
+        ...(isSubagentTool && isNonTerminal
+          ? {}
+          : block.toolCall.params
+            ? { params: block.toolCall.params }
+            : {}),
+        ...(block.calledBy ? { calledBy: block.calledBy } : {}),
+        ...(block.toolCall.displayTitle
+          ? {
+              display: {
+                title: block.toolCall.displayTitle,
+              },
+            }
+          : {}),
+      }
+
+      return {
+        type: MothershipStreamV1EventType.tool,
+        phase: MothershipStreamV1ToolPhase.call,
+        toolCall,
+      }
+    }
+    default:
+      return { type: MothershipStreamV1EventType.text, content: block.content }
+  }
+}
+
+export function buildPersistedAssistantMessage(
+  result: OrchestratorResult,
+  requestId?: string
+): PersistedMessage {
+  const message: PersistedMessage = {
+    id: generateId(),
+    role: 'assistant',
+    content: redactSensitiveContent(result.content),
+    timestamp: new Date().toISOString(),
+  }
+
+  if (requestId || result.requestId) {
+    message.requestId = requestId || result.requestId
+  }
+
+  if (result.contentBlocks.length > 0) {
+    message.contentBlocks = mergeAndRedactPersistedBlocks(result.contentBlocks.map(mapContentBlock))
+  }
+
+  return message
+}
+
+export function withStoppedContentBlock(message: PersistedMessage): PersistedMessage {
+  const contentBlocks = message.contentBlocks ?? []
+  const hasAssistantText = contentBlocks.some(
+    (block) =>
+      block.type === MothershipStreamV1EventType.text &&
+      block.channel !== MothershipStreamV1TextChannel.thinking &&
+      block.content?.trim()
+  )
+  if (
+    contentBlocks.some(
+      (block) =>
+        block.type === MothershipStreamV1EventType.complete &&
+        block.status === MothershipStreamV1CompletionStatus.cancelled
+    )
+  ) {
+    return message
+  }
+
+  return normalizeMessage({
+    ...message,
+    contentBlocks: [
+      ...(hasAssistantText || !message.content.trim()
+        ? []
+        : [
+            {
+              type: MothershipStreamV1EventType.text,
+              channel: MothershipStreamV1TextChannel.assistant,
+              content: message.content,
+            },
+          ]),
+      ...contentBlocks,
+      {
+        type: MothershipStreamV1EventType.complete,
+        status: MothershipStreamV1CompletionStatus.cancelled,
+      },
+    ],
+  })
+}
+
+export interface UserMessageParams {
+  id: string
+  content: string
+  fileAttachments?: PersistedFileAttachment[]
+  contexts?: PersistedMessageContext[]
+}
+
+export function buildPersistedUserMessage(params: UserMessageParams): PersistedMessage {
+  const message: PersistedMessage = {
+    id: params.id,
+    role: 'user',
+    content: params.content,
+    timestamp: new Date().toISOString(),
+  }
+
+  if (params.fileAttachments && params.fileAttachments.length > 0) {
+    message.fileAttachments = params.fileAttachments
+  }
+
+  if (params.contexts && params.contexts.length > 0) {
+    message.contexts = params.contexts.map((c) => ({
+      kind: c.kind,
+      label: c.label,
+      ...(c.workflowId ? { workflowId: c.workflowId } : {}),
+      ...(c.knowledgeId ? { knowledgeId: c.knowledgeId } : {}),
+      ...(c.tableId ? { tableId: c.tableId } : {}),
+      ...(c.fileId ? { fileId: c.fileId } : {}),
+      ...(c.folderId ? { folderId: c.folderId } : {}),
+      ...(c.chatId ? { chatId: c.chatId } : {}),
+    }))
+  }
+
+  return message
+}
+
+// ---------------------------------------------------------------------------
+// Read: raw JSONB → PersistedMessage
+// Handles both canonical (type: 'tool', 'text', 'span', 'complete') and
+// legacy (type: 'tool_call', 'thinking', 'subagent', 'stopped') blocks.
+// ---------------------------------------------------------------------------
+
+const CANONICAL_BLOCK_TYPES: Set<string> = new Set(Object.values(MothershipStreamV1EventType))
+
+interface RawBlock {
+  type: string
+  lane?: string
+  content?: string
+  /** Go persists text blocks with key "text" instead of "content" */
+  text?: string
+  channel?: string
+  phase?: string
+  kind?: string
+  lifecycle?: string
+  status?: string
+  timestamp?: number
+  endedAt?: number
+  parentToolCallId?: string
+  spanId?: string
+  parentSpanId?: string
+  toolCall?: {
+    id?: string
+    name?: string
+    state?: string
+    params?: Record<string, unknown>
+    result?: { success: boolean; output?: unknown; error?: string }
+    display?: { text?: string; title?: string; phaseLabel?: string }
+    calledBy?: string
+    durationMs?: number
+    error?: string
+  } | null
+}
+
+interface LegacyToolCall {
+  id: string
+  name: string
+  status: string
+  params?: Record<string, unknown>
+  result?: unknown
+  error?: string
+  durationMs?: number
+}
+
+const OUTCOME_NORMALIZATION: Record<string, PersistedToolState> = {
+  [MothershipStreamV1ToolOutcome.success]: MothershipStreamV1ToolOutcome.success,
+  [MothershipStreamV1ToolOutcome.error]: MothershipStreamV1ToolOutcome.error,
+  [MothershipStreamV1ToolOutcome.cancelled]: MothershipStreamV1ToolOutcome.cancelled,
+  [MothershipStreamV1ToolOutcome.skipped]: MothershipStreamV1ToolOutcome.skipped,
+  [MothershipStreamV1ToolOutcome.rejected]: MothershipStreamV1ToolOutcome.rejected,
+  aborted: MothershipStreamV1ToolOutcome.cancelled,
+  failed: MothershipStreamV1ToolOutcome.error,
+  interrupted: 'interrupted',
+  pending: 'pending',
+  executing: 'executing',
+}
+
+function normalizeToolState(state: string | undefined): PersistedToolState {
+  if (!state) return 'pending'
+  return OUTCOME_NORMALIZATION[state] ?? MothershipStreamV1ToolOutcome.error
+}
+
+function isCanonicalBlock(block: RawBlock): boolean {
+  return CANONICAL_BLOCK_TYPES.has(block.type)
+}
+
+function normalizeCanonicalBlock(block: RawBlock): PersistedContentBlock {
+  const result: PersistedContentBlock = {
+    type: block.type as MothershipStreamV1EventType,
+  }
+  if (block.lane === 'subagent') {
+    result.lane = block.lane
+  }
+  const blockContent = block.content ?? block.text
+  if (blockContent !== undefined) result.content = blockContent
+  if (block.channel) result.channel = block.channel as MothershipStreamV1TextChannel
+  if (block.phase) result.phase = block.phase as MothershipStreamV1ToolPhase
+  if (block.kind) result.kind = block.kind as MothershipStreamV1SpanPayloadKind
+  if (block.lifecycle) result.lifecycle = block.lifecycle as MothershipStreamV1SpanLifecycleEvent
+  if (block.status) result.status = block.status as MothershipStreamV1CompletionStatus
+  if (block.parentToolCallId) result.parentToolCallId = block.parentToolCallId
+  if (block.toolCall) {
+    result.toolCall = {
+      id: block.toolCall.id ?? '',
+      name: block.toolCall.name ?? '',
+      state: normalizeToolState(block.toolCall.state),
+      ...(block.toolCall.params ? { params: block.toolCall.params } : {}),
+      ...(block.toolCall.result ? { result: block.toolCall.result } : {}),
+      ...(block.toolCall.calledBy ? { calledBy: block.toolCall.calledBy } : {}),
+      ...(block.toolCall.error ? { error: block.toolCall.error } : {}),
+      ...(block.toolCall.durationMs ? { durationMs: block.toolCall.durationMs } : {}),
+      ...(block.toolCall.display
+        ? {
+            display: {
+              title:
+                block.toolCall.display.title ??
+                block.toolCall.display.text ??
+                block.toolCall.display.phaseLabel,
+            },
+          }
+        : {}),
+    }
+  }
+  return result
+}
+
+function normalizeLegacyBlock(block: RawBlock): PersistedContentBlock {
+  if (block.type === 'tool_call' && block.toolCall) {
+    return {
+      type: MothershipStreamV1EventType.tool,
+      phase: MothershipStreamV1ToolPhase.call,
+      toolCall: {
+        id: block.toolCall.id ?? '',
+        name: block.toolCall.name ?? '',
+        state: normalizeToolState(block.toolCall.state),
+        ...(block.toolCall.params ? { params: block.toolCall.params } : {}),
+        ...(block.toolCall.result ? { result: block.toolCall.result } : {}),
+        ...(block.toolCall.calledBy ? { calledBy: block.toolCall.calledBy } : {}),
+        ...(block.toolCall.display
+          ? {
+              display: {
+                title:
+                  block.toolCall.display.title ??
+                  block.toolCall.display.text ??
+                  block.toolCall.display.phaseLabel,
+              },
+            }
+          : {}),
+      },
+    }
+  }
+
+  if (block.type === 'thinking') {
+    return {
+      type: MothershipStreamV1EventType.text,
+      channel: MothershipStreamV1TextChannel.thinking,
+      content: block.content,
+    }
+  }
+
+  if (block.type === 'subagent' || block.type === 'subagent_text') {
+    if (block.type === 'subagent_text') {
+      return {
+        type: MothershipStreamV1EventType.text,
+        lane: 'subagent',
+        channel: MothershipStreamV1TextChannel.assistant,
+        content: block.content,
+      }
+    }
+    return {
+      type: MothershipStreamV1EventType.span,
+      kind: MothershipStreamV1SpanPayloadKind.subagent,
+      lifecycle: MothershipStreamV1SpanLifecycleEvent.start,
+      content: block.content,
+    }
+  }
+
+  if (block.type === 'subagent_thinking') {
+    return {
+      type: MothershipStreamV1EventType.text,
+      lane: 'subagent',
+      channel: MothershipStreamV1TextChannel.thinking,
+      content: block.content,
+    }
+  }
+
+  if (block.type === 'subagent_end') {
+    return {
+      type: MothershipStreamV1EventType.span,
+      kind: MothershipStreamV1SpanPayloadKind.subagent,
+      lifecycle: MothershipStreamV1SpanLifecycleEvent.end,
+    }
+  }
+
+  if (block.type === 'stopped') {
+    return {
+      type: MothershipStreamV1EventType.complete,
+      status: MothershipStreamV1CompletionStatus.cancelled,
+    }
+  }
+
+  return {
+    type: MothershipStreamV1EventType.text,
+    channel: MothershipStreamV1TextChannel.assistant,
+    content: block.content ?? block.text,
+  }
+}
+
+function normalizeBlock(block: RawBlock): PersistedContentBlock {
+  const result = isCanonicalBlock(block)
+    ? normalizeCanonicalBlock(block)
+    : normalizeLegacyBlock(block)
+  if (typeof block.timestamp === 'number' && result.timestamp === undefined) {
+    result.timestamp = block.timestamp
+  }
+  if (typeof block.endedAt === 'number' && result.endedAt === undefined) {
+    result.endedAt = block.endedAt
+  }
+  if (block.parentToolCallId && result.parentToolCallId === undefined) {
+    result.parentToolCallId = block.parentToolCallId
+  }
+  if (block.spanId && result.spanId === undefined) {
+    result.spanId = block.spanId
+  }
+  if (block.parentSpanId && result.parentSpanId === undefined) {
+    result.parentSpanId = block.parentSpanId
+  }
+  return result
+}
+
+function normalizeLegacyToolCall(tc: LegacyToolCall): PersistedContentBlock {
+  const state = normalizeToolState(tc.status)
+  return {
+    type: MothershipStreamV1EventType.tool,
+    phase: MothershipStreamV1ToolPhase.call,
+    toolCall: {
+      id: tc.id,
+      name: tc.name,
+      state,
+      ...(tc.params ? { params: tc.params } : {}),
+      ...(tc.result != null
+        ? {
+            result: {
+              success: tc.status === MothershipStreamV1ToolOutcome.success,
+              output: tc.result,
+              ...(tc.error ? { error: tc.error } : {}),
+            },
+          }
+        : {}),
+      ...(tc.durationMs ? { durationMs: tc.durationMs } : {}),
+    },
+  }
+}
+
+function blocksContainTools(blocks: RawBlock[]): boolean {
+  return blocks.some((b) => b.type === 'tool_call' || b.type === MothershipStreamV1EventType.tool)
+}
+
+function normalizeBlocks(rawBlocks: RawBlock[], messageContent: string): PersistedContentBlock[] {
+  const blocks = rawBlocks.map(normalizeBlock)
+  const hasAssistantText = blocks.some(
+    (b) =>
+      b.type === MothershipStreamV1EventType.text &&
+      b.channel !== MothershipStreamV1TextChannel.thinking &&
+      b.content?.trim()
+  )
+  if (!hasAssistantText && messageContent.trim()) {
+    blocks.push({
+      type: MothershipStreamV1EventType.text,
+      channel: MothershipStreamV1TextChannel.assistant,
+      content: messageContent,
+    })
+  }
+  return blocks
+}
+
+export function normalizeMessage(raw: Record<string, unknown>): PersistedMessage {
+  const msg: PersistedMessage = {
+    id: (raw.id as string) ?? generateId(),
+    role: (raw.role as 'user' | 'assistant') ?? 'assistant',
+    content: (raw.content as string) ?? '',
+    timestamp: (raw.timestamp as string) ?? new Date().toISOString(),
+  }
+
+  if (raw.requestId && typeof raw.requestId === 'string') {
+    msg.requestId = raw.requestId
+  }
+
+  const rawBlocks = raw.contentBlocks as RawBlock[] | undefined
+  const rawToolCalls = raw.toolCalls as LegacyToolCall[] | undefined
+  const hasBlocks = Array.isArray(rawBlocks) && rawBlocks.length > 0
+  const hasToolCalls = Array.isArray(rawToolCalls) && rawToolCalls.length > 0
+
+  if (hasBlocks) {
+    msg.contentBlocks = normalizeBlocks(rawBlocks!, msg.content)
+    const contentBlocksAlreadyContainTools = blocksContainTools(rawBlocks!)
+    if (hasToolCalls && !contentBlocksAlreadyContainTools) {
+      msg.contentBlocks.push(...rawToolCalls!.map(normalizeLegacyToolCall))
+    }
+  } else if (hasToolCalls) {
+    msg.contentBlocks = rawToolCalls!.map(normalizeLegacyToolCall)
+    if (msg.content.trim()) {
+      msg.contentBlocks.push({
+        type: MothershipStreamV1EventType.text,
+        channel: MothershipStreamV1TextChannel.assistant,
+        content: msg.content,
+      })
+    }
+  }
+
+  const rawAttachments = raw.fileAttachments as PersistedFileAttachment[] | undefined
+  if (Array.isArray(rawAttachments) && rawAttachments.length > 0) {
+    msg.fileAttachments = rawAttachments
+  }
+
+  const rawContexts = raw.contexts as PersistedMessageContext[] | undefined
+  if (Array.isArray(rawContexts) && rawContexts.length > 0) {
+    msg.contexts = rawContexts.map((c) => ({
+      kind: c.kind,
+      label: c.label,
+      ...(c.workflowId ? { workflowId: c.workflowId } : {}),
+      ...(c.knowledgeId ? { knowledgeId: c.knowledgeId } : {}),
+      ...(c.tableId ? { tableId: c.tableId } : {}),
+      ...(c.fileId ? { fileId: c.fileId } : {}),
+      ...(c.folderId ? { folderId: c.folderId } : {}),
+      ...(c.chatId ? { chatId: c.chatId } : {}),
+    }))
+  }
+
+  return msg
+}

apps/sim/lib/copilot/chat/post.test.ts

@@ -0,0 +1,449 @@
+/**
+ * @vitest-environment node
+ */
+
+import {
+  authMockFns,
+  permissionsMock,
+  permissionsMockFns,
+  workflowsUtilsMock,
+  workflowsUtilsMockFns,
+} from '@sim/testing'
+import { NextRequest } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const resolveWorkflowIdForUser = workflowsUtilsMockFns.mockResolveWorkflowIdForUser
+const getUserEntityPermissions = permissionsMockFns.mockGetUserEntityPermissions
+
+const {
+  getEffectiveDecryptedEnv,
+  generateWorkspaceSnapshot,
+  processContextsServer,
+  resolveActiveResourceContext,
+  buildCopilotRequestPayload,
+  createSSEStream,
+  acquirePendingChatStream,
+  getPendingChatStreamId,
+  releasePendingChatStream,
+  resolveOrCreateChat,
+  finalizeAssistantTurn,
+  appendCopilotChatMessages,
+  mockPublishStatusChanged,
+} = vi.hoisted(() => ({
+  getEffectiveDecryptedEnv: vi.fn(),
+  generateWorkspaceSnapshot: vi.fn(),
+  processContextsServer: vi.fn(),
+  resolveActiveResourceContext: vi.fn(),
+  buildCopilotRequestPayload: vi.fn(),
+  createSSEStream: vi.fn(),
+  acquirePendingChatStream: vi.fn(),
+  getPendingChatStreamId: vi.fn(),
+  releasePendingChatStream: vi.fn(),
+  resolveOrCreateChat: vi.fn(),
+  finalizeAssistantTurn: vi.fn(),
+  appendCopilotChatMessages: vi.fn(),
+  mockPublishStatusChanged: vi.fn(),
+}))
+
+const getSession = authMockFns.mockGetSession
+
+vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock)
+
+vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock)
+
+vi.mock('@/lib/environment/utils', () => ({
+  getEffectiveDecryptedEnv,
+}))
+
+vi.mock('@/lib/copilot/chat/workspace-context', () => ({
+  generateWorkspaceSnapshot,
+}))
+
+vi.mock('@/lib/copilot/chat/process-contents', () => ({
+  processContextsServer,
+  resolveActiveResourceContext,
+}))
+
+vi.mock('@/lib/copilot/chat/payload', () => ({
+  buildCopilotRequestPayload,
+}))
+
+vi.mock('@/lib/copilot/request/lifecycle/start', () => ({
+  createSSEStream,
+  SSE_RESPONSE_HEADERS: { 'Content-Type': 'text/event-stream' },
+}))
+
+vi.mock('@/lib/copilot/request/session', () => ({
+  acquirePendingChatStream,
+  getPendingChatStreamId,
+  releasePendingChatStream,
+}))
+
+vi.mock('@/lib/copilot/chat/lifecycle', () => ({
+  resolveOrCreateChat,
+}))
+
+vi.mock('@/lib/copilot/chat/terminal-state', () => ({
+  finalizeAssistantTurn,
+}))
+
+vi.mock('@/lib/copilot/chat/messages-store', () => ({
+  appendCopilotChatMessages,
+}))
+
+vi.mock('@/lib/copilot/chat-status', () => ({
+  chatPubSub: {
+    publishStatusChanged: mockPublishStatusChanged,
+  },
+}))
+
+vi.mock('@sim/db', () => {
+  const update = vi.fn(() => ({
+    set: vi.fn(() => ({
+      where: vi.fn(() => ({
+        returning: vi.fn().mockResolvedValue([]),
+      })),
+    })),
+  }))
+  const select = vi.fn(() => ({
+    from: vi.fn(() => ({
+      where: vi.fn(() => ({
+        limit: vi.fn().mockResolvedValue([{ permissionType: 'write' }]),
+      })),
+    })),
+  }))
+  return {
+    db: {
+      update,
+      select,
+      transaction: async (cb: (tx: { update: typeof update; select: typeof select }) => unknown) =>
+        cb({ update, select }),
+    },
+  }
+})
+
+vi.mock('drizzle-orm', () => ({
+  and: vi.fn(() => ({})),
+  eq: vi.fn(() => ({})),
+  sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ strings, values }),
+}))
+
+import { handleUnifiedChatPost } from './post'
+
+describe('handleUnifiedChatPost', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    getSession.mockResolvedValue({ user: { id: 'user-1' } })
+    resolveWorkflowIdForUser.mockResolvedValue({
+      status: 'resolved',
+      workflowId: 'wf-1',
+      workspaceId: 'ws-1',
+      workflowName: 'Workflow One',
+    })
+    getUserEntityPermissions.mockResolvedValue('write')
+    getEffectiveDecryptedEnv.mockResolvedValue({ API_KEY: 'secret' })
+    generateWorkspaceSnapshot.mockResolvedValue({
+      markdown: 'workspace context',
+      snapshot: { workflows: [{ id: 'wf-1', name: 'Alpha', path: 'workflows/Alpha' }] },
+    })
+    processContextsServer.mockResolvedValue([])
+    resolveActiveResourceContext.mockResolvedValue(null)
+    buildCopilotRequestPayload.mockImplementation(async (params: Record<string, unknown>) => params)
+    createSSEStream.mockReturnValue(new ReadableStream())
+    acquirePendingChatStream.mockResolvedValue(true)
+    getPendingChatStreamId.mockResolvedValue(null)
+    releasePendingChatStream.mockResolvedValue(undefined)
+    resolveOrCreateChat.mockResolvedValue({
+      chatId: 'chat-1',
+      chat: { id: 'chat-1' },
+      conversationHistory: [],
+      isNew: true,
+    })
+    finalizeAssistantTurn.mockResolvedValue({
+      found: true,
+      updated: true,
+      appendedAssistant: true,
+      workspaceId: 'ws-1',
+      outcome: 'appended_assistant',
+    })
+  })
+
+  it('routes workflow-attached chat requests through the copilot backend path', async () => {
+    const response = await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workflowId: 'wf-1',
+          workspaceId: 'ws-1',
+        }),
+      })
+    )
+
+    expect(response.status).toBe(200)
+    expect(generateWorkspaceSnapshot).toHaveBeenCalledWith('ws-1', 'user-1')
+    expect(buildCopilotRequestPayload).toHaveBeenCalledWith(
+      expect.objectContaining({
+        model: 'claude-opus-4-8',
+        workspaceContext: 'workspace context',
+        // Regression guard: the branch must forward the typed snapshot, not drop it.
+        vfs: expect.objectContaining({ workflows: expect.any(Array) }),
+      }),
+      { selectedModel: 'claude-opus-4-8' }
+    )
+    expect(createSSEStream).toHaveBeenCalledWith(
+      expect.objectContaining({
+        titleModel: 'claude-opus-4-8',
+        workspaceId: 'ws-1',
+        orchestrateOptions: expect.objectContaining({
+          workflowId: 'wf-1',
+          goRoute: '/api/copilot',
+          executionContext: expect.objectContaining({
+            userId: 'user-1',
+            workflowId: 'wf-1',
+            workspaceId: 'ws-1',
+            requestMode: 'agent',
+          }),
+        }),
+      })
+    )
+  })
+
+  it('routes workspace chat requests through the mothership backend path', async () => {
+    const response = await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workspaceId: 'ws-1',
+          createNewChat: true,
+        }),
+      })
+    )
+
+    expect(response.status).toBe(200)
+    expect(buildCopilotRequestPayload).toHaveBeenCalledWith(
+      expect.objectContaining({
+        workspaceId: 'ws-1',
+        workspaceContext: 'workspace context',
+        // Regression guard: the branch must forward the typed snapshot, not drop it.
+        vfs: expect.objectContaining({ workflows: expect.any(Array) }),
+      }),
+      { selectedModel: '' }
+    )
+    expect(createSSEStream).toHaveBeenCalledWith(
+      expect.objectContaining({
+        titleModel: 'claude-opus-4-8',
+        workspaceId: 'ws-1',
+        orchestrateOptions: expect.objectContaining({
+          workspaceId: 'ws-1',
+          goRoute: '/api/mothership',
+          executionContext: expect.objectContaining({
+            userId: 'user-1',
+            workflowId: '',
+            workspaceId: 'ws-1',
+            requestMode: 'agent',
+          }),
+        }),
+      })
+    )
+  })
+
+  it('accepts tagged skill contexts and forwards them to context resolution', async () => {
+    const response = await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workspaceId: 'ws-1',
+          createNewChat: true,
+          contexts: [{ kind: 'skill', skillId: 'sk-1', label: 'my-skill' }],
+        }),
+      })
+    )
+
+    expect(response.status).toBe(200)
+    expect(processContextsServer).toHaveBeenCalledWith(
+      expect.arrayContaining([
+        expect.objectContaining({ kind: 'skill', skillId: 'sk-1', label: 'my-skill' }),
+      ]),
+      'user-1',
+      'Hello',
+      'ws-1',
+      expect.anything()
+    )
+  })
+
+  it('persists cancelled partial responses from the server lifecycle', async () => {
+    await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workspaceId: 'ws-1',
+          createNewChat: true,
+        }),
+      })
+    )
+
+    const streamArgs = createSSEStream.mock.calls[0]?.[0]
+    const onComplete = streamArgs?.orchestrateOptions?.onComplete
+    expect(onComplete).toBeTypeOf('function')
+
+    await onComplete({
+      success: false,
+      cancelled: true,
+      content: 'partial answer',
+      contentBlocks: [],
+      toolCalls: [],
+      chatId: 'chat-1',
+      requestId: 'request-1',
+    })
+
+    expect(finalizeAssistantTurn).toHaveBeenCalledWith(
+      expect.objectContaining({
+        chatId: 'chat-1',
+        userMessageId: expect.any(String),
+        streamMarkerPolicy: 'active-or-cleared',
+        assistantMessage: expect.objectContaining({
+          role: 'assistant',
+          content: 'partial answer',
+          contentBlocks: expect.arrayContaining([
+            expect.objectContaining({ type: 'complete', status: 'cancelled' }),
+          ]),
+        }),
+      })
+    )
+  })
+
+  it('persists partial responses when the server lifecycle throws (onError)', async () => {
+    await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workspaceId: 'ws-1',
+          createNewChat: true,
+        }),
+      })
+    )
+
+    const streamArgs = createSSEStream.mock.calls[0]?.[0]
+    const onError = streamArgs?.orchestrateOptions?.onError
+    expect(onError).toBeTypeOf('function')
+
+    await onError(new Error('bedrock overloaded'), {
+      success: false,
+      cancelled: false,
+      content: 'partial answer',
+      contentBlocks: [],
+      toolCalls: [],
+      chatId: 'chat-1',
+      requestId: 'request-1',
+    })
+
+    expect(finalizeAssistantTurn).toHaveBeenCalledWith(
+      expect.objectContaining({
+        chatId: 'chat-1',
+        userMessageId: expect.any(String),
+        streamMarkerPolicy: 'active-or-cleared',
+        assistantMessage: expect.objectContaining({
+          role: 'assistant',
+          content: 'partial answer',
+        }),
+      })
+    )
+  })
+
+  it('clears the stream marker without an assistant message when nothing streamed before the throw', async () => {
+    await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workspaceId: 'ws-1',
+          createNewChat: true,
+        }),
+      })
+    )
+
+    const streamArgs = createSSEStream.mock.calls[0]?.[0]
+    const onError = streamArgs?.orchestrateOptions?.onError
+    expect(onError).toBeTypeOf('function')
+
+    await onError(new Error('immediate failure'), {
+      success: false,
+      cancelled: false,
+      content: '',
+      contentBlocks: [],
+      toolCalls: [],
+      chatId: 'chat-1',
+      requestId: 'request-1',
+    })
+
+    const lastCall = finalizeAssistantTurn.mock.calls.at(-1)?.[0]
+    expect(lastCall).toMatchObject({
+      chatId: 'chat-1',
+      streamMarkerPolicy: 'active-or-cleared',
+    })
+    expect(lastCall?.assistantMessage).toBeUndefined()
+  })
+
+  it('republishes completed status when cancelled lifecycle persistence already ran', async () => {
+    await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+          workspaceId: 'ws-1',
+          createNewChat: true,
+        }),
+      })
+    )
+
+    const streamArgs = createSSEStream.mock.calls[0]?.[0]
+    const onComplete = streamArgs?.orchestrateOptions?.onComplete
+    expect(onComplete).toBeTypeOf('function')
+
+    finalizeAssistantTurn.mockResolvedValueOnce({
+      found: true,
+      updated: false,
+      appendedAssistant: false,
+      workspaceId: 'ws-1',
+      outcome: 'assistant_already_persisted',
+    })
+
+    await onComplete({
+      success: false,
+      cancelled: true,
+      content: 'partial answer',
+      contentBlocks: [],
+      toolCalls: [],
+      chatId: 'chat-1',
+      requestId: 'request-1',
+    })
+
+    expect(mockPublishStatusChanged).toHaveBeenCalledWith({
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      type: 'completed',
+      streamId: streamArgs?.streamId,
+    })
+  })
+
+  it('rejects requests that have neither workflow nor workspace attachment', async () => {
+    const response = await handleUnifiedChatPost(
+      new NextRequest('http://localhost/api/copilot/chat', {
+        method: 'POST',
+        body: JSON.stringify({
+          message: 'Hello',
+        }),
+      })
+    )
+
+    expect(response.status).toBe(400)
+    await expect(response.json()).resolves.toMatchObject({
+      error: 'workspaceId is required when workflowId is not provided',
+    })
+  })
+})

apps/sim/lib/copilot/chat/process-contents.test.ts

@@ -0,0 +1,258 @@
+/**
+ * @vitest-environment node
+ */
+
+import { dbChainMock, dbChainMockFns, workflowAuthzMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { ChatContext } from '@/stores/panel'
+
+const { getSkillById } = vi.hoisted(() => ({ getSkillById: vi.fn() }))
+
+vi.mock('@/lib/workflows/skills/operations', () => ({ getSkillById }))
+/**
+ * Overrides the global `@sim/db` mock: the logs-context tests below need
+ * controllable row data, which the stable `dbChainMockFns.limit` provides.
+ */
+vi.mock('@sim/db', () => dbChainMock)
+
+import { processContextsServer } from './process-contents'
+
+describe('processContextsServer - skill contexts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('resolves a tagged skill to full content + encoded VFS path', async () => {
+    getSkillById.mockResolvedValue({
+      id: 'sk-1',
+      name: 'My Skill — PostHog',
+      description: 'desc',
+      content: '# My Skill\n\nDo the thing.',
+    })
+
+    const result = await processContextsServer(
+      [{ kind: 'skill', skillId: 'sk-1', label: 'My Skill — PostHog' } as ChatContext],
+      'user-1',
+      'hello',
+      'ws-1'
+    )
+
+    expect(getSkillById).toHaveBeenCalledWith({ skillId: 'sk-1', workspaceId: 'ws-1' })
+    expect(result).toEqual([
+      {
+        type: 'skill',
+        tag: '@My Skill — PostHog',
+        content: '# My Skill\n\nDo the thing.',
+        path: 'agent/skills/My%20Skill%20%E2%80%94%20PostHog.json',
+      },
+    ])
+  })
+
+  it('drops a skill that does not resolve (unknown or cross-workspace)', async () => {
+    getSkillById.mockResolvedValue(null)
+
+    const result = await processContextsServer(
+      [{ kind: 'skill', skillId: 'missing', label: 'x' } as ChatContext],
+      'user-1',
+      'hello',
+      'ws-1'
+    )
+
+    expect(result).toEqual([])
+  })
+
+  it('drops a skill when no workspace is in scope', async () => {
+    const result = await processContextsServer(
+      [{ kind: 'skill', skillId: 'sk-1', label: 'x' } as ChatContext],
+      'user-1',
+      'hello',
+      undefined
+    )
+
+    expect(getSkillById).not.toHaveBeenCalled()
+    expect(result).toEqual([])
+  })
+})
+
+describe('processContextsServer - logs contexts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('resolves a tagged run to a compact summary with a block overview, never raw input/output', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      {
+        id: 'log-1',
+        workflowId: 'wf-1',
+        workspaceId: 'ws-1',
+        executionId: 'exec-1',
+        level: 'error',
+        trigger: 'manual',
+        startedAt: new Date('2026-01-01T00:00:00.000Z'),
+        endedAt: new Date('2026-01-01T00:00:01.000Z'),
+        totalDurationMs: 1000,
+        executionData: {
+          traceSpans: [
+            {
+              id: 'span-1',
+              blockId: 'block-1',
+              name: 'Agent 1',
+              type: 'agent',
+              status: 'failed',
+              duration: 500,
+              input: { prompt: 'do the thing' },
+              output: { error: '429 No active subscription' },
+            },
+          ],
+        },
+        costTotal: '0.05',
+        workflowName: 'My Flow',
+      },
+    ])
+    workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+      allowed: true,
+      workflow: { workspaceId: 'ws-1' },
+    })
+
+    const result = await processContextsServer(
+      [{ kind: 'logs', executionId: 'exec-1', label: 'My Flow' } as ChatContext],
+      'user-1',
+      'hello',
+      'ws-1'
+    )
+
+    expect(result).toHaveLength(1)
+    expect(result[0].type).toBe('logs')
+    expect(result[0].tag).toBe('@My Flow')
+
+    const summary = JSON.parse(result[0].content)
+    expect(summary).toMatchObject({
+      executionId: 'exec-1',
+      workflowId: 'wf-1',
+      workflowName: 'My Flow',
+      level: 'error',
+      trigger: 'manual',
+      totalDurationMs: 1000,
+      cost: { total: 0.05 },
+      overview: [
+        {
+          id: 'span-1',
+          blockId: 'block-1',
+          name: 'Agent 1',
+          type: 'agent',
+          status: 'failed',
+          durationMs: 500,
+        },
+      ],
+    })
+    const serialized = JSON.stringify(summary)
+    expect(serialized).not.toContain('do the thing')
+    expect(serialized).not.toContain('429 No active subscription')
+    expect(summary.note).toContain('query_logs')
+    expect(summary.note).toContain('exec-1')
+  })
+
+  it('drops the overview (keeping the rest of the summary) when it exceeds the size cap', async () => {
+    const traceSpans = Array.from({ length: 2000 }, (_, i) => ({
+      id: `span-${i}`,
+      blockId: `block-${i}`,
+      name: `Block ${i}`,
+      type: 'agent',
+      status: 'success',
+      duration: 10,
+    }))
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      {
+        id: 'log-1',
+        workflowId: 'wf-1',
+        workspaceId: 'ws-1',
+        executionId: 'exec-1',
+        level: 'error',
+        trigger: 'manual',
+        startedAt: new Date('2026-01-01T00:00:00.000Z'),
+        endedAt: null,
+        totalDurationMs: null,
+        executionData: { traceSpans },
+        costTotal: null,
+        workflowName: 'My Flow',
+      },
+    ])
+    workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+      allowed: true,
+      workflow: { workspaceId: 'ws-1' },
+    })
+
+    const result = await processContextsServer(
+      [{ kind: 'logs', executionId: 'exec-1', label: 'My Flow' } as ChatContext],
+      'user-1',
+      'hello',
+      'ws-1'
+    )
+
+    const summary = JSON.parse(result[0].content)
+    expect(summary.overview).toBeUndefined()
+    expect(summary.executionId).toBe('exec-1')
+    expect(summary.note).toContain('query_logs')
+  })
+
+  it('drops a log context when the workflow is outside the current workspace', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      {
+        id: 'log-1',
+        workflowId: 'wf-1',
+        workspaceId: 'ws-other',
+        executionId: 'exec-1',
+        level: 'error',
+        trigger: 'manual',
+        startedAt: new Date('2026-01-01T00:00:00.000Z'),
+        endedAt: null,
+        totalDurationMs: null,
+        costTotal: null,
+        workflowName: 'My Flow',
+      },
+    ])
+    workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+      allowed: true,
+      workflow: { workspaceId: 'ws-other' },
+    })
+
+    const result = await processContextsServer(
+      [{ kind: 'logs', executionId: 'exec-1', label: 'My Flow' } as ChatContext],
+      'user-1',
+      'hello',
+      'ws-1'
+    )
+
+    expect(result).toEqual([])
+  })
+
+  it('drops a log context the user is not authorized to read', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      {
+        id: 'log-1',
+        workflowId: 'wf-1',
+        workspaceId: 'ws-1',
+        executionId: 'exec-1',
+        level: 'error',
+        trigger: 'manual',
+        startedAt: new Date('2026-01-01T00:00:00.000Z'),
+        endedAt: null,
+        totalDurationMs: null,
+        costTotal: null,
+        workflowName: 'My Flow',
+      },
+    ])
+    workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({
+      allowed: false,
+    })
+
+    const result = await processContextsServer(
+      [{ kind: 'logs', executionId: 'exec-1', label: 'My Flow' } as ChatContext],
+      'user-1',
+      'hello',
+      'ws-1'
+    )
+
+    expect(result).toEqual([])
+  })
+})

apps/sim/lib/copilot/chat/process-contents.ts

@@ -0,0 +1,843 @@
+import { db, dbReplica } from '@sim/db'
+import { knowledgeBase, workflowSchedule } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import {
+  authorizeWorkflowByWorkspacePermission,
+  getActiveWorkflowRecord,
+} from '@sim/platform-authz/workflow'
+import { and, eq, isNull, ne } from 'drizzle-orm'
+import { QueryLogs } from '@/lib/copilot/generated/tool-catalog-v1'
+import { normalizeVfsSegment } from '@/lib/copilot/vfs/normalize-segment'
+import {
+  buildVfsFolderPathMap,
+  canonicalBlockVfsPath,
+  canonicalKnowledgeBaseVfsDir,
+  canonicalTableVfsPath,
+  canonicalWorkflowVfsDir,
+  canonicalWorkspaceFilePath,
+  encodeVfsPathSegments,
+  encodeVfsSegment,
+} from '@/lib/copilot/vfs/path-utils'
+import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags'
+import { toOverview } from '@/lib/logs/log-views'
+import type { TraceSpan } from '@/lib/logs/types'
+import { getTableById } from '@/lib/table/service'
+import { getWorkspaceFileFolderPath } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
+import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+import { getSkillById } from '@/lib/workflows/skills/operations'
+import { listFolders } from '@/lib/workflows/utils'
+import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
+import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check'
+import { escapeRegExp } from '@/executor/constants'
+import type { ChatContext } from '@/stores/panel'
+
+type AgentContextType =
+  | 'past_chat'
+  | 'workflow'
+  | 'current_workflow'
+  | 'blocks'
+  | 'logs'
+  | 'knowledge'
+  | 'table'
+  | 'file'
+  | 'workflow_block'
+  | 'docs'
+  | 'folder'
+  | 'filefolder'
+  | 'active_resource'
+  | 'skill'
+
+interface AgentContext {
+  type: AgentContextType
+  tag: string
+  content: string
+  /**
+   * Canonical, URL-encoded VFS path for the tagged resource (e.g.
+   * `agent/skills/My%20Skill.json`). Tagged resources are sent as path
+   * pointers so the model reads them on demand via VFS tools instead of the
+   * full body bloating the request. Skills are the exception: they carry both
+   * `path` and the full `content` so the skill is autoloaded.
+   */
+  path?: string
+}
+
+const logger = createLogger('ProcessContents')
+
+// Server-side variant (recommended for use in API routes)
+export async function processContextsServer(
+  contexts: ChatContext[] | undefined,
+  userId: string,
+  userMessage?: string,
+  currentWorkspaceId?: string,
+  chatId?: string
+): Promise<AgentContext[]> {
+  if (!Array.isArray(contexts) || contexts.length === 0) return []
+  const tasks = contexts.map(async (ctx) => {
+    try {
+      if (ctx.kind === 'skill' && ctx.skillId && currentWorkspaceId) {
+        return await processSkillFromDb(
+          ctx.skillId,
+          currentWorkspaceId,
+          ctx.label ? `@${ctx.label}` : '@'
+        )
+      }
+      if (ctx.kind === 'past_chat' && ctx.chatId) {
+        return await processPastChatFromDb(
+          ctx.chatId,
+          userId,
+          ctx.label ? `@${ctx.label}` : '@',
+          currentWorkspaceId
+        )
+      }
+      if ((ctx.kind === 'workflow' || ctx.kind === 'current_workflow') && ctx.workflowId) {
+        return await processWorkflowFromDb(
+          ctx.workflowId,
+          userId,
+          ctx.label ? `@${ctx.label}` : '@',
+          ctx.kind,
+          currentWorkspaceId,
+          chatId
+        )
+      }
+      if (ctx.kind === 'knowledge' && ctx.knowledgeId) {
+        return await processKnowledgeFromDb(
+          ctx.knowledgeId,
+          userId,
+          ctx.label ? `@${ctx.label}` : '@',
+          currentWorkspaceId
+        )
+      }
+      if (ctx.kind === 'blocks' && ctx.blockIds?.length > 0) {
+        return await processBlockMetadata(
+          ctx.blockIds[0],
+          ctx.label ? `@${ctx.label}` : '@',
+          userId,
+          currentWorkspaceId
+        )
+      }
+      if (ctx.kind === 'logs' && ctx.executionId) {
+        return await processExecutionLogFromDb(
+          ctx.executionId,
+          userId,
+          ctx.label ? `@${ctx.label}` : '@',
+          currentWorkspaceId
+        )
+      }
+      if (ctx.kind === 'workflow_block' && ctx.workflowId && ctx.blockId) {
+        return await processWorkflowBlockFromDb(
+          ctx.workflowId,
+          userId,
+          ctx.blockId,
+          ctx.label,
+          currentWorkspaceId
+        )
+      }
+      if (ctx.kind === 'table' && ctx.tableId && currentWorkspaceId) {
+        const result = await resolveTableResource(ctx.tableId, currentWorkspaceId)
+        if (!result) return null
+        return {
+          type: 'table',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: result.content,
+          path: result.path,
+        }
+      }
+      if (ctx.kind === 'file' && ctx.fileId && currentWorkspaceId) {
+        const result = await resolveFileResource(ctx.fileId, currentWorkspaceId)
+        if (!result) return null
+        return {
+          type: 'file',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: result.content,
+          path: result.path,
+        }
+      }
+      if (ctx.kind === 'folder' && 'folderId' in ctx && ctx.folderId && currentWorkspaceId) {
+        const result = await resolveFolderResource(ctx.folderId, currentWorkspaceId)
+        if (!result) return null
+        return {
+          type: 'folder',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: result.content,
+          path: result.path,
+        }
+      }
+      if (ctx.kind === 'filefolder' && ctx.fileFolderId && currentWorkspaceId) {
+        const result = await resolveFileFolderResource(ctx.fileFolderId, currentWorkspaceId)
+        if (!result) return null
+        return {
+          type: 'filefolder',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: result.content,
+          path: result.path,
+        }
+      }
+      if (ctx.kind === 'scheduledtask' && ctx.scheduleId && currentWorkspaceId) {
+        const result = await resolveScheduledTaskResource(ctx.scheduleId, currentWorkspaceId)
+        if (!result) return null
+        return {
+          type: 'active_resource',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: result.content,
+          path: result.path,
+        }
+      }
+      if (ctx.kind === 'docs') {
+        try {
+          const { searchDocumentationServerTool } = await import(
+            '@/lib/copilot/tools/server/docs/search-documentation'
+          )
+          const rawQuery = (userMessage || '').trim() || ctx.label || 'Sim documentation'
+          const query = sanitizeMessageForDocs(rawQuery, contexts)
+          const res = await searchDocumentationServerTool.execute({ query, topK: 10 })
+          const content = JSON.stringify(res?.results || [])
+          return { type: 'docs', tag: ctx.label ? `@${ctx.label}` : '@', content }
+        } catch (e) {
+          logger.error('Failed to process docs context', e)
+          return null
+        }
+      }
+      return null
+    } catch (error) {
+      logger.error('Failed processing context (server)', { ctx, error })
+      return null
+    }
+  })
+  const results = await Promise.all(tasks)
+  const filtered = results.filter(
+    (r): r is AgentContext =>
+      !!r &&
+      ((typeof r.content === 'string' && r.content.trim().length > 0) ||
+        (typeof r.path === 'string' && r.path.length > 0))
+  )
+  logger.info('Processed contexts (server)', {
+    totalRequested: contexts.length,
+    totalProcessed: filtered.length,
+    kinds: Array.from(filtered.reduce((s, r) => s.add(r.type), new Set<string>())),
+  })
+  return filtered
+}
+
+function sanitizeMessageForDocs(rawMessage: string, contexts: ChatContext[] | undefined): string {
+  if (!rawMessage) return ''
+  if (!Array.isArray(contexts) || contexts.length === 0) {
+    // No context mapping; conservatively strip all @mentions-like tokens
+    const stripped = rawMessage
+      .replace(/(^|\s)@([^\s]+)/g, ' ')
+      .replace(/\s{2,}/g, ' ')
+      .trim()
+    return stripped
+  }
+
+  // Gather labels by kind
+  const blockLabels = new Set(
+    contexts
+      .filter((c) => c.kind === 'blocks')
+      .map((c) => c.label)
+      .filter((l): l is string => typeof l === 'string' && l.length > 0)
+  )
+  const nonBlockLabels = new Set(
+    contexts
+      .filter((c) => c.kind !== 'blocks')
+      .map((c) => c.label)
+      .filter((l): l is string => typeof l === 'string' && l.length > 0)
+  )
+
+  let result = rawMessage
+
+  // 1) Remove all non-block mentions entirely
+  for (const label of nonBlockLabels) {
+    const pattern = new RegExp(`(^|\\s)@${escapeRegExp(label)}(?!\\S)`, 'g')
+    result = result.replace(pattern, ' ')
+  }
+
+  // 2) For block mentions, strip the '@' but keep the block name
+  for (const label of blockLabels) {
+    const pattern = new RegExp(`@${escapeRegExp(label)}(?!\\S)`, 'g')
+    result = result.replace(pattern, label)
+  }
+
+  // 3) Remove any remaining @mentions (unknown or not in contexts)
+  result = result.replace(/(^|\s)@([^\s]+)/g, ' ')
+
+  // Normalize whitespace
+  result = result.replace(/\s{2,}/g, ' ').trim()
+  return result
+}
+
+async function processSkillFromDb(
+  skillId: string,
+  workspaceId: string,
+  tag: string
+): Promise<AgentContext | null> {
+  try {
+    const s = await getSkillById({ skillId, workspaceId })
+    if (!s) return null
+    // Skills are autoloaded: carry the full SKILL.md body so the Go side can
+    // inject it into the dynamic system message for the turn. The path lets the
+    // model re-read the canonical VFS file if it needs to.
+    const path = `agent/skills/${encodeVfsSegment(s.name)}.json`
+    return { type: 'skill', tag, content: s.content, path }
+  } catch (error) {
+    logger.error('Error processing skill context (db)', { skillId, error })
+    return null
+  }
+}
+
+async function processPastChatFromDb(
+  chatId: string,
+  userId: string,
+  tag: string,
+  currentWorkspaceId?: string
+): Promise<AgentContext | null> {
+  try {
+    const { getAccessibleCopilotChatWithMessages } = await import('./lifecycle')
+    const chat = await getAccessibleCopilotChatWithMessages(chatId, userId)
+    if (!chat) {
+      return null
+    }
+
+    if (currentWorkspaceId) {
+      if (chat.workspaceId && chat.workspaceId !== currentWorkspaceId) {
+        return null
+      }
+      if (chat.workflowId) {
+        const activeWorkflow = await getActiveWorkflowRecord(chat.workflowId)
+        if (!activeWorkflow || activeWorkflow.workspaceId !== currentWorkspaceId) {
+          return null
+        }
+      }
+    }
+    const messages = Array.isArray(chat.messages) ? (chat as any).messages : []
+    const content = messages
+      .map((m: any) => {
+        const role = m.role || 'user'
+        let text = ''
+        if (Array.isArray(m.contentBlocks) && m.contentBlocks.length > 0) {
+          text = m.contentBlocks
+            .filter((b: any) => b?.type === 'text')
+            .map((b: any) => String(b.content || ''))
+            .join('')
+            .trim()
+        }
+        if (!text && typeof m.content === 'string') text = m.content
+        return `${role}: ${text}`.trim()
+      })
+      .filter((s: string) => s.length > 0)
+      .join('\n')
+    logger.info('Processed past_chat context from DB', {
+      chatId,
+      length: content.length,
+      lines: content ? content.split('\n').length : 0,
+    })
+    return { type: 'past_chat', tag, content }
+  } catch (error) {
+    logger.error('Error processing past chat from db', { chatId, error })
+    return null
+  }
+}
+
+/**
+ * Resolve a workflow folder id to its canonical, per-segment-encoded VFS folder
+ * path. Returns null for root-level workflows or when the folder can't be
+ * resolved. Uses the shared {@link buildVfsFolderPathMap} so the pointer path
+ * matches what the workspace VFS serves.
+ */
+async function resolveWorkflowFolderPath(
+  workspaceId: string | null | undefined,
+  folderId: string | null | undefined
+): Promise<string | null> {
+  if (!folderId || !workspaceId) return null
+  try {
+    const folders = await listFolders(workspaceId)
+    return buildVfsFolderPathMap(folders).get(folderId) ?? null
+  } catch (error) {
+    logger.warn('Failed to resolve workflow folder path', { workspaceId, folderId, error })
+    return null
+  }
+}
+
+async function processWorkflowFromDb(
+  workflowId: string,
+  userId: string | undefined,
+  tag: string,
+  kind: 'workflow' | 'current_workflow' = 'workflow',
+  currentWorkspaceId?: string,
+  _chatId?: string
+): Promise<AgentContext | null> {
+  try {
+    let workflowRecord: Awaited<ReturnType<typeof getActiveWorkflowRecord>> = null
+
+    if (userId) {
+      const authorization = await authorizeWorkflowByWorkspacePermission({
+        workflowId,
+        userId,
+        action: 'read',
+      })
+      if (!authorization.allowed) {
+        return null
+      }
+      if (currentWorkspaceId && authorization.workflow?.workspaceId !== currentWorkspaceId) {
+        return null
+      }
+      workflowRecord = authorization.workflow ?? null
+    }
+
+    if (!workflowRecord) {
+      workflowRecord = await getActiveWorkflowRecord(workflowId)
+    }
+    if (!workflowRecord) return null
+
+    // Emit a VFS-path pointer instead of the full (potentially huge) workflow
+    // state/meta. `current_workflow` points at the live state; a plain
+    // `workflow` mention points at the lighter metadata file.
+    const folderPath = await resolveWorkflowFolderPath(
+      workflowRecord.workspaceId ?? currentWorkspaceId,
+      workflowRecord.folderId
+    )
+    const dir = canonicalWorkflowVfsDir({ name: workflowRecord.name, folderPath })
+    const path = kind === 'current_workflow' ? `${dir}/state.json` : `${dir}/meta.json`
+    return { type: kind, tag, content: '', path }
+  } catch (error) {
+    logger.error('Error processing workflow context', { workflowId, error })
+    return null
+  }
+}
+
+async function processPastChat(chatId: string, tagOverride?: string): Promise<AgentContext | null> {
+  try {
+    // boundary-raw-fetch: GET /api/mothership/chat?chatId=... has no defineRouteContract;
+    // the route forwards to the copilot chat handler and emits a free-form chat envelope
+    // that isn't covered by mothershipChatGetQuerySchema or copilotChatGetContract.
+    const resp = await fetch(`/api/mothership/chat?chatId=${encodeURIComponent(chatId)}`)
+    if (!resp.ok) {
+      logger.error('Failed to fetch past chat', { chatId, status: resp.status })
+      return null
+    }
+    const data = await resp.json()
+    const messages = Array.isArray(data?.chat?.messages) ? data.chat.messages : []
+    const content = messages
+      .map((m: any) => {
+        const role = m.role || 'user'
+        // Prefer contentBlocks text if present (joins text blocks), else use content
+        let text = ''
+        if (Array.isArray(m.contentBlocks) && m.contentBlocks.length > 0) {
+          text = m.contentBlocks
+            .filter((b: any) => b?.type === 'text')
+            .map((b: any) => String(b.content || ''))
+            .join('')
+            .trim()
+        }
+        if (!text && typeof m.content === 'string') text = m.content
+        return `${role}: ${text}`.trim()
+      })
+      .filter((s: string) => s.length > 0)
+      .join('\n')
+    logger.info('Processed past_chat context via API', { chatId, length: content.length })
+
+    return { type: 'past_chat', tag: tagOverride || '@', content }
+  } catch (error) {
+    logger.error('Error processing past chat', { chatId, error })
+    return null
+  }
+}
+
+// Back-compat alias; used by processContexts above
+async function processPastChatViaApi(chatId: string, tag?: string) {
+  return processPastChat(chatId, tag)
+}
+
+async function processKnowledgeFromDb(
+  knowledgeBaseId: string,
+  userId: string | undefined,
+  tag: string,
+  currentWorkspaceId?: string
+): Promise<AgentContext | null> {
+  try {
+    if (userId) {
+      const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, userId)
+      if (!accessCheck.hasAccess) {
+        return null
+      }
+      if (currentWorkspaceId && accessCheck.knowledgeBase?.workspaceId !== currentWorkspaceId) {
+        return null
+      }
+    }
+
+    const conditions = [eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)]
+    if (currentWorkspaceId) {
+      conditions.push(eq(knowledgeBase.workspaceId, currentWorkspaceId))
+    }
+    const kbRows = await dbReplica
+      .select({
+        id: knowledgeBase.id,
+        name: knowledgeBase.name,
+      })
+      .from(knowledgeBase)
+      .where(and(...conditions))
+      .limit(1)
+    const kb = kbRows?.[0]
+    if (!kb) return null
+
+    return {
+      type: 'knowledge',
+      tag,
+      content: '',
+      path: `${canonicalKnowledgeBaseVfsDir(kb.name)}/meta.json`,
+    }
+  } catch (error) {
+    logger.error('Error processing knowledge context (db)', { knowledgeBaseId, error })
+    return null
+  }
+}
+
+async function processBlockMetadata(
+  blockId: string,
+  tag: string,
+  userId?: string,
+  workspaceId?: string
+): Promise<AgentContext | null> {
+  try {
+    const permissionConfig =
+      userId && workspaceId ? await getUserPermissionConfig(userId, workspaceId) : null
+    const allowedIntegrations =
+      permissionConfig?.allowedIntegrations ?? getAllowedIntegrationsFromEnv()
+    if (allowedIntegrations != null && !allowedIntegrations.includes(blockId.toLowerCase())) {
+      logger.debug('Block not allowed by integration allowlist', { blockId, userId })
+      return null
+    }
+
+    const { registry: blockRegistry } = await import('@/blocks/registry')
+    if (!(blockRegistry as any)[blockId]) {
+      return null
+    }
+
+    return { type: 'blocks', tag, content: '', path: canonicalBlockVfsPath(blockId) }
+  } catch (error) {
+    logger.error('Error processing block metadata', { blockId, error })
+    return null
+  }
+}
+
+async function processWorkflowBlockFromDb(
+  workflowId: string,
+  userId: string | undefined,
+  blockId: string,
+  label?: string,
+  currentWorkspaceId?: string
+): Promise<AgentContext | null> {
+  try {
+    let workflowRecord: Awaited<ReturnType<typeof getActiveWorkflowRecord>> = null
+    if (userId) {
+      const authorization = await authorizeWorkflowByWorkspacePermission({
+        workflowId,
+        userId,
+        action: 'read',
+      })
+      if (!authorization.allowed) {
+        return null
+      }
+      if (currentWorkspaceId && authorization.workflow?.workspaceId !== currentWorkspaceId) {
+        return null
+      }
+      workflowRecord = authorization.workflow ?? null
+    }
+
+    if (!workflowRecord) {
+      workflowRecord = await getActiveWorkflowRecord(workflowId)
+    }
+    if (!workflowRecord) return null
+
+    const folderPath = await resolveWorkflowFolderPath(
+      workflowRecord.workspaceId ?? currentWorkspaceId,
+      workflowRecord.folderId
+    )
+    const dir = canonicalWorkflowVfsDir({ name: workflowRecord.name, folderPath })
+    const tag = label ? `@${label} in Workflow` : `@${blockId} in Workflow`
+    // Point at the workflow state; the block id tells the model which node to
+    // look up inside state.json without inlining the full block definition.
+    return {
+      type: 'workflow_block',
+      tag,
+      content: `Block id: ${blockId}`,
+      path: `${dir}/state.json`,
+    }
+  } catch (error) {
+    logger.error('Error processing workflow_block context', { workflowId, blockId, error })
+    return null
+  }
+}
+
+/**
+ * Cap on the serialized summary (including the block overview tree) sent for
+ * a tagged run. `toOverview` already excludes every block's input/output, so
+ * this is a safety net against pathological span counts, not the primary
+ * defense — mirrors `MAX_FULL_RESULT_BYTES` in `query-logs.ts`, scaled down
+ * since this lands in the prompt unconditionally rather than behind an
+ * explicit tool call.
+ */
+const MAX_LOG_SUMMARY_BYTES = 64 * 1024
+
+/**
+ * Resolve a tagged run to a compact summary instead of its full execution
+ * trace. A run's trace can carry every block's input/output plus nested
+ * tool-call spans, which is unbounded and would repeatedly blow the context
+ * window if inlined directly. The summary includes the block-level overview
+ * tree (name/type/status/timing/cost, no input or output — the same
+ * projection `query_logs`'s `overview` view returns) so the model can see
+ * which block failed without a round trip, and points it at `query_logs` for
+ * that block's actual input/output/error, or to grep the trace.
+ *
+ * `materializeExecutionData` only unwraps a top-level object-storage pointer,
+ * for runs whose whole trace was offloaded as one blob — a no-op for the
+ * common inline case. Individual span input/output stay as large-value refs;
+ * `toOverview` never resolves those.
+ */
+async function processExecutionLogFromDb(
+  executionId: string,
+  userId: string | undefined,
+  tag: string,
+  currentWorkspaceId?: string
+): Promise<AgentContext | null> {
+  try {
+    const { workflowExecutionLogs, workflow } = await import('@sim/db/schema')
+    const rows = await db
+      .select({
+        id: workflowExecutionLogs.id,
+        workflowId: workflowExecutionLogs.workflowId,
+        workspaceId: workflowExecutionLogs.workspaceId,
+        executionId: workflowExecutionLogs.executionId,
+        level: workflowExecutionLogs.level,
+        trigger: workflowExecutionLogs.trigger,
+        startedAt: workflowExecutionLogs.startedAt,
+        endedAt: workflowExecutionLogs.endedAt,
+        totalDurationMs: workflowExecutionLogs.totalDurationMs,
+        executionData: workflowExecutionLogs.executionData,
+        costTotal: workflowExecutionLogs.costTotal,
+        workflowName: workflow.name,
+      })
+      .from(workflowExecutionLogs)
+      .innerJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id))
+      .where(eq(workflowExecutionLogs.executionId, executionId))
+      .limit(1)
+
+    const log = rows?.[0] as any
+    if (!log) return null
+
+    if (userId) {
+      const authorization = await authorizeWorkflowByWorkspacePermission({
+        workflowId: log.workflowId,
+        userId,
+        action: 'read',
+      })
+      if (!authorization.allowed) {
+        return null
+      }
+      if (currentWorkspaceId && authorization.workflow?.workspaceId !== currentWorkspaceId) {
+        return null
+      }
+    }
+
+    const { materializeExecutionData } = await import('@/lib/logs/execution/trace-store')
+    const executionData = (await materializeExecutionData(
+      log.executionData as Record<string, unknown> | null,
+      { workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId }
+    )) as { traceSpans?: TraceSpan[] } | undefined
+    const overview = executionData?.traceSpans?.length
+      ? toOverview(executionData.traceSpans)
+      : undefined
+
+    const summary = {
+      id: log.id,
+      workflowId: log.workflowId,
+      executionId: log.executionId,
+      level: log.level,
+      trigger: log.trigger,
+      startedAt: log.startedAt?.toISOString?.() || String(log.startedAt),
+      endedAt: log.endedAt?.toISOString?.() || (log.endedAt ? String(log.endedAt) : null),
+      totalDurationMs: log.totalDurationMs ?? null,
+      workflowName: log.workflowName || '',
+      cost: log.costTotal != null ? { total: Number(log.costTotal) } : undefined,
+      overview,
+      note: `For a block's input/output/error, or to grep the trace, call ${QueryLogs.id} with executionId: '${log.executionId}' — view: 'full' (scope with blockId or blockName), or pattern to grep.`,
+    }
+
+    if (overview && JSON.stringify(summary).length > MAX_LOG_SUMMARY_BYTES) {
+      summary.overview = undefined
+    }
+
+    const content = JSON.stringify(summary)
+    return { type: 'logs', tag, content }
+  } catch (error) {
+    logger.error('Error processing execution log context (db)', { executionId, error })
+    return null
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Active resource context resolution (direct DB lookups, workspace-scoped)
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolves the content of the currently active resource tab via direct DB
+ * queries. Each resource type has a dedicated handler that fetches only the
+ * single resource needed — avoiding the full VFS materialisation overhead.
+ */
+export async function resolveActiveResourceContext(
+  resourceType: string,
+  resourceId: string,
+  workspaceId: string,
+  userId: string,
+  chatId?: string
+): Promise<AgentContext | null> {
+  try {
+    switch (resourceType) {
+      case 'workflow': {
+        const ctx = await processWorkflowFromDb(
+          resourceId,
+          userId,
+          '@active_resource',
+          'current_workflow',
+          workspaceId,
+          chatId
+        )
+        if (!ctx) return null
+        return {
+          type: 'active_resource',
+          tag: '@active_resource',
+          content: ctx.content,
+          path: ctx.path,
+        }
+      }
+      case 'knowledgebase': {
+        const ctx = await processKnowledgeFromDb(
+          resourceId,
+          userId,
+          '@active_resource',
+          workspaceId
+        )
+        if (!ctx) return null
+        return {
+          type: 'active_resource',
+          tag: '@active_resource',
+          content: ctx.content,
+          path: ctx.path,
+        }
+      }
+      case 'table': {
+        return await resolveTableResource(resourceId, workspaceId)
+      }
+      case 'file': {
+        return await resolveFileResource(resourceId, workspaceId)
+      }
+      case 'folder': {
+        return await resolveFolderResource(resourceId, workspaceId)
+      }
+      case 'filefolder': {
+        return await resolveFileFolderResource(resourceId, workspaceId)
+      }
+      case 'scheduledtask': {
+        return await resolveScheduledTaskResource(resourceId, workspaceId)
+      }
+      default:
+        return null
+    }
+  } catch (error) {
+    logger.error('Failed to resolve active resource context', { resourceType, resourceId, error })
+    return null
+  }
+}
+async function resolveTableResource(
+  tableId: string,
+  workspaceId: string
+): Promise<AgentContext | null> {
+  const table = await getTableById(tableId)
+  if (!table) return null
+  if (table.workspaceId !== workspaceId) return null
+  return {
+    type: 'active_resource',
+    tag: '@active_resource',
+    content: '',
+    path: canonicalTableVfsPath(table.name),
+  }
+}
+
+async function resolveScheduledTaskResource(
+  scheduleId: string,
+  workspaceId: string
+): Promise<AgentContext | null> {
+  const [row] = await db
+    .select({ id: workflowSchedule.id, jobTitle: workflowSchedule.jobTitle })
+    .from(workflowSchedule)
+    .where(
+      and(
+        eq(workflowSchedule.id, scheduleId),
+        eq(workflowSchedule.sourceWorkspaceId, workspaceId),
+        eq(workflowSchedule.sourceType, 'job'),
+        isNull(workflowSchedule.archivedAt),
+        // Mirror the VFS materializer (workspace-vfs `materializeJobs`), which
+        // excludes completed jobs — otherwise we'd point at a meta.json it never
+        // wrote and the agent's read would dangle.
+        ne(workflowSchedule.status, 'completed')
+      )
+    )
+    .limit(1)
+  if (!row) return null
+  // The VFS materializes jobs at `jobs/{sanitized title}/meta.json` (see
+  // workspace-vfs `materializeJobs`); emit the same lightweight path pointer so
+  // the agent reads it via the VFS instead of us inlining the (heavy) row.
+  return {
+    type: 'active_resource',
+    tag: '@active_resource',
+    content: '',
+    path: `jobs/${normalizeVfsSegment(row.jobTitle || row.id)}/meta.json`,
+  }
+}
+
+async function resolveFileResource(
+  fileId: string,
+  workspaceId: string
+): Promise<AgentContext | null> {
+  const record = await getWorkspaceFile(workspaceId, fileId)
+  if (!record) return null
+  return {
+    type: 'active_resource',
+    tag: '@active_resource',
+    content: '',
+    path: canonicalWorkspaceFilePath({ folderPath: record.folderPath, name: record.name }),
+  }
+}
+
+async function resolveFileFolderResource(
+  folderId: string,
+  workspaceId: string
+): Promise<AgentContext | null> {
+  try {
+    const rawPath = await getWorkspaceFileFolderPath(workspaceId, folderId)
+    if (!rawPath) return null
+    const encoded = encodeVfsPathSegments(rawPath.split('/').filter(Boolean))
+    return {
+      type: 'active_resource',
+      tag: '@active_resource',
+      content: '',
+      path: `files/${encoded}`,
+    }
+  } catch (error) {
+    logger.error('Failed to resolve file folder resource', { folderId, error })
+    return null
+  }
+}
+
+async function resolveFolderResource(
+  folderId: string,
+  workspaceId: string
+): Promise<AgentContext | null> {
+  const folderPath = await resolveWorkflowFolderPath(workspaceId, folderId)
+  if (!folderPath) return null
+  return {
+    type: 'active_resource',
+    tag: '@active_resource',
+    content: '',
+    path: `workflows/${folderPath}`,
+  }
+}

apps/sim/lib/copilot/chat/sim-key-redaction.test.ts

@@ -0,0 +1,154 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import type { ChatMessage } from '@/app/workspace/[workspaceId]/home/types'
+import {
+  captureRevealedSimKeys,
+  extractRevealedSimKeys,
+  restoreRevealedSimKeysForMessage,
+} from './sim-key-redaction'
+
+const credential = (value: string) =>
+  `<credential>${JSON.stringify({ value, type: 'sim_key' })}</credential>`
+const redacted = `<credential>${JSON.stringify({ type: 'sim_key', redacted: true })}</credential>`
+
+describe('sim-key-redaction', () => {
+  describe('extractRevealedSimKeys', () => {
+    it('returns sim_key values in document order', () => {
+      const text = `first ${credential('sk-sim-A')} mid ${credential('sk-sim-B')}`
+      expect(extractRevealedSimKeys(text)).toEqual(['sk-sim-A', 'sk-sim-B'])
+    })
+
+    it('skips redacted entries and non-sim_key tags', () => {
+      const link = `<credential>${JSON.stringify({ value: 'https://x', type: 'link', provider: 'slack' })}</credential>`
+      const text = `${link} ${credential('sk-sim-A')} ${redacted}`
+      expect(extractRevealedSimKeys(text)).toEqual(['sk-sim-A'])
+    })
+  })
+
+  describe('captureRevealedSimKeys', () => {
+    it('records new keys under each provided key', () => {
+      const cache = new Map<string, string[]>()
+      captureRevealedSimKeys(cache, ['msg-1', 'req-1'], credential('sk-sim-A'))
+      expect(cache.get('msg-1')).toEqual(['sk-sim-A'])
+      expect(cache.get('req-1')).toEqual(['sk-sim-A'])
+    })
+
+    it('extends but never shrinks the captured list across calls', () => {
+      const cache = new Map<string, string[]>()
+      captureRevealedSimKeys(
+        cache,
+        ['msg-1'],
+        `${credential('sk-sim-A')} ${credential('sk-sim-B')}`
+      )
+      captureRevealedSimKeys(cache, ['msg-1'], credential('sk-sim-A'))
+      expect(cache.get('msg-1')).toEqual(['sk-sim-A', 'sk-sim-B'])
+    })
+
+    it('skips undefined keys without throwing', () => {
+      const cache = new Map<string, string[]>()
+      captureRevealedSimKeys(cache, ['msg-1', undefined], credential('sk-sim-A'))
+      expect(cache.get('msg-1')).toEqual(['sk-sim-A'])
+      expect(cache.size).toBe(1)
+    })
+
+    it('ignores content with no credential tag', () => {
+      const cache = new Map<string, string[]>()
+      captureRevealedSimKeys(cache, ['msg-1'], 'plain assistant text')
+      expect(cache.has('msg-1')).toBe(false)
+    })
+  })
+
+  describe('restoreRevealedSimKeysForMessage', () => {
+    it('substitutes the live key back into a redacted message', () => {
+      const cache = new Map<string, string[]>([['msg-1', ['sk-sim-A']]])
+      const msg: ChatMessage = {
+        id: 'msg-1',
+        role: 'assistant',
+        content: `Here is your key: ${redacted} save it.`,
+        contentBlocks: [{ type: 'text', content: `Here is your key: ${redacted} save it.` }],
+      }
+      const restored = restoreRevealedSimKeysForMessage(msg, cache)
+      expect(restored.content).toContain('"sk-sim-A"')
+      expect(restored.content).not.toContain('"redacted":true')
+      expect(restored.contentBlocks?.[0].content).toContain('"sk-sim-A"')
+    })
+
+    it('substitutes multiple keys in stream order', () => {
+      const cache = new Map<string, string[]>([['msg-1', ['sk-sim-A', 'sk-sim-B']]])
+      const msg: ChatMessage = {
+        id: 'msg-1',
+        role: 'assistant',
+        content: `first ${redacted} second ${redacted}`,
+      }
+      const restored = restoreRevealedSimKeysForMessage(msg, cache)
+      expect(restored.content).toBe(
+        `first ${credential('sk-sim-A')} second ${credential('sk-sim-B')}`
+      )
+    })
+
+    it('leaves a redacted tag in place if no live value is captured for that slot', () => {
+      const cache = new Map<string, string[]>([['msg-1', ['sk-sim-A']]])
+      const msg: ChatMessage = {
+        id: 'msg-1',
+        role: 'assistant',
+        content: `first ${redacted} second ${redacted}`,
+      }
+      const restored = restoreRevealedSimKeysForMessage(msg, cache)
+      expect(restored.content).toBe(`first ${credential('sk-sim-A')} second ${redacted}`)
+    })
+
+    it('returns the same message reference when nothing to restore', () => {
+      const cache = new Map<string, string[]>()
+      const msg: ChatMessage = {
+        id: 'msg-1',
+        role: 'assistant',
+        content: 'no credentials here',
+      }
+      expect(restoreRevealedSimKeysForMessage(msg, cache)).toBe(msg)
+    })
+
+    it('does nothing for user messages', () => {
+      const cache = new Map<string, string[]>([['msg-1', ['sk-sim-A']]])
+      const msg: ChatMessage = {
+        id: 'msg-1',
+        role: 'user',
+        content: redacted,
+      }
+      expect(restoreRevealedSimKeysForMessage(msg, cache)).toBe(msg)
+    })
+
+    it('threads the cursor across separate content blocks so each block gets its matching key', () => {
+      const cache = new Map<string, string[]>([['msg-1', ['sk-sim-A', 'sk-sim-B']]])
+      const msg: ChatMessage = {
+        id: 'msg-1',
+        role: 'assistant',
+        content: `first ${redacted} (tool ran) second ${redacted}`,
+        contentBlocks: [
+          { type: 'text', content: `first ${redacted}` },
+          { type: 'tool_call', content: '' },
+          { type: 'text', content: `second ${redacted}` },
+        ],
+      }
+      const restored = restoreRevealedSimKeysForMessage(msg, cache)
+      expect(restored.contentBlocks?.[0].content).toContain('"sk-sim-A"')
+      expect(restored.contentBlocks?.[0].content).not.toContain('"sk-sim-B"')
+      expect(restored.contentBlocks?.[2].content).toContain('"sk-sim-B"')
+      expect(restored.contentBlocks?.[2].content).not.toContain('"sk-sim-A"')
+    })
+
+    it('isolates revealed values by message id (multiple keys across messages)', () => {
+      const cache = new Map<string, string[]>([
+        ['msg-1', ['sk-sim-A']],
+        ['msg-2', ['sk-sim-B']],
+      ])
+      const msg1: ChatMessage = { id: 'msg-1', role: 'assistant', content: redacted }
+      const msg2: ChatMessage = { id: 'msg-2', role: 'assistant', content: redacted }
+      expect(restoreRevealedSimKeysForMessage(msg1, cache).content).toContain('sk-sim-A')
+      expect(restoreRevealedSimKeysForMessage(msg2, cache).content).toContain('sk-sim-B')
+      expect(restoreRevealedSimKeysForMessage(msg1, cache).content).not.toContain('sk-sim-B')
+    })
+  })
+})

apps/sim/lib/copilot/chat/sim-key-redaction.ts

@@ -0,0 +1,263 @@
+import type { PersistedContentBlock } from '@/lib/copilot/chat/persisted-message'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1TextChannel,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { GenerateApiKey } from '@/lib/copilot/generated/tool-catalog-v1'
+import { REDACTED_MARKER } from '@/lib/core/security/redaction'
+import type { ChatMessage, ContentBlock } from '@/app/workspace/[workspaceId]/home/types'
+
+/**
+ * Two-sided handling of `sim_key` API keys in the Mothership chat:
+ *
+ * - **Write side** (server, runs in `buildPersistedAssistantMessage`):
+ *   strip every revealed `<credential type="sim_key">` value before the row
+ *   hits Postgres. Reloading a chat days later — or pulling the row from the
+ *   DB directly — never re-exposes the key.
+ *
+ * - **Read side** (client, runs in `useChat`'s message selector): an in-memory
+ *   page-session cache captures revealed values during the live SSE stream.
+ *   When the post-stream refetch returns the redacted persisted message, the
+ *   selector re-injects the captured values so the user can still copy the
+ *   key they just generated. Cache is dropped on page unload.
+ */
+
+const CREDENTIAL_TAG_PATTERN = /<credential>([\s\S]*?)<\/credential>/g
+const REDACTED_TAG_PATTERN = /<credential>[^<]*"redacted"\s*:\s*true[^<]*<\/credential>/
+const SIM_KEY_TYPE = 'sim_key'
+const REDACTED_SIM_KEY_TAG = `<credential>${JSON.stringify({
+  type: SIM_KEY_TYPE,
+  redacted: true,
+})}</credential>`
+
+interface CredentialTagBody {
+  type?: unknown
+  value?: unknown
+  redacted?: unknown
+}
+
+function parseCredentialBody(body: string): CredentialTagBody | null {
+  try {
+    return JSON.parse(body) as CredentialTagBody
+  } catch {
+    return null
+  }
+}
+
+function hasRedactedSimKeyTag(content: string | undefined): boolean {
+  return typeof content === 'string' && REDACTED_TAG_PATTERN.test(content)
+}
+
+// Write side ---------------------------------------------------------------
+
+/**
+ * Replace every revealed `<credential type="sim_key">` tag in `content` with a
+ * placeholder marked `redacted: true`. Other credential types (e.g. OAuth
+ * `link`) and malformed bodies pass through unchanged.
+ */
+export function redactSensitiveContent<T extends string | undefined>(content: T): T {
+  if (typeof content !== 'string' || !content.includes('<credential>')) return content
+  return content.replace(CREDENTIAL_TAG_PATTERN, (match, body: string) => {
+    const parsed = parseCredentialBody(body)
+    return parsed?.type === SIM_KEY_TYPE ? REDACTED_SIM_KEY_TAG : match
+  }) as T
+}
+
+/**
+ * Replace the raw `key` field in a `generate_api_key` tool result with the
+ * shared redaction marker. The persisted tool result still records the
+ * call's outcome and metadata; only the secret is stripped.
+ */
+export function redactToolCallResult(
+  toolName: string | undefined,
+  result: { success: boolean; output?: unknown; error?: string } | undefined
+): { success: boolean; output?: unknown; error?: string } | undefined {
+  if (!result || toolName !== GenerateApiKey.id) return result
+  const output = result.output
+  if (!output || typeof output !== 'object') return result
+  const record = output as Record<string, unknown>
+  if (typeof record.key !== 'string') return result
+  return {
+    ...result,
+    output: { ...record, key: REDACTED_MARKER, redacted: true },
+  }
+}
+
+function isMergeableAssistantTextBlock(block: PersistedContentBlock): boolean {
+  return (
+    block.type === MothershipStreamV1EventType.text &&
+    block.channel === MothershipStreamV1TextChannel.assistant &&
+    block.toolCall === undefined
+  )
+}
+
+/**
+ * Streaming produces one assistant-text block per token chunk, which means a
+ * `<credential>...</credential>` tag can straddle dozens of blocks. Per-block
+ * redaction can't see across that boundary and would persist the secret. So
+ * coalesce consecutive same-lane assistant-text blocks into a single block,
+ * then redact the merged content.
+ *
+ * Block timestamps for assistant text aren't user-visible (only `thinking`
+ * blocks drive the "Thought for Ns" chip), so collapsing the run is safe.
+ */
+export function mergeAndRedactPersistedBlocks(
+  blocks: PersistedContentBlock[]
+): PersistedContentBlock[] {
+  const out: PersistedContentBlock[] = []
+  let runStart = -1
+  let runLane: PersistedContentBlock['lane']
+
+  const flushRun = (endExclusive: number) => {
+    if (runStart < 0) return
+    const run = blocks.slice(runStart, endExclusive)
+    runStart = -1
+    if (run.length === 0) return
+    if (run.length === 1) {
+      const single = run[0]
+      out.push({ ...single, content: redactSensitiveContent(single.content) })
+      return
+    }
+    const head = run[0]
+    const tail = run[run.length - 1]
+    out.push({
+      ...head,
+      content: redactSensitiveContent(run.map((b) => b.content ?? '').join('')),
+      ...(tail.endedAt !== undefined ? { endedAt: tail.endedAt } : {}),
+    })
+  }
+
+  for (let i = 0; i < blocks.length; i++) {
+    const block = blocks[i]
+    const sameRun = runStart >= 0 && isMergeableAssistantTextBlock(block) && runLane === block.lane
+    if (sameRun) continue
+    flushRun(i)
+    if (isMergeableAssistantTextBlock(block)) {
+      runStart = i
+      runLane = block.lane
+    } else {
+      out.push(block)
+    }
+  }
+  flushRun(blocks.length)
+
+  return out
+}
+
+// Read side ----------------------------------------------------------------
+
+/**
+ * Page-session cache of `sim_key` credential values revealed during the live
+ * SSE stream, keyed by either the synthetic live-assistant id (used while
+ * streaming) or the persisted message's `requestId` (used after refetch).
+ * Lives in a `useRef`; never persisted; dropped on unload.
+ */
+export type RevealedSimKeysByMessage = Map<string, string[]>
+
+/**
+ * Scan an assembled assistant message for `<credential type="sim_key">` tags
+ * and return their values in stream order, skipping anything already redacted.
+ */
+export function extractRevealedSimKeys(content: string): string[] {
+  if (!content || !content.includes('<credential>')) return []
+  const values: string[] = []
+  for (const match of content.matchAll(CREDENTIAL_TAG_PATTERN)) {
+    const parsed = parseCredentialBody(match[1])
+    if (parsed?.type === SIM_KEY_TYPE && !parsed.redacted && typeof parsed.value === 'string') {
+      values.push(parsed.value)
+    }
+  }
+  return values
+}
+
+/**
+ * Extend the cache entries for the given keys with any newly-revealed values.
+ * Each key in `keys` is written the same array — passing both the live-stream
+ * id and the persisted `requestId` lets the post-finalize refetch hit the
+ * cache after the message is renamed to its real UUID. The longest captured
+ * list wins so a rerun that surfaces fewer values can't shrink the entry.
+ */
+export function captureRevealedSimKeys(
+  cache: RevealedSimKeysByMessage,
+  keys: ReadonlyArray<string | undefined>,
+  content: string
+): void {
+  if (!content.includes('<credential>')) return
+  const next = extractRevealedSimKeys(content)
+  if (next.length === 0) return
+  for (const key of keys) {
+    if (!key) continue
+    const existing = cache.get(key)
+    if (!existing || next.length > existing.length) cache.set(key, next)
+  }
+}
+
+function restoreInString(
+  content: string,
+  revealedValues: string[],
+  startCursor: number
+): {
+  next: string
+  changed: boolean
+  cursor: number
+} {
+  if (!content.includes('<credential>') || revealedValues.length === 0) {
+    return { next: content, changed: false, cursor: startCursor }
+  }
+  let cursor = startCursor
+  let changed = false
+  const next = content.replace(CREDENTIAL_TAG_PATTERN, (match, body: string) => {
+    const parsed = parseCredentialBody(body)
+    if (parsed?.type === SIM_KEY_TYPE && parsed.redacted === true) {
+      const value = revealedValues[cursor]
+      cursor += 1
+      if (typeof value === 'string') {
+        changed = true
+        return `<credential>${JSON.stringify({ value, type: SIM_KEY_TYPE })}</credential>`
+      }
+    }
+    return match
+  })
+  return { next, changed, cursor }
+}
+
+/**
+ * Replace redacted `sim_key` tags in a single message with the live values
+ * captured for that message. Returns the original message reference unchanged
+ * when there's nothing to substitute, so memoized children keep their identity.
+ */
+export function restoreRevealedSimKeysForMessage(
+  message: ChatMessage,
+  cache: RevealedSimKeysByMessage
+): ChatMessage {
+  if (message.role !== 'assistant') return message
+  const revealed =
+    cache.get(message.id) ?? (message.requestId ? cache.get(message.requestId) : undefined)
+  if (!revealed || revealed.length === 0) return message
+  if (
+    !hasRedactedSimKeyTag(message.content) &&
+    !message.contentBlocks?.some((b) => hasRedactedSimKeyTag(b.content))
+  ) {
+    return message
+  }
+
+  const restoredContent = restoreInString(message.content, revealed, 0)
+  let blocksChanged = false
+  let blockCursor = 0
+  const nextBlocks: ContentBlock[] | undefined = message.contentBlocks?.map((block) => {
+    if (!hasRedactedSimKeyTag(block.content)) return block
+    const restored = restoreInString(block.content as string, revealed, blockCursor)
+    blockCursor = restored.cursor
+    if (!restored.changed) return block
+    blocksChanged = true
+    return { ...block, content: restored.next }
+  })
+
+  if (!restoredContent.changed && !blocksChanged) return message
+
+  return {
+    ...message,
+    content: restoredContent.next,
+    ...(nextBlocks ? { contentBlocks: nextBlocks } : {}),
+  }
+}

apps/sim/lib/copilot/chat/stream-liveness.test.ts

@@ -0,0 +1,154 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockAnd, mockEq, mockGetChatStreamLockOwners, mockSet, mockUpdate, mockWhere } = vi.hoisted(
+  () => ({
+    mockAnd: vi.fn((...conditions: unknown[]) => ({ type: 'and', conditions })),
+    mockEq: vi.fn((field: unknown, value: unknown) => ({ type: 'eq', field, value })),
+    mockGetChatStreamLockOwners: vi.fn(),
+    mockSet: vi.fn(),
+    mockUpdate: vi.fn(),
+    mockWhere: vi.fn(),
+  })
+)
+
+vi.mock('@sim/db', () => ({
+  db: { update: mockUpdate },
+}))
+
+vi.mock('@sim/db/schema', () => ({
+  copilotChats: {
+    id: 'copilotChats.id',
+    conversationId: 'copilotChats.conversationId',
+  },
+}))
+
+vi.mock('drizzle-orm', () => ({
+  and: mockAnd,
+  eq: mockEq,
+}))
+
+vi.mock('@/lib/copilot/request/session', () => ({
+  getChatStreamLockOwners: mockGetChatStreamLockOwners,
+}))
+
+import { reconcileChatStreamMarkers } from '@/lib/copilot/chat/stream-liveness'
+
+describe('reconcileChatStreamMarkers', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockSet.mockReturnValue({ where: mockWhere })
+    mockUpdate.mockReturnValue({ set: mockSet })
+    mockWhere.mockResolvedValue(undefined)
+    mockGetChatStreamLockOwners.mockResolvedValue({
+      status: 'verified',
+      ownersByChatId: new Map<string, string>(),
+    })
+  })
+
+  it('clears a persisted stream marker when Redis verifies no lock owner exists', async () => {
+    const markers = await reconcileChatStreamMarkers([
+      { chatId: 'chat-stuck', streamId: 'stream-orphaned' },
+    ])
+
+    expect(mockGetChatStreamLockOwners).toHaveBeenCalledWith(['chat-stuck'])
+    expect(markers.get('chat-stuck')).toEqual({
+      chatId: 'chat-stuck',
+      streamId: null,
+      status: 'inactive',
+    })
+  })
+
+  it('repairs a verified stale persisted stream marker when requested', async () => {
+    await reconcileChatStreamMarkers([{ chatId: 'chat-stuck', streamId: 'stream-orphaned' }], {
+      repairVerifiedStaleMarkers: true,
+    })
+
+    expect(mockUpdate).toHaveBeenCalled()
+    expect(mockSet).toHaveBeenCalledWith({ conversationId: null })
+    expect(mockWhere).toHaveBeenCalledWith(
+      mockAnd(
+        mockEq('copilotChats.id', 'chat-stuck'),
+        mockEq('copilotChats.conversationId', 'stream-orphaned')
+      )
+    )
+  })
+
+  it('uses the canonical Redis owner when the persisted stream marker is stale', async () => {
+    mockGetChatStreamLockOwners.mockResolvedValueOnce({
+      status: 'verified',
+      ownersByChatId: new Map([['chat-mismatch', 'stream-live']]),
+    })
+
+    const markers = await reconcileChatStreamMarkers([
+      { chatId: 'chat-mismatch', streamId: 'stream-stale' },
+    ])
+
+    expect(markers.get('chat-mismatch')).toEqual({
+      chatId: 'chat-mismatch',
+      streamId: 'stream-live',
+      status: 'active',
+    })
+  })
+
+  it('preserves persisted stream markers when Redis state is unknown', async () => {
+    mockGetChatStreamLockOwners.mockResolvedValueOnce({
+      status: 'unknown',
+      ownersByChatId: new Map<string, string>(),
+    })
+
+    const markers = await reconcileChatStreamMarkers([
+      { chatId: 'chat-remote', streamId: 'stream-remote' },
+    ])
+
+    expect(markers.get('chat-remote')).toEqual({
+      chatId: 'chat-remote',
+      streamId: 'stream-remote',
+      status: 'unknown',
+    })
+  })
+
+  it('preserves a persisted marker when unknown local owner differs', async () => {
+    mockGetChatStreamLockOwners.mockResolvedValueOnce({
+      status: 'unknown',
+      ownersByChatId: new Map([['chat-mismatch', 'stream-local']]),
+    })
+
+    const markers = await reconcileChatStreamMarkers([
+      { chatId: 'chat-mismatch', streamId: 'stream-persisted' },
+    ])
+
+    expect(markers.get('chat-mismatch')).toEqual({
+      chatId: 'chat-mismatch',
+      streamId: 'stream-persisted',
+      status: 'unknown',
+    })
+  })
+
+  it('treats a null persisted marker as inactive even when Redis still holds a lock (post-completion teardown window)', async () => {
+    mockGetChatStreamLockOwners.mockResolvedValueOnce({
+      status: 'verified',
+      ownersByChatId: new Map([['chat-starting', 'stream-starting']]),
+    })
+
+    const markers = await reconcileChatStreamMarkers([{ chatId: 'chat-starting', streamId: null }])
+
+    expect(markers.get('chat-starting')).toEqual({
+      chatId: 'chat-starting',
+      streamId: null,
+      status: 'inactive',
+    })
+  })
+
+  it('does not query locks when no chats have persisted stream markers', async () => {
+    const markers = await reconcileChatStreamMarkers([{ chatId: 'chat-idle', streamId: null }])
+
+    expect(markers.get('chat-idle')).toEqual({
+      chatId: 'chat-idle',
+      streamId: null,
+      status: 'inactive',
+    })
+  })
+})

apps/sim/lib/copilot/chat/stream-liveness.ts

@@ -0,0 +1,135 @@
+import { db } from '@sim/db'
+import { copilotChats } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { and, eq } from 'drizzle-orm'
+import { getChatStreamLockOwners } from '@/lib/copilot/request/session'
+
+const logger = createLogger('ChatStreamLiveness')
+
+export interface ChatStreamMarkerCandidate {
+  chatId: string
+  streamId: string | null
+}
+
+export interface ReconciledChatStreamMarker {
+  chatId: string
+  streamId: string | null
+  status: 'active' | 'inactive' | 'unknown'
+}
+
+interface ReconcileChatStreamMarkersOptions {
+  repairVerifiedStaleMarkers?: boolean
+}
+
+/**
+ * Reconciles persisted chat stream markers against the runtime stream lock.
+ *
+ * Redis lock ownership is the canonical live-stream signal. When the lookup is
+ * verified, missing owners clear stale persisted markers and present owners win
+ * over stale DB values. When Redis state is unknown, persisted markers are
+ * preserved so a transient Redis failure in a multi-pod deployment does not
+ * incorrectly hide a live stream owned by another pod.
+ */
+export async function reconcileChatStreamMarkers(
+  candidates: ChatStreamMarkerCandidate[],
+  options: ReconcileChatStreamMarkersOptions = {}
+): Promise<Map<string, ReconciledChatStreamMarker>> {
+  const results = new Map<string, ReconciledChatStreamMarker>()
+
+  for (const candidate of candidates) {
+    if (candidate.streamId === null) {
+      results.set(candidate.chatId, {
+        chatId: candidate.chatId,
+        streamId: null,
+        status: 'inactive',
+      })
+      continue
+    }
+    results.set(candidate.chatId, {
+      chatId: candidate.chatId,
+      streamId: candidate.streamId,
+      status: 'unknown',
+    })
+  }
+
+  const candidatesWithMarkers = candidates.filter((candidate) => candidate.streamId !== null)
+  if (candidatesWithMarkers.length === 0) {
+    return results
+  }
+
+  const { status, ownersByChatId } = await getChatStreamLockOwners(
+    candidatesWithMarkers.map((candidate) => candidate.chatId)
+  )
+
+  for (const candidate of candidatesWithMarkers) {
+    const owner = ownersByChatId.get(candidate.chatId)
+    if (owner && (status === 'verified' || owner === candidate.streamId)) {
+      results.set(candidate.chatId, {
+        chatId: candidate.chatId,
+        streamId: owner,
+        status: 'active',
+      })
+      continue
+    }
+
+    if (status === 'verified') {
+      results.set(candidate.chatId, {
+        chatId: candidate.chatId,
+        streamId: null,
+        status: 'inactive',
+      })
+      continue
+    }
+
+    results.set(candidate.chatId, {
+      chatId: candidate.chatId,
+      streamId: candidate.streamId,
+      status: 'unknown',
+    })
+  }
+
+  if (options.repairVerifiedStaleMarkers) {
+    await repairVerifiedStaleMarkers(candidates, results)
+  }
+
+  return results
+}
+
+async function repairVerifiedStaleMarkers(
+  candidates: ChatStreamMarkerCandidate[],
+  results: Map<string, ReconciledChatStreamMarker>
+): Promise<void> {
+  const staleCandidates = candidates.filter(
+    (candidate): candidate is { chatId: string; streamId: string } => {
+      const result = results.get(candidate.chatId)
+      return (
+        candidate.streamId !== null && result?.status === 'inactive' && result.streamId === null
+      )
+    }
+  )
+
+  if (staleCandidates.length === 0) return
+
+  await Promise.all(
+    staleCandidates.map(async (candidate) => {
+      try {
+        await db
+          .update(copilotChats)
+          .set({ conversationId: null })
+          .where(
+            and(
+              eq(copilotChats.id, candidate.chatId),
+              eq(copilotChats.conversationId, candidate.streamId)
+            )
+          )
+      } catch (error) {
+        logger.warn('Failed to repair stale chat stream marker', {
+          chatId: candidate.chatId,
+          streamId: candidate.streamId,
+          error: toError(error).message,
+        })
+      }
+    })
+  )
+}

apps/sim/lib/copilot/chat/stream-tool-outcome.ts

@@ -0,0 +1,47 @@
+import { isRecordLike } from '@sim/utils/object'
+import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1'
+
+type TerminalToolOutcome =
+  | typeof MothershipStreamV1ToolOutcome.success
+  | typeof MothershipStreamV1ToolOutcome.error
+  | typeof MothershipStreamV1ToolOutcome.cancelled
+  | typeof MothershipStreamV1ToolOutcome.skipped
+  | typeof MothershipStreamV1ToolOutcome.rejected
+
+interface ResolveStreamToolOutcomeParams {
+  output?: unknown
+  status?: string
+  success?: boolean
+}
+
+export function resolveStreamToolOutcome({
+  output,
+  status,
+  success,
+}: ResolveStreamToolOutcomeParams): TerminalToolOutcome {
+  const outputRecord = isRecordLike(output) ? output : undefined
+  const isCancelled =
+    outputRecord?.reason === 'user_cancelled' ||
+    outputRecord?.cancelledByUser === true ||
+    status === MothershipStreamV1ToolOutcome.cancelled
+
+  if (isCancelled) {
+    return MothershipStreamV1ToolOutcome.cancelled
+  }
+
+  switch (status) {
+    case MothershipStreamV1ToolOutcome.success:
+    case MothershipStreamV1ToolOutcome.error:
+    case MothershipStreamV1ToolOutcome.skipped:
+    case MothershipStreamV1ToolOutcome.rejected:
+      return status
+    case 'aborted':
+      return MothershipStreamV1ToolOutcome.cancelled
+    case 'failed':
+      return MothershipStreamV1ToolOutcome.error
+    default:
+      return success === true
+        ? MothershipStreamV1ToolOutcome.success
+        : MothershipStreamV1ToolOutcome.error
+  }
+}

apps/sim/lib/copilot/chat/terminal-state.test.ts

@@ -0,0 +1,147 @@
+/**
+ * @vitest-environment node
+ */
+
+import { copilotChats } from '@sim/db/schema'
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { eq } from 'drizzle-orm'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+const { mockAppendCopilotChatMessages } = vi.hoisted(() => ({
+  mockAppendCopilotChatMessages: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/chat/messages-store', () => ({
+  appendCopilotChatMessages: mockAppendCopilotChatMessages,
+}))
+
+import { finalizeAssistantTurn } from './terminal-state'
+
+const assistantMessage = {
+  id: 'assistant-1',
+  role: 'assistant' as const,
+  content: 'hi',
+  timestamp: '2024-01-01T00:00:00.000Z',
+}
+
+/**
+ * Sequence the two in-tx reads: the chat row (`FOR UPDATE ... LIMIT 1`) and the
+ * last-message lookup that drives dedup — both terminate on `.limit(1)`.
+ */
+function mockReads(opts: {
+  chat: Record<string, unknown> | null
+  last?: { messageId: string; role: string }
+}) {
+  dbChainMockFns.limit.mockResolvedValueOnce(opts.chat ? [opts.chat] : [])
+  dbChainMockFns.limit.mockResolvedValueOnce(opts.last ? [opts.last] : [])
+}
+
+describe('finalizeAssistantTurn', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    // Drain the once-queue (clearAllMocks/resetDbChainMock don't), then restore defaults.
+    dbChainMockFns.limit.mockReset()
+    resetDbChainMock()
+  })
+
+  it('appends the assistant message when the user turn has no reply yet', async () => {
+    mockReads({
+      chat: { conversationId: 'user-1', workspaceId: 'ws-1', model: null },
+      last: { messageId: 'user-1', role: 'user' },
+    })
+
+    const result = await finalizeAssistantTurn({
+      chatId: 'chat-1',
+      userMessageId: 'user-1',
+      assistantMessage,
+    })
+
+    expect(result.appendedAssistant).toBe(true)
+    const updateArg = dbChainMockFns.set.mock.calls[0]?.[0] as Record<string, unknown>
+    expect(updateArg).toEqual(
+      expect.objectContaining({ updatedAt: expect.any(Date), conversationId: null })
+    )
+    expect(Object.hasOwn(updateArg, 'messages')).toBe(false)
+    expect(dbChainMockFns.where).toHaveBeenCalledWith(eq(copilotChats.id, 'chat-1'))
+    expect(mockAppendCopilotChatMessages).toHaveBeenCalledTimes(1)
+    expect(mockAppendCopilotChatMessages).toHaveBeenCalledWith(
+      'chat-1',
+      [assistantMessage],
+      { streamId: 'user-1', chatModel: null },
+      expect.anything()
+    )
+  })
+
+  it('only clears the active stream marker when a response is already persisted', async () => {
+    mockReads({
+      chat: { conversationId: 'user-1', workspaceId: 'ws-1', model: null },
+      last: { messageId: 'assistant-1', role: 'assistant' },
+    })
+
+    const result = await finalizeAssistantTurn({
+      chatId: 'chat-1',
+      userMessageId: 'user-1',
+      assistantMessage: { ...assistantMessage, id: 'assistant-2' },
+    })
+
+    expect(result.outcome).toBe('assistant_already_persisted')
+    const updateArg = dbChainMockFns.set.mock.calls[0]?.[0] as Record<string, unknown>
+    expect(updateArg).toEqual(
+      expect.objectContaining({ updatedAt: expect.any(Date), conversationId: null })
+    )
+    expect(Object.hasOwn(updateArg, 'messages')).toBe(false)
+    expect(mockAppendCopilotChatMessages).not.toHaveBeenCalled()
+  })
+
+  it('appends a stopped assistant when the stream marker was already cleared', async () => {
+    mockReads({
+      chat: { conversationId: null, workspaceId: 'ws-1', model: null },
+      last: { messageId: 'user-1', role: 'user' },
+    })
+
+    const result = await finalizeAssistantTurn({
+      chatId: 'chat-1',
+      userMessageId: 'user-1',
+      streamMarkerPolicy: 'active-or-cleared',
+      assistantMessage,
+    })
+
+    expect(result.appendedAssistant).toBe(true)
+    expect(mockAppendCopilotChatMessages).toHaveBeenCalledTimes(1)
+  })
+
+  it('does not append on a cleared marker unless the policy allows it', async () => {
+    mockReads({ chat: { conversationId: null, workspaceId: 'ws-1', model: null } })
+
+    const result = await finalizeAssistantTurn({
+      chatId: 'chat-1',
+      userMessageId: 'user-1',
+      assistantMessage,
+    })
+
+    expect(result.updated).toBe(false)
+    expect(dbChainMockFns.set).not.toHaveBeenCalled()
+    expect(mockAppendCopilotChatMessages).not.toHaveBeenCalled()
+  })
+
+  it('reports already persisted when a cleared marker races with a duplicate stop', async () => {
+    mockReads({
+      chat: { conversationId: null, workspaceId: 'ws-1', model: null },
+      last: { messageId: 'assistant-1', role: 'assistant' },
+    })
+
+    const result = await finalizeAssistantTurn({
+      chatId: 'chat-1',
+      userMessageId: 'user-1',
+      streamMarkerPolicy: 'active-or-cleared',
+      assistantMessage: { ...assistantMessage, id: 'assistant-2' },
+    })
+
+    expect(result.updated).toBe(false)
+    expect(result.outcome).toBe('assistant_already_persisted')
+    expect(dbChainMockFns.set).not.toHaveBeenCalled()
+    expect(mockAppendCopilotChatMessages).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/copilot/chat/terminal-state.ts

@@ -0,0 +1,160 @@
+import { db } from '@sim/db'
+import { copilotChats, copilotMessages } from '@sim/db/schema'
+import { and, desc, eq, isNull, sql } from 'drizzle-orm'
+import { appendCopilotChatMessages } from '@/lib/copilot/chat/messages-store'
+import type { PersistedMessage } from '@/lib/copilot/chat/persisted-message'
+import { CopilotChatFinalizeOutcome } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { withCopilotSpan } from '@/lib/copilot/request/otel'
+
+type StreamMarkerPolicy = 'active-only' | 'active-or-cleared'
+
+interface FinalizeAssistantTurnParams {
+  chatId: string
+  userMessageId: string
+  userId?: string
+  assistantMessage?: PersistedMessage
+  streamMarkerPolicy?: StreamMarkerPolicy
+}
+
+export interface FinalizeAssistantTurnResult {
+  found: boolean
+  updated: boolean
+  appendedAssistant: boolean
+  workspaceId?: string | null
+  outcome: (typeof CopilotChatFinalizeOutcome)[keyof typeof CopilotChatFinalizeOutcome]
+}
+
+/**
+ * Clear the active stream marker for a chat and optionally append the assistant
+ * message if a response has not already been persisted immediately after the
+ * triggering user message.
+ */
+export async function finalizeAssistantTurn({
+  chatId,
+  userMessageId,
+  userId,
+  assistantMessage,
+  streamMarkerPolicy = 'active-only',
+}: FinalizeAssistantTurnParams): Promise<FinalizeAssistantTurnResult> {
+  return withCopilotSpan(
+    TraceSpan.CopilotChatFinalizeAssistantTurn,
+    {
+      [TraceAttr.DbSystem]: 'postgresql',
+      [TraceAttr.DbSqlTable]: 'copilot_chats',
+      [TraceAttr.ChatId]: chatId,
+      [TraceAttr.ChatUserMessageId]: userMessageId,
+      [TraceAttr.ChatHasAssistantMessage]: !!assistantMessage,
+    },
+    async (span) => {
+      const result = await db.transaction(async (tx) => {
+        const where = userId
+          ? and(eq(copilotChats.id, chatId), eq(copilotChats.userId, userId))
+          : eq(copilotChats.id, chatId)
+        const [row] = await tx
+          .select({
+            conversationId: copilotChats.conversationId,
+            workspaceId: copilotChats.workspaceId,
+            model: copilotChats.model,
+          })
+          .from(copilotChats)
+          .where(where)
+          .for('update')
+          .limit(1)
+
+        if (!row) {
+          return {
+            found: false,
+            updated: false,
+            appendedAssistant: false,
+            workspaceId: null,
+            outcome: CopilotChatFinalizeOutcome.StaleUserMessage,
+          }
+        }
+
+        const chatModel = row.model ?? null
+
+        const markerMatches = row.conversationId === userMessageId
+        const markerAlreadyCleared = row.conversationId === null
+        const ownsTurn =
+          markerMatches || (streamMarkerPolicy === 'active-or-cleared' && markerAlreadyCleared)
+        if (!ownsTurn) {
+          return {
+            found: true,
+            updated: false,
+            appendedAssistant: false,
+            workspaceId: row.workspaceId,
+            outcome: CopilotChatFinalizeOutcome.StaleUserMessage,
+          }
+        }
+
+        // Append only when the user message is still the last row: anything
+        // after it means the turn already has a response (dedup under the lock).
+        const [lastMessage] = await tx
+          .select({ messageId: copilotMessages.messageId, role: copilotMessages.role })
+          .from(copilotMessages)
+          .where(and(eq(copilotMessages.chatId, chatId), isNull(copilotMessages.deletedAt)))
+          .orderBy(
+            sql`${copilotMessages.seq} desc nulls last`,
+            desc(copilotMessages.createdAt),
+            desc(copilotMessages.id)
+          )
+          .limit(1)
+        const canAppendAssistant = lastMessage?.messageId === userMessageId
+        const alreadyHasResponse = lastMessage?.role === 'assistant'
+
+        const updateWhere = userId
+          ? and(eq(copilotChats.id, chatId), eq(copilotChats.userId, userId))
+          : eq(copilotChats.id, chatId)
+        const baseUpdate = {
+          conversationId: null,
+          updatedAt: new Date(),
+        }
+
+        if (assistantMessage && canAppendAssistant) {
+          await tx.update(copilotChats).set(baseUpdate).where(updateWhere)
+          await appendCopilotChatMessages(
+            chatId,
+            [assistantMessage],
+            { streamId: userMessageId, chatModel },
+            tx
+          )
+          return {
+            found: true,
+            updated: true,
+            appendedAssistant: true,
+            workspaceId: row.workspaceId,
+            outcome: CopilotChatFinalizeOutcome.AppendedAssistant,
+          }
+        }
+
+        if (markerMatches) {
+          await tx.update(copilotChats).set(baseUpdate).where(updateWhere)
+          return {
+            found: true,
+            updated: true,
+            appendedAssistant: false,
+            workspaceId: row.workspaceId,
+            outcome: assistantMessage
+              ? CopilotChatFinalizeOutcome.AssistantAlreadyPersisted
+              : CopilotChatFinalizeOutcome.ClearedStreamMarkerOnly,
+          }
+        }
+
+        return {
+          found: true,
+          updated: false,
+          appendedAssistant: false,
+          workspaceId: row.workspaceId,
+          outcome: alreadyHasResponse
+            ? CopilotChatFinalizeOutcome.AssistantAlreadyPersisted
+            : CopilotChatFinalizeOutcome.StaleUserMessage,
+        }
+      })
+
+      span.setAttribute(TraceAttr.ChatFinalizeOutcome, result.outcome)
+      return result
+    }
+  )
+}

apps/sim/lib/copilot/chat/workspace-context.test.ts

@@ -0,0 +1,295 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => ({ db: {} }))
+vi.mock('@sim/db/schema', () => ({
+  knowledgeBase: {},
+  knowledgeConnector: {},
+  mcpServers: {},
+  userTableDefinitions: {},
+  userTableRows: {},
+  workflow: {},
+  workflowFolder: {},
+  workflowSchedule: {},
+}))
+
+import { canonicalWorkflowVfsDir } from '@/lib/copilot/vfs/path-utils'
+import { buildVfsSnapshot, buildWorkspaceMd, type WorkspaceMdData } from './workspace-context'
+
+function baseData(overrides: Partial<WorkspaceMdData> = {}): WorkspaceMdData {
+  return {
+    workspace: { id: 'ws-1', name: 'WS', ownerId: 'u-1' },
+    members: [],
+    workflows: [],
+    knowledgeBases: [],
+    tables: [],
+    files: [],
+    oauthIntegrations: [],
+    envVariables: [],
+    ...overrides,
+  }
+}
+
+describe('buildWorkspaceMd - workflow VFS state paths', () => {
+  // `workflows[].folderPath` arrives ALREADY per-segment percent-encoded (it is
+  // the value from buildVfsFolderPathMap / resolveFolderPath that also builds the
+  // stored VFS keys). The advertised path must not re-encode it.
+  it('emits a single-encoded state path for a folder name with a space', () => {
+    const md = buildWorkspaceMd(
+      baseData({
+        workflows: [
+          { id: 'wf-1', name: 'The Elder', isDeployed: false, folderPath: 'The%20Elder' },
+        ],
+      })
+    )
+
+    expect(md).toContain('workflows/The%20Elder/The%20Elder/state.json')
+    // The exact double-encoding regression: `%20` -> `%2520`.
+    expect(md).not.toContain('The%2520Elder')
+  })
+
+  it('matches the canonical VFS dir helper the materializer/pointers use', () => {
+    const folderPath = 'My%20Folder/Sub%20Folder'
+    const md = buildWorkspaceMd(
+      baseData({
+        workflows: [{ id: 'wf-1', name: 'My Flow', isDeployed: false, folderPath }],
+      })
+    )
+
+    const expected = `${canonicalWorkflowVfsDir({ name: 'My Flow', folderPath })}/state.json`
+    expect(expected).toBe('workflows/My%20Folder/Sub%20Folder/My%20Flow/state.json')
+    expect(md).toContain(expected)
+  })
+
+  it('advertises canonical encoded VFS paths for root-level workflows', () => {
+    const md = buildWorkspaceMd(
+      baseData({
+        workflows: [{ id: 'wf-1', name: 'Root Flow', isDeployed: false, folderPath: null }],
+      })
+    )
+
+    expect(md).toContain('VFS dir: `workflows/Root%20Flow`')
+    expect(md).toContain('VFS state path: `workflows/Root%20Flow/state.json`')
+  })
+})
+
+describe('buildWorkspaceMd - connected integrations / credentials', () => {
+  it('lists each connected account with its credentialId and never leaks tokens', () => {
+    const md = buildWorkspaceMd(
+      baseData({
+        oauthIntegrations: [
+          {
+            id: 'cred-abc',
+            providerId: 'google-email',
+            displayName: 'alice@example.com',
+            role: 'admin',
+          },
+          { id: 'cred-def', providerId: 'slack', displayName: 'Workspace Bot', role: 'member' },
+        ],
+      })
+    )
+
+    // credentialId must be present so the superagent can pass it without reading credentials.json.
+    expect(md).toContain('credentialId: `cred-abc`')
+    expect(md).toContain('credentialId: `cred-def`')
+    expect(md).toContain('google-email')
+    expect(md).toContain('slack')
+
+    // No OAuth secrets/tokens may ever appear in the workspace context.
+    for (const secret of [
+      'accessToken',
+      'refreshToken',
+      'idToken',
+      'clientSecret',
+      'access_token',
+      'refresh_token',
+    ]) {
+      expect(md).not.toContain(secret)
+    }
+  })
+
+  it('renders (none) when no integrations are connected', () => {
+    const md = buildWorkspaceMd(baseData({ oauthIntegrations: [] }))
+    expect(md).toContain('## Connected Integrations\n(none)')
+  })
+})
+
+describe('buildWorkspaceMd - determinism (prompt-cache stability)', () => {
+  it('is byte-identical regardless of input row order', () => {
+    const a = buildWorkspaceMd(
+      baseData({
+        members: [
+          { name: 'Bob', email: 'bob@x.com', permissionType: 'admin' },
+          { name: 'Amy', email: 'amy@x.com', permissionType: 'write' },
+        ],
+        workflows: [
+          { id: 'wf-2', name: 'Zeta', isDeployed: false, folderPath: null },
+          { id: 'wf-1', name: 'Alpha', isDeployed: true, folderPath: null },
+        ],
+        tables: [
+          { id: 't-2', name: 'Orders', description: null, rowCount: 5 },
+          { id: 't-1', name: 'Customers', description: null, rowCount: 9 },
+        ],
+        knowledgeBases: [
+          { id: 'kb-2', name: 'Docs', connectorTypes: ['notion', 'github'] },
+          { id: 'kb-1', name: 'Articles', connectorTypes: ['github', 'notion'] },
+        ],
+        oauthIntegrations: [
+          { id: 'c-2', providerId: 'slack', displayName: null, role: null },
+          { id: 'c-1', providerId: 'github', displayName: null, role: null },
+        ],
+        envVariables: ['ZED', 'API_KEY'],
+        customTools: [
+          { id: 'ct-2', name: 'Beta Tool' },
+          { id: 'ct-1', name: 'Alpha Tool' },
+        ],
+        mcpServers: [
+          { id: 'mcp-2', name: 'Zulu', url: null, enabled: false },
+          { id: 'mcp-1', name: 'Mike', url: 'https://x', enabled: true },
+        ],
+        skills: [
+          { id: 'sk-2', name: 'Writer', description: 'writes' },
+          { id: 'sk-1', name: 'Editor', description: 'edits' },
+        ],
+        jobs: [
+          {
+            id: 'j-2',
+            title: 'Nightly',
+            prompt: 'run nightly',
+            cronExpression: '0 0 * * *',
+            status: 'active',
+            lifecycle: 'persistent',
+            sourceTaskName: null,
+          },
+          {
+            id: 'j-1',
+            title: 'Hourly',
+            prompt: 'run hourly',
+            cronExpression: '0 * * * *',
+            status: 'active',
+            lifecycle: 'persistent',
+            sourceTaskName: null,
+          },
+        ],
+      })
+    )
+    const b = buildWorkspaceMd(
+      baseData({
+        members: [
+          { name: 'Amy', email: 'amy@x.com', permissionType: 'write' },
+          { name: 'Bob', email: 'bob@x.com', permissionType: 'admin' },
+        ],
+        workflows: [
+          { id: 'wf-1', name: 'Alpha', isDeployed: true, folderPath: null },
+          { id: 'wf-2', name: 'Zeta', isDeployed: false, folderPath: null },
+        ],
+        tables: [
+          { id: 't-1', name: 'Customers', description: null, rowCount: 9 },
+          { id: 't-2', name: 'Orders', description: null, rowCount: 5 },
+        ],
+        knowledgeBases: [
+          { id: 'kb-1', name: 'Articles', connectorTypes: ['notion', 'github'] },
+          { id: 'kb-2', name: 'Docs', connectorTypes: ['github', 'notion'] },
+        ],
+        oauthIntegrations: [
+          { id: 'c-1', providerId: 'github', displayName: null, role: null },
+          { id: 'c-2', providerId: 'slack', displayName: null, role: null },
+        ],
+        envVariables: ['API_KEY', 'ZED'],
+        customTools: [
+          { id: 'ct-1', name: 'Alpha Tool' },
+          { id: 'ct-2', name: 'Beta Tool' },
+        ],
+        mcpServers: [
+          { id: 'mcp-1', name: 'Mike', url: 'https://x', enabled: true },
+          { id: 'mcp-2', name: 'Zulu', url: null, enabled: false },
+        ],
+        skills: [
+          { id: 'sk-1', name: 'Editor', description: 'edits' },
+          { id: 'sk-2', name: 'Writer', description: 'writes' },
+        ],
+        jobs: [
+          {
+            id: 'j-1',
+            title: 'Hourly',
+            prompt: 'run hourly',
+            cronExpression: '0 * * * *',
+            status: 'active',
+            lifecycle: 'persistent',
+            sourceTaskName: null,
+          },
+          {
+            id: 'j-2',
+            title: 'Nightly',
+            prompt: 'run nightly',
+            cronExpression: '0 0 * * *',
+            status: 'active',
+            lifecycle: 'persistent',
+            sourceTaskName: null,
+          },
+        ],
+      })
+    )
+    expect(a).toBe(b)
+  })
+
+  it('ignores volatile workflow run timestamps', () => {
+    const withRun = buildWorkspaceMd(
+      baseData({
+        workflows: [
+          {
+            id: 'wf-1',
+            name: 'Alpha',
+            isDeployed: false,
+            folderPath: null,
+            lastRunAt: new Date('2026-06-18T12:00:00Z'),
+          },
+        ],
+      })
+    )
+    const withoutRun = buildWorkspaceMd(
+      baseData({
+        workflows: [{ id: 'wf-1', name: 'Alpha', isDeployed: false, folderPath: null }],
+      })
+    )
+    expect(withRun).toBe(withoutRun)
+    expect(withRun).not.toContain('last run')
+  })
+
+  it('ignores volatile table row counts', () => {
+    const a = buildWorkspaceMd(
+      baseData({ tables: [{ id: 't-1', name: 'Customers', description: null, rowCount: 1 }] })
+    )
+    const b = buildWorkspaceMd(
+      baseData({ tables: [{ id: 't-1', name: 'Customers', description: null, rowCount: 9999 }] })
+    )
+    expect(a).toBe(b)
+    expect(a).not.toContain('rows')
+  })
+})
+
+describe('custom blocks', () => {
+  const customBlocks = [
+    { type: 'custom_block_abc', name: 'Invoice Parser', description: 'Parses invoices' },
+  ]
+
+  it('renders a Custom Blocks section in the workspace markdown', () => {
+    const md = buildWorkspaceMd(baseData({ customBlocks }))
+    expect(md).toContain('## Custom Blocks (1)')
+    expect(md).toContain('- **Invoice Parser** (custom_block_abc) — Parses invoices')
+  })
+
+  it('omits the section when there are no custom blocks', () => {
+    expect(buildWorkspaceMd(baseData())).not.toContain('## Custom Blocks')
+  })
+
+  it('never leaks custom blocks into the typed snapshot Go diffs (diff-safety)', () => {
+    const withBlocks = buildVfsSnapshot(baseData({ customBlocks }))
+    const without = buildVfsSnapshot(baseData())
+    expect('customBlocks' in withBlocks).toBe(false)
+    expect(JSON.stringify(withBlocks)).toBe(JSON.stringify(without))
+  })
+})

apps/sim/lib/copilot/chat/workspace-context.ts

@@ -0,0 +1,659 @@
+import { db } from '@sim/db'
+import {
+  knowledgeBase,
+  knowledgeConnector,
+  mcpServers,
+  userTableDefinitions,
+  workflow,
+  workflowFolder,
+  workflowSchedule,
+} from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { truncate } from '@sim/utils/string'
+import { and, eq, inArray, isNull } from 'drizzle-orm'
+import type {
+  VfsSnapshotV1,
+  VfsSnapshotV1Job,
+  VfsSnapshotV1Workflow,
+} from '@/lib/copilot/generated/vfs-snapshot-v1'
+import { normalizeVfsSegment } from '@/lib/copilot/vfs/normalize-segment'
+import { canonicalWorkflowVfsDir, canonicalWorkspaceFilePath } from '@/lib/copilot/vfs/path-utils'
+import { getAccessibleOAuthCredentials } from '@/lib/credentials/environment'
+import { listWorkspaceFiles } from '@/lib/uploads/contexts/workspace'
+import { listCustomBlockSummariesForWorkspace } from '@/lib/workflows/custom-blocks/operations'
+import { listCustomTools } from '@/lib/workflows/custom-tools/operations'
+import { listSkills } from '@/lib/workflows/skills/operations'
+import {
+  assertActiveWorkspaceAccess,
+  getUsersWithPermissions,
+  getWorkspaceWithOwner,
+} from '@/lib/workspaces/permissions/utils'
+
+const logger = createLogger('WorkspaceContext')
+
+const PROVIDER_SERVICES: Record<string, string[]> = {
+  google: ['Gmail', 'Sheets', 'Calendar', 'Drive'],
+  'google-service-account': ['Gmail', 'Sheets', 'Calendar', 'Drive'],
+  slack: ['Slack'],
+  github: ['GitHub'],
+  microsoft: ['Outlook', 'OneDrive'],
+  linear: ['Linear'],
+  notion: ['Notion'],
+  stripe: ['Stripe'],
+  airtable: ['Airtable'],
+  jira: ['Jira'],
+  confluence: ['Confluence'],
+}
+
+export interface WorkspaceMdData {
+  workspace: { id: string; name: string; ownerId: string } | null
+  members: Array<{ name: string; email: string; permissionType: string }>
+  workflows: Array<{
+    id: string
+    name: string
+    description?: string | null
+    isDeployed: boolean
+    lastRunAt?: Date | null
+    folderPath?: string | null
+  }>
+  knowledgeBases: Array<{
+    id: string
+    name: string
+    description?: string | null
+    connectorTypes?: string[]
+  }>
+  // rowCount is no longer rendered (it is volatile and would bust the cached
+  // prompt prefix); kept optional so callers that still have it cheaply (the VFS
+  // materializer via listTables) need not change, while generateWorkspaceContext
+  // skips the per-table COUNT query entirely.
+  tables: Array<{ id: string; name: string; description?: string | null; rowCount?: number }>
+  files: Array<{ id: string; name: string; type: string; size: number; folderPath?: string | null }>
+  oauthIntegrations: Array<{
+    id: string
+    providerId: string
+    displayName?: string | null
+    role?: string | null
+  }>
+  envVariables: string[]
+  tasks?: Array<{ id: string; title: string; updatedAt: Date }>
+  customTools?: Array<{ id: string; name: string }>
+  customBlocks?: Array<{ type: string; name: string; description?: string }>
+  mcpServers?: Array<{ id: string; name: string; url?: string | null; enabled: boolean }>
+  skills?: Array<{ id: string; name: string; description: string }>
+  jobs?: Array<{
+    id: string
+    title: string | null
+    prompt: string
+    cronExpression: string | null
+    status: string
+    lifecycle: string
+    sourceTaskName: string | null
+  }>
+}
+
+/**
+ * Deterministic string ordering. The workspace inventory is placed in the
+ * prompt-cache prefix (mothership), so its bytes must be identical for identical
+ * workspace state regardless of DB row order — otherwise the cache silently
+ * busts every turn. `localeCompare` with a pinned locale gives stable, readable
+ * ordering across Sim instances (all run the same Node/ICU build).
+ */
+function stableCompare(a: string, b: string): number {
+  return a.localeCompare(b, 'en')
+}
+
+/** Stable order by display name, tie-broken by id, for inventory listings. */
+function byNameThenId(a: { name: string; id: string }, b: { name: string; id: string }): number {
+  return stableCompare(a.name, b.name) || stableCompare(a.id, b.id)
+}
+
+/**
+ * Pure formatting: build WORKSPACE.md content from pre-fetched data.
+ * No DB access — callers are responsible for providing the data.
+ *
+ * Output is deterministic: every collection is sorted by a stable key and
+ * volatile fields (run timestamps, mutable row counts) are omitted, so the
+ * rendered inventory only changes when the workspace structurally changes. This
+ * is what lets the mothership cache it in the prompt prefix across turns.
+ */
+export function buildWorkspaceMd(data: WorkspaceMdData): string {
+  const sections: string[] = []
+
+  if (data.workspace) {
+    sections.push(
+      `## Workspace\n- **Name**: ${data.workspace.name}\n- **ID**: ${data.workspace.id}\n- **Owner**: ${data.workspace.ownerId}`
+    )
+  }
+
+  if (data.members.length > 0) {
+    const lines = [...data.members]
+      .sort((a, b) => stableCompare(a.email, b.email))
+      .map((m) => {
+        const display = m.name ? `${m.name} (${m.email})` : m.email
+        return `- ${display} — ${m.permissionType}`
+      })
+    sections.push(`## Members\n${lines.join('\n')}`)
+  }
+
+  if (data.workflows.length > 0) {
+    const rootWorkflows: typeof data.workflows = []
+    const folderWorkflows = new Map<string, typeof data.workflows>()
+
+    for (const wf of data.workflows) {
+      if (wf.folderPath) {
+        const existing = folderWorkflows.get(wf.folderPath) ?? []
+        existing.push(wf)
+        folderWorkflows.set(wf.folderPath, existing)
+      } else {
+        rootWorkflows.push(wf)
+      }
+    }
+
+    const formatWf = (wf: (typeof data.workflows)[0], indent: string) => {
+      const parts = [`${indent}- **${wf.name}** (${wf.id})`]
+      const workflowDir = canonicalWorkflowVfsDir({ name: wf.name, folderPath: wf.folderPath })
+      parts.push(`${indent}  VFS dir: \`${workflowDir}\``)
+      parts.push(`${indent}  VFS state path: \`${workflowDir}/state.json\``)
+      if (wf.description) parts.push(`${indent}  ${wf.description}`)
+      // `deployed` is a structural flag (kept); `lastRunAt` is intentionally
+      // omitted — it changes on every run and would bust the cached prompt
+      // prefix that carries this inventory. Current run data lives in
+      // workflows/{name}/executions.json.
+      if (wf.isDeployed) parts[0] += ' — deployed'
+      return parts.join('\n')
+    }
+
+    const lines: string[] = []
+    lines.push(
+      'Use the canonical VFS dir/state path shown under each workflow. Paths are percent-encoded per segment; copy them verbatim and do not infer paths from display names.'
+    )
+    for (const wf of [...rootWorkflows].sort(byNameThenId)) {
+      lines.push(formatWf(wf, ''))
+    }
+    const sortedFolders = [...folderWorkflows.entries()].sort((a, b) => stableCompare(a[0], b[0]))
+    for (const [folder, wfs] of sortedFolders) {
+      lines.push(`- 📁 **${folder}/**`)
+      for (const wf of [...wfs].sort(byNameThenId)) {
+        lines.push(formatWf(wf, '  '))
+      }
+    }
+    sections.push(`## Workflows (${data.workflows.length})\n${lines.join('\n')}`)
+  } else {
+    sections.push('## Workflows (0)\n(none)')
+  }
+
+  if (data.knowledgeBases.length > 0) {
+    const lines = [...data.knowledgeBases].sort(byNameThenId).map((kb) => {
+      let line = `- **${kb.name}** (${kb.id})`
+      if (kb.description) line += ` — ${kb.description}`
+      if (kb.connectorTypes && kb.connectorTypes.length > 0) {
+        line += ` | connectors: ${[...kb.connectorTypes].sort(stableCompare).join(', ')}`
+      }
+      return line
+    })
+    sections.push(`## Knowledge Bases (${data.knowledgeBases.length})\n${lines.join('\n')}`)
+  } else {
+    sections.push('## Knowledge Bases (0)\n(none)')
+  }
+
+  if (data.tables.length > 0) {
+    // rowCount is omitted: it changes on every row write and would bust the
+    // cached prompt prefix. Live counts are in tables/{name}/meta.json.
+    const lines = [...data.tables].sort(byNameThenId).map((t) => {
+      let line = `- **${t.name}** (${t.id})`
+      if (t.description) line += ` — ${t.description}`
+      return line
+    })
+    sections.push(`## Tables (${data.tables.length})\n${lines.join('\n')}`)
+  } else {
+    sections.push('## Tables (0)\n(none)')
+  }
+
+  if (data.files.length > 0) {
+    const rootFiles: typeof data.files = []
+    const folderFiles = new Map<string, typeof data.files>()
+    for (const f of data.files) {
+      if (f.folderPath) {
+        const existing = folderFiles.get(f.folderPath) ?? []
+        existing.push(f)
+        folderFiles.set(f.folderPath, existing)
+      } else {
+        rootFiles.push(f)
+      }
+    }
+    const fileLine = (f: (typeof data.files)[0], indent: string) => {
+      const vfsPath = canonicalWorkspaceFilePath({ folderPath: f.folderPath, name: f.name })
+      return `${indent}- **${f.name}** (${f.id}) — ${f.type}, ${formatSize(f.size)} — \`${vfsPath}\``
+    }
+    const lines: string[] = [
+      'Read or edit a file by the exact VFS path shown in backticks below — copy it verbatim (it is already percent-encoded) and append `/content` to read the contents. Do not retype the display name or re-encode the path.',
+    ]
+    for (const f of [...rootFiles].sort(byNameThenId)) {
+      lines.push(fileLine(f, ''))
+    }
+    const sortedFolders = [...folderFiles.entries()].sort((a, b) => stableCompare(a[0], b[0]))
+    for (const [folder, folderFileList] of sortedFolders) {
+      lines.push(`- 📁 **${folder}/**`)
+      for (const f of [...folderFileList].sort(byNameThenId)) {
+        lines.push(fileLine(f, '  '))
+      }
+    }
+    sections.push(`## Files (${data.files.length})\n${lines.join('\n')}`)
+  } else {
+    sections.push('## Files (0)\n(none)')
+  }
+
+  if (data.oauthIntegrations.length > 0) {
+    const lines = [...data.oauthIntegrations]
+      .sort((a, b) => stableCompare(a.providerId, b.providerId) || stableCompare(a.id, b.id))
+      .map((c) => {
+        const services = PROVIDER_SERVICES[c.providerId]
+        const svc = services ? ` (${services.join(', ')})` : ''
+        const who = c.displayName ? ` — ${c.displayName}` : ''
+        const role = c.role ? `, ${c.role}` : ''
+        return `- ${c.providerId}${svc}${who}${role} — credentialId: \`${c.id}\``
+      })
+    sections.push(
+      `## Connected Integrations\nPass these credentialId values directly on OAuth tool calls — no need to read environment/credentials.json for them.\n${lines.join('\n')}`
+    )
+  } else {
+    sections.push('## Connected Integrations\n(none)')
+  }
+
+  if (data.envVariables.length > 0) {
+    const lines = [...data.envVariables].sort(stableCompare).map((v) => `- ${v}`)
+    sections.push(`## Environment Variables (${data.envVariables.length})\n${lines.join('\n')}`)
+  }
+
+  if (data.customTools && data.customTools.length > 0) {
+    const lines = [...data.customTools].sort(byNameThenId).map((t) => `- **${t.name}** (${t.id})`)
+    sections.push(`## Custom Tools (${data.customTools.length})\n${lines.join('\n')}`)
+  }
+
+  if (data.customBlocks && data.customBlocks.length > 0) {
+    const lines = [...data.customBlocks]
+      .sort((a, b) => a.name.localeCompare(b.name))
+      .map((b) => `- **${b.name}** (${b.type})${b.description ? ` — ${b.description}` : ''}`)
+    sections.push(`## Custom Blocks (${data.customBlocks.length})\n${lines.join('\n')}`)
+  }
+
+  if (data.mcpServers && data.mcpServers.length > 0) {
+    const lines = [...data.mcpServers].sort(byNameThenId).map((s) => {
+      const status = s.enabled ? 'enabled' : 'disabled'
+      return `- **${s.name}** (${s.id}) — ${status}${s.url ? `, ${s.url}` : ''}`
+    })
+    sections.push(`## MCP Servers (${data.mcpServers.length})\n${lines.join('\n')}`)
+  }
+
+  if (data.skills && data.skills.length > 0) {
+    const lines = [...data.skills]
+      .sort(byNameThenId)
+      .map((s) => `- **${s.name}** (${s.id}) — ${s.description}`)
+    sections.push(
+      `## Skills (${data.skills.length})\n` +
+        'To use a skill, call the load_user_skill tool with its name to load the full instructions, then follow them. The descriptions below only say when each skill applies — they are not the instructions.\n' +
+        lines.join('\n')
+    )
+  }
+
+  if (data.jobs && data.jobs.length > 0) {
+    const lines = [...data.jobs]
+      .sort((a, b) => stableCompare(a.title || a.id, b.title || b.id) || stableCompare(a.id, b.id))
+      .map((j) => {
+        const displayName = j.title || j.id
+        let line = `- **${displayName}** (${j.id}) — ${j.status}`
+        if (j.lifecycle !== 'persistent') line += ` [${j.lifecycle}]`
+        if (j.cronExpression) line += `, cron: ${j.cronExpression}`
+        if (j.sourceTaskName) line += `, task: ${j.sourceTaskName}`
+        const promptPreview = j.prompt.length > 80 ? truncate(j.prompt, 77) : j.prompt
+        line += `\n  ${promptPreview}`
+        return line
+      })
+    sections.push(`## Jobs (${data.jobs.length})\n${lines.join('\n')}`)
+  }
+
+  return sections.join('\n\n')
+}
+
+export function buildWorkspaceContextMd(data: WorkspaceMdData): string {
+  return ['# Workspace Context', '', buildWorkspaceMd(data)].join('\n\n')
+}
+
+/**
+ * Generate WORKSPACE.md content from actual database state.
+ * Served as a top-level VFS file. The Go system prompt keeps only stable
+ * discovery rules; the LLM reads dynamic workspace state from VFS files.
+ * The LLM never writes this file directly.
+ */
+// Fetch + assemble the workspace inventory data once, from the PRIMARY db
+// (read-your-writes: a just-edited workflow is visible immediately, so the
+// injected snapshot can't lag behind a `glob`). Both the markdown inventory and
+// the typed VFS snapshot are built from this single fetch. Returns null when the
+// workspace is unavailable or a fetch fails.
+async function buildWorkspaceMdData(
+  workspaceId: string,
+  userId: string
+): Promise<WorkspaceMdData | null> {
+  try {
+    await assertActiveWorkspaceAccess(workspaceId, userId)
+    const wsRow = await getWorkspaceWithOwner(workspaceId)
+    if (!wsRow) {
+      return null
+    }
+
+    const [
+      members,
+      workflows,
+      folderRows,
+      kbs,
+      tables,
+      files,
+      credentials,
+      customTools,
+      mcpServerRows,
+      skillRows,
+      jobRows,
+      customBlockSummaries,
+    ] = await Promise.all([
+      getUsersWithPermissions(workspaceId),
+
+      db
+        .select({
+          id: workflow.id,
+          name: workflow.name,
+          description: workflow.description,
+          isDeployed: workflow.isDeployed,
+          lastRunAt: workflow.lastRunAt,
+          folderId: workflow.folderId,
+        })
+        .from(workflow)
+        .where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt))),
+
+      db
+        .select({
+          id: workflowFolder.id,
+          name: workflowFolder.name,
+          parentId: workflowFolder.parentId,
+        })
+        .from(workflowFolder)
+        .where(and(eq(workflowFolder.workspaceId, workspaceId), isNull(workflowFolder.archivedAt))),
+
+      db
+        .select({
+          id: knowledgeBase.id,
+          name: knowledgeBase.name,
+          description: knowledgeBase.description,
+        })
+        .from(knowledgeBase)
+        .where(and(eq(knowledgeBase.workspaceId, workspaceId), isNull(knowledgeBase.deletedAt))),
+
+      db
+        .select({
+          id: userTableDefinitions.id,
+          name: userTableDefinitions.name,
+          description: userTableDefinitions.description,
+        })
+        .from(userTableDefinitions)
+        .where(
+          and(
+            eq(userTableDefinitions.workspaceId, workspaceId),
+            isNull(userTableDefinitions.archivedAt)
+          )
+        ),
+
+      listWorkspaceFiles(workspaceId),
+
+      getAccessibleOAuthCredentials(workspaceId, userId),
+
+      listCustomTools({ userId, workspaceId }),
+
+      db
+        .select({
+          id: mcpServers.id,
+          name: mcpServers.name,
+          url: mcpServers.url,
+          enabled: mcpServers.enabled,
+        })
+        .from(mcpServers)
+        .where(and(eq(mcpServers.workspaceId, workspaceId), isNull(mcpServers.deletedAt))),
+
+      listSkills({ workspaceId, includeBuiltins: false }),
+
+      db
+        .select({
+          id: workflowSchedule.id,
+          jobTitle: workflowSchedule.jobTitle,
+          prompt: workflowSchedule.prompt,
+          cronExpression: workflowSchedule.cronExpression,
+          status: workflowSchedule.status,
+          lifecycle: workflowSchedule.lifecycle,
+          sourceTaskName: workflowSchedule.sourceTaskName,
+        })
+        .from(workflowSchedule)
+        .where(
+          and(
+            eq(workflowSchedule.sourceWorkspaceId, workspaceId),
+            eq(workflowSchedule.sourceType, 'job'),
+            isNull(workflowSchedule.archivedAt)
+          )
+        ),
+
+      listCustomBlockSummariesForWorkspace(workspaceId),
+    ])
+
+    const kbIds = kbs.map((kb) => kb.id)
+    const connectorRows =
+      kbIds.length > 0
+        ? await db
+            .select({
+              knowledgeBaseId: knowledgeConnector.knowledgeBaseId,
+              connectorType: knowledgeConnector.connectorType,
+            })
+            .from(knowledgeConnector)
+            .where(
+              and(
+                inArray(knowledgeConnector.knowledgeBaseId, kbIds),
+                isNull(knowledgeConnector.archivedAt),
+                isNull(knowledgeConnector.deletedAt)
+              )
+            )
+        : []
+    const connectorTypesByKb = new Map<string, string[]>()
+    for (const row of connectorRows) {
+      const types = connectorTypesByKb.get(row.knowledgeBaseId) ?? []
+      if (!types.includes(row.connectorType)) {
+        types.push(row.connectorType)
+      }
+      connectorTypesByKb.set(row.knowledgeBaseId, types)
+    }
+
+    const folderPathMap = new Map<string, string>()
+    const folderById = new Map(folderRows.map((f) => [f.id, f]))
+    function resolveFolderPath(id: string): string {
+      const cached = folderPathMap.get(id)
+      if (cached !== undefined) return cached
+      const folder = folderById.get(id)
+      if (!folder) return id
+      const parentPath = folder.parentId ? resolveFolderPath(folder.parentId) : ''
+      const normalizedName = normalizeVfsSegment(folder.name)
+      const path = parentPath ? `${parentPath}/${normalizedName}` : normalizedName
+      folderPathMap.set(id, path)
+      return path
+    }
+
+    return {
+      workspace: wsRow,
+      members,
+      workflows: workflows.map((wf) => ({
+        ...wf,
+        folderPath: wf.folderId ? resolveFolderPath(wf.folderId) : null,
+      })),
+      knowledgeBases: kbs.map((kb) => ({
+        ...kb,
+        // Sort connector types so the snapshot is order-stable: the DB query has
+        // no ORDER BY, and the Go delta engine compares item JSON byte-wise, so
+        // an unsorted (but unchanged) list would emit a spurious "modified"
+        // delta and needlessly bust the prompt cache.
+        connectorTypes: connectorTypesByKb.get(kb.id)?.sort(stableCompare),
+      })),
+      tables: tables.map((t) => ({ id: t.id, name: t.name, description: t.description })),
+      files: files.map((f) => ({
+        id: f.id,
+        name: f.name,
+        type: f.type,
+        size: f.size,
+        folderPath: f.folderPath ?? null,
+      })),
+      oauthIntegrations: credentials.map((c) => ({
+        id: c.id,
+        providerId: c.providerId,
+        displayName: c.displayName,
+        role: c.role,
+      })),
+      envVariables: [],
+      customTools: customTools.map((t) => ({ id: t.id, name: t.title })),
+      customBlocks: customBlockSummaries,
+      mcpServers: mcpServerRows,
+      skills: skillRows.map((s) => ({ id: s.id, name: s.name, description: s.description })),
+      jobs: jobRows
+        .filter((j) => j.status !== 'completed')
+        .map((j) => ({
+          id: j.id,
+          title: j.jobTitle,
+          prompt: j.prompt || '',
+          cronExpression: j.cronExpression,
+          status: j.status,
+          lifecycle: j.lifecycle,
+          sourceTaskName: j.sourceTaskName,
+        })),
+    }
+  } catch (err) {
+    logger.error('Failed to build workspace data', {
+      workspaceId,
+      error: toError(err).message,
+    })
+    return null
+  }
+}
+
+const WORKSPACE_CONTEXT_UNAVAILABLE_MD =
+  '## Workspace\n(unavailable)\n\n## Workflows\n(unavailable)\n\n## Knowledge Bases\n(unavailable)\n\n## Tables\n(unavailable)\n\n## Files\n(unavailable)\n\n## Connected Integrations\n(unavailable)'
+
+/**
+ * Generate WORKSPACE.md markdown from current DB state (primary db). The LLM
+ * reads dynamic workspace state from VFS files; it never writes this file.
+ */
+export async function generateWorkspaceContext(
+  workspaceId: string,
+  userId: string
+): Promise<string> {
+  const data = await buildWorkspaceMdData(workspaceId, userId)
+  return data ? buildWorkspaceMd(data) : WORKSPACE_CONTEXT_UNAVAILABLE_MD
+}
+
+/**
+ * Build BOTH the markdown inventory and the typed VFS snapshot from a single
+ * primary-db fetch. The snapshot is the structured form Go diffs into
+ * baseline+delta messages; the markdown is the transition fallback. Returns null
+ * when the workspace is unavailable.
+ */
+export async function generateWorkspaceSnapshot(
+  workspaceId: string,
+  userId: string
+): Promise<{ markdown: string; snapshot: VfsSnapshotV1 } | null> {
+  const data = await buildWorkspaceMdData(workspaceId, userId)
+  if (!data) return null
+  return { markdown: buildWorkspaceMd(data), snapshot: buildVfsSnapshot(data) }
+}
+
+/**
+ * Map the workspace inventory data to the typed VFS snapshot contract. Pure;
+ * mirrors buildWorkspaceMd's field selection. Resource order is irrelevant — Go
+ * diffs by stable id, not position.
+ */
+export function buildVfsSnapshot(data: WorkspaceMdData): VfsSnapshotV1 {
+  const workflows: VfsSnapshotV1Workflow[] = data.workflows.map((wf) => ({
+    id: wf.id,
+    name: wf.name,
+    path: canonicalWorkflowVfsDir({ name: wf.name, folderPath: wf.folderPath }),
+    ...(wf.description ? { description: wf.description } : {}),
+    ...(wf.isDeployed ? { isDeployed: true } : {}),
+    ...(wf.folderPath ? { folderPath: wf.folderPath } : {}),
+  }))
+  const jobs: VfsSnapshotV1Job[] = (data.jobs ?? [])
+    .filter((j) => j.status !== 'completed')
+    .map((j) => ({
+      id: j.id,
+      ...(j.title ? { title: j.title } : {}),
+      ...(j.prompt ? { prompt: j.prompt } : {}),
+      ...(j.cronExpression ? { cronExpression: j.cronExpression } : {}),
+      ...(j.status ? { status: j.status } : {}),
+      ...(j.lifecycle ? { lifecycle: j.lifecycle } : {}),
+      ...(j.sourceTaskName ? { sourceTaskName: j.sourceTaskName } : {}),
+    }))
+  return {
+    ...(data.workspace
+      ? {
+          workspace: {
+            id: data.workspace.id,
+            name: data.workspace.name,
+            ...(data.workspace.ownerId ? { ownerId: data.workspace.ownerId } : {}),
+          },
+        }
+      : {}),
+    members: data.members.map((m) => ({
+      ...(m.name ? { name: m.name } : {}),
+      email: m.email,
+      ...(m.permissionType ? { permissionType: m.permissionType } : {}),
+    })),
+    workflows,
+    knowledgeBases: data.knowledgeBases.map((kb) => ({
+      id: kb.id,
+      name: kb.name,
+      ...(kb.description ? { description: kb.description } : {}),
+      ...(kb.connectorTypes && kb.connectorTypes.length > 0
+        ? { connectorTypes: kb.connectorTypes }
+        : {}),
+    })),
+    tables: data.tables.map((t) => ({
+      id: t.id,
+      name: t.name,
+      ...(t.description ? { description: t.description } : {}),
+    })),
+    files: data.files.map((f) => ({
+      id: f.id,
+      name: f.name,
+      path: canonicalWorkspaceFilePath({ folderPath: f.folderPath, name: f.name }),
+      ...(f.type ? { type: f.type } : {}),
+      ...(f.size ? { size: f.size } : {}),
+      ...(f.folderPath ? { folderPath: f.folderPath } : {}),
+    })),
+    integrations: data.oauthIntegrations.map((c) => ({
+      id: c.id,
+      providerId: c.providerId,
+      ...(c.displayName ? { displayName: c.displayName } : {}),
+      ...(c.role ? { role: c.role } : {}),
+    })),
+    envVars: data.envVariables,
+    customTools: (data.customTools ?? []).map((t) => ({ id: t.id, name: t.name })),
+    mcpServers: (data.mcpServers ?? []).map((s) => ({
+      id: s.id,
+      name: s.name,
+      ...(s.url ? { url: s.url } : {}),
+      ...(s.enabled ? { enabled: true } : {}),
+    })),
+    skills: (data.skills ?? []).map((s) => ({
+      id: s.id,
+      name: s.name,
+      ...(s.description ? { description: s.description } : {}),
+    })),
+    jobs,
+  }
+}
+
+function formatSize(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`
+}

apps/sim/lib/copilot/constants.ts

@@ -0,0 +1,63 @@
+import { env } from '@/lib/core/config/env'
+
+export const SIM_AGENT_API_URL_DEFAULT = 'https://www.copilot.sim.ai'
+export const SIM_AGENT_VERSION = '3.0.0'
+
+/** Resolved copilot backend URL — reads from env with fallback to default. */
+const rawAgentUrl = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
+export const SIM_AGENT_API_URL =
+  rawAgentUrl.startsWith('http://') || rawAgentUrl.startsWith('https://')
+    ? rawAgentUrl
+    : SIM_AGENT_API_URL_DEFAULT
+
+/** Default timeout for the copilot orchestration stream loop (60 min). */
+export const ORCHESTRATION_TIMEOUT_MS = 3_600_000
+
+/**
+ * Watchdog cap for a single sim-executed copilot tool. A tool that neither
+ * resolves nor rejects within its cap is failed with a timeout error so the
+ * checkpoint loop can resume Go with an error result instead of wedging the
+ * chat (and its pending-stream lock) behind a hung await forever.
+ */
+export const TOOL_WATCHDOG_DEFAULT_MS = 60_000
+
+/**
+ * Watchdog cap for tool classes with legitimately long runtimes (workflow
+ * executions, media/image generation, sandboxed code, deep research). Those
+ * tools carry their own inner budgets (plan execution timeouts, sandbox
+ * timeouts), so this cap only backstops a true hang and sits above all of
+ * them — matching ORCHESTRATION_TIMEOUT_MS so it never undercuts a legal run.
+ */
+export const TOOL_WATCHDOG_LONG_RUNNING_MS = ORCHESTRATION_TIMEOUT_MS
+
+/** Extra slack the resume gate allows past the slowest pending tool's watchdog. */
+export const TOOL_WATCHDOG_RESUME_GRACE_MS = 30_000
+
+/** Timeout for the client-side streaming response handler (60 min). */
+export const STREAM_TIMEOUT_MS = 3_600_000
+
+/** SessionStorage key for persisting active stream metadata across page reloads. */
+export const STREAM_STORAGE_KEY = 'copilot_active_stream'
+
+/** POST — send a chat message through the unified mothership chat surface. */
+export const MOTHERSHIP_CHAT_API_PATH = '/api/mothership/chat'
+
+/** POST — confirm or reject a tool call. */
+export const COPILOT_CONFIRM_API_PATH = '/api/copilot/confirm'
+
+/** Maximum entries in the in-memory SSE tool-event dedup cache. */
+export const STREAM_BUFFER_MAX_DEDUP_ENTRIES = 1_000
+
+/** Approximate max inline tool-result budget before artifact/error handling takes over. */
+export const TOOL_RESULT_MAX_INLINE_TOKENS = 50_000
+
+/** Rough chars-per-token estimate used when only serialized text length is available. */
+export const TOOL_RESULT_ESTIMATED_CHARS_PER_TOKEN = 4
+
+/** Approximate max inline tool-result size in characters. */
+export const TOOL_RESULT_MAX_INLINE_CHARS =
+  TOOL_RESULT_MAX_INLINE_TOKENS * TOOL_RESULT_ESTIMATED_CHARS_PER_TOKEN
+
+export const COPILOT_MODES = ['ask', 'build', 'plan'] as const
+
+export const COPILOT_REQUEST_MODES = ['ask', 'build', 'plan', 'agent'] as const

apps/sim/lib/copilot/entitlements.ts

@@ -0,0 +1,72 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { LRUCache } from 'lru-cache'
+import { isCustomBlocksEligible } from '@/lib/workflows/custom-blocks/operations'
+
+const logger = createLogger('CopilotEntitlements')
+
+/**
+ * Cross-repo contract: the mothership (Go) matches these exact strings against
+ * its `core.Entitlement*` constants to gate agent surfaces.
+ */
+export const CUSTOM_BLOCKS_ENTITLEMENT = 'custom-blocks'
+
+/**
+ * Workspace entitlements — plan/flag-gated org capabilities sent to the
+ * mothership as the chat payload's `entitlements` array. The Go side hides the
+ * matching tools, skills, and prompt sections when an entitlement is absent, so
+ * a non-entitled org's agents never hear of the feature.
+ *
+ * Adding an entitlement:
+ * 1. Add the kebab-case name and a fail-closed evaluator here. Every payload
+ *    site (interactive chat, headless execute, inbox) picks it up automatically.
+ * 2. Go repo: add the matching `Entitlement*` constant in `internal/core` and
+ *    gate surfaces declaratively — `RequiredEntitlement` on tool definitions,
+ *    `entitlement:` frontmatter on skills, a conditional section in
+ *    `BuildAgentEnvelope`, or a variant swap in `Capabilities`.
+ * 3. Keep enforcement in sim: the Go gating is advertisement-only (the payload
+ *    is forgeable), so the sim-side tool handler must re-check the same
+ *    predicate at execution time.
+ */
+const ENTITLEMENT_EVALUATORS: Record<
+  string,
+  (workspaceId: string, userId?: string) => Promise<boolean>
+> = {
+  [CUSTOM_BLOCKS_ENTITLEMENT]: isCustomBlocksEligible,
+}
+
+const entitlementsCache = new LRUCache<string, Promise<string[]>>({
+  max: 500,
+  ttl: 5_000,
+})
+
+/**
+ * The entitlements to send to the mothership for a request in this workspace.
+ * Each evaluator fails closed (an error means the entitlement is absent).
+ * Cached briefly so the several per-message callers collapse to one evaluation.
+ */
+export function computeWorkspaceEntitlements(
+  workspaceId: string,
+  userId?: string
+): Promise<string[]> {
+  const cacheKey = `${workspaceId}:${userId ?? ''}`
+  const cached = entitlementsCache.get(cacheKey)
+  if (cached) return cached
+
+  const promise = Promise.all(
+    Object.entries(ENTITLEMENT_EVALUATORS).map(async ([name, evaluate]) => {
+      try {
+        return (await evaluate(workspaceId, userId)) ? name : null
+      } catch (error) {
+        logger.warn('Entitlement evaluation failed; treating as absent', {
+          entitlement: name,
+          workspaceId,
+          error: getErrorMessage(error),
+        })
+        return null
+      }
+    })
+  ).then((names) => names.filter((name): name is string => name !== null))
+  entitlementsCache.set(cacheKey, promise)
+  return promise
+}

apps/sim/lib/copilot/generated/metrics-v1.ts

@@ -0,0 +1,60 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+// Source: copilot/copilot/contracts/metrics-v1.schema.json
+// Regenerate with: bun run metrics-contract:generate
+//
+// Canonical mothership OTel metric names. Call sites should reference
+// `Metric.<Identifier>` (e.g. `Metric.CopilotToolDuration`) rather than raw
+// string literals, so the Go-side contract is the single source of truth and
+// typos become compile errors.
+//
+// NAMES ONLY. Label keys and histogram bucket boundaries are NOT in this
+// contract — Go owns the label-cardinality allowlist and the shared bucket
+// constant, and the Sim emitter MUST mirror those by hand so the Go∪Sim metric
+// union is queryable as one series set.
+
+export const Metric = {
+  CopilotCacheAttempted: 'copilot.cache.attempted',
+  CopilotCacheHit: 'copilot.cache.hit',
+  CopilotCacheWrite: 'copilot.cache.write',
+  CopilotFileReadDuration: 'copilot.file.read.duration',
+  CopilotFileReadSize: 'copilot.file.read.size',
+  CopilotMessagesSerializeDuration: 'copilot.messages.serialize.duration',
+  CopilotRequestCount: 'copilot.request.count',
+  CopilotRequestDuration: 'copilot.request.duration',
+  CopilotToolCalls: 'copilot.tool.calls',
+  CopilotToolDuration: 'copilot.tool.duration',
+  CopilotVfsDelta: 'copilot.vfs.delta',
+  CopilotVfsMaterializeDuration: 'copilot.vfs.materialize.duration',
+  GenAiClientCacheTokenUsage: 'gen_ai.client.cache.token.usage',
+  GenAiClientTokenUsage: 'gen_ai.client.token.usage',
+  LlmClientErrors: 'llm.client.errors',
+  LlmClientOutputCutoff: 'llm.client.output_cutoff',
+  LlmClientStreamDuration: 'llm.client.stream.duration',
+  LlmClientTimeToFirstToken: 'llm.client.time_to_first_token',
+} as const
+
+export type MetricKey = keyof typeof Metric
+export type MetricValue = (typeof Metric)[MetricKey]
+
+/** Readonly sorted list of every canonical mothership metric name. */
+export const MetricValues: readonly MetricValue[] = [
+  'copilot.cache.attempted',
+  'copilot.cache.hit',
+  'copilot.cache.write',
+  'copilot.file.read.duration',
+  'copilot.file.read.size',
+  'copilot.messages.serialize.duration',
+  'copilot.request.count',
+  'copilot.request.duration',
+  'copilot.tool.calls',
+  'copilot.tool.duration',
+  'copilot.vfs.delta',
+  'copilot.vfs.materialize.duration',
+  'gen_ai.client.cache.token.usage',
+  'gen_ai.client.token.usage',
+  'llm.client.errors',
+  'llm.client.output_cutoff',
+  'llm.client.stream.duration',
+  'llm.client.time_to_first_token',
+] as const

apps/sim/lib/copilot/generated/mothership-stream-v1.ts

@@ -0,0 +1,555 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+
+/**
+ * Shared execution-oriented mothership stream contract from Go to Sim.
+ */
+export type MothershipStreamV1EventEnvelope =
+  | MothershipStreamV1SessionStartEventEnvelope
+  | MothershipStreamV1SessionChatEventEnvelope
+  | MothershipStreamV1SessionTitleEventEnvelope
+  | MothershipStreamV1SessionTraceEventEnvelope
+  | MothershipStreamV1TextEventEnvelope
+  | MothershipStreamV1ToolCallEventEnvelope
+  | MothershipStreamV1ToolArgsDeltaEventEnvelope
+  | MothershipStreamV1ToolResultEventEnvelope
+  | MothershipStreamV1SubagentSpanStartEventEnvelope
+  | MothershipStreamV1SubagentSpanEndEventEnvelope
+  | MothershipStreamV1StructuredResultSpanEventEnvelope
+  | MothershipStreamV1SubagentResultSpanEventEnvelope
+  | MothershipStreamV1ResourceUpsertEventEnvelope
+  | MothershipStreamV1ResourceRemoveEventEnvelope
+  | MothershipStreamV1CheckpointPauseEventEnvelope
+  | MothershipStreamV1RunResumedEventEnvelope
+  | MothershipStreamV1CompactionStartEventEnvelope
+  | MothershipStreamV1CompactionDoneEventEnvelope
+  | MothershipStreamV1ErrorEventEnvelope
+  | MothershipStreamV1CompleteEventEnvelope
+export type MothershipStreamV1TextChannel = 'assistant' | 'thinking'
+export type MothershipStreamV1ToolExecutor = 'go' | 'sim' | 'client'
+export type MothershipStreamV1ToolMode = 'sync' | 'async'
+export type MothershipStreamV1ToolStatus =
+  | 'generating'
+  | 'executing'
+  | 'success'
+  | 'error'
+  | 'cancelled'
+  | 'skipped'
+  | 'rejected'
+export type MothershipStreamV1CompletionStatus = 'complete' | 'error' | 'cancelled'
+
+export interface MothershipStreamV1SessionStartEventEnvelope {
+  payload: MothershipStreamV1SessionStartPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'session'
+  v: 1
+}
+export interface MothershipStreamV1SessionStartPayload {
+  data?: MothershipStreamV1SessionStartData
+  kind: 'start'
+}
+export interface MothershipStreamV1SessionStartData {
+  responseId?: string
+}
+export interface MothershipStreamV1StreamScope {
+  agentId?: string
+  lane: 'subagent'
+  parentSpanId?: string
+  parentToolCallId?: string
+  spanId?: string
+}
+export interface MothershipStreamV1StreamRef {
+  chatId?: string
+  cursor?: string
+  streamId: string
+}
+export interface MothershipStreamV1Trace {
+  /**
+   * OTel trace ID from the first Go ingress. May differ from requestId when Sim assigns the canonical request identity.
+   */
+  goTraceId?: string
+  requestId: string
+  spanId?: string
+}
+export interface MothershipStreamV1SessionChatEventEnvelope {
+  payload: MothershipStreamV1SessionChatPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'session'
+  v: 1
+}
+export interface MothershipStreamV1SessionChatPayload {
+  chatId: string
+  kind: 'chat'
+}
+export interface MothershipStreamV1SessionTitleEventEnvelope {
+  payload: MothershipStreamV1SessionTitlePayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'session'
+  v: 1
+}
+export interface MothershipStreamV1SessionTitlePayload {
+  kind: 'title'
+  title: string
+}
+export interface MothershipStreamV1SessionTraceEventEnvelope {
+  payload: MothershipStreamV1SessionTracePayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'session'
+  v: 1
+}
+export interface MothershipStreamV1SessionTracePayload {
+  kind: 'trace'
+  requestId: string
+  spanId?: string
+}
+export interface MothershipStreamV1TextEventEnvelope {
+  payload: MothershipStreamV1TextPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'text'
+  v: 1
+}
+export interface MothershipStreamV1TextPayload {
+  channel: MothershipStreamV1TextChannel
+  text: string
+}
+export interface MothershipStreamV1ToolCallEventEnvelope {
+  payload: MothershipStreamV1ToolCallDescriptor
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'tool'
+  v: 1
+}
+export interface MothershipStreamV1ToolCallDescriptor {
+  arguments?: MothershipStreamV1AdditionalPropertiesMap
+  executor: MothershipStreamV1ToolExecutor
+  mode: MothershipStreamV1ToolMode
+  partial?: boolean
+  phase: 'call'
+  status?: MothershipStreamV1ToolStatus
+  toolCallId: string
+  toolName: string
+  ui?: MothershipStreamV1ToolUI
+}
+export interface MothershipStreamV1AdditionalPropertiesMap {
+  [k: string]: unknown
+}
+export interface MothershipStreamV1ToolUI {
+  clientExecutable?: boolean
+  hidden?: boolean
+  internal?: boolean
+}
+export interface MothershipStreamV1ToolArgsDeltaEventEnvelope {
+  payload: MothershipStreamV1ToolArgsDeltaPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'tool'
+  v: 1
+}
+export interface MothershipStreamV1ToolArgsDeltaPayload {
+  argumentsDelta: string
+  executor: MothershipStreamV1ToolExecutor
+  mode: MothershipStreamV1ToolMode
+  phase: 'args_delta'
+  toolCallId: string
+  toolName: string
+}
+export interface MothershipStreamV1ToolResultEventEnvelope {
+  payload: MothershipStreamV1ToolResultPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'tool'
+  v: 1
+}
+export interface MothershipStreamV1ToolResultPayload {
+  error?: string
+  executor: MothershipStreamV1ToolExecutor
+  mode: MothershipStreamV1ToolMode
+  output?: unknown
+  phase: 'result'
+  status?: MothershipStreamV1ToolStatus
+  success: boolean
+  toolCallId: string
+  toolName: string
+}
+export interface MothershipStreamV1SubagentSpanStartEventEnvelope {
+  payload: MothershipStreamV1SubagentSpanStartPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'span'
+  v: 1
+}
+export interface MothershipStreamV1SubagentSpanStartPayload {
+  agent?: string
+  data?: unknown
+  event: 'start'
+  kind: 'subagent'
+}
+export interface MothershipStreamV1SubagentSpanEndEventEnvelope {
+  payload: MothershipStreamV1SubagentSpanEndPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'span'
+  v: 1
+}
+export interface MothershipStreamV1SubagentSpanEndPayload {
+  agent?: string
+  data?: unknown
+  event: 'end'
+  kind: 'subagent'
+}
+export interface MothershipStreamV1StructuredResultSpanEventEnvelope {
+  payload: MothershipStreamV1StructuredResultSpanPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'span'
+  v: 1
+}
+export interface MothershipStreamV1StructuredResultSpanPayload {
+  agent?: string
+  data?: unknown
+  kind: 'structured_result'
+}
+export interface MothershipStreamV1SubagentResultSpanEventEnvelope {
+  payload: MothershipStreamV1SubagentResultSpanPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'span'
+  v: 1
+}
+export interface MothershipStreamV1SubagentResultSpanPayload {
+  agent?: string
+  data?: unknown
+  kind: 'subagent_result'
+}
+export interface MothershipStreamV1ResourceUpsertEventEnvelope {
+  payload: MothershipStreamV1ResourceUpsertPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'resource'
+  v: 1
+}
+export interface MothershipStreamV1ResourceUpsertPayload {
+  op: 'upsert'
+  resource: MothershipStreamV1ResourceDescriptor
+}
+export interface MothershipStreamV1ResourceDescriptor {
+  id: string
+  title?: string
+  type: string
+}
+export interface MothershipStreamV1ResourceRemoveEventEnvelope {
+  payload: MothershipStreamV1ResourceRemovePayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'resource'
+  v: 1
+}
+export interface MothershipStreamV1ResourceRemovePayload {
+  op: 'remove'
+  resource: MothershipStreamV1ResourceDescriptor
+}
+export interface MothershipStreamV1CheckpointPauseEventEnvelope {
+  payload: MothershipStreamV1CheckpointPausePayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'run'
+  v: 1
+}
+export interface MothershipStreamV1CheckpointPausePayload {
+  checkpointId: string
+  executionId: string
+  frames?: MothershipStreamV1CheckpointPauseFrame[]
+  kind: 'checkpoint_pause'
+  pendingToolCallIds: string[]
+  runId: string
+}
+export interface MothershipStreamV1CheckpointPauseFrame {
+  checkpointId?: string
+  parentToolCallId: string
+  parentToolName: string
+  pendingToolIds: string[]
+}
+export interface MothershipStreamV1RunResumedEventEnvelope {
+  payload: MothershipStreamV1RunResumedPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'run'
+  v: 1
+}
+export interface MothershipStreamV1RunResumedPayload {
+  kind: 'resumed'
+}
+export interface MothershipStreamV1CompactionStartEventEnvelope {
+  payload: MothershipStreamV1CompactionStartPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'run'
+  v: 1
+}
+export interface MothershipStreamV1CompactionStartPayload {
+  kind: 'compaction_start'
+}
+export interface MothershipStreamV1CompactionDoneEventEnvelope {
+  payload: MothershipStreamV1CompactionDonePayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'run'
+  v: 1
+}
+export interface MothershipStreamV1CompactionDonePayload {
+  data?: MothershipStreamV1CompactionDoneData
+  kind: 'compaction_done'
+}
+export interface MothershipStreamV1CompactionDoneData {
+  summary_chars: number
+}
+export interface MothershipStreamV1ErrorEventEnvelope {
+  payload: MothershipStreamV1ErrorPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'error'
+  v: 1
+}
+export interface MothershipStreamV1ErrorPayload {
+  code?: string
+  data?: unknown
+  displayMessage?: string
+  error?: string
+  message: string
+  provider?: string
+}
+export interface MothershipStreamV1CompleteEventEnvelope {
+  payload: MothershipStreamV1CompletePayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'complete'
+  v: 1
+}
+export interface MothershipStreamV1CompletePayload {
+  cost?: MothershipStreamV1CostData
+  reason?: string
+  response?: unknown
+  status: MothershipStreamV1CompletionStatus
+  usage?: MothershipStreamV1UsageData
+}
+export interface MothershipStreamV1CostData {
+  input?: number
+  output?: number
+  total?: number
+}
+export interface MothershipStreamV1UsageData {
+  cache_creation_input_tokens?: number
+  cache_read_input_tokens?: number
+  input_tokens?: number
+  model?: string
+  output_tokens?: number
+  total_tokens?: number
+}
+
+export type MothershipStreamV1AsyncToolRecordStatus =
+  | 'pending'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'cancelled'
+  | 'delivered'
+
+export const MothershipStreamV1AsyncToolRecordStatus = {
+  pending: 'pending',
+  running: 'running',
+  completed: 'completed',
+  failed: 'failed',
+  cancelled: 'cancelled',
+  delivered: 'delivered',
+} as const
+
+export const MothershipStreamV1CompletionStatus = {
+  complete: 'complete',
+  error: 'error',
+  cancelled: 'cancelled',
+} as const
+
+export type MothershipStreamV1EventType =
+  | 'session'
+  | 'text'
+  | 'tool'
+  | 'span'
+  | 'resource'
+  | 'run'
+  | 'error'
+  | 'complete'
+
+export const MothershipStreamV1EventType = {
+  session: 'session',
+  text: 'text',
+  tool: 'tool',
+  span: 'span',
+  resource: 'resource',
+  run: 'run',
+  error: 'error',
+  complete: 'complete',
+} as const
+
+export type MothershipStreamV1ResourceOp = 'upsert' | 'remove'
+
+export const MothershipStreamV1ResourceOp = {
+  upsert: 'upsert',
+  remove: 'remove',
+} as const
+
+export type MothershipStreamV1RunKind =
+  | 'checkpoint_pause'
+  | 'resumed'
+  | 'compaction_start'
+  | 'compaction_done'
+
+export const MothershipStreamV1RunKind = {
+  checkpoint_pause: 'checkpoint_pause',
+  resumed: 'resumed',
+  compaction_start: 'compaction_start',
+  compaction_done: 'compaction_done',
+} as const
+
+export type MothershipStreamV1SessionKind = 'trace' | 'chat' | 'title' | 'start'
+
+export const MothershipStreamV1SessionKind = {
+  trace: 'trace',
+  chat: 'chat',
+  title: 'title',
+  start: 'start',
+} as const
+
+export type MothershipStreamV1SpanKind = 'subagent'
+
+export const MothershipStreamV1SpanKind = {
+  subagent: 'subagent',
+} as const
+
+export type MothershipStreamV1SpanLifecycleEvent = 'start' | 'end'
+
+export const MothershipStreamV1SpanLifecycleEvent = {
+  start: 'start',
+  end: 'end',
+} as const
+
+export type MothershipStreamV1SpanPayloadKind = 'subagent' | 'structured_result' | 'subagent_result'
+
+export const MothershipStreamV1SpanPayloadKind = {
+  subagent: 'subagent',
+  structured_result: 'structured_result',
+  subagent_result: 'subagent_result',
+} as const
+
+export const MothershipStreamV1TextChannel = {
+  assistant: 'assistant',
+  thinking: 'thinking',
+} as const
+
+export const MothershipStreamV1ToolExecutor = {
+  go: 'go',
+  sim: 'sim',
+  client: 'client',
+} as const
+
+export const MothershipStreamV1ToolMode = {
+  sync: 'sync',
+  async: 'async',
+} as const
+
+export type MothershipStreamV1ToolOutcome =
+  | 'success'
+  | 'error'
+  | 'cancelled'
+  | 'skipped'
+  | 'rejected'
+
+export const MothershipStreamV1ToolOutcome = {
+  success: 'success',
+  error: 'error',
+  cancelled: 'cancelled',
+  skipped: 'skipped',
+  rejected: 'rejected',
+} as const
+
+export type MothershipStreamV1ToolPhase = 'call' | 'args_delta' | 'result'
+
+export const MothershipStreamV1ToolPhase = {
+  call: 'call',
+  args_delta: 'args_delta',
+  result: 'result',
+} as const
+
+export const MothershipStreamV1ToolStatus = {
+  generating: 'generating',
+  executing: 'executing',
+  success: 'success',
+  error: 'error',
+  cancelled: 'cancelled',
+  skipped: 'skipped',
+  rejected: 'rejected',
+} as const

apps/sim/lib/copilot/generated/request-trace-v1.ts

@@ -0,0 +1,139 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1Outcome".
+ */
+export type RequestTraceV1Outcome = 'success' | 'error' | 'cancelled'
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1SpanSource".
+ */
+export type RequestTraceV1SpanSource = 'sim' | 'go'
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1SpanStatus".
+ */
+export type RequestTraceV1SpanStatus = 'ok' | 'error' | 'cancelled' | 'pending'
+
+/**
+ * Trace report sent from Sim to Go after a request completes.
+ */
+export interface RequestTraceV1SimReport {
+  chatId?: string
+  cost?: RequestTraceV1CostSummary
+  durationMs: number
+  endMs: number
+  executionId?: string
+  goTraceId?: string
+  outcome: RequestTraceV1Outcome
+  runId?: string
+  simRequestId: string
+  spans: RequestTraceV1Span[]
+  startMs: number
+  streamId?: string
+  usage?: RequestTraceV1UsageSummary
+  userMessage?: string
+}
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1CostSummary".
+ */
+export interface RequestTraceV1CostSummary {
+  billedTotalCost?: number
+  rawTotalCost?: number
+}
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1Span".
+ */
+export interface RequestTraceV1Span {
+  attributes?: MothershipStreamV1AdditionalPropertiesMap
+  durationMs: number
+  endMs: number
+  kind?: string
+  name: string
+  parentName?: string
+  source?: RequestTraceV1SpanSource
+  startMs: number
+  status: RequestTraceV1SpanStatus
+}
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "MothershipStreamV1AdditionalPropertiesMap".
+ */
+export interface MothershipStreamV1AdditionalPropertiesMap {
+  [k: string]: unknown
+}
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1UsageSummary".
+ */
+export interface RequestTraceV1UsageSummary {
+  cacheAttemptedRequests?: number
+  cacheHitRequests?: number
+  cacheReadTokens?: number
+  cacheSavingsRate?: number
+  cacheWriteRequests?: number
+  cacheWriteTokens?: number
+  inputTokens?: number
+  outputTokens?: number
+}
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1MergedTrace".
+ */
+export interface RequestTraceV1MergedTrace {
+  chatId?: string
+  cost?: RequestTraceV1CostSummary
+  durationMs: number
+  endMs: number
+  goTraceId: string
+  outcome: RequestTraceV1Outcome
+  serviceCharges?: MothershipStreamV1AdditionalPropertiesMap
+  simRequestId?: string
+  spans: RequestTraceV1Span[]
+  startMs: number
+  streamId?: string
+  usage?: RequestTraceV1UsageSummary
+  userId?: string
+}
+/**
+ * This interface was referenced by `RequestTraceV1SimReport`'s JSON-Schema
+ * via the `definition` "RequestTraceV1SimReport".
+ */
+export interface RequestTraceV1SimReport1 {
+  chatId?: string
+  cost?: RequestTraceV1CostSummary
+  durationMs: number
+  endMs: number
+  executionId?: string
+  goTraceId?: string
+  outcome: RequestTraceV1Outcome
+  runId?: string
+  simRequestId: string
+  spans: RequestTraceV1Span[]
+  startMs: number
+  streamId?: string
+  usage?: RequestTraceV1UsageSummary
+  userMessage?: string
+}
+
+export const RequestTraceV1Outcome = {
+  success: 'success',
+  error: 'error',
+  cancelled: 'cancelled',
+} as const
+
+export const RequestTraceV1SpanSource = {
+  sim: 'sim',
+  go: 'go',
+} as const
+
+export const RequestTraceV1SpanStatus = {
+  ok: 'ok',
+  error: 'error',
+  cancelled: 'cancelled',
+  pending: 'pending',
+} as const

apps/sim/lib/copilot/generated/trace-attribute-values-v1.ts

@@ -0,0 +1,372 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+// Source: copilot/copilot/contracts/trace-attribute-values-v1.schema.json
+// Regenerate with: bun run trace-attribute-values-contract:generate
+//
+// Canonical closed-set value vocabularies for mothership OTel
+// attributes. Call sites should reference e.g.
+// `CopilotRequestCancelReason.ExplicitStop` rather than the raw
+// string literal, so typos become compile errors and the Go contract
+// remains the single source of truth.
+
+export const AbortBackend = {
+  InProcess: 'in_process',
+  Redis: 'redis',
+} as const
+
+export type AbortBackendKey = keyof typeof AbortBackend
+export type AbortBackendValue = (typeof AbortBackend)[AbortBackendKey]
+
+export const AbortRedisResult = {
+  Error: 'error',
+  Ok: 'ok',
+  Slow: 'slow',
+} as const
+
+export type AbortRedisResultKey = keyof typeof AbortRedisResult
+export type AbortRedisResultValue = (typeof AbortRedisResult)[AbortRedisResultKey]
+
+export const AuthKeyMatch = {
+  None: 'none',
+  User: 'user',
+} as const
+
+export type AuthKeyMatchKey = keyof typeof AuthKeyMatch
+export type AuthKeyMatchValue = (typeof AuthKeyMatch)[AuthKeyMatchKey]
+
+export const BillingAnalyticsOutcome = {
+  Duplicate: 'duplicate',
+  RetriesExhausted: 'retries_exhausted',
+  Success: 'success',
+  Unknown: 'unknown',
+} as const
+
+export type BillingAnalyticsOutcomeKey = keyof typeof BillingAnalyticsOutcome
+export type BillingAnalyticsOutcomeValue =
+  (typeof BillingAnalyticsOutcome)[BillingAnalyticsOutcomeKey]
+
+export const BillingFlushOutcome = {
+  CheckpointAlreadyClaimed: 'checkpoint_already_claimed',
+  CheckpointLoadFailed: 'checkpoint_load_failed',
+  Flushed: 'flushed',
+  NoCheckpoint: 'no_checkpoint',
+  NoSnapshot: 'no_snapshot',
+  SkippedUnconfigured: 'skipped_unconfigured',
+} as const
+
+export type BillingFlushOutcomeKey = keyof typeof BillingFlushOutcome
+export type BillingFlushOutcomeValue = (typeof BillingFlushOutcome)[BillingFlushOutcomeKey]
+
+export const BillingRouteOutcome = {
+  AuthFailed: 'auth_failed',
+  Billed: 'billed',
+  BillingDisabled: 'billing_disabled',
+  DuplicateIdempotencyKey: 'duplicate_idempotency_key',
+  InternalError: 'internal_error',
+  InvalidBody: 'invalid_body',
+} as const
+
+export type BillingRouteOutcomeKey = keyof typeof BillingRouteOutcome
+export type BillingRouteOutcomeValue = (typeof BillingRouteOutcome)[BillingRouteOutcomeKey]
+
+export const CopilotAbortOutcome = {
+  BadRequest: 'bad_request',
+  FallbackPersistFailed: 'fallback_persist_failed',
+  ForceReleased: 'force_released',
+  MissingMessageId: 'missing_message_id',
+  MissingStreamId: 'missing_stream_id',
+  NoChatId: 'no_chat_id',
+  Ok: 'ok',
+  SettleTimeout: 'settle_timeout',
+  Settled: 'settled',
+  Unauthorized: 'unauthorized',
+} as const
+
+export type CopilotAbortOutcomeKey = keyof typeof CopilotAbortOutcome
+export type CopilotAbortOutcomeValue = (typeof CopilotAbortOutcome)[CopilotAbortOutcomeKey]
+
+export const CopilotBranchKind = {
+  Workflow: 'workflow',
+  Workspace: 'workspace',
+} as const
+
+export type CopilotBranchKindKey = keyof typeof CopilotBranchKind
+export type CopilotBranchKindValue = (typeof CopilotBranchKind)[CopilotBranchKindKey]
+
+export const CopilotChatFinalizeOutcome = {
+  AppendedAssistant: 'appended_assistant',
+  AssistantAlreadyPersisted: 'assistant_already_persisted',
+  ClearedStreamMarkerOnly: 'cleared_stream_marker_only',
+  StaleUserMessage: 'stale_user_message',
+} as const
+
+export type CopilotChatFinalizeOutcomeKey = keyof typeof CopilotChatFinalizeOutcome
+export type CopilotChatFinalizeOutcomeValue =
+  (typeof CopilotChatFinalizeOutcome)[CopilotChatFinalizeOutcomeKey]
+
+export const CopilotChatPersistOutcome = {
+  Appended: 'appended',
+  ChatNotFound: 'chat_not_found',
+} as const
+
+export type CopilotChatPersistOutcomeKey = keyof typeof CopilotChatPersistOutcome
+export type CopilotChatPersistOutcomeValue =
+  (typeof CopilotChatPersistOutcome)[CopilotChatPersistOutcomeKey]
+
+export const CopilotConfirmOutcome = {
+  Delivered: 'delivered',
+  Forbidden: 'forbidden',
+  InternalError: 'internal_error',
+  RunNotFound: 'run_not_found',
+  ToolCallNotFound: 'tool_call_not_found',
+  Unauthorized: 'unauthorized',
+  UpdateFailed: 'update_failed',
+  ValidationError: 'validation_error',
+} as const
+
+export type CopilotConfirmOutcomeKey = keyof typeof CopilotConfirmOutcome
+export type CopilotConfirmOutcomeValue = (typeof CopilotConfirmOutcome)[CopilotConfirmOutcomeKey]
+
+export const CopilotFinalizeOutcome = {
+  Aborted: 'aborted',
+  Error: 'error',
+  Success: 'success',
+} as const
+
+export type CopilotFinalizeOutcomeKey = keyof typeof CopilotFinalizeOutcome
+export type CopilotFinalizeOutcomeValue = (typeof CopilotFinalizeOutcome)[CopilotFinalizeOutcomeKey]
+
+export const CopilotLeg = {
+  SimToGo: 'sim_to_go',
+} as const
+
+export type CopilotLegKey = keyof typeof CopilotLeg
+export type CopilotLegValue = (typeof CopilotLeg)[CopilotLegKey]
+
+export const CopilotOutputFileOutcome = {
+  Failed: 'failed',
+  Uploaded: 'uploaded',
+} as const
+
+export type CopilotOutputFileOutcomeKey = keyof typeof CopilotOutputFileOutcome
+export type CopilotOutputFileOutcomeValue =
+  (typeof CopilotOutputFileOutcome)[CopilotOutputFileOutcomeKey]
+
+export const CopilotRecoveryOutcome = {
+  GapDetected: 'gap_detected',
+  InRange: 'in_range',
+} as const
+
+export type CopilotRecoveryOutcomeKey = keyof typeof CopilotRecoveryOutcome
+export type CopilotRecoveryOutcomeValue = (typeof CopilotRecoveryOutcome)[CopilotRecoveryOutcomeKey]
+
+export const CopilotRequestCancelReason = {
+  ClientDisconnect: 'client_disconnect',
+  ExplicitStop: 'explicit_stop',
+  Timeout: 'timeout',
+  Unknown: 'unknown',
+} as const
+
+export type CopilotRequestCancelReasonKey = keyof typeof CopilotRequestCancelReason
+export type CopilotRequestCancelReasonValue =
+  (typeof CopilotRequestCancelReason)[CopilotRequestCancelReasonKey]
+
+export const CopilotResourcesOp = {
+  Delete: 'delete',
+  None: 'none',
+  Upsert: 'upsert',
+} as const
+
+export type CopilotResourcesOpKey = keyof typeof CopilotResourcesOp
+export type CopilotResourcesOpValue = (typeof CopilotResourcesOp)[CopilotResourcesOpKey]
+
+export const CopilotResumeOutcome = {
+  BatchDelivered: 'batch_delivered',
+  ClientDisconnected: 'client_disconnected',
+  EndedWithoutTerminal: 'ended_without_terminal',
+  StreamNotFound: 'stream_not_found',
+  TerminalDelivered: 'terminal_delivered',
+} as const
+
+export type CopilotResumeOutcomeKey = keyof typeof CopilotResumeOutcome
+export type CopilotResumeOutcomeValue = (typeof CopilotResumeOutcome)[CopilotResumeOutcomeKey]
+
+export const CopilotSseCloseReason = {
+  Aborted: 'aborted',
+  BackendError: 'backend_error',
+  BillingLimit: 'billing_limit',
+  ClosedNoTerminal: 'closed_no_terminal',
+  Error: 'error',
+  Terminal: 'terminal',
+  Timeout: 'timeout',
+} as const
+
+export type CopilotSseCloseReasonKey = keyof typeof CopilotSseCloseReason
+export type CopilotSseCloseReasonValue = (typeof CopilotSseCloseReason)[CopilotSseCloseReasonKey]
+
+export const CopilotStopOutcome = {
+  ChatNotFound: 'chat_not_found',
+  InternalError: 'internal_error',
+  NoMatchingRow: 'no_matching_row',
+  Persisted: 'persisted',
+  Unauthorized: 'unauthorized',
+  ValidationError: 'validation_error',
+} as const
+
+export type CopilotStopOutcomeKey = keyof typeof CopilotStopOutcome
+export type CopilotStopOutcomeValue = (typeof CopilotStopOutcome)[CopilotStopOutcomeKey]
+
+export const CopilotSurface = {
+  Copilot: 'copilot',
+  Mothership: 'mothership',
+} as const
+
+export type CopilotSurfaceKey = keyof typeof CopilotSurface
+export type CopilotSurfaceValue = (typeof CopilotSurface)[CopilotSurfaceKey]
+
+export const CopilotTableOutcome = {
+  EmptyContent: 'empty_content',
+  EmptyRows: 'empty_rows',
+  Failed: 'failed',
+  Imported: 'imported',
+  InvalidJsonShape: 'invalid_json_shape',
+  InvalidShape: 'invalid_shape',
+  RowLimitExceeded: 'row_limit_exceeded',
+  TableNotFound: 'table_not_found',
+  Wrote: 'wrote',
+} as const
+
+export type CopilotTableOutcomeKey = keyof typeof CopilotTableOutcome
+export type CopilotTableOutcomeValue = (typeof CopilotTableOutcome)[CopilotTableOutcomeKey]
+
+export const CopilotTableSourceFormat = {
+  Csv: 'csv',
+  Json: 'json',
+} as const
+
+export type CopilotTableSourceFormatKey = keyof typeof CopilotTableSourceFormat
+export type CopilotTableSourceFormatValue =
+  (typeof CopilotTableSourceFormat)[CopilotTableSourceFormatKey]
+
+export const CopilotTransport = {
+  Batch: 'batch',
+  Headless: 'headless',
+  Stream: 'stream',
+} as const
+
+export type CopilotTransportKey = keyof typeof CopilotTransport
+export type CopilotTransportValue = (typeof CopilotTransport)[CopilotTransportKey]
+
+export const CopilotValidateOutcome = {
+  InternalAuthFailed: 'internal_auth_failed',
+  InternalError: 'internal_error',
+  InvalidBody: 'invalid_body',
+  Ok: 'ok',
+  UsageExceeded: 'usage_exceeded',
+  UserNotFound: 'user_not_found',
+} as const
+
+export type CopilotValidateOutcomeKey = keyof typeof CopilotValidateOutcome
+export type CopilotValidateOutcomeValue = (typeof CopilotValidateOutcome)[CopilotValidateOutcomeKey]
+
+export const CopilotVfsOutcome = {
+  PassthroughFitsBudget: 'passthrough_fits_budget',
+  PassthroughNoMetadata: 'passthrough_no_metadata',
+  PassthroughNoSharp: 'passthrough_no_sharp',
+  RejectedNoMetadata: 'rejected_no_metadata',
+  RejectedNoSharp: 'rejected_no_sharp',
+  RejectedTooLargeAfterResize: 'rejected_too_large_after_resize',
+  Resized: 'resized',
+} as const
+
+export type CopilotVfsOutcomeKey = keyof typeof CopilotVfsOutcome
+export type CopilotVfsOutcomeValue = (typeof CopilotVfsOutcome)[CopilotVfsOutcomeKey]
+
+export const CopilotVfsReadOutcome = {
+  BinaryPlaceholder: 'binary_placeholder',
+  DocumentParsed: 'document_parsed',
+  DocumentTooLarge: 'document_too_large',
+  ImagePrepared: 'image_prepared',
+  ImageTooLarge: 'image_too_large',
+  ParseFailed: 'parse_failed',
+  ReadFailed: 'read_failed',
+  TextRead: 'text_read',
+  TextTooLarge: 'text_too_large',
+} as const
+
+export type CopilotVfsReadOutcomeKey = keyof typeof CopilotVfsReadOutcome
+export type CopilotVfsReadOutcomeValue = (typeof CopilotVfsReadOutcome)[CopilotVfsReadOutcomeKey]
+
+export const CopilotVfsReadPath = {
+  Binary: 'binary',
+  Image: 'image',
+  ParseableDocument: 'parseable_document',
+  Text: 'text',
+} as const
+
+export type CopilotVfsReadPathKey = keyof typeof CopilotVfsReadPath
+export type CopilotVfsReadPathValue = (typeof CopilotVfsReadPath)[CopilotVfsReadPathKey]
+
+export const LlmErrorStage = {
+  BuildRequest: 'build_request',
+  Decode: 'decode',
+  HttpBuild: 'http_build',
+  HttpStatus: 'http_status',
+  Invoke: 'invoke',
+  MarshalRequest: 'marshal_request',
+  StreamClose: 'stream_close',
+} as const
+
+export type LlmErrorStageKey = keyof typeof LlmErrorStage
+export type LlmErrorStageValue = (typeof LlmErrorStage)[LlmErrorStageKey]
+
+export const RateLimitOutcome = {
+  Allowed: 'allowed',
+  IncrError: 'incr_error',
+  Limited: 'limited',
+} as const
+
+export type RateLimitOutcomeKey = keyof typeof RateLimitOutcome
+export type RateLimitOutcomeValue = (typeof RateLimitOutcome)[RateLimitOutcomeKey]
+
+export const ToolAsyncWaiterResolution = {
+  ContextCancelled: 'context_cancelled',
+  Poll: 'poll',
+  Pubsub: 'pubsub',
+  StoredAfterClose: 'stored_after_close',
+  StoredBeforeSubscribe: 'stored_before_subscribe',
+  StoredPostSubscribe: 'stored_post_subscribe',
+  SubscriptionClosed: 'subscription_closed',
+  Unknown: 'unknown',
+} as const
+
+export type ToolAsyncWaiterResolutionKey = keyof typeof ToolAsyncWaiterResolution
+export type ToolAsyncWaiterResolutionValue =
+  (typeof ToolAsyncWaiterResolution)[ToolAsyncWaiterResolutionKey]
+
+export const ToolErrorKind = {
+  Dispatch: 'dispatch',
+  NotFound: 'not_found',
+} as const
+
+export type ToolErrorKindKey = keyof typeof ToolErrorKind
+export type ToolErrorKindValue = (typeof ToolErrorKind)[ToolErrorKindKey]
+
+export const ToolExecutor = {
+  Client: 'client',
+  Go: 'go',
+  Sim: 'sim',
+} as const
+
+export type ToolExecutorKey = keyof typeof ToolExecutor
+export type ToolExecutorValue = (typeof ToolExecutor)[ToolExecutorKey]
+
+export const ToolStoreStatus = {
+  Cancelled: 'cancelled',
+  Completed: 'completed',
+  Failed: 'failed',
+  Pending: 'pending',
+} as const
+
+export type ToolStoreStatusKey = keyof typeof ToolStoreStatus
+export type ToolStoreStatusValue = (typeof ToolStoreStatus)[ToolStoreStatusKey]

apps/sim/lib/copilot/generated/trace-events-v1.ts

@@ -0,0 +1,52 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+// Source: copilot/copilot/contracts/trace-events-v1.schema.json
+// Regenerate with: bun run trace-events-contract:generate
+//
+// Canonical mothership OTel span event names. Call sites should
+// reference `TraceEvent.<Identifier>` (e.g.
+// `TraceEvent.RequestCancelled`) rather than raw string literals,
+// so the Go-side contract is the single source of truth and typos
+// become compile errors.
+
+export const TraceEvent = {
+  ContextTransform: 'context.transform',
+  CopilotOutputFileError: 'copilot.output_file.error',
+  CopilotSseFirstEvent: 'copilot.sse.first_event',
+  CopilotSseIdleGapExceeded: 'copilot.sse.idle_gap_exceeded',
+  CopilotSseTerminalEventReceived: 'copilot.sse.terminal_event_received',
+  CopilotTableError: 'copilot.table.error',
+  CopilotVfsParseFailed: 'copilot.vfs.parse_failed',
+  CopilotVfsResizeAttempt: 'copilot.vfs.resize_attempt',
+  CopilotVfsResizeAttemptFailed: 'copilot.vfs.resize_attempt_failed',
+  GenAiPromptCacheBreakpoint: 'gen_ai.prompt_cache.breakpoint',
+  LlmInvokeSent: 'llm.invoke.sent',
+  LlmStreamFirstChunk: 'llm.stream.first_chunk',
+  LlmStreamOpened: 'llm.stream.opened',
+  PgNotifyFailed: 'pg_notify_failed',
+  RedisSubscribed: 'redis.subscribed',
+  RequestCancelled: 'request.cancelled',
+} as const
+
+export type TraceEventKey = keyof typeof TraceEvent
+export type TraceEventValue = (typeof TraceEvent)[TraceEventKey]
+
+/** Readonly sorted list of every canonical event name. */
+export const TraceEventValues: readonly TraceEventValue[] = [
+  'context.transform',
+  'copilot.output_file.error',
+  'copilot.sse.first_event',
+  'copilot.sse.idle_gap_exceeded',
+  'copilot.sse.terminal_event_received',
+  'copilot.table.error',
+  'copilot.vfs.parse_failed',
+  'copilot.vfs.resize_attempt',
+  'copilot.vfs.resize_attempt_failed',
+  'gen_ai.prompt_cache.breakpoint',
+  'llm.invoke.sent',
+  'llm.stream.first_chunk',
+  'llm.stream.opened',
+  'pg_notify_failed',
+  'redis.subscribed',
+  'request.cancelled',
+] as const

apps/sim/lib/copilot/generated/trace-spans-v1.ts

@@ -0,0 +1,155 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+// Source: copilot/copilot/contracts/trace-spans-v1.schema.json
+// Regenerate with: bun run trace-spans-contract:generate
+//
+// Canonical mothership OTel span names. Call sites should reference
+// `TraceSpan.<Identifier>` (e.g. `TraceSpan.CopilotVfsReadFile`)
+// rather than raw string literals, so the Go-side contract is the
+// single source of truth and typos become compile errors.
+
+export const TraceSpan = {
+  AnthropicCountTokens: 'anthropic.count_tokens',
+  AsyncToolStoreSet: 'async_tool_store.set',
+  AuthRateLimitRecord: 'auth.rate_limit.record',
+  AuthValidateKey: 'auth.validate_key',
+  ChatContinueWithToolResults: 'chat.continue_with_tool_results',
+  ChatExplicitAbortConsume: 'chat.explicit_abort.consume',
+  ChatExplicitAbortFlushPausedBilling: 'chat.explicit_abort.flush_paused_billing',
+  ChatExplicitAbortHandle: 'chat.explicit_abort.handle',
+  ChatExplicitAbortMark: 'chat.explicit_abort.mark',
+  ChatExplicitAbortPeek: 'chat.explicit_abort.peek',
+  ChatGateAcquire: 'chat.gate.acquire',
+  ChatPersistAfterDone: 'chat.persist_after_done',
+  ChatSetup: 'chat.setup',
+  ContextReduce: 'context.reduce',
+  ContextSummarizeChunk: 'context.summarize_chunk',
+  CopilotAnalyticsFlush: 'copilot.analytics.flush',
+  CopilotAnalyticsSaveRequest: 'copilot.analytics.save_request',
+  CopilotAnalyticsUpdateBilling: 'copilot.analytics.update_billing',
+  CopilotAsyncRunsClaimCompleted: 'copilot.async_runs.claim_completed',
+  CopilotAsyncRunsCreateRunCheckpoint: 'copilot.async_runs.create_run_checkpoint',
+  CopilotAsyncRunsCreateRunSegment: 'copilot.async_runs.create_run_segment',
+  CopilotAsyncRunsGetAsyncToolCall: 'copilot.async_runs.get_async_tool_call',
+  CopilotAsyncRunsGetLatestForExecution: 'copilot.async_runs.get_latest_for_execution',
+  CopilotAsyncRunsGetLatestForStream: 'copilot.async_runs.get_latest_for_stream',
+  CopilotAsyncRunsGetMany: 'copilot.async_runs.get_many',
+  CopilotAsyncRunsGetRunSegment: 'copilot.async_runs.get_run_segment',
+  CopilotAsyncRunsListForRun: 'copilot.async_runs.list_for_run',
+  CopilotAsyncRunsMarkAsyncToolStatus: 'copilot.async_runs.mark_async_tool_status',
+  CopilotAsyncRunsReleaseClaim: 'copilot.async_runs.release_claim',
+  CopilotAsyncRunsUpdateRunStatus: 'copilot.async_runs.update_run_status',
+  CopilotAsyncRunsUpsertAsyncToolCall: 'copilot.async_runs.upsert_async_tool_call',
+  CopilotAuthValidateApiKey: 'copilot.auth.validate_api_key',
+  CopilotBillingUpdateCost: 'copilot.billing.update_cost',
+  CopilotChatAbortActiveStream: 'copilot.chat.abort_active_stream',
+  CopilotChatAbortStream: 'copilot.chat.abort_stream',
+  CopilotChatAbortWaitSettle: 'copilot.chat.abort_wait_settle',
+  CopilotChatAcquirePendingStreamLock: 'copilot.chat.acquire_pending_stream_lock',
+  CopilotChatBuildExecutionContext: 'copilot.chat.build_execution_context',
+  CopilotChatBuildPayload: 'copilot.chat.build_payload',
+  CopilotChatBuildWorkspaceContext: 'copilot.chat.build_workspace_context',
+  CopilotChatFinalizeAssistantTurn: 'copilot.chat.finalize_assistant_turn',
+  CopilotChatPersistUserMessage: 'copilot.chat.persist_user_message',
+  CopilotChatResolveAgentContexts: 'copilot.chat.resolve_agent_contexts',
+  CopilotChatResolveBranch: 'copilot.chat.resolve_branch',
+  CopilotChatResolveOrCreateChat: 'copilot.chat.resolve_or_create_chat',
+  CopilotChatStopStream: 'copilot.chat.stop_stream',
+  CopilotConfirmToolResult: 'copilot.confirm.tool_result',
+  CopilotFinalizeStream: 'copilot.finalize_stream',
+  CopilotRecoveryCheckReplayGap: 'copilot.recovery.check_replay_gap',
+  CopilotResumeRequest: 'copilot.resume.request',
+  CopilotSseReadLoop: 'copilot.sse.read_loop',
+  CopilotSubagentExecute: 'copilot.subagent.execute',
+  CopilotToolWaitForClientResult: 'copilot.tool.wait_for_client_result',
+  CopilotToolsHandleResourceSideEffects: 'copilot.tools.handle_resource_side_effects',
+  CopilotToolsWriteCsvToTable: 'copilot.tools.write_csv_to_table',
+  CopilotToolsWriteOutputFile: 'copilot.tools.write_output_file',
+  CopilotToolsWriteOutputTable: 'copilot.tools.write_output_table',
+  CopilotVfsMaterialize: 'copilot.vfs.materialize',
+  CopilotVfsPrepareImage: 'copilot.vfs.prepare_image',
+  CopilotVfsReadFile: 'copilot.vfs.read_file',
+  GenAiAgentExecute: 'gen_ai.agent.execute',
+  LlmStream: 'llm.stream',
+  ProviderRouterCountTokens: 'provider.router.count_tokens',
+  ProviderRouterRoute: 'provider.router.route',
+  SimUpdateCost: 'sim.update_cost',
+  SimValidateApiKey: 'sim.validate_api_key',
+  ToolAsyncWaiterWait: 'tool.async_waiter.wait',
+  ToolExecute: 'tool.execute',
+} as const
+
+export type TraceSpanKey = keyof typeof TraceSpan
+export type TraceSpanValue = (typeof TraceSpan)[TraceSpanKey]
+
+/** Readonly sorted list of every canonical span name. */
+export const TraceSpanValues: readonly TraceSpanValue[] = [
+  'anthropic.count_tokens',
+  'async_tool_store.set',
+  'auth.rate_limit.record',
+  'auth.validate_key',
+  'chat.continue_with_tool_results',
+  'chat.explicit_abort.consume',
+  'chat.explicit_abort.flush_paused_billing',
+  'chat.explicit_abort.handle',
+  'chat.explicit_abort.mark',
+  'chat.explicit_abort.peek',
+  'chat.gate.acquire',
+  'chat.persist_after_done',
+  'chat.setup',
+  'context.reduce',
+  'context.summarize_chunk',
+  'copilot.analytics.flush',
+  'copilot.analytics.save_request',
+  'copilot.analytics.update_billing',
+  'copilot.async_runs.claim_completed',
+  'copilot.async_runs.create_run_checkpoint',
+  'copilot.async_runs.create_run_segment',
+  'copilot.async_runs.get_async_tool_call',
+  'copilot.async_runs.get_latest_for_execution',
+  'copilot.async_runs.get_latest_for_stream',
+  'copilot.async_runs.get_many',
+  'copilot.async_runs.get_run_segment',
+  'copilot.async_runs.list_for_run',
+  'copilot.async_runs.mark_async_tool_status',
+  'copilot.async_runs.release_claim',
+  'copilot.async_runs.update_run_status',
+  'copilot.async_runs.upsert_async_tool_call',
+  'copilot.auth.validate_api_key',
+  'copilot.billing.update_cost',
+  'copilot.chat.abort_active_stream',
+  'copilot.chat.abort_stream',
+  'copilot.chat.abort_wait_settle',
+  'copilot.chat.acquire_pending_stream_lock',
+  'copilot.chat.build_execution_context',
+  'copilot.chat.build_payload',
+  'copilot.chat.build_workspace_context',
+  'copilot.chat.finalize_assistant_turn',
+  'copilot.chat.persist_user_message',
+  'copilot.chat.resolve_agent_contexts',
+  'copilot.chat.resolve_branch',
+  'copilot.chat.resolve_or_create_chat',
+  'copilot.chat.stop_stream',
+  'copilot.confirm.tool_result',
+  'copilot.finalize_stream',
+  'copilot.recovery.check_replay_gap',
+  'copilot.resume.request',
+  'copilot.sse.read_loop',
+  'copilot.subagent.execute',
+  'copilot.tool.wait_for_client_result',
+  'copilot.tools.handle_resource_side_effects',
+  'copilot.tools.write_csv_to_table',
+  'copilot.tools.write_output_file',
+  'copilot.tools.write_output_table',
+  'copilot.vfs.materialize',
+  'copilot.vfs.prepare_image',
+  'copilot.vfs.read_file',
+  'gen_ai.agent.execute',
+  'llm.stream',
+  'provider.router.count_tokens',
+  'provider.router.route',
+  'sim.update_cost',
+  'sim.validate_api_key',
+  'tool.async_waiter.wait',
+  'tool.execute',
+] as const

apps/sim/lib/copilot/generated/vfs-snapshot-v1.ts

@@ -0,0 +1,131 @@
+// AUTO-GENERATED FILE. DO NOT EDIT.
+//
+
+/**
+ * Structured workspace inventory snapshot Sim sends to Go; Go diffs successive snapshots into baseline+delta messages.
+ */
+export interface VfsSnapshotV1 {
+  customTools?: VfsSnapshotV1NamedResource[]
+  envVars?: string[]
+  files?: VfsSnapshotV1File[]
+  integrations?: VfsSnapshotV1Integration[]
+  jobs?: VfsSnapshotV1Job[]
+  knowledgeBases?: VfsSnapshotV1KnowledgeBase[]
+  mcpServers?: VfsSnapshotV1McpServer[]
+  members?: VfsSnapshotV1Member[]
+  skills?: VfsSnapshotV1Skill[]
+  tables?: VfsSnapshotV1Table[]
+  workflows?: VfsSnapshotV1Workflow[]
+  workspace?: VfsSnapshotV1Workspace
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1NamedResource".
+ */
+export interface VfsSnapshotV1NamedResource {
+  id: string
+  name: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1File".
+ */
+export interface VfsSnapshotV1File {
+  folderPath?: string
+  id: string
+  name: string
+  path: string
+  size?: number
+  type?: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Integration".
+ */
+export interface VfsSnapshotV1Integration {
+  displayName?: string
+  id: string
+  providerId: string
+  role?: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Job".
+ */
+export interface VfsSnapshotV1Job {
+  cronExpression?: string
+  id: string
+  lifecycle?: string
+  prompt?: string
+  sourceTaskName?: string
+  status?: string
+  title?: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1KnowledgeBase".
+ */
+export interface VfsSnapshotV1KnowledgeBase {
+  connectorTypes?: string[]
+  description?: string
+  id: string
+  name: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1McpServer".
+ */
+export interface VfsSnapshotV1McpServer {
+  enabled?: boolean
+  id: string
+  name: string
+  url?: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Member".
+ */
+export interface VfsSnapshotV1Member {
+  email: string
+  name?: string
+  permissionType?: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Skill".
+ */
+export interface VfsSnapshotV1Skill {
+  description?: string
+  id: string
+  name: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Table".
+ */
+export interface VfsSnapshotV1Table {
+  description?: string
+  id: string
+  name: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Workflow".
+ */
+export interface VfsSnapshotV1Workflow {
+  description?: string
+  folderPath?: string
+  id: string
+  isDeployed?: boolean
+  name: string
+  path: string
+}
+/**
+ * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema
+ * via the `definition` "VfsSnapshotV1Workspace".
+ */
+export interface VfsSnapshotV1Workspace {
+  id: string
+  name: string
+  ownerId?: string
+}

apps/sim/lib/copilot/integration-tools.ts

@@ -0,0 +1,103 @@
+import type { BlockVisibilityState } from '@/lib/core/config/block-visibility'
+import { BLOCK_REGISTRY } from '@/blocks/registry-maps'
+import { isHiddenUnder } from '@/blocks/visibility/context'
+import { tools as toolRegistry } from '@/tools/registry'
+import type { ToolConfig } from '@/tools/types'
+import { getLatestVersionTools, stripVersionSuffix } from '@/tools/utils'
+
+export interface ExposedIntegrationTool {
+  /**
+   * Full registry tool id — also the agent-callable id and the schema `id`
+   * field (e.g. gmail_read_v2). No stripping: discovery, the schema id, and the
+   * callable id are all this exact value, matching the block's tools.access.
+   */
+  toolId: string
+  config: ToolConfig
+  /** Service directory name, e.g. "gmail". */
+  service: string
+  /** Operation stem within the service (used for the VFS path filename), e.g. "read". */
+  operation: string
+  /** Owning block's registry type — the key block-visibility rules gate on. */
+  blockType: string
+  /** Owning block's static `preview` marker, for the per-viewer filter. */
+  preview?: boolean
+}
+
+let cached: ExposedIntegrationTool[] | null = null
+
+/**
+ * Returns the UNGATED universe of integration tools exposable to the copilot
+ * agent: the latest version of each operation owned by a non-`hideFromToolbar`
+ * block — INCLUDING unreleased `preview` blocks.
+ *
+ * Deliberately sourced from the raw `BLOCK_REGISTRY` (never the visibility-
+ * projected `getAllBlocks`) so this process-global memo is deterministic and
+ * can never be poisoned by whichever viewer's gated projection ran first.
+ * Every per-viewer consumer MUST apply {@link filterExposedIntegrationTools}
+ * before exposing the set.
+ *
+ * This is the single source of truth shared by VFS discovery
+ * (components/integrations/**) and the deferred callable-tool payload, so the
+ * agent can call exactly what it can discover — no orphan callable tools, and no
+ * version drift between what the VFS shows and what is loadable.
+ */
+export function getExposedIntegrationTools(): ExposedIntegrationTool[] {
+  if (cached) return cached
+
+  // Map the tool ids each visible block exposes (both the raw id and its
+  // version-stripped base name) to that block's service directory + type.
+  const toolToBlock = new Map<string, { service: string; blockType: string; preview?: boolean }>()
+  for (const block of Object.values(BLOCK_REGISTRY)) {
+    if (block.hideFromToolbar) continue
+    if (!block.tools?.access) continue
+    const service = stripVersionSuffix(block.type)
+    const owner = { service, blockType: block.type, preview: block.preview }
+    for (const toolId of block.tools.access) {
+      toolToBlock.set(toolId, owner)
+      toolToBlock.set(stripVersionSuffix(toolId), owner)
+    }
+  }
+
+  const exposed: ExposedIntegrationTool[] = []
+  const seen = new Set<string>()
+  for (const [toolId, config] of Object.entries(getLatestVersionTools(toolRegistry))) {
+    const baseName = stripVersionSuffix(toolId)
+    const owner = toolToBlock.get(toolId) ?? toolToBlock.get(baseName)
+    if (!owner) continue
+    if (seen.has(baseName)) continue
+    seen.add(baseName)
+    const prefix = `${owner.service}_`
+    const operation = baseName.startsWith(prefix) ? baseName.slice(prefix.length) : baseName
+    exposed.push({
+      toolId,
+      config,
+      service: owner.service,
+      operation,
+      blockType: owner.blockType,
+      preview: owner.preview,
+    })
+  }
+
+  cached = exposed
+  return exposed
+}
+
+/**
+ * Per-viewer projection of the exposed set: drops tools whose owning block is
+ * hidden under `vis` (unrevealed preview blocks — including with a null state —
+ * and kill-switched types). Apply at every surface that hands the set to a
+ * viewer: VFS stamping, the deferred tool payload, `list_integration_tools`.
+ */
+export function filterExposedIntegrationTools(
+  tools: ExposedIntegrationTool[],
+  vis: BlockVisibilityState | null
+): ExposedIntegrationTool[] {
+  return tools.filter(
+    (tool) => !isHiddenUnder(vis, { type: tool.blockType, preview: tool.preview })
+  )
+}
+
+/** Test-only: clears the memoized set so registry changes are picked up. */
+export function resetExposedIntegrationToolsCache(): void {
+  cached = null
+}

apps/sim/lib/copilot/persistence/tool-confirm/index.ts

@@ -0,0 +1,178 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import {
+  ASYNC_TOOL_STATUS,
+  type AsyncCompletionEnvelope,
+  type AsyncConfirmationState,
+  isAsyncEphemeralConfirmationStatus,
+} from '@/lib/copilot/async-runs/lifecycle'
+import { getAsyncToolCalls } from '@/lib/copilot/async-runs/repository'
+import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1'
+import { getRedisClient } from '@/lib/core/config/redis'
+import { createPubSubChannel, type PubSubChannel } from '@/lib/events/pubsub'
+
+const logger = createLogger('CopilotOrchestratorPersistence')
+const TOOL_CONFIRMATION_TTL_SECONDS = 60 * 10
+const toolConfirmationKey = (toolCallId: string) => `copilot:tool-confirmation:${toolCallId}`
+
+type ToolConfirmGlobal = typeof globalThis & {
+  _toolConfirmationChannel?: PubSubChannel<AsyncCompletionEnvelope>
+}
+
+const _g = globalThis as ToolConfirmGlobal
+if (!_g._toolConfirmationChannel) {
+  _g._toolConfirmationChannel = createPubSubChannel<AsyncCompletionEnvelope>({
+    channel: 'copilot:tool-confirmation',
+    label: 'CopilotToolConfirmation',
+  })
+}
+const toolConfirmationChannel = _g._toolConfirmationChannel
+
+/**
+ * Get a tool call confirmation state from the durable async tool row.
+ *
+ * The durable row reconstructs only the live execution lifecycle plus the
+ * `background` detach signal.
+ */
+export async function getToolConfirmation(
+  toolCallId: string
+): Promise<AsyncConfirmationState | null> {
+  const [row] = await getAsyncToolCalls([toolCallId]).catch((err) => {
+    logger.warn('Failed to fetch async tool calls', {
+      toolCallId,
+      error: toError(err).message,
+    })
+    return []
+  })
+  if (!row) return null
+  if (row.status === ASYNC_TOOL_STATUS.delivered) {
+    logger.warn('Delivered async tool rows are outside request confirmation flow', {
+      toolCallId,
+    })
+    return null
+  }
+  return {
+    status:
+      row.status === ASYNC_TOOL_STATUS.completed
+        ? MothershipStreamV1ToolOutcome.success
+        : row.status === ASYNC_TOOL_STATUS.failed
+          ? MothershipStreamV1ToolOutcome.error
+          : row.status === ASYNC_TOOL_STATUS.cancelled
+            ? MothershipStreamV1ToolOutcome.cancelled
+            : row.status,
+    message: row.error || undefined,
+    ...(row.result !== undefined ? { data: row.result } : {}),
+    timestamp: row.updatedAt?.toISOString?.(),
+  }
+}
+
+export function publishToolConfirmation(event: AsyncCompletionEnvelope): void {
+  logger.info('Publishing tool confirmation event', {
+    toolCallId: event.toolCallId,
+    status: event.status,
+  })
+  const redis = getRedisClient()
+  if (redis) {
+    void redis
+      .set(
+        toolConfirmationKey(event.toolCallId),
+        JSON.stringify(event),
+        'EX',
+        TOOL_CONFIRMATION_TTL_SECONDS
+      )
+      .then(() => {
+        logger.info('Persisted tool confirmation in Redis', {
+          toolCallId: event.toolCallId,
+          status: event.status,
+          redisKey: toolConfirmationKey(event.toolCallId),
+        })
+      })
+      .catch((error) => {
+        logger.warn('Failed to persist tool confirmation in Redis', {
+          toolCallId: event.toolCallId,
+          error: toError(error).message,
+        })
+      })
+  } else {
+    logger.warn('Redis unavailable while publishing tool confirmation', {
+      toolCallId: event.toolCallId,
+      status: event.status,
+    })
+  }
+  toolConfirmationChannel.publish(event)
+}
+
+/**
+ * Wait for an async tool confirmation update.
+ *
+ * `background` still arrives as a request-local detach signal, so it is checked
+ * from pubsub before falling back to the durable async tool row.
+ */
+export async function waitForToolConfirmation(
+  toolCallId: string,
+  timeoutMs: number,
+  abortSignal?: AbortSignal,
+  options: {
+    acceptStatus?: (status: AsyncConfirmationState['status']) => boolean
+  } = {}
+): Promise<AsyncConfirmationState | null> {
+  const acceptStatus = options.acceptStatus ?? (() => true)
+  return new Promise((resolve) => {
+    let settled = false
+    let timeoutId: ReturnType<typeof setTimeout> | null = null
+    let unsubscribe: (() => void) | null = null
+
+    const cleanup = () => {
+      if (timeoutId) clearTimeout(timeoutId)
+      if (unsubscribe) unsubscribe()
+      abortSignal?.removeEventListener('abort', onAbort)
+    }
+
+    const settle = (value: AsyncConfirmationState | null) => {
+      if (settled) return
+      settled = true
+      cleanup()
+      resolve(value)
+    }
+
+    const onAbort = () => settle(null)
+
+    unsubscribe = toolConfirmationChannel.subscribe((event) => {
+      if (event.toolCallId !== toolCallId) return
+      if (isAsyncEphemeralConfirmationStatus(event.status) && acceptStatus(event.status)) {
+        settle({
+          status: event.status,
+          ...(event.message !== undefined ? { message: event.message } : {}),
+          ...(event.data !== undefined ? { data: event.data } : {}),
+          ...(event.timestamp !== undefined ? { timestamp: event.timestamp } : {}),
+        })
+        return
+      }
+      void getToolConfirmation(toolCallId).then((latest) => {
+        if (!latest || !acceptStatus(latest.status)) return
+        logger.info('Resolved tool confirmation from pubsub', {
+          toolCallId,
+          status: latest.status,
+        })
+        settle(latest)
+      })
+    })
+
+    timeoutId = setTimeout(() => settle(null), timeoutMs)
+    if (abortSignal?.aborted) {
+      settle(null)
+      return
+    }
+    abortSignal?.addEventListener('abort', onAbort, { once: true })
+
+    void getToolConfirmation(toolCallId).then((latest) => {
+      if (latest && acceptStatus(latest.status)) {
+        logger.info('Resolved tool confirmation after subscribe', {
+          toolCallId,
+          status: latest.status,
+        })
+        settle(latest)
+      }
+    })
+  })
+}

apps/sim/lib/copilot/persistence/tool-confirm/tool-confirm.test.ts

@@ -0,0 +1,166 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { getAsyncToolCalls } = vi.hoisted(() => ({
+  getAsyncToolCalls: vi.fn(),
+}))
+
+const channelHandlers = new Set<(event: any) => void>()
+
+vi.mock('@/lib/copilot/async-runs/repository', () => ({
+  getAsyncToolCalls,
+}))
+
+vi.mock('@/lib/events/pubsub', () => ({
+  createPubSubChannel: () => ({
+    publish(event: any) {
+      for (const handler of channelHandlers) handler(event)
+    },
+    subscribe(handler: (event: any) => void) {
+      channelHandlers.add(handler)
+      return () => {
+        channelHandlers.delete(handler)
+      }
+    },
+    dispose() {},
+  }),
+}))
+
+import {
+  getToolConfirmation,
+  publishToolConfirmation,
+  waitForToolConfirmation,
+} from '@/lib/copilot/persistence/tool-confirm'
+
+describe('copilot orchestrator persistence', () => {
+  let row: {
+    status: string
+    error?: string | null
+    result?: unknown
+    updatedAt: Date
+  } | null
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    channelHandlers.clear()
+    row = null
+    getAsyncToolCalls.mockImplementation(async () => (row ? [row] : []))
+  })
+
+  it('reads the durable DB row as the source of truth', async () => {
+    row = {
+      status: 'completed',
+      result: { ok: true },
+      error: null,
+      updatedAt: new Date('2026-01-01T00:00:00.000Z'),
+    }
+
+    await expect(getToolConfirmation('tool-1')).resolves.toEqual({
+      status: 'success',
+      message: undefined,
+      data: { ok: true },
+      timestamp: '2026-01-01T00:00:00.000Z',
+    })
+  })
+
+  it('preserves primitive durable results in confirmations', async () => {
+    row = {
+      status: 'completed',
+      result: 'done',
+      error: null,
+      updatedAt: new Date('2026-01-01T00:00:00.000Z'),
+    }
+
+    await expect(getToolConfirmation('tool-1')).resolves.toEqual({
+      status: 'success',
+      message: undefined,
+      data: 'done',
+      timestamp: '2026-01-01T00:00:00.000Z',
+    })
+  })
+
+  it('ignores delivered rows in request confirmation flow', async () => {
+    row = {
+      status: 'delivered',
+      result: { ok: true },
+      error: null,
+      updatedAt: new Date('2026-01-01T00:00:00.000Z'),
+    }
+
+    await expect(getToolConfirmation('tool-1')).resolves.toBeNull()
+  })
+
+  it('ignores background when waiting for a foreground terminal status', async () => {
+    row = {
+      status: 'pending',
+      error: null,
+      result: null,
+      updatedAt: new Date('2026-01-01T00:00:00.000Z'),
+    }
+
+    const waitPromise = waitForToolConfirmation('tool-1', 5_000, undefined, {
+      acceptStatus: (status) =>
+        status === 'success' || status === 'error' || status === 'cancelled',
+    })
+
+    publishToolConfirmation({
+      toolCallId: 'tool-1',
+      status: 'background',
+      message: 'Client disconnected, execution continuing server-side',
+      timestamp: '2026-01-01T00:00:01.000Z',
+    })
+
+    await Promise.resolve()
+
+    row = {
+      status: 'completed',
+      error: null,
+      result: { ok: true },
+      updatedAt: new Date('2026-01-01T00:00:02.000Z'),
+    }
+
+    publishToolConfirmation({
+      toolCallId: 'tool-1',
+      status: 'success',
+      timestamp: '2026-01-01T00:00:02.000Z',
+    })
+
+    await expect(waitPromise).resolves.toEqual({
+      status: 'success',
+      message: undefined,
+      data: { ok: true },
+      timestamp: '2026-01-01T00:00:02.000Z',
+    })
+  })
+
+  it('resolves background from the pubsub event when the durable row stays pending', async () => {
+    row = {
+      status: 'pending',
+      error: null,
+      result: null,
+      updatedAt: new Date('2026-01-01T00:00:00.000Z'),
+    }
+
+    const waitPromise = waitForToolConfirmation('tool-1', 5_000, undefined, {
+      acceptStatus: (status) => status === 'background',
+    })
+
+    await Promise.resolve()
+
+    publishToolConfirmation({
+      toolCallId: 'tool-1',
+      status: 'background',
+      message: 'Client disconnected, execution continuing server-side',
+      timestamp: '2026-01-01T00:00:01.000Z',
+    })
+
+    await expect(waitPromise).resolves.toEqual({
+      status: 'background',
+      message: 'Client disconnected, execution continuing server-side',
+      timestamp: '2026-01-01T00:00:01.000Z',
+    })
+  })
+})

apps/sim/lib/copilot/request/context/request-context.ts

@@ -0,0 +1,33 @@
+import { generateId } from '@sim/utils/id'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+import type { StreamingContext } from '@/lib/copilot/request/types'
+
+/**
+ * Create a fresh StreamingContext.
+ */
+export function createStreamingContext(overrides?: Partial<StreamingContext>): StreamingContext {
+  return {
+    chatId: undefined,
+    executionId: undefined,
+    runId: undefined,
+    messageId: generateId(),
+    accumulatedContent: '',
+    finalAssistantContent: '',
+    sawMainToolCall: false,
+    contentBlocks: [],
+    toolCalls: new Map(),
+    pendingToolPromises: new Map(),
+    currentThinkingBlock: null,
+    subagentThinkingBlocks: new Map(),
+    isInThinkingBlock: false,
+    subAgentContent: {},
+    subAgentToolCalls: {},
+    pendingContent: '',
+    streamComplete: false,
+    wasAborted: false,
+    errors: [],
+    activeFileIntents: new Map(),
+    trace: new TraceCollector(),
+    ...overrides,
+  }
+}

apps/sim/lib/copilot/request/context/result.test.ts

@@ -0,0 +1,97 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1'
+import { FunctionExecute } from '@/lib/copilot/generated/tool-catalog-v1'
+import { buildToolCallSummaries } from '@/lib/copilot/request/context/result'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+import type { StreamingContext } from '@/lib/copilot/request/types'
+
+function makeContext(): StreamingContext {
+  return {
+    chatId: undefined,
+    requestId: undefined,
+    executionId: undefined,
+    runId: undefined,
+    messageId: 'msg-1',
+    accumulatedContent: '',
+    finalAssistantContent: '',
+    sawMainToolCall: false,
+    contentBlocks: [],
+    toolCalls: new Map(),
+    pendingToolPromises: new Map(),
+    awaitingAsyncContinuation: undefined,
+    currentThinkingBlock: null,
+    subagentThinkingBlocks: new Map(),
+    isInThinkingBlock: false,
+    subAgentContent: {},
+    subAgentToolCalls: {},
+    pendingContent: '',
+    streamComplete: false,
+    wasAborted: false,
+    errors: [],
+    trace: new TraceCollector(),
+  }
+}
+
+describe('buildToolCallSummaries', () => {
+  it.concurrent('keeps pending tools as pending instead of defaulting to success', () => {
+    const context = makeContext()
+    context.toolCalls.set('tool-1', {
+      id: 'tool-1',
+      name: 'download_to_workspace_file',
+      status: 'pending',
+      startTime: 1,
+    })
+
+    const summaries = buildToolCallSummaries(context)
+
+    expect(summaries).toHaveLength(1)
+    expect(summaries[0]?.status).toBe('pending')
+  })
+
+  it.concurrent('keeps executing tools as executing when no result exists yet', () => {
+    const context = makeContext()
+    context.toolCalls.set('tool-2', {
+      id: 'tool-2',
+      name: FunctionExecute.id,
+      status: 'executing',
+      startTime: 1,
+    })
+
+    const summaries = buildToolCallSummaries(context)
+
+    expect(summaries).toHaveLength(1)
+    expect(summaries[0]?.status).toBe('executing')
+  })
+
+  it.concurrent(
+    'preserves canonical terminal statuses instead of deriving them from result success',
+    () => {
+      const context = makeContext()
+      context.toolCalls.set('tool-3', {
+        id: 'tool-3',
+        name: 'download_to_workspace_file',
+        status: MothershipStreamV1ToolOutcome.cancelled,
+        result: { success: false },
+        error: 'Stopped by user',
+        startTime: 1,
+        endTime: 2,
+      })
+
+      const summaries = buildToolCallSummaries(context)
+
+      expect(summaries).toHaveLength(1)
+      expect(summaries[0]).toEqual({
+        id: 'tool-3',
+        name: 'download_to_workspace_file',
+        status: MothershipStreamV1ToolOutcome.cancelled,
+        params: undefined,
+        result: undefined,
+        error: 'Stopped by user',
+        durationMs: 1,
+      })
+    }
+  )
+})

apps/sim/lib/copilot/request/context/result.ts

@@ -0,0 +1,18 @@
+import { getToolCallStateOutput } from '@/lib/copilot/request/tool-call-state'
+import type { StreamingContext, ToolCallSummary } from '@/lib/copilot/request/types'
+
+/**
+ * Build a ToolCallSummary array from the streaming context.
+ */
+export function buildToolCallSummaries(context: StreamingContext): ToolCallSummary[] {
+  return Array.from(context.toolCalls.values()).map((toolCall) => ({
+    id: toolCall.id,
+    name: toolCall.name,
+    status: toolCall.status,
+    params: toolCall.params,
+    result: getToolCallStateOutput(toolCall),
+    error: toolCall.error,
+    durationMs:
+      toolCall.endTime && toolCall.startTime ? toolCall.endTime - toolCall.startTime : undefined,
+  }))
+}

apps/sim/lib/copilot/request/go/fetch.test.ts

@@ -0,0 +1,79 @@
+import { trace } from '@opentelemetry/api'
+import {
+  BasicTracerProvider,
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} from '@opentelemetry/sdk-trace-base'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { fetchGo } from '@/lib/copilot/request/go/fetch'
+
+describe('fetchGo', () => {
+  const exporter = new InMemorySpanExporter()
+  const provider = new BasicTracerProvider({
+    spanProcessors: [new SimpleSpanProcessor(exporter)],
+  })
+
+  beforeEach(() => {
+    exporter.reset()
+    trace.setGlobalTracerProvider(provider)
+    vi.restoreAllMocks()
+  })
+
+  it('emits a client span with http.* attrs and injects traceparent', async () => {
+    const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {
+      const headers = init.headers as Record<string, string>
+      expect(headers.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
+      return new Response('ok', {
+        status: 200,
+        headers: { 'content-length': '2' },
+      })
+    })
+    vi.stubGlobal('fetch', fetchMock)
+
+    const res = await fetchGo('https://backend.example.com/api/copilot', {
+      method: 'POST',
+      body: 'payload',
+      operation: 'stream',
+      attributes: { 'copilot.leg': 'sim_to_go' },
+    })
+    expect(res.status).toBe(200)
+
+    const spans = exporter.getFinishedSpans()
+    expect(spans).toHaveLength(1)
+    const attrs = spans[0].attributes
+    expect(spans[0].name).toBe('sim → go /api/copilot')
+    expect(attrs['http.method']).toBe('POST')
+    expect(attrs['http.url']).toBe('https://backend.example.com/api/copilot')
+    expect(attrs['http.target']).toBe('/api/copilot')
+    expect(attrs['http.status_code']).toBe(200)
+    expect(attrs['copilot.operation']).toBe('stream')
+    expect(attrs['copilot.leg']).toBe('sim_to_go')
+    expect(typeof attrs['http.response.headers_ms']).toBe('number')
+  })
+
+  it('marks span as error on non-2xx response', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('nope', { status: 500 })))
+
+    const res = await fetchGo('https://backend.example.com/api/tools/resume', {
+      method: 'POST',
+    })
+    expect(res.status).toBe(500)
+
+    const spans = exporter.getFinishedSpans()
+    expect(spans).toHaveLength(1)
+    expect(spans[0].status.code).toBe(2)
+  })
+
+  it('records exceptions when fetch throws', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network boom')))
+
+    await expect(
+      fetchGo('https://backend.example.com/api/traces', { method: 'POST' })
+    ).rejects.toThrow('network boom')
+
+    const spans = exporter.getFinishedSpans()
+    expect(spans).toHaveLength(1)
+    expect(spans[0].status.code).toBe(2)
+    expect(spans[0].events.some((e) => e.name === 'exception')).toBe(true)
+  })
+})

apps/sim/lib/copilot/request/go/fetch.ts

@@ -0,0 +1,112 @@
+import { type Context, context, SpanStatusCode, trace } from '@opentelemetry/api'
+import { CopilotLeg } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { traceHeaders } from '@/lib/copilot/request/go/propagation'
+import { isActionableErrorStatus, markSpanForError } from '@/lib/copilot/request/otel'
+
+// Lazy tracer resolution: module-level `trace.getTracer()` can be evaluated
+// before `instrumentation-node.ts` installs the TracerProvider under
+// Next.js 16 + Turbopack dev, freezing a NoOp tracer and silently dropping
+// every outbound Sim → Go span. Resolving per-call avoids the race.
+const getTracer = () => trace.getTracer('sim-copilot-http', '1.0.0')
+
+interface OutboundFetchOptions extends RequestInit {
+  otelContext?: Context
+  spanName?: string
+  operation?: string
+  attributes?: Record<string, string | number | boolean>
+}
+
+/**
+ * Perform an outbound Sim → Go fetch wrapped in an OTel child span so each
+ * call shows up as a distinct segment in Jaeger, and propagates the W3C
+ * traceparent so the Go-side span joins the same trace.
+ *
+ * The span captures generic attributes (method, status, duration, response
+ * size, error code) so any future latency investigation — not just images or
+ * Bedrock — has uniform metadata to work with.
+ */
+export async function fetchGo(url: string, options: OutboundFetchOptions = {}): Promise<Response> {
+  const {
+    otelContext,
+    spanName,
+    operation,
+    attributes,
+    headers: providedHeaders,
+    ...init
+  } = options
+
+  const parsed = safeParseUrl(url)
+  const pathname = parsed?.pathname ?? url
+  const method = (init.method ?? 'GET').toUpperCase()
+  const parentContext = otelContext ?? context.active()
+
+  const span = getTracer().startSpan(
+    spanName ?? `sim → go ${pathname}`,
+    {
+      attributes: {
+        [TraceAttr.HttpMethod]: method,
+        [TraceAttr.HttpUrl]: url,
+        [TraceAttr.HttpTarget]: pathname,
+        [TraceAttr.NetPeerName]: parsed?.host ?? '',
+        [TraceAttr.CopilotLeg]: CopilotLeg.SimToGo,
+        ...(operation ? { [TraceAttr.CopilotOperation]: operation } : {}),
+        ...(attributes ?? {}),
+      },
+    },
+    parentContext
+  )
+
+  const activeContext = trace.setSpan(parentContext, span)
+  const propagatedHeaders = traceHeaders({}, activeContext)
+  const mergedHeaders = {
+    ...(providedHeaders as Record<string, string> | undefined),
+    ...propagatedHeaders,
+  }
+
+  const start = performance.now()
+  try {
+    const response = await context.with(activeContext, () =>
+      fetch(url, {
+        ...init,
+        method,
+        headers: mergedHeaders,
+      })
+    )
+    const elapsedMs = performance.now() - start
+    const contentLength = Number(response.headers.get('content-length') ?? 0)
+    span.setAttribute(TraceAttr.HttpStatusCode, response.status)
+    span.setAttribute(TraceAttr.HttpResponseHeadersMs, Math.round(elapsedMs))
+    if (contentLength > 0) {
+      span.setAttribute(TraceAttr.HttpResponseContentLength, contentLength)
+    }
+    // Only mark ERROR for actionable status codes. 4xx that represent
+    // normal auth/validation rejections (400/401/403/404/405/422/etc.)
+    // stay UNSET so error dashboards don't drown in expected rejection
+    // paths. See `isActionableErrorStatus` in Go's telemetry middleware
+    // for the mirror rule (5xx + 402/409/429).
+    if (isActionableErrorStatus(response.status)) {
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: `HTTP ${response.status}`,
+      })
+    } else {
+      span.setStatus({ code: SpanStatusCode.OK })
+    }
+    return response
+  } catch (error) {
+    span.setAttribute(TraceAttr.HttpResponseHeadersMs, Math.round(performance.now() - start))
+    markSpanForError(span, error)
+    throw error
+  } finally {
+    span.end()
+  }
+}
+
+function safeParseUrl(url: string): URL | null {
+  try {
+    return new URL(url)
+  } catch {
+    return null
+  }
+}

apps/sim/lib/copilot/request/go/file-preview-adapter.ts

@@ -0,0 +1,761 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { MothershipStreamV1EventType } from '@/lib/copilot/generated/mothership-stream-v1'
+import {
+  createFilePreviewSession,
+  type FilePreviewContentMode,
+  type FilePreviewSession,
+  type FilePreviewTargetKind,
+  isToolArgsDeltaStreamEvent,
+  isToolCallStreamEvent,
+  isToolResultStreamEvent,
+  type SyntheticFilePreviewPayload,
+  upsertFilePreviewSession,
+} from '@/lib/copilot/request/session'
+import type {
+  ActiveFileIntent,
+  ExecutionContext,
+  OrchestratorOptions,
+  StreamEvent,
+  StreamingContext,
+} from '@/lib/copilot/request/types'
+import { peekFileIntent } from '@/lib/copilot/tools/server/files/file-intent-store'
+import {
+  buildFilePreviewText,
+  loadWorkspaceFileTextForPreview,
+} from '@/lib/copilot/tools/server/files/file-preview'
+import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+
+const logger = createLogger('CopilotFilePreviewAdapter')
+
+type JsonRecord = Record<string, unknown>
+type FileIntent = ActiveFileIntent
+
+type EditContentStreamState = {
+  raw: string
+  lastContentSnapshot?: string
+}
+
+type FilePreviewStreamState = {
+  session: FilePreviewSession
+  lastEmittedPreviewText: string
+  lastSnapshotAt: number
+}
+
+type ParsedWorkspaceFileArgs = {
+  operation: string
+  target: FileIntent['target']
+  title?: string
+  contentType?: string
+  edit?: JsonRecord
+}
+
+const PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS = 80
+const DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS = 1000
+
+function asJsonRecord(value: unknown): JsonRecord | undefined {
+  return value && typeof value === 'object' && !Array.isArray(value)
+    ? (value as JsonRecord)
+    : undefined
+}
+
+function toPreviewTargetKind(kind: string | undefined): FilePreviewTargetKind | undefined {
+  return kind === 'new_file' || kind === 'file_id' ? kind : undefined
+}
+
+async function resolvePreviewTarget(args: {
+  workspaceId?: string
+  target: FileIntent['target']
+}): Promise<FileIntent['target']> {
+  if (args.target.kind !== 'path' || !args.workspaceId || !args.target.path) {
+    return args.target
+  }
+
+  const file = await resolveWorkspaceFileReference(args.workspaceId, args.target.path)
+  if (!file) {
+    return args.target
+  }
+
+  return {
+    kind: 'file_id',
+    fileId: file.id,
+    fileName: args.target.fileName ?? file.name,
+    path: args.target.path,
+  }
+}
+
+function parseWorkspaceFileArgs(value: unknown): ParsedWorkspaceFileArgs | undefined {
+  const args = asJsonRecord(value)
+  if (!args) {
+    return undefined
+  }
+
+  const operation = typeof args.operation === 'string' ? args.operation : undefined
+  const target = asJsonRecord(args.target)
+  const targetKind = typeof target?.kind === 'string' ? target.kind : undefined
+  if (!operation || !target || !targetKind) {
+    return undefined
+  }
+
+  const fileId = typeof target.fileId === 'string' ? target.fileId : undefined
+  const fileName = typeof target.fileName === 'string' ? target.fileName : undefined
+  const path = typeof target.path === 'string' ? target.path : undefined
+  const title = typeof args.title === 'string' ? args.title : undefined
+  const contentType = typeof args.contentType === 'string' ? args.contentType : undefined
+  const edit = asJsonRecord(args.edit)
+
+  return {
+    operation,
+    target: {
+      kind: targetKind,
+      ...(fileId ? { fileId } : {}),
+      ...(fileName ? { fileName } : {}),
+      ...(path ? { path } : {}),
+    },
+    ...(title ? { title } : {}),
+    ...(contentType ? { contentType } : {}),
+    ...(edit ? { edit } : {}),
+  }
+}
+
+function extractWorkspaceFileResult(output: unknown): { fileId?: string; fileName?: string } {
+  const candidates: JsonRecord[] = []
+  const root = asJsonRecord(output)
+  if (root) {
+    candidates.push(root)
+    const rootData = asJsonRecord(root.data)
+    if (rootData) candidates.push(rootData)
+    const rootOutput = asJsonRecord(root.output)
+    if (rootOutput) {
+      candidates.push(rootOutput)
+      const outputData = asJsonRecord(rootOutput.data)
+      if (outputData) candidates.push(outputData)
+    }
+  }
+
+  for (const candidate of candidates) {
+    const fileId =
+      typeof candidate.id === 'string'
+        ? candidate.id
+        : typeof candidate.fileId === 'string'
+          ? candidate.fileId
+          : undefined
+    if (!fileId) continue
+
+    const fileName =
+      typeof candidate.name === 'string'
+        ? candidate.name
+        : typeof candidate.fileName === 'string'
+          ? candidate.fileName
+          : undefined
+    return { fileId, fileName }
+  }
+  return {}
+}
+
+export function decodeJsonStringPrefix(input: string): string {
+  let output = ''
+  for (let i = 0; i < input.length; i++) {
+    const ch = input[i]
+    if (ch !== '\\') {
+      output += ch
+      continue
+    }
+    const next = input[i + 1]
+    if (!next) break
+    if (next === 'n') {
+      output += '\n'
+      i++
+      continue
+    }
+    if (next === 't') {
+      output += '\t'
+      i++
+      continue
+    }
+    if (next === 'r') {
+      output += '\r'
+      i++
+      continue
+    }
+    if (next === '"') {
+      output += '"'
+      i++
+      continue
+    }
+    if (next === '\\') {
+      output += '\\'
+      i++
+      continue
+    }
+    if (next === '/') {
+      output += '/'
+      i++
+      continue
+    }
+    if (next === 'b') {
+      output += '\b'
+      i++
+      continue
+    }
+    if (next === 'f') {
+      output += '\f'
+      i++
+      continue
+    }
+    if (next === 'u') {
+      const hex = input.slice(i + 2, i + 6)
+      if (hex.length < 4 || !/^[0-9a-fA-F]{4}$/.test(hex)) break
+      output += String.fromCharCode(Number.parseInt(hex, 16))
+      i += 5
+      continue
+    }
+    break
+  }
+  return output
+}
+
+export function extractEditContent(raw: string): string {
+  const marker = '"content":'
+  const idx = raw.indexOf(marker)
+  if (idx === -1) return ''
+  const rest = raw.slice(idx + marker.length).trimStart()
+  if (!rest.startsWith('"')) return rest
+  let end = -1
+  for (let i = 1; i < rest.length; i++) {
+    if (rest[i] === '\\') {
+      i++
+      continue
+    }
+    if (rest[i] === '"') {
+      end = i
+      break
+    }
+  }
+  const inner = end === -1 ? rest.slice(1) : rest.slice(1, end)
+  return decodeJsonStringPrefix(inner)
+}
+
+function isContentOperation(
+  operation: string | undefined
+): operation is 'append' | 'update' | 'patch' {
+  return operation === 'append' || operation === 'update' || operation === 'patch'
+}
+
+function isDocFormat(fileName: string | undefined): boolean {
+  return /\.(pptx|docx|pdf)$/i.test(fileName ?? '')
+}
+
+function buildPreviewSessionFromIntent(
+  streamId: string,
+  intent: FileIntent,
+  current?: FilePreviewSession
+): FilePreviewSession {
+  const targetKind = toPreviewTargetKind(intent.target.kind)
+
+  return createFilePreviewSession({
+    streamId,
+    toolCallId: intent.toolCallId,
+    fileName: intent.target.fileName ?? current?.fileName,
+    ...(intent.target.fileId ? { fileId: intent.target.fileId } : {}),
+    ...(targetKind ? { targetKind } : {}),
+    operation: intent.operation,
+    ...(intent.edit ? { edit: intent.edit } : {}),
+    ...(typeof current?.baseContent === 'string' ? { baseContent: current.baseContent } : {}),
+    previewText: current?.previewText ?? '',
+    previewVersion: current?.previewVersion ?? 0,
+    status: current?.status ?? 'pending',
+    completedAt: current?.completedAt,
+  })
+}
+
+async function persistFilePreviewSession(session: FilePreviewSession): Promise<void> {
+  try {
+    await upsertFilePreviewSession(session)
+  } catch (error) {
+    logger.warn('Failed to persist file preview session', {
+      streamId: session.streamId,
+      toolCallId: session.toolCallId,
+      previewVersion: session.previewVersion,
+      error: toError(error).message,
+    })
+  }
+}
+
+export function buildPreviewContentUpdate(
+  previousText: string,
+  nextText: string,
+  lastSnapshotAt: number,
+  now: number,
+  operation: string | undefined
+): { content: string; contentMode: FilePreviewContentMode; lastSnapshotAt: number } {
+  const shouldForceSnapshot =
+    previousText.length === 0 ||
+    !nextText.startsWith(previousText) ||
+    operation === 'patch' ||
+    operation === 'append' ||
+    now - lastSnapshotAt >= DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS
+
+  if (shouldForceSnapshot) {
+    return {
+      content: nextText,
+      contentMode: 'snapshot',
+      lastSnapshotAt: now,
+    }
+  }
+
+  return {
+    content: nextText.slice(previousText.length),
+    contentMode: 'delta',
+    lastSnapshotAt,
+  }
+}
+
+export interface FilePreviewAdapterState {
+  editContentState: Map<string, EditContentStreamState>
+  filePreviewState: Map<string, FilePreviewStreamState>
+}
+
+export function createFilePreviewAdapterState(): FilePreviewAdapterState {
+  return {
+    editContentState: new Map<string, EditContentStreamState>(),
+    filePreviewState: new Map<string, FilePreviewStreamState>(),
+  }
+}
+
+async function emitPreviewEvent(
+  streamEvent: StreamEvent,
+  options: Pick<OrchestratorOptions, 'onEvent'>,
+  payload: SyntheticFilePreviewPayload
+): Promise<void> {
+  await options.onEvent?.({
+    type: MothershipStreamV1EventType.tool,
+    payload,
+    ...(streamEvent.scope ? { scope: streamEvent.scope } : {}),
+  })
+}
+
+export async function processFilePreviewStreamEvent(input: {
+  streamId: string
+  streamEvent: StreamEvent
+  context: StreamingContext
+  execContext: ExecutionContext
+  options: Pick<OrchestratorOptions, 'onEvent'>
+  state: FilePreviewAdapterState
+}): Promise<void> {
+  const { streamId, streamEvent, context, execContext, options, state } = input
+  const { editContentState, filePreviewState } = state
+
+  // Scope the in-flight intent to the invoking file subagent's channel (its
+  // outer tool_use id) so two file agents streaming concurrently never read or
+  // overwrite each other's intent. workspace_file and edit_content from the same
+  // file agent share this channel id, so they pair up; siblings stay isolated.
+  const channelId = streamEvent.scope?.parentToolCallId ?? ''
+  const getIntent = (): FileIntent | null => context.activeFileIntents.get(channelId) ?? null
+  const setIntent = (intent: FileIntent): void => {
+    context.activeFileIntents.set(channelId, intent)
+  }
+  const clearIntent = (): void => {
+    context.activeFileIntents.delete(channelId)
+  }
+
+  if (isToolCallStreamEvent(streamEvent) && streamEvent.payload.toolName === 'workspace_file') {
+    const toolCallId = streamEvent.payload.toolCallId
+    const parsedArgs = parseWorkspaceFileArgs(streamEvent.payload.arguments)
+    if (toolCallId && parsedArgs) {
+      const { operation, title, contentType, edit } = parsedArgs
+      const target = await resolvePreviewTarget({
+        workspaceId: execContext.workspaceId,
+        target: parsedArgs.target,
+      })
+      const previewTargetKind = toPreviewTargetKind(target.kind)
+      const { fileId, fileName } = target
+
+      const isContentOp = isContentOperation(operation)
+      // Per-channel: a re-declared workspace_file just overwrites THIS channel's
+      // slot. No cross-message intent clearing — that would wipe a concurrent
+      // sibling file agent's pending intent.
+      const intent: FileIntent = {
+        toolCallId,
+        operation,
+        target,
+        ...(title ? { title } : {}),
+        ...(contentType ? { contentType } : {}),
+        ...(edit ? { edit } : {}),
+      }
+      setIntent(intent)
+
+      if (isContentOp && previewTargetKind) {
+        let previewBaseContent: string | undefined
+        if (
+          execContext.workspaceId &&
+          fileId &&
+          (operation === 'append' || operation === 'patch')
+        ) {
+          previewBaseContent = await loadWorkspaceFileTextForPreview(
+            execContext.workspaceId,
+            fileId
+          )
+        }
+
+        let session = buildPreviewSessionFromIntent(streamId, intent)
+        if (previewBaseContent !== undefined) {
+          session = { ...session, baseContent: previewBaseContent }
+        }
+        filePreviewState.set(toolCallId, {
+          session,
+          lastEmittedPreviewText: '',
+          lastSnapshotAt: 0,
+        })
+        await persistFilePreviewSession(session)
+
+        await emitPreviewEvent(streamEvent, options, {
+          toolCallId,
+          toolName: 'workspace_file',
+          previewPhase: 'file_preview_start',
+        })
+        await emitPreviewEvent(streamEvent, options, {
+          toolCallId,
+          toolName: 'workspace_file',
+          previewPhase: 'file_preview_target',
+          operation,
+          target: {
+            kind: previewTargetKind,
+            ...(fileId ? { fileId } : {}),
+            ...(fileName ? { fileName } : {}),
+          },
+          ...(title ? { title } : {}),
+        })
+        if (edit) {
+          await emitPreviewEvent(streamEvent, options, {
+            toolCallId,
+            toolName: 'workspace_file',
+            previewPhase: 'file_preview_edit_meta',
+            edit,
+          })
+        }
+      }
+    }
+  }
+
+  const workspaceResultIntent = getIntent()
+  if (
+    isToolResultStreamEvent(streamEvent) &&
+    streamEvent.payload.toolName === 'workspace_file' &&
+    workspaceResultIntent &&
+    isContentOperation(workspaceResultIntent.operation)
+  ) {
+    const result = extractWorkspaceFileResult(streamEvent.payload.output)
+    if (result.fileId && workspaceResultIntent.target.kind === 'path') {
+      const intent: FileIntent = {
+        ...workspaceResultIntent,
+        target: {
+          kind: 'file_id',
+          fileId: result.fileId,
+          fileName: result.fileName ?? workspaceResultIntent.target.fileName,
+          path: workspaceResultIntent.target.path,
+        },
+      }
+      setIntent(intent)
+
+      let previewBaseContent: string | undefined
+      if (
+        execContext.workspaceId &&
+        (intent.operation === 'append' || intent.operation === 'patch')
+      ) {
+        previewBaseContent = await loadWorkspaceFileTextForPreview(
+          execContext.workspaceId,
+          result.fileId
+        )
+      }
+
+      let session = buildPreviewSessionFromIntent(streamId, intent)
+      if (previewBaseContent !== undefined) {
+        session = { ...session, baseContent: previewBaseContent }
+      }
+      filePreviewState.set(intent.toolCallId, {
+        session,
+        lastEmittedPreviewText: '',
+        lastSnapshotAt: 0,
+      })
+      await persistFilePreviewSession(session)
+
+      await emitPreviewEvent(streamEvent, options, {
+        toolCallId: intent.toolCallId,
+        toolName: 'workspace_file',
+        previewPhase: 'file_preview_start',
+      })
+      await emitPreviewEvent(streamEvent, options, {
+        toolCallId: intent.toolCallId,
+        toolName: 'workspace_file',
+        previewPhase: 'file_preview_target',
+        operation: intent.operation,
+        target: {
+          kind: 'file_id',
+          fileId: result.fileId,
+          ...(result.fileName ? { fileName: result.fileName } : {}),
+        },
+        ...(intent.title ? { title: intent.title } : {}),
+      })
+      if (intent.edit) {
+        await emitPreviewEvent(streamEvent, options, {
+          toolCallId: intent.toolCallId,
+          toolName: 'workspace_file',
+          previewPhase: 'file_preview_edit_meta',
+          edit: intent.edit,
+        })
+      }
+    }
+  }
+
+  const patchDeleteIntent = getIntent()
+  if (
+    isToolResultStreamEvent(streamEvent) &&
+    streamEvent.payload.toolName === 'workspace_file' &&
+    patchDeleteIntent &&
+    isContentOperation(patchDeleteIntent.operation) &&
+    patchDeleteIntent.operation === 'patch' &&
+    patchDeleteIntent.edit?.strategy === 'anchored' &&
+    patchDeleteIntent.edit?.mode === 'delete_between' &&
+    execContext.workspaceId &&
+    patchDeleteIntent.target.fileId &&
+    !isDocFormat(patchDeleteIntent.target.fileName)
+  ) {
+    const currentPreview = filePreviewState.get(patchDeleteIntent.toolCallId)
+    const previewText = buildFilePreviewText({
+      operation: 'patch',
+      streamedContent: '',
+      existingContent: currentPreview?.session.baseContent,
+      edit: currentPreview?.session.edit,
+    })
+
+    if (previewText !== undefined) {
+      const baseSession = buildPreviewSessionFromIntent(
+        streamId,
+        patchDeleteIntent,
+        currentPreview?.session
+      )
+      const nextSession: FilePreviewSession = {
+        ...baseSession,
+        status: 'streaming',
+        previewText,
+        previewVersion: (currentPreview?.session.previewVersion ?? 0) + 1,
+        updatedAt: new Date().toISOString(),
+      }
+      filePreviewState.set(patchDeleteIntent.toolCallId, {
+        session: nextSession,
+        lastEmittedPreviewText: previewText,
+        lastSnapshotAt: Date.now(),
+      })
+      await persistFilePreviewSession(nextSession)
+      await emitPreviewEvent(streamEvent, options, {
+        toolCallId: nextSession.toolCallId,
+        toolName: 'workspace_file',
+        previewPhase: 'file_preview_content',
+        content: previewText,
+        contentMode: 'snapshot',
+        previewVersion: nextSession.previewVersion,
+        fileName: nextSession.fileName,
+        ...(nextSession.fileId ? { fileId: nextSession.fileId } : {}),
+        ...(nextSession.targetKind ? { targetKind: nextSession.targetKind } : {}),
+        ...(nextSession.operation ? { operation: nextSession.operation } : {}),
+        ...(nextSession.edit ? { edit: nextSession.edit } : {}),
+      })
+    }
+  }
+
+  if (isToolArgsDeltaStreamEvent(streamEvent) && streamEvent.payload.toolName === 'edit_content') {
+    const toolCallId = streamEvent.payload.toolCallId
+    const delta = streamEvent.payload.argumentsDelta
+    const stateForTool = editContentState.get(toolCallId) ?? { raw: '' }
+    stateForTool.raw += delta
+
+    const editIntent = getIntent()
+    if (editIntent) {
+      const streamedContent = extractEditContent(stateForTool.raw)
+      if (streamedContent !== (stateForTool.lastContentSnapshot ?? '')) {
+        stateForTool.lastContentSnapshot = streamedContent
+        let currentPreview = filePreviewState.get(editIntent.toolCallId) ?? {
+          session: buildPreviewSessionFromIntent(streamId, editIntent),
+          lastEmittedPreviewText: '',
+          lastSnapshotAt: 0,
+        }
+
+        if (
+          currentPreview.session.baseContent === undefined &&
+          (editIntent.operation === 'append' || editIntent.operation === 'patch') &&
+          execContext.workspaceId &&
+          editIntent.target.fileId
+        ) {
+          const intentBase = await peekFileIntent(
+            execContext.workspaceId,
+            editIntent.target.fileId,
+            {
+              chatId: execContext.chatId,
+              messageId: execContext.messageId,
+              channelId,
+            }
+          )
+          if (typeof intentBase?.existingContent === 'string') {
+            const seededSession: FilePreviewSession = {
+              ...currentPreview.session,
+              baseContent: intentBase.existingContent,
+              ...(intentBase.edit ? { edit: intentBase.edit } : {}),
+            }
+            currentPreview = {
+              ...currentPreview,
+              session: seededSession,
+            }
+            filePreviewState.set(editIntent.toolCallId, currentPreview)
+            await persistFilePreviewSession(seededSession)
+          }
+        }
+
+        const previewText = isContentOperation(editIntent.operation)
+          ? buildFilePreviewText({
+              operation: editIntent.operation,
+              streamedContent,
+              existingContent: currentPreview.session.baseContent,
+              edit: currentPreview.session.edit,
+            })
+          : undefined
+
+        if (previewText !== undefined) {
+          const baseSession = buildPreviewSessionFromIntent(
+            streamId,
+            editIntent,
+            currentPreview.session
+          )
+          const now = Date.now()
+          const nextSession: FilePreviewSession = {
+            ...baseSession,
+            status: 'streaming',
+            previewText,
+            previewVersion: (currentPreview.session.previewVersion ?? 0) + 1,
+            updatedAt: new Date(now).toISOString(),
+          }
+
+          await persistFilePreviewSession(nextSession)
+
+          if (
+            nextSession.operation === 'patch' &&
+            now - currentPreview.lastSnapshotAt < PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS
+          ) {
+            filePreviewState.set(editIntent.toolCallId, {
+              session: nextSession,
+              lastEmittedPreviewText: currentPreview.lastEmittedPreviewText,
+              lastSnapshotAt: currentPreview.lastSnapshotAt,
+            })
+          } else {
+            const previewUpdate = buildPreviewContentUpdate(
+              currentPreview.lastEmittedPreviewText,
+              nextSession.previewText,
+              currentPreview.lastSnapshotAt,
+              now,
+              nextSession.operation
+            )
+
+            filePreviewState.set(editIntent.toolCallId, {
+              session: nextSession,
+              lastEmittedPreviewText: nextSession.previewText,
+              lastSnapshotAt: previewUpdate.lastSnapshotAt,
+            })
+
+            await emitPreviewEvent(streamEvent, options, {
+              toolCallId: nextSession.toolCallId,
+              toolName: 'workspace_file',
+              previewPhase: 'file_preview_content',
+              content: previewUpdate.content,
+              contentMode: previewUpdate.contentMode,
+              previewVersion: nextSession.previewVersion,
+              fileName: nextSession.fileName,
+              ...(nextSession.fileId ? { fileId: nextSession.fileId } : {}),
+              ...(nextSession.targetKind ? { targetKind: nextSession.targetKind } : {}),
+              ...(nextSession.operation ? { operation: nextSession.operation } : {}),
+              ...(nextSession.edit ? { edit: nextSession.edit } : {}),
+            })
+          }
+        } else {
+          filePreviewState.set(editIntent.toolCallId, {
+            session: currentPreview.session,
+            lastEmittedPreviewText: currentPreview.lastEmittedPreviewText,
+            lastSnapshotAt: currentPreview.lastSnapshotAt,
+          })
+        }
+      }
+    }
+
+    editContentState.set(toolCallId, stateForTool)
+  }
+
+  if (isToolCallStreamEvent(streamEvent) && streamEvent.payload.toolName === 'edit_content') {
+    const toolCallId = streamEvent.payload.toolCallId
+    if (toolCallId) {
+      editContentState.delete(toolCallId)
+    }
+  }
+
+  const editResultIntent = getIntent()
+  if (
+    isToolResultStreamEvent(streamEvent) &&
+    streamEvent.payload.toolName === 'edit_content' &&
+    editResultIntent
+  ) {
+    const currentPreview = filePreviewState.get(editResultIntent.toolCallId)
+    const completedAt = new Date().toISOString()
+
+    if (
+      currentPreview &&
+      currentPreview.lastEmittedPreviewText !== currentPreview.session.previewText &&
+      currentPreview.session.previewText.length > 0
+    ) {
+      filePreviewState.set(editResultIntent.toolCallId, {
+        session: currentPreview.session,
+        lastEmittedPreviewText: currentPreview.session.previewText,
+        lastSnapshotAt: Date.now(),
+      })
+      await emitPreviewEvent(streamEvent, options, {
+        toolCallId: currentPreview.session.toolCallId,
+        toolName: 'workspace_file',
+        previewPhase: 'file_preview_content',
+        content: currentPreview.session.previewText,
+        contentMode: 'snapshot',
+        previewVersion: currentPreview.session.previewVersion,
+        fileName: currentPreview.session.fileName,
+        ...(currentPreview.session.fileId ? { fileId: currentPreview.session.fileId } : {}),
+        ...(currentPreview.session.targetKind
+          ? { targetKind: currentPreview.session.targetKind }
+          : {}),
+        ...(currentPreview.session.operation
+          ? { operation: currentPreview.session.operation }
+          : {}),
+        ...(currentPreview.session.edit ? { edit: currentPreview.session.edit } : {}),
+      })
+    }
+
+    if (currentPreview) {
+      const completedSession: FilePreviewSession = {
+        ...currentPreview.session,
+        status: 'complete',
+        updatedAt: completedAt,
+        completedAt,
+      }
+      filePreviewState.set(editResultIntent.toolCallId, {
+        session: completedSession,
+        lastEmittedPreviewText: completedSession.previewText,
+        lastSnapshotAt: Date.now(),
+      })
+      await persistFilePreviewSession(completedSession)
+    }
+
+    await emitPreviewEvent(streamEvent, options, {
+      toolCallId: editResultIntent.toolCallId,
+      toolName: 'workspace_file',
+      previewPhase: 'file_preview_complete',
+      fileId: editResultIntent.target.fileId,
+      output: streamEvent.payload.output,
+      ...(currentPreview ? { previewVersion: currentPreview.session.previewVersion } : {}),
+    })
+    clearIntent()
+  }
+}

apps/sim/lib/copilot/request/go/parser.ts

@@ -0,0 +1,129 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+
+const logger = createLogger('CopilotSseParser')
+
+export class FatalSseEventError extends Error {}
+
+function createParseFailure(message: string, preview: string): FatalSseEventError {
+  logger.error(message, { preview })
+  return new FatalSseEventError(message)
+}
+
+function normalizeSseLine(line: string): string {
+  return line.endsWith('\r') ? line.slice(0, -1) : line
+}
+
+/**
+ * Processes an SSE stream by calling onEvent for each parsed event.
+ *
+ * @param onEvent Called per parsed event. Return true to stop processing.
+ */
+export async function processSSEStream(
+  reader: ReadableStreamDefaultReader<Uint8Array>,
+  decoder: TextDecoder,
+  abortSignal: AbortSignal | undefined,
+  onEvent: (event: unknown) => boolean | undefined | Promise<boolean | undefined>
+): Promise<void> {
+  let buffer = ''
+
+  try {
+    try {
+      while (true) {
+        if (abortSignal?.aborted) {
+          logger.info('SSE stream aborted by signal')
+          break
+        }
+
+        const { done, value } = await reader.read()
+        if (done) break
+
+        buffer += decoder.decode(value, { stream: true })
+        const lines = buffer.split('\n')
+        buffer = lines.pop() || ''
+
+        let stopped = false
+        for (const line of lines) {
+          const normalizedLine = normalizeSseLine(line)
+          if (abortSignal?.aborted) {
+            logger.info('SSE stream aborted mid-chunk (between events)')
+            return
+          }
+          if (!normalizedLine.trim()) continue
+          if (!normalizedLine.startsWith('data: ')) continue
+
+          const jsonStr = normalizedLine.slice(6)
+          if (jsonStr === '[DONE]') continue
+
+          let parsed: unknown
+          try {
+            parsed = JSON.parse(jsonStr)
+          } catch (error) {
+            const preview = jsonStr.slice(0, 200)
+            const detail = toError(error).message
+            throw createParseFailure(`Failed to parse SSE event JSON: ${detail}`, preview)
+          }
+
+          try {
+            if (await onEvent(parsed)) {
+              stopped = true
+              break
+            }
+          } catch (error) {
+            if (error instanceof FatalSseEventError) {
+              throw error
+            }
+            logger.warn('Failed to handle SSE event', {
+              preview: jsonStr.slice(0, 200),
+              error: toError(error).message,
+            })
+          }
+        }
+        if (stopped) break
+      }
+    } catch (error) {
+      const aborted =
+        abortSignal?.aborted || (error instanceof DOMException && error.name === 'AbortError')
+      if (aborted) {
+        logger.info('SSE stream read aborted')
+        return
+      }
+      throw error
+    }
+
+    const normalizedBuffer = normalizeSseLine(buffer)
+    if (normalizedBuffer.trim() && normalizedBuffer.startsWith('data: ')) {
+      const jsonStr = normalizedBuffer.slice(6)
+      if (jsonStr === '[DONE]') {
+        return
+      }
+
+      let parsed: unknown
+      try {
+        parsed = JSON.parse(jsonStr)
+      } catch (error) {
+        const preview = normalizedBuffer.slice(0, 200)
+        const detail = toError(error).message
+        throw createParseFailure(`Failed to parse final SSE buffer JSON: ${detail}`, preview)
+      }
+
+      try {
+        await onEvent(parsed)
+      } catch (error) {
+        if (error instanceof FatalSseEventError) {
+          throw error
+        }
+        logger.warn('Failed to handle final SSE event', {
+          preview: normalizedBuffer.slice(0, 200),
+          error: toError(error).message,
+        })
+      }
+    }
+  } finally {
+    try {
+      reader.releaseLock()
+    } catch {
+      logger.warn('Failed to release SSE reader lock')
+    }
+  }
+}

apps/sim/lib/copilot/request/go/propagation.ts

@@ -0,0 +1,57 @@
+import { type Context, context } from '@opentelemetry/api'
+import { W3CTraceContextPropagator } from '@opentelemetry/core'
+
+const propagator = new W3CTraceContextPropagator()
+const headerSetter = {
+  set(carrier: Record<string, string>, key: string, value: string) {
+    carrier[key] = value
+  },
+}
+
+const headerGetter = {
+  keys(carrier: Headers): string[] {
+    const out: string[] = []
+    carrier.forEach((_, key) => {
+      out.push(key)
+    })
+    return out
+  },
+  get(carrier: Headers, key: string): string | undefined {
+    return carrier.get(key) ?? undefined
+  },
+}
+
+/**
+ * Injects W3C trace context (traceparent, tracestate) into outbound HTTP
+ * headers so Go-side spans join the same OTel trace tree as the calling
+ * Sim span.
+ *
+ * Usage: spread the result into your fetch headers:
+ *   fetch(url, { headers: { ...myHeaders, ...traceHeaders() } })
+ */
+export function traceHeaders(
+  carrier?: Record<string, string>,
+  otelContext?: Context
+): Record<string, string> {
+  const headers: Record<string, string> = carrier ?? {}
+  propagator.inject(otelContext ?? context.active(), headers, headerSetter)
+  return headers
+}
+
+/**
+ * Extracts W3C trace context from incoming request headers (traceparent /
+ * tracestate) and returns an OTel Context seeded with the upstream span.
+ *
+ * Use this at the top of inbound Sim route handlers that Go calls into
+ * (e.g. /api/billing/update-cost, /api/copilot/api-keys/validate) so the
+ * Sim-side span becomes a proper child of the Go-side client span in the
+ * same trace — closing the round trip in Jaeger.
+ *
+ * When no traceparent is present (e.g. calls from a browser or a client
+ * that hasn't been instrumented), this returns `context.active()`
+ * unchanged, and any span started under it becomes a new root — the same
+ * behavior as before this helper existed.
+ */
+export function contextFromRequestHeaders(headers: Headers): Context {
+  return propagator.extract(context.active(), headers, headerGetter)
+}

apps/sim/lib/copilot/request/go/stream.test.ts

@@ -0,0 +1,869 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+  MothershipStreamV1ToolExecutor,
+  MothershipStreamV1ToolMode,
+  MothershipStreamV1ToolOutcome,
+  MothershipStreamV1ToolPhase,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+
+vi.mock('@/lib/copilot/request/session', async () => {
+  const actual = await vi.importActual<typeof import('@/lib/copilot/request/session')>(
+    '@/lib/copilot/request/session'
+  )
+  return {
+    ...actual,
+    hasAbortMarker: vi.fn().mockResolvedValue(false),
+    upsertFilePreviewSession: vi.fn(async (session) => session),
+  }
+})
+
+const resolveWorkspaceFileReferenceMock = vi.hoisted(() => vi.fn())
+
+vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
+  resolveWorkspaceFileReference: resolveWorkspaceFileReferenceMock,
+}))
+
+vi.mock('@/lib/copilot/tools/server/files/file-preview', async () => {
+  const actual = await vi.importActual<
+    typeof import('@/lib/copilot/tools/server/files/file-preview')
+  >('@/lib/copilot/tools/server/files/file-preview')
+  return {
+    ...actual,
+    loadWorkspaceFileTextForPreview: vi.fn().mockResolvedValue(''),
+  }
+})
+
+import {
+  buildPreviewContentUpdate,
+  decodeJsonStringPrefix,
+  extractEditContent,
+  runStreamLoop,
+} from '@/lib/copilot/request/go/stream'
+import { AbortReason, createEvent, hasAbortMarker } from '@/lib/copilot/request/session'
+import { RequestTraceV1Outcome, TraceCollector } from '@/lib/copilot/request/trace'
+import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types'
+
+function createSseResponse(events: unknown[]): Response {
+  const payload = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join('')
+
+  return new Response(
+    new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(new TextEncoder().encode(payload))
+        controller.close()
+      },
+    }),
+    {
+      status: 200,
+      headers: {
+        'Content-Type': 'text/event-stream',
+      },
+    }
+  )
+}
+
+function createRawSseResponse(payload: string): Response {
+  return new Response(
+    new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(new TextEncoder().encode(payload))
+        controller.close()
+      },
+    }),
+    {
+      status: 200,
+      headers: {
+        'Content-Type': 'text/event-stream',
+      },
+    }
+  )
+}
+
+function createStreamingContext(): StreamingContext {
+  return {
+    messageId: 'msg-1',
+    accumulatedContent: '',
+    finalAssistantContent: '',
+    sawMainToolCall: false,
+    contentBlocks: [],
+    toolCalls: new Map(),
+    pendingToolPromises: new Map(),
+    currentThinkingBlock: null,
+    subagentThinkingBlocks: new Map(),
+    isInThinkingBlock: false,
+    subAgentContent: {},
+    subAgentToolCalls: {},
+    pendingContent: '',
+    streamComplete: false,
+    wasAborted: false,
+    errors: [],
+    activeFileIntents: new Map(),
+    trace: new TraceCollector(),
+  }
+}
+
+describe('copilot go stream helpers', () => {
+  beforeEach(() => {
+    vi.stubGlobal('fetch', vi.fn())
+    resolveWorkspaceFileReferenceMock.mockReset()
+    resolveWorkspaceFileReferenceMock.mockResolvedValue(null)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('decodes complete escapes and stops at incomplete unicode escapes', () => {
+    expect(decodeJsonStringPrefix('hello\\nworld')).toBe('hello\nworld')
+    expect(decodeJsonStringPrefix('emoji \\u263A')).toBe('emoji ☺')
+    expect(decodeJsonStringPrefix('partial \\u26')).toBe('partial ')
+  })
+
+  it('extracts the streamed edit_content prefix from partial JSON', () => {
+    expect(extractEditContent('{"content":"hello\\nwor')).toBe('hello\nwor')
+    expect(extractEditContent('{"content":"tab\\tvalue"}')).toBe('tab\tvalue')
+  })
+
+  it('emits full snapshots for append (sidebar viewer uses replace mode; no delta merge)', () => {
+    expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'append')).toEqual({
+      content: 'hello world',
+      contentMode: 'snapshot',
+      lastSnapshotAt: 200,
+    })
+  })
+
+  it('emits deltas for update when the preview extends the previous text', () => {
+    expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'update')).toEqual({
+      content: ' world',
+      contentMode: 'delta',
+      lastSnapshotAt: 100,
+    })
+  })
+
+  it('falls back to snapshots for patches and divergent content', () => {
+    expect(buildPreviewContentUpdate('hello', 'goodbye', 100, 200, 'update')).toEqual({
+      content: 'goodbye',
+      contentMode: 'snapshot',
+      lastSnapshotAt: 200,
+    })
+
+    expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'patch')).toEqual({
+      content: 'hello world',
+      contentMode: 'snapshot',
+      lastSnapshotAt: 200,
+    })
+  })
+
+  it('hydrates path-based workspace_file edits into file preview events before edit_content streams', async () => {
+    resolveWorkspaceFileReferenceMock.mockResolvedValue({
+      id: 'file-1',
+      name: 'notes.md',
+    })
+
+    const workspaceFileCall = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'workspace-file-path-1',
+        toolName: 'workspace_file',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.call,
+        arguments: {
+          operation: 'update',
+          target: { kind: 'path', path: 'files/notes.md' },
+          title: 'Update notes',
+        },
+      },
+    })
+    const workspaceFileResult = createEvent({
+      streamId: 'stream-1',
+      cursor: '2',
+      seq: 2,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'workspace-file-path-1',
+        toolName: 'workspace_file',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.result,
+        success: true,
+        output: {
+          success: true,
+          data: { id: 'file-1', name: 'notes.md', operation: 'update' },
+        },
+      },
+    })
+    const editContentDelta = createEvent({
+      streamId: 'stream-1',
+      cursor: '3',
+      seq: 3,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'edit-content-path-1',
+        toolName: 'edit_content',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.args_delta,
+        argumentsDelta: '{"content":"hello world',
+      },
+    })
+    const editContentResult = createEvent({
+      streamId: 'stream-1',
+      cursor: '4',
+      seq: 4,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'edit-content-path-1',
+        toolName: 'edit_content',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.result,
+        success: true,
+        output: {
+          success: true,
+          data: { id: 'file-1', name: 'notes.md' },
+        },
+      },
+    })
+    const complete = createEvent({
+      streamId: 'stream-1',
+      cursor: '5',
+      seq: 5,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.complete,
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(
+      createSseResponse([
+        workspaceFileCall,
+        workspaceFileResult,
+        editContentDelta,
+        editContentResult,
+        complete,
+      ])
+    )
+
+    const onEvent = vi.fn()
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+      workspaceId: 'workspace-1',
+      messageId: 'msg-1',
+    }
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      onEvent,
+      timeout: 1000,
+    })
+
+    const previewEvents = onEvent.mock.calls
+      .map(([event]) => event)
+      .filter(
+        (event) =>
+          event.type === MothershipStreamV1EventType.tool && 'previewPhase' in event.payload
+      )
+
+    expect(previewEvents.map((event) => event.payload.previewPhase)).toEqual([
+      'file_preview_start',
+      'file_preview_target',
+      'file_preview_content',
+      'file_preview_complete',
+    ])
+    expect(previewEvents[1].payload).toMatchObject({
+      previewPhase: 'file_preview_target',
+      target: { kind: 'file_id', fileId: 'file-1', fileName: 'notes.md' },
+    })
+    expect(previewEvents[2].payload).toMatchObject({
+      previewPhase: 'file_preview_content',
+      fileId: 'file-1',
+      targetKind: 'file_id',
+      content: 'hello world',
+    })
+    expect(previewEvents[3].payload).toMatchObject({
+      previewPhase: 'file_preview_complete',
+      fileId: 'file-1',
+    })
+    expect(resolveWorkspaceFileReferenceMock).toHaveBeenCalledWith('workspace-1', 'files/notes.md')
+  })
+
+  it('resolves workflow alias paths to the backing file before streaming previews', async () => {
+    resolveWorkspaceFileReferenceMock.mockResolvedValue({
+      id: 'changelog-file-1',
+      name: 'workflow-1.md',
+    })
+
+    const workspaceFileCall = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'workspace-file-alias-1',
+        toolName: 'workspace_file',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.call,
+        arguments: {
+          operation: 'append',
+          target: { kind: 'path', path: 'workflows/My%20Workflow/changelog.md' },
+          title: 'Update changelog',
+        },
+      },
+    })
+    const editContentDelta = createEvent({
+      streamId: 'stream-1',
+      cursor: '2',
+      seq: 2,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'edit-content-alias-1',
+        toolName: 'edit_content',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.args_delta,
+        argumentsDelta: '{"content":"\\n- Added a workflow step',
+      },
+    })
+    const editContentResult = createEvent({
+      streamId: 'stream-1',
+      cursor: '3',
+      seq: 3,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'edit-content-alias-1',
+        toolName: 'edit_content',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.result,
+        success: true,
+        output: {
+          success: true,
+          data: { id: 'changelog-file-1', name: 'workflow-1.md' },
+        },
+      },
+    })
+    const complete = createEvent({
+      streamId: 'stream-1',
+      cursor: '4',
+      seq: 4,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.complete,
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(
+      createSseResponse([workspaceFileCall, editContentDelta, editContentResult, complete])
+    )
+
+    const onEvent = vi.fn()
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+      workspaceId: 'workspace-1',
+      messageId: 'msg-1',
+    }
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      onEvent,
+      timeout: 1000,
+    })
+
+    const previewEvents = onEvent.mock.calls
+      .map(([event]) => event)
+      .filter(
+        (event) =>
+          event.type === MothershipStreamV1EventType.tool && 'previewPhase' in event.payload
+      )
+
+    expect(previewEvents.map((event) => event.payload.previewPhase)).toEqual([
+      'file_preview_start',
+      'file_preview_target',
+      'file_preview_content',
+      'file_preview_complete',
+    ])
+    expect(previewEvents[1].payload).toMatchObject({
+      previewPhase: 'file_preview_target',
+      target: { kind: 'file_id', fileId: 'changelog-file-1', fileName: 'workflow-1.md' },
+    })
+    expect(previewEvents[2].payload).toMatchObject({
+      previewPhase: 'file_preview_content',
+      fileId: 'changelog-file-1',
+      targetKind: 'file_id',
+      content: '\n- Added a workflow step',
+    })
+    expect(previewEvents[3].payload).toMatchObject({
+      previewPhase: 'file_preview_complete',
+      fileId: 'changelog-file-1',
+    })
+    expect(resolveWorkspaceFileReferenceMock).toHaveBeenCalledWith(
+      'workspace-1',
+      'workflows/My%20Workflow/changelog.md'
+    )
+  })
+
+  it('drops duplicate tool_result events before forwarding them', async () => {
+    const toolResult = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'tool-result-dedupe',
+        toolName: 'search_online',
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.result,
+        success: true,
+        output: { value: 'ok' },
+      },
+    })
+    const complete = createEvent({
+      streamId: 'stream-1',
+      cursor: '2',
+      seq: 2,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.complete,
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([toolResult, toolResult, complete]))
+
+    const onEvent = vi.fn()
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      onEvent,
+      timeout: 1000,
+    })
+
+    expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
+      MothershipStreamV1EventType.tool,
+      MothershipStreamV1EventType.complete,
+    ])
+    expect(onEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.tool,
+        payload: expect.objectContaining({
+          toolCallId: 'tool-result-dedupe',
+          phase: MothershipStreamV1ToolPhase.result,
+        }),
+      })
+    )
+    expect(context.toolCalls.get('tool-result-dedupe')).toEqual(
+      expect.objectContaining({
+        id: 'tool-result-dedupe',
+        name: 'search_online',
+        status: MothershipStreamV1ToolOutcome.success,
+        result: { success: true, output: { value: 'ok' } },
+      })
+    )
+  })
+
+  it('does not retry transient backend statuses because stream requests are not idempotent', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(new Response('bad gateway', { status: 502 }))
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toMatchObject({
+      name: 'CopilotBackendError',
+      status: 502,
+      body: 'bad gateway',
+    })
+
+    expect(fetch).toHaveBeenCalledTimes(1)
+  })
+
+  it('does not retry non-transient backend statuses before the SSE stream opens', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(new Response('limit reached', { status: 402 }))
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toThrow('Usage limit reached')
+
+    expect(fetch).toHaveBeenCalledTimes(1)
+  })
+
+  it('does not retry network errors because Go may already be executing the request', async () => {
+    vi.mocked(fetch).mockRejectedValueOnce(new TypeError('fetch failed'))
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toThrow('fetch failed')
+
+    expect(fetch).toHaveBeenCalledTimes(1)
+  })
+
+  it('fails closed when the shared stream ends before a terminal event', async () => {
+    const textEvent = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.text,
+      payload: {
+        channel: 'assistant',
+        text: 'partial response',
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toThrow('Copilot backend stream ended before a terminal event')
+    expect(
+      context.errors.some((message) =>
+        message.includes('Copilot backend stream ended before a terminal event')
+      )
+    ).toBe(true)
+  })
+
+  it('reclassifies as aborted when the body closes without terminal but the abort marker is set', async () => {
+    const textEvent = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.text,
+      payload: {
+        channel: 'assistant',
+        text: 'partial response',
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
+    vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      timeout: 1000,
+    })
+
+    expect(hasAbortMarker).toHaveBeenCalledWith(context.messageId)
+    expect(context.wasAborted).toBe(true)
+    expect(
+      context.errors.some((message) =>
+        message.includes('Copilot backend stream ended before a terminal event')
+      )
+    ).toBe(false)
+  })
+
+  it('invokes onAbortObserved with MarkerObservedAtBodyClose when reclassifying via the abort marker', async () => {
+    const textEvent = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.text,
+      payload: {
+        channel: 'assistant',
+        text: 'partial response',
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
+    vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+    const onAbortObserved = vi.fn()
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      timeout: 1000,
+      onAbortObserved,
+    })
+
+    expect(onAbortObserved).toHaveBeenCalledTimes(1)
+    expect(onAbortObserved).toHaveBeenCalledWith(AbortReason.MarkerObservedAtBodyClose)
+    expect(context.wasAborted).toBe(true)
+  })
+
+  it('does not invoke onAbortObserved when no abort marker is present at body close', async () => {
+    const textEvent = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.text,
+      payload: {
+        channel: 'assistant',
+        text: 'partial response',
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
+    vi.mocked(hasAbortMarker).mockResolvedValueOnce(false)
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+    const onAbortObserved = vi.fn()
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+        onAbortObserved,
+      })
+    ).rejects.toThrow('Copilot backend stream ended before a terminal event')
+
+    expect(onAbortObserved).not.toHaveBeenCalled()
+  })
+
+  it('still fails closed when the body closes without terminal and the abort marker check throws', async () => {
+    const textEvent = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.text,
+      payload: {
+        channel: 'assistant',
+        text: 'partial response',
+      },
+    })
+
+    vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
+    vi.mocked(hasAbortMarker).mockRejectedValueOnce(new Error('redis unavailable'))
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toThrow('Copilot backend stream ended before a terminal event')
+    expect(context.wasAborted).toBe(false)
+  })
+
+  it('fails closed when the shared stream receives an invalid event', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      createSseResponse([
+        {
+          v: 1,
+          type: MothershipStreamV1EventType.tool,
+          seq: 1,
+          ts: '2026-01-01T00:00:00.000Z',
+          stream: { streamId: 'stream-1', cursor: '1' },
+          payload: {
+            phase: MothershipStreamV1ToolPhase.result,
+          },
+        },
+      ])
+    )
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toThrow('Received invalid stream event on shared path')
+    expect(
+      context.errors.some((message) =>
+        message.includes('Received invalid stream event on shared path')
+      )
+    ).toBe(true)
+  })
+
+  it('fails closed when the shared stream receives malformed JSON', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      createRawSseResponse('data: {"v":1,"type":"text","payload":\n\n')
+    )
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await expect(
+      runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+        timeout: 1000,
+      })
+    ).rejects.toThrow('Failed to parse SSE event JSON')
+    expect(
+      context.errors.some((message) => message.includes('Failed to parse SSE event JSON'))
+    ).toBe(true)
+  })
+
+  it('records a split canonical request id and go trace id from the stream envelope', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      createSseResponse([
+        {
+          v: 1,
+          type: MothershipStreamV1EventType.text,
+          seq: 1,
+          ts: '2026-01-01T00:00:00.000Z',
+          stream: { streamId: 'stream-1', cursor: '1' },
+          trace: {
+            requestId: 'sim-request-1',
+            goTraceId: 'go-trace-1',
+          },
+          payload: {
+            channel: 'assistant',
+            text: 'hello',
+          },
+        },
+        createEvent({
+          streamId: 'stream-1',
+          cursor: '2',
+          seq: 2,
+          requestId: 'sim-request-1',
+          type: MothershipStreamV1EventType.complete,
+          payload: {
+            status: MothershipStreamV1CompletionStatus.complete,
+          },
+        }),
+      ])
+    )
+
+    const context = createStreamingContext()
+    context.requestId = 'sim-request-1'
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      timeout: 1000,
+    })
+
+    expect(context.requestId).toBe('sim-request-1')
+    expect(
+      context.trace.build({
+        outcome: RequestTraceV1Outcome.success,
+        simRequestId: 'sim-request-1',
+      }).goTraceId
+    ).toBe('go-trace-1')
+  })
+
+  it('records span identity on the subagent block from the scope', async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      createSseResponse([
+        createEvent({
+          streamId: 'stream-1',
+          cursor: '1',
+          seq: 1,
+          requestId: 'req-1',
+          type: MothershipStreamV1EventType.span,
+          scope: {
+            lane: 'subagent',
+            agentId: 'deploy',
+            parentToolCallId: 'tc-deploy-inner',
+            spanId: 'S2',
+            parentSpanId: 'S1',
+          },
+          payload: {
+            kind: 'subagent',
+            event: 'start',
+            agent: 'deploy',
+            data: { tool_call_id: 'tc-deploy-inner', nested: true },
+          },
+        }),
+        createEvent({
+          streamId: 'stream-1',
+          cursor: '2',
+          seq: 2,
+          requestId: 'req-1',
+          type: MothershipStreamV1EventType.complete,
+          payload: { status: MothershipStreamV1CompletionStatus.complete },
+        }),
+      ])
+    )
+
+    const context = createStreamingContext()
+    const execContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+
+    await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
+      timeout: 1000,
+    })
+
+    const subagentBlock = context.contentBlocks.find((block) => block.type === 'subagent')
+    expect(subagentBlock).toBeDefined()
+    expect(subagentBlock?.spanId).toBe('S2')
+    expect(subagentBlock?.parentSpanId).toBe('S1')
+    expect(subagentBlock?.parentToolCallId).toBe('tc-deploy-inner')
+  })
+})

apps/sim/lib/copilot/request/go/stream.ts

@@ -0,0 +1,708 @@
+import { type Context, SpanStatusCode } from '@opentelemetry/api'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { ORCHESTRATION_TIMEOUT_MS } from '@/lib/copilot/constants'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1SpanLifecycleEvent,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { CopilotSseCloseReason } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceEvent } from '@/lib/copilot/generated/trace-events-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { fetchGo } from '@/lib/copilot/request/go/fetch'
+import {
+  buildPreviewContentUpdate,
+  createFilePreviewAdapterState,
+  decodeJsonStringPrefix,
+  extractEditContent,
+  processFilePreviewStreamEvent,
+} from '@/lib/copilot/request/go/file-preview-adapter'
+import { FatalSseEventError, processSSEStream } from '@/lib/copilot/request/go/parser'
+import {
+  handleSubagentRouting,
+  prePersistClientExecutableToolCall,
+  sseHandlers,
+  subAgentHandlers,
+} from '@/lib/copilot/request/handlers'
+import {
+  flushSubagentThinkingBlock,
+  flushThinkingBlock,
+} from '@/lib/copilot/request/handlers/types'
+import { getCopilotTracer } from '@/lib/copilot/request/otel'
+import {
+  AbortReason,
+  eventToStreamEvent,
+  hasAbortMarker,
+  isSubagentSpanStreamEvent,
+  parsePersistedStreamEventEnvelope,
+} from '@/lib/copilot/request/session'
+import { shouldSkipToolCallEvent, shouldSkipToolResultEvent } from '@/lib/copilot/request/sse-utils'
+import type {
+  ExecutionContext,
+  OrchestratorOptions,
+  StreamEvent,
+  StreamingContext,
+} from '@/lib/copilot/request/types'
+
+const logger = createLogger('CopilotGoStream')
+
+export { buildPreviewContentUpdate, decodeJsonStringPrefix, extractEditContent }
+
+type JsonRecord = Record<string, unknown>
+
+type SubagentSpanData = {
+  pending?: boolean
+  toolCallId?: string
+}
+
+function asJsonRecord(value: unknown): JsonRecord | undefined {
+  return value && typeof value === 'object' && !Array.isArray(value)
+    ? (value as JsonRecord)
+    : undefined
+}
+
+function parseSubagentSpanData(value: unknown): SubagentSpanData | undefined {
+  const data = asJsonRecord(value)
+  if (!data) {
+    return undefined
+  }
+
+  const toolCallId = typeof data.tool_call_id === 'string' ? data.tool_call_id : undefined
+  const pending = typeof data.pending === 'boolean' ? data.pending : undefined
+
+  return {
+    ...(toolCallId ? { toolCallId } : {}),
+    ...(pending !== undefined ? { pending } : {}),
+  }
+}
+
+export class CopilotBackendError extends Error {
+  status?: number
+  body?: string
+
+  constructor(message: string, options?: { status?: number; body?: string }) {
+    super(message)
+    this.name = 'CopilotBackendError'
+    this.status = options?.status
+    this.body = options?.body
+  }
+}
+
+export class BillingLimitError extends Error {
+  constructor(public readonly userId: string) {
+    super('Usage limit reached')
+    this.name = 'BillingLimitError'
+  }
+}
+
+/**
+ * Options for the shared stream processing loop.
+ */
+export interface StreamLoopOptions extends OrchestratorOptions {
+  /**
+   * Called for each normalized event BEFORE standard handler dispatch.
+   * Return true to skip the default handler for this event.
+   */
+  onBeforeDispatch?: (event: StreamEvent, context: StreamingContext) => boolean | undefined
+  /**
+   * Called when the Go backend's trace ID (go_trace_id) is first received via SSE.
+   */
+  onGoTraceId?: (goTraceId: string) => void
+  otelContext?: Context
+}
+
+/**
+ * Run the SSE stream processing loop against the Go backend.
+ *
+ * Handles: fetch -> parse -> normalize -> dedupe -> subagent routing -> handler dispatch.
+ * Callers provide the fetch URL/options and can intercept events via onBeforeDispatch.
+ * Feature-specific normalization runs through dedicated adapters before the raw event is forwarded.
+ */
+export async function runStreamLoop(
+  fetchUrl: string,
+  fetchOptions: RequestInit,
+  context: StreamingContext,
+  execContext: ExecutionContext,
+  options: StreamLoopOptions
+): Promise<void> {
+  const { timeout = ORCHESTRATION_TIMEOUT_MS, abortSignal } = options
+  const filePreviewAdapterState = createFilePreviewAdapterState()
+
+  const pathname = new URL(fetchUrl).pathname
+  const requestBodyBytes = estimateBodyBytes(fetchOptions.body)
+  const fetchSpan = context.trace.startSpan(`HTTP Request → ${pathname}`, 'sim.http.fetch', {
+    url: fetchUrl,
+    method: fetchOptions.method ?? 'GET',
+    requestBodyBytes,
+  })
+  const fetchStart = performance.now()
+  let response: Response
+  try {
+    response = await fetchGo(fetchUrl, {
+      ...fetchOptions,
+      signal: abortSignal,
+      otelContext: options.otelContext,
+      spanName: `sim → go ${pathname}`,
+      operation: 'stream',
+      attributes: {
+        [TraceAttr.CopilotStream]: true,
+        ...(requestBodyBytes ? { [TraceAttr.HttpRequestContentLength]: requestBodyBytes } : {}),
+      },
+    })
+  } catch (error) {
+    fetchSpan.attributes = {
+      ...(fetchSpan.attributes ?? {}),
+      headersMs: Math.round(performance.now() - fetchStart),
+    }
+    context.trace.endSpan(fetchSpan, abortSignal?.aborted ? 'cancelled' : 'error')
+    throw error
+  }
+  const headersElapsedMs = Math.round(performance.now() - fetchStart)
+  fetchSpan.attributes = {
+    ...(fetchSpan.attributes ?? {}),
+    status: response.status,
+    headersMs: headersElapsedMs,
+  }
+
+  if (!response.ok) {
+    context.trace.endSpan(fetchSpan, 'error')
+    const errorText = await response.text().catch(() => '')
+
+    if (response.status === 402) {
+      throw new BillingLimitError(execContext.userId)
+    }
+
+    throw new CopilotBackendError(
+      `Copilot backend error (${response.status}): ${errorText || response.statusText}`,
+      { status: response.status, body: errorText || response.statusText }
+    )
+  }
+
+  if (!response.body) {
+    context.trace.endSpan(fetchSpan, 'error')
+    throw new CopilotBackendError('Copilot backend response missing body')
+  }
+
+  context.trace.endSpan(fetchSpan)
+
+  const bodySpan = context.trace.startSpan(`SSE Body → ${pathname}`, 'sim.http.stream_body', {
+    url: fetchUrl,
+    method: fetchOptions.method ?? 'GET',
+  })
+
+  // Aggregate counters populated inline by the reader wrapper + onEvent
+  // dispatcher below and flushed to both the legacy TraceCollector span
+  // and the OTel read-loop span when the loop terminates. Kept as plain
+  // JS variables (not span attrs) so incrementing them is free — we
+  // only pay OTel cost once at span End().
+  //
+  // Idle-gap tracking is split two ways so we can tell apart
+  // upstream-silent from we-were-busy:
+  //
+  //   - `longestInboundGapMs`: biggest time between consecutive
+  //     `reader.read()` calls returning bytes. Upper bound on
+  //     "Go silent". Actually also includes Node waiting for main
+  //     thread free, so see dispatchMs below.
+  //   - `longestDispatchMs`: biggest time any single event handler
+  //     took between "event received" and "returned control". Upper
+  //     bound on "Sim was CPU-bound on a handler". If this is high
+  //     AND inbound gap is high at the same time, it's Sim. If only
+  //     inbound gap is high, it's upstream.
+  //   - `totalDispatchMs`: sum of all handler times. Helps gauge
+  //     whether handlers in aggregate ate a meaningful fraction of
+  //     the read loop.
+  const counters = {
+    bytes: 0,
+    chunks: 0,
+    events: 0,
+    eventsByType: {
+      session: 0,
+      text: 0,
+      tool: 0,
+      span: 0,
+      resource: 0,
+      run: 0,
+      error: 0,
+      complete: 0,
+    } as Record<MothershipStreamV1EventType, number>,
+    firstEventMs: undefined as number | undefined,
+    lastChunkMs: performance.now(),
+    longestInboundGapMs: 0,
+    longestDispatchMs: 0,
+    totalDispatchMs: 0,
+  }
+  const bodyStart = performance.now()
+  let endedOn: string = CopilotSseCloseReason.Terminal
+
+  // Wrap the body's reader so we can track per-chunk bytes and the gap
+  // between chunks. `processSSEStream` consumes this reader exactly as
+  // it would the raw one — no API changes there.
+  const IDLE_GAP_EVENT_THRESHOLD_MS = 10000
+  const rawReader = response.body.getReader()
+  const reader: ReadableStreamDefaultReader<Uint8Array> = {
+    async read() {
+      const result = await rawReader.read()
+      if (!result.done && result.value) {
+        const now = performance.now()
+        const gap = now - counters.lastChunkMs
+        if (gap > counters.longestInboundGapMs) counters.longestInboundGapMs = gap
+        counters.lastChunkMs = now
+        counters.chunks += 1
+        counters.bytes += result.value.byteLength
+      }
+      return result
+    },
+    cancel: (reason) => rawReader.cancel(reason),
+    releaseLock: () => rawReader.releaseLock(),
+    get closed() {
+      return rawReader.closed
+    },
+  }
+  const decoder = new TextDecoder()
+
+  const timeoutId = setTimeout(() => {
+    context.errors.push('Request timed out')
+    context.streamComplete = true
+    endedOn = CopilotSseCloseReason.Timeout
+    reader.cancel().catch(() => {})
+  }, timeout)
+
+  try {
+    await processSSEStream(reader, decoder, abortSignal, async (raw) => {
+      // Track how long THIS handler invocation takes so we can tell
+      // apart "Go was silent" from "we were CPU-bound on a handler".
+      // `longestInboundGapMs` includes handler time (the next reader.read
+      // doesn't run until the previous handler returns), so dispatch
+      // time is the correction needed to isolate upstream silence.
+      const dispatchStart = performance.now()
+      try {
+        if (counters.events === 0) {
+          counters.firstEventMs = Math.round(performance.now() - bodyStart)
+        }
+        counters.events += 1
+        if (abortSignal?.aborted) {
+          context.wasAborted = true
+          return true
+        }
+
+        const parsedEvent = parsePersistedStreamEventEnvelope(raw)
+        if (!parsedEvent.ok) {
+          const detail = [parsedEvent.message, ...(parsedEvent.errors ?? [])]
+            .filter(Boolean)
+            .join('; ')
+          const failureMessage = `Received invalid stream event on shared path: ${detail}`
+          context.errors.push(failureMessage)
+          logger.error('Received invalid stream event on shared path', {
+            reason: parsedEvent.reason,
+            message: parsedEvent.message,
+            errors: parsedEvent.errors,
+          })
+          throw new FatalSseEventError(failureMessage)
+        }
+
+        const envelope = parsedEvent.event
+        const streamEvent = eventToStreamEvent(envelope)
+        if (envelope.trace?.requestId) {
+          const goTraceId = envelope.trace.goTraceId || envelope.trace.requestId
+          context.trace.setGoTraceId(goTraceId)
+          options.onGoTraceId?.(goTraceId)
+        }
+
+        // Per-type counters for the copilot.sse.read_loop span. Bound set
+        // (8 types) so this can never blow up into high cardinality.
+        if (streamEvent.type in counters.eventsByType) {
+          counters.eventsByType[streamEvent.type as MothershipStreamV1EventType] += 1
+        }
+
+        // Surface the full error payload the moment it arrives on the wire. This
+        // is the single chokepoint every error event passes through (main AND
+        // subagent lanes), before subagent routing — which has no `error`
+        // handler — would otherwise swallow it. The client only renders
+        // `message`/`displayMessage`, so log `code`/`provider`/`data` (the raw
+        // upstream provider error) here to explain a client-side "Stream error".
+        if (streamEvent.type === MothershipStreamV1EventType.error) {
+          const errorPayload = streamEvent.payload
+          logger.error('Received error event from Go copilot stream', {
+            path: pathname,
+            lane: streamEvent.scope?.lane ?? 'main',
+            parentToolCallId: streamEvent.scope?.parentToolCallId,
+            agentId: streamEvent.scope?.agentId,
+            code: errorPayload.code,
+            provider: errorPayload.provider,
+            message: errorPayload.message,
+            error: errorPayload.error,
+            displayMessage: errorPayload.displayMessage,
+            data: errorPayload.data,
+            requestId: context.requestId,
+            messageId: context.messageId,
+          })
+        }
+
+        if (shouldSkipToolCallEvent(streamEvent) || shouldSkipToolResultEvent(streamEvent)) {
+          return
+        }
+
+        await processFilePreviewStreamEvent({
+          streamId: envelope.stream.streamId,
+          streamEvent,
+          context,
+          execContext,
+          options,
+          state: filePreviewAdapterState,
+        })
+
+        await prePersistClientExecutableToolCall(streamEvent, context)
+
+        try {
+          await options.onEvent?.(streamEvent)
+        } catch (error) {
+          logger.warn('Failed to forward stream event', {
+            type: streamEvent.type,
+            error: getErrorMessage(error),
+          })
+        }
+
+        // Yield a macrotask so Node.js flushes the HTTP response buffer to
+        // the browser. Microtask yields (await Promise.resolve()) are not
+        // enough — the I/O layer needs a full event loop tick to write.
+        await new Promise<void>((resolve) => setImmediate(resolve))
+
+        if (options.onBeforeDispatch?.(streamEvent, context)) {
+          return context.streamComplete || undefined
+        }
+
+        if (isSubagentSpanStreamEvent(streamEvent)) {
+          const spanData = parseSubagentSpanData(streamEvent.payload.data)
+          const toolCallId = streamEvent.scope?.parentToolCallId || spanData?.toolCallId
+          // Deterministic nesting identity. spanId / parentSpanId are the
+          // primary keys; the toolCallId-keyed stack below is the legacy
+          // fallback for streams that predate span identity.
+          const spanId = streamEvent.scope?.spanId
+          const parentSpanId = streamEvent.scope?.parentSpanId
+          const subagentName = streamEvent.payload.agent
+          const spanEvt = streamEvent.payload.event
+          const isPendingPause = spanData?.pending === true
+          // A subagent lifecycle boundary breaks the main thinking stream.
+          // Flush any open thinking block into contentBlocks BEFORE we push
+          // the `subagent` marker, or the persisted order ends up
+          // [subagent, thinking] and the UI renders the subagent group
+          // above a thinking block that actually happened first.
+          flushSubagentThinkingBlock(context)
+          flushThinkingBlock(context)
+          if (spanEvt === MothershipStreamV1SpanLifecycleEvent.start) {
+            if (toolCallId) {
+              context.subAgentContent[toolCallId] ??= ''
+              context.subAgentToolCalls[toolCallId] ??= []
+            }
+            if (toolCallId && subagentName) {
+              const openParents = (context.openSubagentParents ??= new Set<string>())
+              if (!openParents.has(toolCallId)) {
+                openParents.add(toolCallId)
+                context.contentBlocks.push({
+                  type: 'subagent',
+                  content: subagentName,
+                  parentToolCallId: toolCallId,
+                  ...(spanId ? { spanId } : {}),
+                  ...(parentSpanId ? { parentSpanId } : {}),
+                  timestamp: Date.now(),
+                })
+              }
+            } else {
+              logger.warn('subagent start missing toolCallId or agent name', {
+                hasToolCallId: Boolean(toolCallId),
+                hasSubagentName: Boolean(subagentName),
+              })
+            }
+            return
+          }
+          if (spanEvt === MothershipStreamV1SpanLifecycleEvent.end) {
+            if (isPendingPause) {
+              return
+            }
+            if (!toolCallId) {
+              logger.warn('subagent end missing toolCallId')
+            }
+            if (toolCallId) {
+              for (let i = context.contentBlocks.length - 1; i >= 0; i--) {
+                const b = context.contentBlocks[i]
+                if (
+                  b.type === 'subagent' &&
+                  b.endedAt === undefined &&
+                  b.parentToolCallId === toolCallId
+                ) {
+                  b.endedAt = Date.now()
+                  break
+                }
+              }
+              context.openSubagentParents?.delete(toolCallId)
+            }
+            return
+          }
+        }
+
+        // Subagent-lane events are routed ONLY by their own scope. A valid one
+        // (has parentToolCallId) goes to the subagent handler; a malformed one
+        // (missing parentToolCallId — Go always stamps it, so this is defensive)
+        // is DROPPED rather than falling through to the main handler, which would
+        // merge foreign subagent text/tools into the durable main assistant
+        // message and mis-attribute it.
+        if (streamEvent.scope?.lane === 'subagent') {
+          if (handleSubagentRouting(streamEvent, context)) {
+            const handler = subAgentHandlers[streamEvent.type]
+            if (handler) {
+              await handler(streamEvent, context, execContext, options)
+            }
+          }
+          return context.streamComplete || undefined
+        }
+
+        const handler = sseHandlers[streamEvent.type]
+        if (handler) {
+          await handler(streamEvent, context, execContext, options)
+        }
+        return context.streamComplete || undefined
+      } finally {
+        const dispatchMs = performance.now() - dispatchStart
+        counters.totalDispatchMs += dispatchMs
+        if (dispatchMs > counters.longestDispatchMs) counters.longestDispatchMs = dispatchMs
+      }
+    })
+
+    if (!context.streamComplete && !abortSignal?.aborted && !context.wasAborted) {
+      let abortRequested = false
+      try {
+        abortRequested = await hasAbortMarker(context.messageId)
+      } catch (error) {
+        logger.warn('Failed to read abort marker at body close', {
+          streamId: context.messageId,
+          error: getErrorMessage(error),
+        })
+      }
+
+      if (abortRequested) {
+        options.onAbortObserved?.(AbortReason.MarkerObservedAtBodyClose)
+        context.wasAborted = true
+        endedOn = CopilotSseCloseReason.Aborted
+      } else {
+        const streamPath = new URL(fetchUrl).pathname
+        const message = `Copilot backend stream ended before a terminal event on ${streamPath}`
+        context.errors.push(message)
+        logger.error('Copilot backend stream ended before a terminal event', {
+          path: streamPath,
+          requestId: context.requestId,
+          messageId: context.messageId,
+        })
+        endedOn = CopilotSseCloseReason.ClosedNoTerminal
+        throw new CopilotBackendError(message, { status: 503 })
+      }
+    }
+  } catch (error) {
+    if (error instanceof FatalSseEventError && !context.errors.includes(error.message)) {
+      context.errors.push(error.message)
+    }
+    if (endedOn === CopilotSseCloseReason.Terminal) {
+      endedOn =
+        error instanceof CopilotBackendError
+          ? CopilotSseCloseReason.BackendError
+          : error instanceof BillingLimitError
+            ? CopilotSseCloseReason.BillingLimit
+            : CopilotSseCloseReason.Error
+    }
+    throw error
+  } finally {
+    if (abortSignal?.aborted) {
+      context.wasAborted = true
+      await reader.cancel().catch(() => {})
+      if (endedOn === CopilotSseCloseReason.Terminal) {
+        endedOn = CopilotSseCloseReason.Aborted
+      }
+    }
+    // An abort or error can tear down the loop mid-thinking. Flush any
+    // open thinking blocks so partial-persistence on /chat/stop sees
+    // them in contentBlocks with endedAt stamped, instead of silently
+    // dropping the in-flight reasoning.
+    flushSubagentThinkingBlock(context)
+    flushThinkingBlock(context)
+    clearTimeout(timeoutId)
+
+    // Legacy TraceCollector span (consumed by the in-memory trace
+    // collector, kept for backwards compatibility with existing
+    // tooling). The real OTel span is stamped below.
+    const bodyDurationMs = Math.round(performance.now() - bodyStart)
+    bodySpan.attributes = {
+      ...(bodySpan.attributes ?? {}),
+      eventsReceived: counters.events,
+      firstEventMs: counters.firstEventMs,
+      endedOn,
+      durationMs: bodyDurationMs,
+    }
+    context.trace.endSpan(
+      bodySpan,
+      endedOn === CopilotSseCloseReason.Terminal
+        ? 'ok'
+        : endedOn === CopilotSseCloseReason.Aborted
+          ? 'cancelled'
+          : 'error'
+    )
+
+    // Real OTel span for Tempo/Grafana. Stamped aggregate-only so
+    // there is no per-chunk OTel cost — one span per read loop with
+    // integer counters, plus a bounded set of events.
+    //
+    // `expectedTerminal` = "the caller considered this leg the FINAL
+    // leg and genuinely expected a terminal event on the wire." We
+    // derive it from `context.streamComplete` MINUS the tool-pause
+    // case: when the server emits a `run.checkpoint_pause`, its
+    // handler also sets `streamComplete=true` to stop the read loop
+    // cleanly, but no `complete` SSE event is ever sent in that
+    // case — that's the tool-pause protocol, not a missing terminal.
+    // `awaitingAsyncContinuation` is set by the same handler, so
+    // its presence distinguishes "tool pause, no terminal expected"
+    // from "caller thought stream was done but server never said so"
+    // (= the real disappeared-response bug class).
+    const expectedTerminal = context.streamComplete && !context.awaitingAsyncContinuation
+    stampSseReadLoopSpan(bodyStart, counters, endedOn, fetchUrl, pathname, {
+      idleGapEventThresholdMs: IDLE_GAP_EVENT_THRESHOLD_MS,
+      expectedTerminal,
+    })
+  }
+}
+
+function estimateBodyBytes(body: BodyInit | null | undefined): number {
+  if (!body) {
+    return 0
+  }
+  if (typeof body === 'string') {
+    return body.length
+  }
+  if (body instanceof ArrayBuffer) {
+    return body.byteLength
+  }
+  if (ArrayBuffer.isView(body)) {
+    return body.byteLength
+  }
+  return 0
+}
+
+type SseReadLoopCounters = {
+  bytes: number
+  chunks: number
+  events: number
+  eventsByType: Record<MothershipStreamV1EventType, number>
+  firstEventMs: number | undefined
+  longestInboundGapMs: number
+  longestDispatchMs: number
+  totalDispatchMs: number
+}
+
+/**
+ * Ship a one-shot `copilot.sse.read_loop` OTel span with the aggregate
+ * counters collected during the read loop. Uses `startTime` so the
+ * span's duration reflects the actual loop wall clock even though we
+ * only talk to OTel once at the end.
+ *
+ * Deliberately synchronous, no per-chunk span calls: total OTel cost
+ * per read loop is fixed (~10 attrs + up to 3 events), independent of
+ * chunk count.
+ */
+function stampSseReadLoopSpan(
+  startPerfMs: number,
+  counters: SseReadLoopCounters,
+  closeReason: string,
+  fetchUrl: string,
+  pathname: string,
+  opts: { idleGapEventThresholdMs: number; expectedTerminal: boolean }
+): void {
+  // Translate performance.now() values into wall-clock Date values so
+  // the span's timestamps land in real time (OTel accepts both, but we
+  // need to pair startTime with a matching "now" for .end()).
+  const nowPerf = performance.now()
+  const nowWall = Date.now()
+  const startWall = nowWall - (nowPerf - startPerfMs)
+
+  const terminalEventSeen = counters.eventsByType.complete > 0 || counters.eventsByType.error > 0
+  // `terminal_event_missing` is the single-attribute dashboard signal
+  // for the "disappeared response" bug class: the caller considered
+  // this leg to be the final one (`context.streamComplete === true`)
+  // but no terminal `complete` or `error` event arrived on the wire.
+  // Tool-pause legs have expectedTerminal=false and never trip this, so
+  // dashboards can filter on `{ .copilot.sse.terminal_event_missing = true }`
+  // without false positives.
+  const terminalEventMissing = opts.expectedTerminal && !terminalEventSeen
+
+  const tracer = getCopilotTracer()
+  const span = tracer.startSpan(TraceSpan.CopilotSseReadLoop, {
+    startTime: startWall,
+    attributes: {
+      [TraceAttr.HttpUrl]: fetchUrl,
+      [TraceAttr.HttpPath]: pathname,
+      [TraceAttr.CopilotSseBytesReceived]: counters.bytes,
+      [TraceAttr.CopilotSseChunksReceived]: counters.chunks,
+      [TraceAttr.CopilotSseEventsReceived]: counters.events,
+      [TraceAttr.CopilotSseEventsSession]: counters.eventsByType.session,
+      [TraceAttr.CopilotSseEventsText]: counters.eventsByType.text,
+      [TraceAttr.CopilotSseEventsTool]: counters.eventsByType.tool,
+      [TraceAttr.CopilotSseEventsSpan]: counters.eventsByType.span,
+      [TraceAttr.CopilotSseEventsResource]: counters.eventsByType.resource,
+      [TraceAttr.CopilotSseEventsRun]: counters.eventsByType.run,
+      [TraceAttr.CopilotSseEventsError]: counters.eventsByType.error,
+      [TraceAttr.CopilotSseEventsComplete]: counters.eventsByType.complete,
+      [TraceAttr.CopilotSseLongestInboundGapMs]: Math.round(counters.longestInboundGapMs),
+      [TraceAttr.CopilotSseLongestDispatchMs]: Math.round(counters.longestDispatchMs),
+      [TraceAttr.CopilotSseTotalDispatchMs]: Math.round(counters.totalDispatchMs),
+      [TraceAttr.CopilotSseCloseReason]: closeReason,
+      [TraceAttr.CopilotSseExpectedTerminal]: opts.expectedTerminal,
+      [TraceAttr.CopilotSseTerminalEventSeen]: terminalEventSeen,
+      [TraceAttr.CopilotSseTerminalEventMissing]: terminalEventMissing,
+    },
+  })
+
+  if (counters.firstEventMs !== undefined) {
+    span.setAttribute(TraceAttr.CopilotSseFirstEventMs, counters.firstEventMs)
+    // Anchor the event to the moment the first SSE event was actually
+    // received (startWall + firstEventMs), not `now`, so a trace
+    // waterfall shows the diamond at the TTFT point — not at span end.
+    span.addEvent(
+      TraceEvent.CopilotSseFirstEvent,
+      { [TraceAttr.CopilotSseFirstEventMs]: counters.firstEventMs },
+      startWall + counters.firstEventMs
+    )
+  }
+  // Fire the idle-gap event when the INBOUND gap (time between TCP
+  // reads returning bytes) exceeds the threshold. This is the
+  // "upstream was silent or Sim was CPU-bound" signal; dispatch time
+  // on its own doesn't warrant an event because it's within our
+  // control and visible on a dedicated attribute.
+  if (counters.longestInboundGapMs >= opts.idleGapEventThresholdMs) {
+    span.addEvent(TraceEvent.CopilotSseIdleGapExceeded, {
+      [TraceAttr.CopilotSseLongestInboundGapMs]: Math.round(counters.longestInboundGapMs),
+      [TraceAttr.CopilotSseLongestDispatchMs]: Math.round(counters.longestDispatchMs),
+    })
+  }
+  if (terminalEventSeen) {
+    span.addEvent(TraceEvent.CopilotSseTerminalEventReceived)
+  }
+
+  // Span status: only mark ERROR for real failures. User aborts and
+  // clean terminals stay UNSET so dashboards filtering `status=error`
+  // don't light up for normal cancellations. Tool-pause legs (caller
+  // didn't set streamComplete) are NOT errors even though they have
+  // no complete event.
+  if (terminalEventMissing) {
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: 'SSE read loop finished without terminal event (caller expected one)',
+    })
+  } else if (
+    closeReason !== CopilotSseCloseReason.Terminal &&
+    closeReason !== CopilotSseCloseReason.Aborted
+  ) {
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: `SSE read loop ended with reason: ${closeReason}`,
+    })
+  }
+
+  span.end(nowWall)
+}

apps/sim/lib/copilot/request/handlers/complete.ts

@@ -0,0 +1,28 @@
+import type { StreamHandler } from './types'
+import { flushSubagentThinkingBlock, flushThinkingBlock } from './types'
+
+export const handleCompleteEvent: StreamHandler = (event, context) => {
+  flushSubagentThinkingBlock(context)
+  flushThinkingBlock(context)
+  if (event.type !== 'complete') {
+    context.streamComplete = true
+    return
+  }
+
+  if (event.payload.usage) {
+    context.usage = {
+      prompt: (context.usage?.prompt || 0) + (event.payload.usage.input_tokens || 0),
+      completion: (context.usage?.completion || 0) + (event.payload.usage.output_tokens || 0),
+    }
+  }
+
+  if (event.payload.cost) {
+    context.cost = {
+      input: (context.cost?.input || 0) + (event.payload.cost.input || 0),
+      output: (context.cost?.output || 0) + (event.payload.cost.output || 0),
+      total: (context.cost?.total || 0) + (event.payload.cost.total || 0),
+    }
+  }
+
+  context.streamComplete = true
+}

apps/sim/lib/copilot/request/handlers/error.ts

@@ -0,0 +1,16 @@
+import type { StreamHandler } from './types'
+import { flushSubagentThinkingBlock, flushThinkingBlock } from './types'
+
+export const handleErrorEvent: StreamHandler = (event, context) => {
+  flushSubagentThinkingBlock(context)
+  flushThinkingBlock(context)
+  if (event.type !== 'error') {
+    context.streamComplete = true
+    return
+  }
+  const message = event.payload.message || event.payload.error
+  if (message) {
+    context.errors.push(message)
+  }
+  context.streamComplete = true
+}

apps/sim/lib/copilot/request/handlers/handlers.test.ts

@@ -0,0 +1,979 @@
+/**
+ * @vitest-environment node
+ */
+
+import { sleep } from '@sim/utils/helpers'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+
+const { isSimExecuted, executeTool, ensureHandlersRegistered } = vi.hoisted(() => ({
+  isSimExecuted: vi.fn().mockReturnValue(true),
+  executeTool: vi.fn().mockResolvedValue({ success: true, output: { ok: true } }),
+  ensureHandlersRegistered: vi.fn(),
+}))
+
+const { upsertAsyncToolCall, markAsyncToolRunning, completeAsyncToolCall, markAsyncToolDelivered } =
+  vi.hoisted(() => ({
+    upsertAsyncToolCall: vi.fn(),
+    markAsyncToolRunning: vi.fn(),
+    completeAsyncToolCall: vi.fn(),
+    markAsyncToolDelivered: vi.fn(),
+  }))
+
+const { waitForToolCompletion } = vi.hoisted(() => ({
+  waitForToolCompletion: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/tool-executor', () => ({
+  isSimExecuted,
+  executeTool,
+  ensureHandlersRegistered,
+  getToolEntry: vi.fn().mockReturnValue(undefined),
+}))
+
+vi.mock('@/lib/copilot/async-runs/repository', () => ({
+  createRunSegment: vi.fn(),
+  updateRunStatus: vi.fn(),
+  getLatestRunForExecution: vi.fn(),
+  getLatestRunForStream: vi.fn(),
+  getRunSegment: vi.fn(),
+  createRunCheckpoint: vi.fn(),
+  getAsyncToolCall: vi.fn(),
+  markAsyncToolStatus: vi.fn(),
+  listAsyncToolCallsForRun: vi.fn(),
+  getAsyncToolCalls: vi.fn(),
+  claimCompletedAsyncToolCall: vi.fn(),
+  releaseCompletedAsyncToolClaim: vi.fn(),
+  upsertAsyncToolCall,
+  markAsyncToolRunning,
+  markAsyncToolDelivered,
+  completeAsyncToolCall,
+}))
+
+vi.mock('@/lib/copilot/request/tools/client', () => ({
+  waitForToolCompletion,
+}))
+
+import {
+  MothershipStreamV1AsyncToolRecordStatus,
+  MothershipStreamV1EventType,
+  MothershipStreamV1ResourceOp,
+  MothershipStreamV1RunKind,
+  MothershipStreamV1TextChannel,
+  MothershipStreamV1ToolExecutor,
+  MothershipStreamV1ToolMode,
+  MothershipStreamV1ToolOutcome,
+  MothershipStreamV1ToolPhase,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { Read as ReadTool } from '@/lib/copilot/generated/tool-catalog-v1'
+import { sseHandlers, subAgentHandlers } from '@/lib/copilot/request/handlers'
+import type { ExecutionContext, StreamEvent, StreamingContext } from '@/lib/copilot/request/types'
+
+describe('sse-handlers tool lifecycle', () => {
+  let context: StreamingContext
+  let execContext: ExecutionContext
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    upsertAsyncToolCall.mockResolvedValue(null)
+    markAsyncToolRunning.mockResolvedValue(null)
+    completeAsyncToolCall.mockResolvedValue(null)
+    markAsyncToolDelivered.mockResolvedValue(null)
+    waitForToolCompletion.mockResolvedValue(null)
+    context = {
+      chatId: undefined,
+      messageId: 'msg-1',
+      accumulatedContent: '',
+      finalAssistantContent: '',
+      sawMainToolCall: false,
+      trace: new TraceCollector(),
+      contentBlocks: [],
+      toolCalls: new Map(),
+      pendingToolPromises: new Map(),
+      currentThinkingBlock: null,
+      subagentThinkingBlocks: new Map(),
+      isInThinkingBlock: false,
+      subAgentContent: {},
+      subAgentToolCalls: {},
+      pendingContent: '',
+      streamComplete: false,
+      wasAborted: false,
+      errors: [],
+    }
+    execContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+    }
+  })
+
+  it('keeps only the latest post-tool assistant text for headless final content', async () => {
+    await sseHandlers.text(
+      {
+        type: MothershipStreamV1EventType.text,
+        payload: {
+          channel: MothershipStreamV1TextChannel.assistant,
+          text: 'I will check that.',
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false }
+    )
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-1',
+          toolName: ReadTool.id,
+          arguments: { path: 'foo.txt' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, autoExecuteTools: false }
+    )
+
+    await sseHandlers.text(
+      {
+        type: MothershipStreamV1EventType.text,
+        payload: {
+          channel: MothershipStreamV1TextChannel.assistant,
+          text: 'Final answer only.',
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false }
+    )
+
+    expect(context.accumulatedContent).toBe('I will check that.Final answer only.')
+    expect(context.finalAssistantContent).toBe('Final answer only.')
+  })
+
+  it('executes tool_call and emits tool_result', async () => {
+    executeTool.mockResolvedValueOnce({ success: true, output: { ok: true } })
+    const onEvent = vi.fn()
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-1',
+          toolName: ReadTool.id,
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+          ui: {},
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: false, timeout: 1000 }
+    )
+
+    // tool_call fires execution without awaiting (fire-and-forget for parallel execution),
+    // so we flush pending microtasks before asserting
+    await sleep(0)
+
+    expect(executeTool).toHaveBeenCalledTimes(1)
+    expect(onEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.tool,
+        payload: expect.objectContaining({
+          toolCallId: 'tool-1',
+          success: true,
+          phase: MothershipStreamV1ToolPhase.result,
+        }),
+      })
+    )
+
+    const updated = context.toolCalls.get('tool-1')
+    expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.success)
+    // Display titles are derived client-side from the tool name (+args), not the
+    // stream; read with no path resolves to the static "Reading file".
+    expect(updated?.displayTitle).toBe('Reading file')
+    expect(updated?.result?.output).toEqual({ ok: true })
+    expect(context.contentBlocks.at(0)).toEqual(
+      expect.objectContaining({
+        type: 'tool_call',
+        toolCall: expect.objectContaining({
+          id: 'tool-1',
+          displayTitle: 'Reading file',
+        }),
+      })
+    )
+  })
+
+  it('preserves primitive tool outputs through async completion persistence', async () => {
+    executeTool.mockResolvedValueOnce({ success: true, output: 'done' })
+    const onEvent = vi.fn()
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-primitive',
+          toolName: ReadTool.id,
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(completeAsyncToolCall).toHaveBeenCalledWith(
+      expect.objectContaining({
+        toolCallId: 'tool-primitive',
+        status: MothershipStreamV1AsyncToolRecordStatus.completed,
+        result: 'done',
+        error: null,
+      })
+    )
+    expect(onEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.tool,
+        payload: expect.objectContaining({
+          toolCallId: 'tool-primitive',
+          phase: MothershipStreamV1ToolPhase.result,
+          success: true,
+          output: 'done',
+        }),
+      })
+    )
+
+    const updated = context.toolCalls.get('tool-primitive')
+    expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.success)
+    expect(updated?.result?.output).toBe('done')
+  })
+
+  it('marks background client workflow tools delivered after synthetic result emission', async () => {
+    waitForToolCompletion.mockResolvedValueOnce({
+      status: 'background',
+      data: { detached: true },
+    })
+    const onEvent = vi.fn()
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-background',
+          toolName: 'run_workflow',
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.client,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: true, timeout: 1000 }
+    )
+
+    await sleep(0)
+    await Promise.allSettled(context.pendingToolPromises.values())
+
+    expect(markAsyncToolDelivered).toHaveBeenCalledWith('tool-background')
+    expect(onEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.tool,
+        payload: expect.objectContaining({
+          toolCallId: 'tool-background',
+          phase: MothershipStreamV1ToolPhase.result,
+          status: MothershipStreamV1ToolOutcome.skipped,
+          success: true,
+          output: { detached: true },
+        }),
+      })
+    )
+    expect(context.toolCalls.get('tool-background')?.status).toBe(
+      MothershipStreamV1ToolOutcome.skipped
+    )
+  })
+
+  it('does not add hidden tool calls to content blocks', async () => {
+    executeTool.mockResolvedValueOnce({ success: true, output: { skill: 'ok' } })
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-hidden',
+          toolName: 'load_agent_skill',
+          arguments: { skill_name: 'markdown-writing' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(executeTool).toHaveBeenCalledTimes(1)
+    expect(context.contentBlocks).toEqual([])
+    expect(context.toolCalls.get('tool-hidden')?.name).toBe('load_agent_skill')
+  })
+
+  it('does not add ui-hidden tool calls to content blocks', async () => {
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-ui-hidden',
+          toolName: 'read',
+          arguments: { path: 'components/integrations/slack/README.md' },
+          executor: MothershipStreamV1ToolExecutor.go,
+          mode: MothershipStreamV1ToolMode.sync,
+          phase: MothershipStreamV1ToolPhase.call,
+          ui: { hidden: true },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    expect(context.contentBlocks).toEqual([])
+    expect(context.toolCalls.get('tool-ui-hidden')?.name).toBe('read')
+  })
+
+  it('removes an existing content block when a later frame marks the tool hidden', async () => {
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-hidden-after-partial',
+          toolName: 'read',
+          executor: MothershipStreamV1ToolExecutor.go,
+          mode: MothershipStreamV1ToolMode.sync,
+          phase: MothershipStreamV1ToolPhase.call,
+          status: 'generating',
+          arguments: { path: 'components/integrations' },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+    expect(context.contentBlocks).toHaveLength(1)
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-hidden-after-partial',
+          toolName: 'read',
+          executor: MothershipStreamV1ToolExecutor.go,
+          mode: MothershipStreamV1ToolMode.sync,
+          phase: MothershipStreamV1ToolPhase.call,
+          arguments: { path: 'components/integrations/slack/README.md' },
+          ui: { hidden: true },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    expect(context.contentBlocks).toEqual([])
+  })
+
+  it('does not show pathless read or glob generating placeholders', async () => {
+    for (const toolName of ['read', 'glob'] as const) {
+      await sseHandlers.tool(
+        {
+          type: MothershipStreamV1EventType.tool,
+          payload: {
+            toolCallId: `${toolName}-generating`,
+            toolName,
+            executor: MothershipStreamV1ToolExecutor.go,
+            mode: MothershipStreamV1ToolMode.sync,
+            phase: MothershipStreamV1ToolPhase.call,
+            status: 'generating',
+          },
+        } satisfies StreamEvent,
+        context,
+        execContext,
+        { interactive: false, timeout: 1000 }
+      )
+    }
+
+    expect(context.contentBlocks).toEqual([])
+    expect(context.toolCalls.has('read-generating')).toBe(false)
+    expect(context.toolCalls.has('glob-generating')).toBe(false)
+  })
+
+  it('updates stored params when a subagent generating event is followed by the final tool call', async () => {
+    executeTool.mockResolvedValueOnce({ success: true, output: { ok: true } })
+    context.toolCalls.set('parent-1', {
+      id: 'parent-1',
+      name: 'workflow',
+      status: 'pending',
+      startTime: Date.now(),
+    })
+
+    await subAgentHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId: 'workflow' },
+        payload: {
+          toolCallId: 'sub-tool-1',
+          toolName: 'create_workflow',
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+          status: 'generating',
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    await subAgentHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId: 'workflow' },
+        payload: {
+          toolCallId: 'sub-tool-1',
+          toolName: 'create_workflow',
+          arguments: { name: 'Example Workflow' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+          status: 'executing',
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(executeTool).toHaveBeenCalledWith(
+      'create_workflow',
+      { name: 'Example Workflow' },
+      expect.any(Object)
+    )
+    expect(context.toolCalls.get('sub-tool-1')?.params).toEqual({ name: 'Example Workflow' })
+    expect(context.subAgentToolCalls['parent-1']?.[0]?.params).toEqual({
+      name: 'Example Workflow',
+    })
+  })
+
+  it('routes subagent text using the event scope parent tool call id', async () => {
+    context.subAgentContent['parent-1'] = ''
+
+    await subAgentHandlers.text(
+      {
+        type: MothershipStreamV1EventType.text,
+        scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId: 'deploy' },
+        payload: {
+          channel: MothershipStreamV1TextChannel.assistant,
+          text: 'hello from deploy',
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    expect(context.subAgentContent['parent-1']).toBe('hello from deploy')
+    expect(context.contentBlocks.at(-1)).toEqual(
+      expect.objectContaining({
+        type: 'subagent_text',
+        content: 'hello from deploy',
+      })
+    )
+  })
+
+  it('routes main assistant text with no scope into accumulatedContent', async () => {
+    await sseHandlers.text(
+      {
+        type: MothershipStreamV1EventType.text,
+        payload: {
+          channel: MothershipStreamV1TextChannel.assistant,
+          text: 'hello from main',
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    expect(context.accumulatedContent).toBe('hello from main')
+    expect(context.contentBlocks.at(-1)).toEqual(
+      expect.objectContaining({
+        type: 'text',
+        content: 'hello from main',
+      })
+    )
+  })
+
+  it('routes subagent tool calls using the event scope parent tool call id', async () => {
+    executeTool.mockResolvedValueOnce({ success: true, output: { ok: true } })
+    context.toolCalls.set('parent-1', {
+      id: 'parent-1',
+      name: 'deploy',
+      status: 'pending',
+      startTime: Date.now(),
+    })
+
+    await subAgentHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId: 'deploy' },
+        payload: {
+          toolCallId: 'sub-tool-scope-1',
+          toolName: 'read',
+          arguments: { path: 'workflow.json' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(context.subAgentToolCalls['parent-1']?.[0]?.id).toBe('sub-tool-scope-1')
+  })
+
+  it('keeps two concurrent subagent lanes separate for text and thinking', async () => {
+    const send = (parent: string, channel: MothershipStreamV1TextChannel, text: string) =>
+      subAgentHandlers.text(
+        {
+          type: MothershipStreamV1EventType.text,
+          scope: {
+            lane: 'subagent',
+            parentToolCallId: parent,
+            spanId: `span-${parent}`,
+            agentId: 'research',
+          },
+          payload: { channel, text },
+        } satisfies StreamEvent,
+        context,
+        execContext,
+        { interactive: false, timeout: 1000 }
+      )
+
+    // Interleaved thinking across two concurrent lanes.
+    await send('A', MothershipStreamV1TextChannel.thinking, 'A-think-1 ')
+    await send('B', MothershipStreamV1TextChannel.thinking, 'B-think-1 ')
+    await send('A', MothershipStreamV1TextChannel.thinking, 'A-think-2')
+
+    // Each lane accumulates its own thinking block — no cross-contamination.
+    expect(context.subagentThinkingBlocks.get('A')?.content).toBe('A-think-1 A-think-2')
+    expect(context.subagentThinkingBlocks.get('B')?.content).toBe('B-think-1 ')
+
+    // Interleaved assistant text across the two lanes.
+    await send('A', MothershipStreamV1TextChannel.assistant, 'A-text')
+    await send('B', MothershipStreamV1TextChannel.assistant, 'B-text')
+
+    expect(context.subAgentContent.A).toBe('A-text')
+    expect(context.subAgentContent.B).toBe('B-text')
+
+    // Assistant text flushed each lane's thinking into contentBlocks, attributed
+    // to the correct parent (not whichever subagent streamed most recently).
+    const thinking = context.contentBlocks.filter((b) => b.type === 'subagent_thinking')
+    expect(thinking.find((b) => b.parentToolCallId === 'A')?.content).toBe('A-think-1 A-think-2')
+    expect(thinking.find((b) => b.parentToolCallId === 'B')?.content).toBe('B-think-1 ')
+  })
+
+  it('drops a subagent text event that is missing its parent tool call id', async () => {
+    const before = context.contentBlocks.length
+    await subAgentHandlers.text(
+      {
+        type: MothershipStreamV1EventType.text,
+        scope: { lane: 'subagent', agentId: 'research' },
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'orphan' },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    // No lane to attribute to — nothing is added rather than mis-attributed.
+    expect(context.contentBlocks.length).toBe(before)
+    expect(Object.keys(context.subAgentContent)).not.toContain('undefined')
+  })
+
+  it('skips duplicate tool_call after result', async () => {
+    executeTool.mockResolvedValueOnce({ success: true, output: { ok: true } })
+
+    const event = {
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'tool-dup',
+        toolName: ReadTool.id,
+        arguments: { workflowId: 'workflow-1' },
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.call,
+      },
+    }
+
+    await sseHandlers.tool(event as StreamEvent, context, execContext, { interactive: false })
+    await sleep(0)
+    await sseHandlers.tool(event as StreamEvent, context, execContext, { interactive: false })
+
+    expect(executeTool).toHaveBeenCalledTimes(1)
+  })
+
+  it('marks an in-flight tool as cancelled when aborted mid-execution', async () => {
+    const abortController = new AbortController()
+    const userStopController = new AbortController()
+    execContext.abortSignal = abortController.signal
+    execContext.userStopSignal = userStopController.signal
+
+    executeTool.mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          setTimeout(() => resolve({ success: true, output: { ok: true } }), 0)
+        })
+    )
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-cancel',
+          toolName: ReadTool.id,
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      {
+        interactive: false,
+        timeout: 1000,
+        abortSignal: abortController.signal,
+        userStopSignal: userStopController.signal,
+      }
+    )
+
+    userStopController.abort()
+    abortController.abort()
+    await sleep(10)
+
+    const updated = context.toolCalls.get('tool-cancel')
+    expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.cancelled)
+    expect(updated?.result).toEqual({ success: false })
+    expect(updated?.error).toBe('Request aborted during tool execution')
+  })
+
+  it('does not replace an in-flight pending promise on duplicate tool_call', async () => {
+    let resolveTool: ((value: { success: boolean; output: { ok: boolean } }) => void) | undefined
+    executeTool.mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveTool = resolve
+        })
+    )
+
+    const event = {
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId: 'tool-inflight',
+        toolName: ReadTool.id,
+        arguments: { workflowId: 'workflow-1' },
+        executor: MothershipStreamV1ToolExecutor.sim,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.call,
+      },
+    }
+
+    await sseHandlers.tool(event as StreamEvent, context, execContext, { interactive: false })
+    await sleep(0)
+
+    const firstPromise = context.pendingToolPromises.get('tool-inflight')
+    expect(firstPromise).toBeDefined()
+
+    await sseHandlers.tool(event as StreamEvent, context, execContext, { interactive: false })
+
+    expect(executeTool).toHaveBeenCalledTimes(1)
+    expect(context.pendingToolPromises.get('tool-inflight')).toBe(firstPromise)
+
+    resolveTool?.({ success: true, output: { ok: true } })
+    await sleep(0)
+
+    expect(context.pendingToolPromises.has('tool-inflight')).toBe(false)
+  })
+
+  it('still executes the tool when async row upsert fails', async () => {
+    upsertAsyncToolCall.mockRejectedValueOnce(new Error('db down'))
+    executeTool.mockResolvedValueOnce({ success: true, output: { ok: true } })
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-upsert-fail',
+          toolName: ReadTool.id,
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent: vi.fn(), interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(executeTool).toHaveBeenCalledTimes(1)
+    expect(context.toolCalls.get('tool-upsert-fail')?.status).toBe(
+      MothershipStreamV1ToolOutcome.success
+    )
+  })
+
+  it('does not execute a tool if a terminal tool_result arrives before local execution starts', async () => {
+    let resolveUpsert: ((value: null) => void) | undefined
+    upsertAsyncToolCall.mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveUpsert = resolve
+        })
+    )
+    const onEvent = vi.fn()
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-race',
+          toolName: ReadTool.id,
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: false, timeout: 1000 }
+    )
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-race',
+          toolName: ReadTool.id,
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.result,
+          success: true,
+          output: { ok: true },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: false, timeout: 1000 }
+    )
+
+    resolveUpsert?.(null)
+    await sleep(0)
+
+    expect(executeTool).not.toHaveBeenCalled()
+    expect(context.toolCalls.get('tool-race')?.status).toBe(MothershipStreamV1ToolOutcome.success)
+    expect(context.toolCalls.get('tool-race')?.result?.output).toEqual({ ok: true })
+  })
+
+  it('does not execute a tool if a tool_result arrives before the tool_call event', async () => {
+    const onEvent = vi.fn()
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-early-result',
+          toolName: ReadTool.id,
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.result,
+          success: true,
+          output: { ok: true },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: false, timeout: 1000 }
+    )
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-early-result',
+          toolName: ReadTool.id,
+          arguments: { workflowId: 'workflow-1' },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent, interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(executeTool).not.toHaveBeenCalled()
+    expect(context.toolCalls.get('tool-early-result')?.status).toBe(
+      MothershipStreamV1ToolOutcome.success
+    )
+  })
+
+  it('reads canonical tool result errors from the error field', async () => {
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-output-only',
+          toolName: ReadTool.id,
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.result,
+          success: false,
+          error: 'output-failure',
+          output: { detail: 'extra-context' },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent: vi.fn(), interactive: false, timeout: 1000 }
+    )
+
+    const updated = context.toolCalls.get('tool-output-only')
+    expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.error)
+    expect(updated?.result?.output).toEqual({ detail: 'extra-context' })
+    expect(updated?.error).toBe('output-failure')
+  })
+
+  it('preserves skipped tool results from the stream contract', async () => {
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-skipped',
+          toolName: ReadTool.id,
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.result,
+          status: MothershipStreamV1ToolOutcome.skipped,
+          success: true,
+          output: { detached: true },
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { onEvent: vi.fn(), interactive: false, timeout: 1000 }
+    )
+
+    const updated = context.toolCalls.get('tool-skipped')
+    expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.skipped)
+    expect(updated?.result?.output).toEqual({ detached: true })
+    expect(updated?.error).toBeUndefined()
+  })
+
+  it('executes dynamic sim tools based on payload executor', async () => {
+    isSimExecuted.mockReturnValueOnce(false)
+    executeTool.mockResolvedValueOnce({ success: true, output: { emails: [] } })
+
+    await sseHandlers.tool(
+      {
+        type: MothershipStreamV1EventType.tool,
+        payload: {
+          toolCallId: 'tool-dynamic-sim',
+          toolName: 'gmail_read',
+          arguments: { maxResults: 10 },
+          executor: MothershipStreamV1ToolExecutor.sim,
+          mode: MothershipStreamV1ToolMode.async,
+          phase: MothershipStreamV1ToolPhase.call,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    await sleep(0)
+
+    expect(executeTool).toHaveBeenCalledWith('gmail_read', { maxResults: 10 }, expect.any(Object))
+    expect(context.toolCalls.get('tool-dynamic-sim')?.status).toBe(
+      MothershipStreamV1ToolOutcome.success
+    )
+  })
+
+  it('clears pending continuation state when a run resumes', async () => {
+    context.awaitingAsyncContinuation = {
+      checkpointId: 'cp-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      pendingToolCallIds: ['tool-1'],
+    }
+    context.streamComplete = true
+
+    await sseHandlers.run(
+      {
+        type: MothershipStreamV1EventType.run,
+        payload: {
+          kind: MothershipStreamV1RunKind.resumed,
+        },
+      } satisfies StreamEvent,
+      context,
+      execContext,
+      { interactive: false, timeout: 1000 }
+    )
+
+    expect(context.awaitingAsyncContinuation).toBeUndefined()
+    expect(context.streamComplete).toBe(false)
+  })
+
+  it('routes resource events through an explicit main-lane handler', async () => {
+    expect(() =>
+      sseHandlers.resource(
+        {
+          type: MothershipStreamV1EventType.resource,
+          payload: {
+            op: MothershipStreamV1ResourceOp.upsert,
+            resource: {
+              type: 'file',
+              id: 'file-1',
+              title: 'Document',
+            },
+          },
+        } satisfies StreamEvent,
+        context,
+        execContext,
+        { interactive: false, timeout: 1000 }
+      )
+    ).not.toThrow()
+  })
+})

apps/sim/lib/copilot/request/handlers/index.ts

@@ -0,0 +1,52 @@
+import { createLogger } from '@sim/logger'
+import { MothershipStreamV1EventType } from '@/lib/copilot/generated/mothership-stream-v1'
+import type { StreamEvent, StreamingContext } from '@/lib/copilot/request/types'
+import { handleCompleteEvent } from './complete'
+import { handleErrorEvent } from './error'
+import { handleResourceEvent } from './resource'
+import { handleRunEvent } from './run'
+import { handleSessionEvent } from './session'
+import { handleSpanEvent } from './span'
+import { handleTextEvent } from './text'
+import { handleToolEvent, prePersistClientExecutableToolCall } from './tool'
+import type { StreamHandler } from './types'
+
+export { prePersistClientExecutableToolCall }
+export type { StreamHandler } from './types'
+
+const logger = createLogger('CopilotHandlerRouting')
+
+export const sseHandlers: Record<string, StreamHandler> = {
+  [MothershipStreamV1EventType.session]: handleSessionEvent,
+  [MothershipStreamV1EventType.tool]: (e, c, ec, o) => handleToolEvent(e, c, ec, o, 'main'),
+  [MothershipStreamV1EventType.text]: handleTextEvent('main'),
+  [MothershipStreamV1EventType.resource]: handleResourceEvent,
+  [MothershipStreamV1EventType.run]: handleRunEvent,
+  [MothershipStreamV1EventType.complete]: handleCompleteEvent,
+  [MothershipStreamV1EventType.error]: handleErrorEvent,
+  [MothershipStreamV1EventType.span]: handleSpanEvent,
+}
+
+export const subAgentHandlers: Record<string, StreamHandler> = {
+  [MothershipStreamV1EventType.text]: handleTextEvent('subagent'),
+  [MothershipStreamV1EventType.tool]: (e, c, ec, o) => handleToolEvent(e, c, ec, o, 'subagent'),
+  [MothershipStreamV1EventType.span]: handleSpanEvent,
+}
+
+export function handleSubagentRouting(event: StreamEvent, _context: StreamingContext): boolean {
+  if (event.scope?.lane !== 'subagent') return false
+
+  // Scope-only attribution: a subagent event MUST carry its own parentToolCallId.
+  // With concurrent subagents there is no single "current" lane to fall back to —
+  // routing by a global pointer would mis-attribute interleaved events to the
+  // last-started subagent. A missing parentToolCallId is a contract violation
+  // (Go always stamps it), so warn and route to the main lane rather than guess.
+  if (!event.scope?.parentToolCallId) {
+    logger.warn('Subagent event missing parent tool call id; routing to main lane', {
+      type: event.type,
+      subagent: event.scope?.agentId,
+    })
+    return false
+  }
+  return true
+}

apps/sim/lib/copilot/request/handlers/resource.ts

@@ -0,0 +1,7 @@
+import type { StreamHandler } from './types'
+
+export const handleResourceEvent: StreamHandler = (event) => {
+  if (event.type !== 'resource') {
+    return
+  }
+}

apps/sim/lib/copilot/request/handlers/run.ts

@@ -0,0 +1,73 @@
+import { createLogger } from '@sim/logger'
+import {
+  MothershipStreamV1RunKind,
+  MothershipStreamV1ToolOutcome,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import type { StreamHandler } from './types'
+import { addContentBlock } from './types'
+
+const logger = createLogger('CopilotRunHandler')
+
+export const handleRunEvent: StreamHandler = (event, context) => {
+  if (event.type !== 'run') {
+    return
+  }
+
+  if (event.payload.kind === MothershipStreamV1RunKind.checkpoint_pause) {
+    const frames = (event.payload.frames ?? []).map((frame) => ({
+      parentToolCallId: frame.parentToolCallId,
+      parentToolName: frame.parentToolName,
+      pendingToolIds: frame.pendingToolIds,
+      // Carried through for the per-subagent resume fan-out; undefined under the
+      // legacy bundled-frame model (all frames share the top-level checkpointId).
+      ...(frame.checkpointId ? { checkpointId: frame.checkpointId } : {}),
+    }))
+
+    context.awaitingAsyncContinuation = {
+      checkpointId: event.payload.checkpointId,
+      executionId: event.payload.executionId || context.executionId,
+      runId: event.payload.runId || context.runId,
+      pendingToolCallIds: event.payload.pendingToolCallIds,
+      frames: frames.length > 0 ? frames : undefined,
+    }
+    logger.info('Received checkpoint pause', {
+      checkpointId: context.awaitingAsyncContinuation.checkpointId,
+      executionId: context.awaitingAsyncContinuation.executionId,
+      runId: context.awaitingAsyncContinuation.runId,
+      pendingToolCallIds: context.awaitingAsyncContinuation.pendingToolCallIds,
+      frameCount: frames.length,
+    })
+    context.streamComplete = true
+    return
+  }
+
+  if (event.payload.kind === MothershipStreamV1RunKind.compaction_start) {
+    addContentBlock(context, {
+      type: 'tool_call',
+      toolCall: {
+        id: `compaction-${Date.now()}`,
+        name: 'context_compaction',
+        status: 'executing',
+      },
+    })
+    return
+  }
+
+  if (event.payload.kind === MothershipStreamV1RunKind.resumed) {
+    context.awaitingAsyncContinuation = undefined
+    context.streamComplete = false
+    logger.info('Received run resumed event')
+    return
+  }
+
+  if (event.payload.kind === MothershipStreamV1RunKind.compaction_done) {
+    addContentBlock(context, {
+      type: 'tool_call',
+      toolCall: {
+        id: `compaction-${Date.now()}`,
+        name: 'context_compaction',
+        status: MothershipStreamV1ToolOutcome.success,
+      },
+    })
+  }
+}

apps/sim/lib/copilot/request/handlers/session.ts

@@ -0,0 +1,14 @@
+import { MothershipStreamV1SessionKind } from '@/lib/copilot/generated/mothership-stream-v1'
+import type { StreamHandler } from './types'
+
+export const handleSessionEvent: StreamHandler = (event, context, execContext) => {
+  if (event.type !== 'session' || event.payload.kind !== MothershipStreamV1SessionKind.chat) {
+    return
+  }
+
+  const chatId = event.payload.chatId
+  context.chatId = chatId
+  if (chatId) {
+    execContext.chatId = chatId
+  }
+}

apps/sim/lib/copilot/request/handlers/span.ts

@@ -0,0 +1,66 @@
+import {
+  MothershipStreamV1SpanLifecycleEvent,
+  MothershipStreamV1SpanPayloadKind,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import type { StreamHandler } from './types'
+
+/**
+ * Mirror Go-emitted span lifecycle events onto the Sim-side TraceCollector.
+ *
+ * Go publishes `span` events for subagent lifecycles and structured-result
+ * payloads. For subagents, the start/end pair is also used for UI routing
+ * elsewhere; here we additionally record a named span on the trace collector
+ * so the final RequestTraceV1 report shows the full nested structure without
+ * requiring the reader to inspect the raw envelope stream.
+ */
+export const handleSpanEvent: StreamHandler = (event, context) => {
+  if (event.type !== 'span') {
+    return
+  }
+
+  const payload = event.payload as {
+    kind?: string
+    event?: string
+    agent?: string
+    data?: unknown
+  }
+  const kind = payload?.kind ?? ''
+  const evt = payload?.event ?? ''
+
+  if (kind === MothershipStreamV1SpanPayloadKind.subagent) {
+    const scopeAgent =
+      typeof payload.agent === 'string' && payload.agent ? payload.agent : 'subagent'
+    // Key by the deterministic spanId so two concurrent runs of the SAME agent
+    // (e.g. two parallel `research` subagents) get distinct trace spans. Fall
+    // back to agent:parentToolCallId for legacy events that predate span ids.
+    const traceKey = event.scope?.spanId || `${scopeAgent}:${event.scope?.parentToolCallId || ''}`
+    if (evt === MothershipStreamV1SpanLifecycleEvent.start) {
+      const span = context.trace.startSpan(`subagent:${scopeAgent}`, 'go.subagent', {
+        agent: scopeAgent,
+        parentToolCallId: event.scope?.parentToolCallId,
+        spanId: event.scope?.spanId,
+      })
+      context.subAgentTraceSpans ??= new Map()
+      context.subAgentTraceSpans.set(traceKey, span)
+    } else if (evt === MothershipStreamV1SpanLifecycleEvent.end) {
+      const span = context.subAgentTraceSpans?.get(traceKey)
+      if (span) {
+        context.trace.endSpan(span, 'ok')
+        context.subAgentTraceSpans?.delete(traceKey)
+      }
+    }
+    return
+  }
+
+  if (
+    kind === MothershipStreamV1SpanPayloadKind.structured_result ||
+    kind === MothershipStreamV1SpanPayloadKind.subagent_result
+  ) {
+    const span = context.trace.startSpan(`${kind}:${payload.agent ?? 'main'}`, `go.${kind}`, {
+      agent: payload.agent,
+      hasData: payload.data !== undefined,
+    })
+    context.trace.endSpan(span, 'ok')
+    return
+  }
+}

apps/sim/lib/copilot/request/handlers/text.ts

@@ -0,0 +1,81 @@
+import { MothershipStreamV1TextChannel } from '@/lib/copilot/generated/mothership-stream-v1'
+import type { StreamHandler, ToolScope } from './types'
+import {
+  addContentBlock,
+  flushSubagentThinkingBlock,
+  flushThinkingBlock,
+  getScopedParentToolCallId,
+  getScopedSpanIdentity,
+} from './types'
+
+export function handleTextEvent(scope: ToolScope): StreamHandler {
+  return (event, context) => {
+    if (event.type !== 'text') {
+      return
+    }
+
+    const chunk = event.payload.text
+    if (!chunk) {
+      return
+    }
+
+    if (scope === 'subagent') {
+      const parentToolCallId = getScopedParentToolCallId(event, context)
+      if (!parentToolCallId) return
+      const spanIdentity = getScopedSpanIdentity(event)
+      if (event.payload.channel === MothershipStreamV1TextChannel.thinking) {
+        // Per-lane thinking: each concurrent subagent accumulates into its own
+        // block keyed by parentToolCallId, so interleaved chunks from a sibling
+        // subagent never flush or corrupt this lane's reasoning.
+        let block = context.subagentThinkingBlocks.get(parentToolCallId)
+        if (!block) {
+          block = {
+            type: 'subagent_thinking',
+            content: '',
+            parentToolCallId,
+            ...spanIdentity,
+            timestamp: Date.now(),
+          }
+          context.subagentThinkingBlocks.set(parentToolCallId, block)
+        }
+        block.content = `${block.content || ''}${chunk}`
+        return
+      }
+      // Real text for this lane: close this lane's thinking block first so the
+      // persisted order is [thinking, text] within the lane.
+      flushSubagentThinkingBlock(context, parentToolCallId)
+      if (context.isInThinkingBlock) {
+        flushThinkingBlock(context)
+      }
+      context.subAgentContent[parentToolCallId] =
+        (context.subAgentContent[parentToolCallId] || '') + chunk
+      addContentBlock(context, {
+        type: 'subagent_text',
+        content: chunk,
+        parentToolCallId,
+        ...spanIdentity,
+      })
+      return
+    }
+
+    if (event.payload.channel === MothershipStreamV1TextChannel.thinking) {
+      if (!context.currentThinkingBlock) {
+        context.currentThinkingBlock = {
+          type: 'thinking',
+          content: '',
+          timestamp: Date.now(),
+        }
+        context.isInThinkingBlock = true
+      }
+      context.currentThinkingBlock.content = `${context.currentThinkingBlock.content || ''}${chunk}`
+      return
+    }
+
+    if (context.isInThinkingBlock) {
+      flushThinkingBlock(context)
+    }
+    context.accumulatedContent += chunk
+    context.finalAssistantContent += chunk
+    addContentBlock(context, { type: 'text', content: chunk })
+  }
+}

apps/sim/lib/copilot/request/handlers/tool.ts

@@ -0,0 +1,560 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { ASYNC_TOOL_CONFIRMATION_STATUS } from '@/lib/copilot/async-runs/lifecycle'
+import { markAsyncToolDelivered, upsertAsyncToolCall } from '@/lib/copilot/async-runs/repository'
+import { STREAM_TIMEOUT_MS } from '@/lib/copilot/constants'
+import {
+  MothershipStreamV1AsyncToolRecordStatus,
+  type MothershipStreamV1ToolCallDescriptor,
+  MothershipStreamV1ToolOutcome,
+  type MothershipStreamV1ToolResultPayload,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { withCopilotSpan } from '@/lib/copilot/request/otel'
+import {
+  isToolArgsDeltaStreamEvent,
+  isToolCallStreamEvent,
+  isToolResultStreamEvent,
+  TOOL_CALL_STATUS,
+} from '@/lib/copilot/request/session'
+import { markToolResultSeen, wasToolResultSeen } from '@/lib/copilot/request/sse-utils'
+import { setTerminalToolCallState } from '@/lib/copilot/request/tool-call-state'
+import { executeToolAndReport, waitForToolCompletion } from '@/lib/copilot/request/tools/executor'
+import type {
+  ExecutionContext,
+  OrchestratorOptions,
+  StreamEvent,
+  StreamingContext,
+  ToolCallState,
+} from '@/lib/copilot/request/types'
+import { getToolEntry, isSimExecuted } from '@/lib/copilot/tool-executor'
+import { isToolHiddenInUi } from '@/lib/copilot/tools/client/hidden-tools'
+import { getToolDisplayTitle } from '@/lib/copilot/tools/tool-display'
+import { isWorkflowToolName } from '@/lib/copilot/tools/workflow-tools'
+import type { ToolScope } from './types'
+import {
+  abortPendingToolIfStreamDead,
+  addContentBlock,
+  emitSyntheticToolResult,
+  ensureTerminalToolCallState,
+  flushSubagentThinkingBlock,
+  flushThinkingBlock,
+  getScopedParentToolCallId,
+  getScopedSpanIdentity,
+  getToolCallUI,
+  getToolResultErrorMessage,
+  handleClientCompletion,
+  inferToolSuccess,
+  registerPendingToolPromise,
+} from './types'
+
+const logger = createLogger('CopilotToolHandler')
+
+function applyToolDisplay(toolCall: ToolCallState | undefined): void {
+  if (!toolCall?.name) return
+  toolCall.displayTitle = getToolDisplayTitle(
+    toolCall.name,
+    toolCall.params as Record<string, unknown> | undefined
+  )
+}
+
+/**
+ * Upsert the durable `async_tool_calls` row before the authoritative tool-call
+ * SSE frame is forwarded to the client, so `/api/copilot/confirm` can never
+ * race ahead of the row that identifies the call. This is the sole
+ * persistence point for client-executable tools; gating mirrors the
+ * client-wait branch in `dispatchToolExecution`.
+ */
+export async function prePersistClientExecutableToolCall(
+  event: StreamEvent,
+  context: StreamingContext
+): Promise<void> {
+  if (event.type !== 'tool') return
+  if (!isToolCallStreamEvent(event)) return
+
+  const data = event.payload
+  const isGenerating = data.status === TOOL_CALL_STATUS.generating
+  const isPartial = data.partial === true || isGenerating
+  if (isPartial) return
+
+  const ui = getToolCallUI(data)
+  if (!ui.clientExecutable) return
+
+  const catalogEntry = getToolEntry(data.toolName)
+  const isInternal = ui.internal === true || catalogEntry?.internal === true
+  if (isInternal) return
+
+  const delegateWorkflowRunToClient = isWorkflowToolName(data.toolName)
+  if (isSimExecuted(data.toolName) && !delegateWorkflowRunToClient) return
+
+  if (!context.runId) return
+
+  await upsertAsyncToolCall({
+    runId: context.runId,
+    toolCallId: data.toolCallId,
+    toolName: data.toolName,
+    args: data.arguments,
+    status: MothershipStreamV1AsyncToolRecordStatus.running,
+  }).catch((err) => {
+    logger.warn('Failed to pre-persist async tool row before forwarding call frame', {
+      toolCallId: data.toolCallId,
+      toolName: data.toolName,
+      error: getErrorMessage(err),
+    })
+  })
+}
+
+/**
+ * Unified tool event handler for both main and subagent scopes.
+ *
+ * The main vs subagent differences are:
+ * - Subagent requires a parentToolCallId and tracks tool calls in subAgentToolCalls
+ * - Subagent result phase also updates the subAgentToolCalls record
+ * - Subagent call phase stores in both subAgentToolCalls and context.toolCalls
+ * - Main call phase only stores in context.toolCalls
+ */
+export async function handleToolEvent(
+  event: StreamEvent,
+  context: StreamingContext,
+  execContext: ExecutionContext,
+  options: OrchestratorOptions,
+  scope: ToolScope
+): Promise<void> {
+  const isSubagent = scope === 'subagent'
+  const parentToolCallId = isSubagent ? getScopedParentToolCallId(event, context) : undefined
+
+  if (isSubagent && !parentToolCallId) return
+
+  if (event.type !== 'tool') {
+    return
+  }
+
+  if (isToolArgsDeltaStreamEvent(event)) {
+    return
+  }
+
+  // A tool event breaks the thinking stream. Flush any open thinking
+  // block into contentBlocks BEFORE we add the tool_call block, or
+  // contentBlocks will end up with tool_call before thinking — which
+  // re-renders on reload in the wrong order (Mothership group above
+  // the Thinking block, even though thinking happened first). A subagent
+  // tool event flushes only its OWN lane so a concurrent sibling's thinking
+  // is left intact; a main tool event flushes all subagent lanes.
+  if (isSubagent && parentToolCallId) {
+    flushSubagentThinkingBlock(context, parentToolCallId)
+  } else {
+    flushSubagentThinkingBlock(context)
+  }
+  flushThinkingBlock(context)
+
+  if (isToolResultStreamEvent(event)) {
+    handleResultPhase(event.payload, context, parentToolCallId)
+    return
+  }
+
+  if (!isToolCallStreamEvent(event)) {
+    return
+  }
+
+  if (!parentToolCallId) {
+    context.sawMainToolCall = true
+    context.finalAssistantContent = ''
+  }
+
+  await handleCallPhase(
+    event.payload,
+    context,
+    execContext,
+    options,
+    parentToolCallId,
+    scope,
+    getScopedSpanIdentity(event)
+  )
+}
+
+function handleResultPhase(
+  data: MothershipStreamV1ToolResultPayload,
+  context: StreamingContext,
+  parentToolCallId: string | undefined
+): void {
+  const { toolCallId, toolName } = data
+  const mainToolCall = ensureTerminalToolCallState(context, toolCallId, toolName)
+  const { success, hasResultData } = inferToolSuccess(data)
+  let status: MothershipStreamV1ToolOutcome
+  if (data.status === MothershipStreamV1ToolOutcome.cancelled) {
+    status = MothershipStreamV1ToolOutcome.cancelled
+  } else if (data.status === MothershipStreamV1ToolOutcome.skipped) {
+    status = MothershipStreamV1ToolOutcome.skipped
+  } else if (data.status === MothershipStreamV1ToolOutcome.rejected) {
+    status = MothershipStreamV1ToolOutcome.rejected
+  } else {
+    status = success ? MothershipStreamV1ToolOutcome.success : MothershipStreamV1ToolOutcome.error
+  }
+  const endTime = Date.now()
+  const errorMessage =
+    !success && status !== MothershipStreamV1ToolOutcome.skipped
+      ? getToolResultErrorMessage(data) ||
+        (status === MothershipStreamV1ToolOutcome.cancelled
+          ? 'Tool cancelled'
+          : status === MothershipStreamV1ToolOutcome.rejected
+            ? 'Tool rejected'
+            : 'Tool failed')
+      : undefined
+
+  if (parentToolCallId) {
+    const toolCalls = context.subAgentToolCalls[parentToolCallId] || []
+    const subAgentToolCall = toolCalls.find((tc) => tc.id === toolCallId)
+    if (subAgentToolCall) {
+      setTerminalToolCallState(subAgentToolCall, {
+        status,
+        ...(hasResultData ? { output: data.output } : {}),
+        ...(errorMessage ? { error: errorMessage } : {}),
+        endTime,
+      })
+    }
+  }
+
+  setTerminalToolCallState(mainToolCall, {
+    status,
+    ...(hasResultData ? { output: data.output } : {}),
+    ...(errorMessage ? { error: errorMessage } : {}),
+    endTime,
+  })
+  stampToolCallBlockEnd(context, toolCallId, endTime)
+  markToolResultSeen(toolCallId)
+}
+
+function stampToolCallBlockEnd(
+  context: StreamingContext,
+  toolCallId: string,
+  endTime: number
+): void {
+  for (let i = context.contentBlocks.length - 1; i >= 0; i--) {
+    const block = context.contentBlocks[i]
+    if (block.type === 'tool_call' && block.toolCall?.id === toolCallId) {
+      if (block.endedAt === undefined) block.endedAt = endTime
+      return
+    }
+  }
+}
+
+async function handleCallPhase(
+  data: MothershipStreamV1ToolCallDescriptor,
+  context: StreamingContext,
+  execContext: ExecutionContext,
+  options: OrchestratorOptions,
+  parentToolCallId: string | undefined,
+  scope: ToolScope,
+  spanIdentity: { spanId?: string; parentSpanId?: string }
+): Promise<void> {
+  const { toolCallId, toolName } = data
+  const args = data.arguments
+  const isGenerating = data.status === TOOL_CALL_STATUS.generating
+  const isPartial = data.partial === true || isGenerating
+  const existing = context.toolCalls.get(toolCallId)
+  const isSubagent = scope === 'subagent'
+  const ui = getToolCallUI(data)
+
+  if (isPartial && shouldDelayVfsPlaceholder(toolName, args)) return
+
+  if (isSubagent) {
+    if (wasToolResultSeen(toolCallId) || existing?.endTime) {
+      if (existing && !existing.name && toolName) existing.name = toolName
+      if (existing && !existing.params && args) existing.params = args
+      applyToolDisplay(existing)
+      return
+    }
+  } else {
+    if (
+      existing?.endTime ||
+      (existing && existing.status !== 'pending' && existing.status !== 'executing')
+    ) {
+      if (!existing.name && toolName) existing.name = toolName
+      if (!existing.params && args) existing.params = args
+      applyToolDisplay(existing)
+      return
+    }
+  }
+
+  if (isSubagent) {
+    registerSubagentToolCall(
+      context,
+      toolCallId,
+      toolName,
+      args,
+      parentToolCallId!,
+      ui,
+      spanIdentity
+    )
+  } else {
+    registerMainToolCall(context, toolCallId, toolName, args, existing, ui)
+  }
+
+  if (isPartial) return
+  if (!isSubagent && wasToolResultSeen(toolCallId)) return
+  if (context.pendingToolPromises.has(toolCallId) || existing?.status === 'executing') {
+    return
+  }
+
+  const toolCall = context.toolCalls.get(toolCallId)
+  if (!toolCall) return
+
+  // Capture the invoking subagent's channel id so the executor can thread it
+  // into the server tool context — this is what scopes the workspace_file ->
+  // edit_content intent handoff to one file subagent under concurrency.
+  if (parentToolCallId) toolCall.parentToolCallId = parentToolCallId
+
+  const readPath = typeof args?.path === 'string' ? args.path : undefined
+  if (toolName === 'read' && readPath?.startsWith('internal/')) return
+
+  const { clientExecutable, simExecutable, internal } = ui
+  const catalogEntry = getToolEntry(toolName)
+  const isInternal = internal || catalogEntry?.internal === true
+  const staticSimExecuted = isSimExecuted(toolName)
+  const willDispatch = !isInternal && (staticSimExecuted || simExecutable || clientExecutable)
+  logger.info('Tool call routing decision', {
+    toolCallId,
+    toolName,
+    scope,
+    isSubagent,
+    parentToolCallId,
+    executor: data.executor,
+    clientExecutable,
+    simExecutable,
+    staticSimExecuted,
+    internal: isInternal,
+    hasPendingPromise: context.pendingToolPromises.has(toolCallId),
+    existingStatus: existing?.status,
+    willDispatch,
+  })
+  if (isInternal) return
+  if (!willDispatch) return
+
+  await dispatchToolExecution(
+    toolCall,
+    toolCallId,
+    toolName,
+    args,
+    context,
+    execContext,
+    options,
+    clientExecutable,
+    scope
+  )
+}
+
+function shouldDelayVfsPlaceholder(
+  toolName: string,
+  args: Record<string, unknown> | undefined
+): boolean {
+  return (toolName === 'read' || toolName === 'glob') && !args
+}
+
+function removeToolCallContentBlock(context: StreamingContext, toolCallId: string): void {
+  for (let i = context.contentBlocks.length - 1; i >= 0; i--) {
+    const block = context.contentBlocks[i]
+    if (block.type === 'tool_call' && block.toolCall?.id === toolCallId) {
+      context.contentBlocks.splice(i, 1)
+    }
+  }
+}
+
+function registerSubagentToolCall(
+  context: StreamingContext,
+  toolCallId: string,
+  toolName: string,
+  args: Record<string, unknown> | undefined,
+  parentToolCallId: string,
+  ui: { title?: string; phaseLabel?: string; hidden?: boolean },
+  spanIdentity: { spanId?: string; parentSpanId?: string }
+): void {
+  if (!context.subAgentToolCalls[parentToolCallId]) {
+    context.subAgentToolCalls[parentToolCallId] = []
+  }
+  const hideFromUi = isToolHiddenInUi(toolName) || ui.hidden === true
+  let toolCall = context.toolCalls.get(toolCallId)
+  if (toolCall) {
+    if (!toolCall.name && toolName) toolCall.name = toolName
+    if (args && !toolCall.params) toolCall.params = args
+    applyToolDisplay(toolCall)
+    if (hideFromUi) removeToolCallContentBlock(context, toolCallId)
+  } else {
+    toolCall = {
+      id: toolCallId,
+      name: toolName,
+      status: 'pending',
+      params: args,
+      startTime: Date.now(),
+    }
+    applyToolDisplay(toolCall)
+    context.toolCalls.set(toolCallId, toolCall)
+    const parentToolCall = context.toolCalls.get(parentToolCallId)
+    if (!hideFromUi) {
+      addContentBlock(context, {
+        type: 'tool_call',
+        toolCall,
+        calledBy: parentToolCall?.name,
+        parentToolCallId,
+        ...spanIdentity,
+      })
+    }
+  }
+
+  const subagentToolCalls = context.subAgentToolCalls[parentToolCallId]
+  const existingSubagentToolCall = subagentToolCalls.find((tc) => tc.id === toolCallId)
+  if (existingSubagentToolCall) {
+    if (!existingSubagentToolCall.name && toolName) existingSubagentToolCall.name = toolName
+    if (args && !existingSubagentToolCall.params) existingSubagentToolCall.params = args
+    applyToolDisplay(existingSubagentToolCall)
+  } else {
+    subagentToolCalls.push(toolCall)
+  }
+}
+
+function registerMainToolCall(
+  context: StreamingContext,
+  toolCallId: string,
+  toolName: string,
+  args: Record<string, unknown> | undefined,
+  existing: ToolCallState | undefined,
+  ui: { title?: string; phaseLabel?: string; hidden?: boolean }
+): void {
+  const hideFromUi = isToolHiddenInUi(toolName) || ui.hidden === true
+  if (existing) {
+    if (args && !existing.params) existing.params = args
+    applyToolDisplay(existing)
+    if (hideFromUi) {
+      removeToolCallContentBlock(context, toolCallId)
+      return
+    }
+    if (
+      !hideFromUi &&
+      !context.contentBlocks.some((b) => b.type === 'tool_call' && b.toolCall?.id === toolCallId)
+    ) {
+      addContentBlock(context, { type: 'tool_call', toolCall: existing })
+    }
+  } else {
+    const created: ToolCallState = {
+      id: toolCallId,
+      name: toolName,
+      status: 'pending',
+      params: args,
+      startTime: Date.now(),
+    }
+    applyToolDisplay(created)
+    context.toolCalls.set(toolCallId, created)
+    if (!hideFromUi) {
+      addContentBlock(context, { type: 'tool_call', toolCall: created })
+    }
+  }
+}
+
+async function dispatchToolExecution(
+  toolCall: ToolCallState,
+  toolCallId: string,
+  toolName: string,
+  args: Record<string, unknown> | undefined,
+  context: StreamingContext,
+  execContext: ExecutionContext,
+  options: OrchestratorOptions,
+  clientExecutable: boolean,
+  scope: ToolScope
+): Promise<void> {
+  const scopeLabel = scope === 'subagent' ? 'subagent ' : ''
+
+  const fireToolExecution = () => {
+    const pendingPromise = (async () => {
+      return executeToolAndReport(toolCallId, context, execContext, options)
+    })().catch((err) => {
+      logger.error(`Parallel ${scopeLabel}tool execution failed`, {
+        toolCallId,
+        toolName,
+        error: toError(err).message,
+      })
+      return {
+        status: MothershipStreamV1ToolOutcome.error,
+        message: 'Tool execution failed',
+        data: { error: 'Tool execution failed' },
+      }
+    })
+    registerPendingToolPromise(context, toolCallId, pendingPromise)
+  }
+
+  if (options.interactive === false) {
+    if (options.autoExecuteTools !== false) {
+      if (!abortPendingToolIfStreamDead(toolCall, toolCallId, options, context)) {
+        fireToolExecution()
+      }
+    }
+    return
+  }
+
+  if (clientExecutable) {
+    const delegateWorkflowRunToClient = isWorkflowToolName(toolName)
+    if (isSimExecuted(toolName) && !delegateWorkflowRunToClient) {
+      if (!abortPendingToolIfStreamDead(toolCall, toolCallId, options, context)) {
+        fireToolExecution()
+      }
+    } else {
+      toolCall.status = 'executing'
+      const pendingPromise = withCopilotSpan(
+        TraceSpan.CopilotToolWaitForClientResult,
+        {
+          [TraceAttr.ToolName]: toolName,
+          [TraceAttr.ToolCallId]: toolCallId,
+          [TraceAttr.ToolTimeoutMs]: options.timeout || STREAM_TIMEOUT_MS,
+          ...(context.runId ? { [TraceAttr.RunId]: context.runId } : {}),
+        },
+        async (span) => {
+          const completion = await waitForToolCompletion(
+            toolCallId,
+            options.timeout || STREAM_TIMEOUT_MS,
+            options.abortSignal
+          )
+          span.setAttribute(TraceAttr.ToolCompletionReceived, completion !== undefined)
+          if (completion) {
+            span.setAttribute(TraceAttr.ToolOutcome, completion.status)
+          }
+          handleClientCompletion(toolCall, toolCallId, completion)
+          if (completion?.status === ASYNC_TOOL_CONFIRMATION_STATUS.background) {
+            await markAsyncToolDelivered(toolCallId).catch((err) => {
+              logger.warn(`Failed to mark background ${scopeLabel}tool delivered`, {
+                toolCallId,
+                toolName,
+                error: toError(err).message,
+              })
+            })
+          }
+          await emitSyntheticToolResult(toolCallId, toolCall.name, completion, options)
+          return (
+            completion ?? {
+              status: MothershipStreamV1ToolOutcome.error,
+              message: 'Tool completion missing',
+              data: { error: 'Tool completion missing' },
+            }
+          )
+        }
+      ).catch((err) => {
+        logger.error(`Client-executable ${scopeLabel}tool wait failed`, {
+          toolCallId,
+          toolName,
+          error: toError(err).message,
+        })
+        return {
+          status: MothershipStreamV1ToolOutcome.error,
+          message: 'Tool wait failed',
+          data: { error: 'Tool wait failed' },
+        }
+      })
+      registerPendingToolPromise(context, toolCallId, pendingPromise)
+    }
+    return
+  }
+
+  if (options.autoExecuteTools !== false) {
+    if (!abortPendingToolIfStreamDead(toolCall, toolCallId, options, context)) {
+      fireToolExecution()
+    }
+  }
+}

apps/sim/lib/copilot/request/handlers/types.ts

@@ -0,0 +1,317 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { isRecordLike } from '@sim/utils/object'
+import type {
+  AsyncCompletionSignal,
+  AsyncTerminalCompletionSnapshot,
+} from '@/lib/copilot/async-runs/lifecycle'
+import { ASYNC_TOOL_CONFIRMATION_STATUS } from '@/lib/copilot/async-runs/lifecycle'
+import {
+  MothershipStreamV1EventType,
+  type MothershipStreamV1StreamScope,
+  type MothershipStreamV1ToolCallDescriptor,
+  MothershipStreamV1ToolExecutor,
+  MothershipStreamV1ToolMode,
+  MothershipStreamV1ToolOutcome,
+  MothershipStreamV1ToolPhase,
+  type MothershipStreamV1ToolResultPayload,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { asRecord, markToolResultSeen } from '@/lib/copilot/request/sse-utils'
+import { setTerminalToolCallState } from '@/lib/copilot/request/tool-call-state'
+import type {
+  ContentBlock,
+  ExecutionContext,
+  OrchestratorOptions,
+  StreamEvent,
+  StreamingContext,
+  ToolCallState,
+} from '@/lib/copilot/request/types'
+
+export type StreamHandler = (
+  event: StreamEvent,
+  context: StreamingContext,
+  execContext: ExecutionContext,
+  options: OrchestratorOptions
+) => void | Promise<void>
+
+export type ToolScope = 'main' | MothershipStreamV1StreamScope['lane']
+
+const logger = createLogger('CopilotHandlerHelpers')
+
+export function addContentBlock(
+  context: StreamingContext,
+  block: Omit<ContentBlock, 'timestamp'>
+): void {
+  context.contentBlocks.push({
+    ...block,
+    timestamp: Date.now(),
+  })
+}
+
+export function stampBlockEnd(block: ContentBlock): void {
+  if (block.endedAt === undefined) block.endedAt = Date.now()
+}
+
+/**
+ * Flush any open thinking block into contentBlocks and clear the thinking state.
+ * Safe to call repeatedly.
+ */
+export function flushThinkingBlock(context: StreamingContext): void {
+  if (context.currentThinkingBlock) {
+    stampBlockEnd(context.currentThinkingBlock)
+    context.contentBlocks.push(context.currentThinkingBlock)
+  }
+  context.isInThinkingBlock = false
+  context.currentThinkingBlock = null
+}
+
+/**
+ * Flush open subagent thinking blocks into contentBlocks. With a parentToolCallId
+ * it flushes only that lane (used when a tool/text event arrives for a specific
+ * subagent); with no argument it flushes ALL open lanes (used at stream end and
+ * at subagent lifecycle boundaries). Safe to call repeatedly.
+ */
+export function flushSubagentThinkingBlock(
+  context: StreamingContext,
+  parentToolCallId?: string
+): void {
+  if (parentToolCallId !== undefined) {
+    const block = context.subagentThinkingBlocks.get(parentToolCallId)
+    if (block) {
+      stampBlockEnd(block)
+      context.contentBlocks.push(block)
+      context.subagentThinkingBlocks.delete(parentToolCallId)
+    }
+    return
+  }
+  for (const block of context.subagentThinkingBlocks.values()) {
+    stampBlockEnd(block)
+    context.contentBlocks.push(block)
+  }
+  context.subagentThinkingBlocks.clear()
+}
+
+/**
+ * Resolve the subagent lane an event belongs to, using ONLY the event's own
+ * scope. The legacy fallback to a single "current subagent" pointer was removed:
+ * with concurrent subagents that pointer reflects whichever subagent started
+ * most recently and would mis-attribute interleaved events. Every subagent-lane
+ * event is guaranteed to carry parentToolCallId (Go stamps it), so a missing one
+ * is a real contract violation — callers warn and drop rather than guess.
+ */
+export function getScopedParentToolCallId(
+  event: StreamEvent,
+  _context: StreamingContext
+): string | undefined {
+  return event.scope?.parentToolCallId
+}
+
+/**
+ * Extract the deterministic span identity from an event's scope. Returns an
+ * empty object for legacy events that predate span identity so callers can
+ * spread it safely and fall back to `parentToolCallId`-based grouping.
+ */
+export function getScopedSpanIdentity(event: StreamEvent): {
+  spanId?: string
+  parentSpanId?: string
+} {
+  const spanId = event.scope?.spanId
+  const parentSpanId = event.scope?.parentSpanId
+  return {
+    ...(spanId ? { spanId } : {}),
+    ...(parentSpanId ? { parentSpanId } : {}),
+  }
+}
+
+export function registerPendingToolPromise(
+  context: StreamingContext,
+  toolCallId: string,
+  pendingPromise: Promise<AsyncCompletionSignal>
+): void {
+  context.pendingToolPromises.set(toolCallId, pendingPromise)
+  pendingPromise.finally(() => {
+    if (context.pendingToolPromises.get(toolCallId) === pendingPromise) {
+      context.pendingToolPromises.delete(toolCallId)
+    }
+  })
+}
+
+/**
+ * When the Sim->Go stream is aborted, avoid starting server-side tool work and
+ * unblock the Go async waiter with a terminal 499 completion.
+ */
+export function abortPendingToolIfStreamDead(
+  toolCall: ToolCallState,
+  toolCallId: string,
+  options: OrchestratorOptions,
+  context: StreamingContext
+): boolean {
+  if (!options.abortSignal?.aborted && !context.wasAborted) {
+    return false
+  }
+  toolCall.status = MothershipStreamV1ToolOutcome.cancelled
+  toolCall.endTime = Date.now()
+  markToolResultSeen(toolCallId)
+  const toolSpan = context.trace.startSpan(toolCall.name || 'unknown_tool', 'tool.execute', {
+    toolCallId,
+    toolName: toolCall.name,
+    cancelReason: 'stream_dead_before_dispatch',
+    abortSignalAborted: options.abortSignal?.aborted ?? false,
+    abortReason: options.abortSignal?.aborted
+      ? String(options.abortSignal.reason ?? 'unknown')
+      : undefined,
+    wasAborted: context.wasAborted ?? false,
+  })
+  context.trace.endSpan(toolSpan, 'cancelled')
+  return true
+}
+
+/**
+ * Extract the behavioral `ui` flags from a typed tool_call payload. The Go
+ * backend enriches tool_call events with `ui: { clientExecutable, internal,
+ * hidden }`; presentation (title/icon) is derived client-side from the tool name.
+ */
+export function getToolCallUI(data: MothershipStreamV1ToolCallDescriptor): {
+  clientExecutable: boolean
+  simExecutable: boolean
+  internal: boolean
+  hidden: boolean
+} {
+  const raw = asRecord(data.ui)
+  return {
+    clientExecutable:
+      raw.clientExecutable === true || data.executor === MothershipStreamV1ToolExecutor.client,
+    simExecutable: data.executor === MothershipStreamV1ToolExecutor.sim,
+    internal: raw.internal === true,
+    hidden: raw.hidden === true,
+  }
+}
+
+/**
+ * Handle the completion signal from a client-executable tool.
+ * Shared by both main and subagent scopes.
+ */
+export function handleClientCompletion(
+  toolCall: ToolCallState,
+  toolCallId: string,
+  completion: AsyncTerminalCompletionSnapshot | null
+): void {
+  if (completion?.status === ASYNC_TOOL_CONFIRMATION_STATUS.background) {
+    setTerminalToolCallState(toolCall, {
+      status: MothershipStreamV1ToolOutcome.skipped,
+      ...(completion.data !== undefined ? { output: completion.data } : {}),
+    })
+    markToolResultSeen(toolCallId)
+    return
+  }
+  if (completion?.status === MothershipStreamV1ToolOutcome.cancelled) {
+    setTerminalToolCallState(toolCall, {
+      status: MothershipStreamV1ToolOutcome.cancelled,
+      ...(completion.data !== undefined ? { output: completion.data } : {}),
+      error: completion.message || 'Tool cancelled',
+    })
+    markToolResultSeen(toolCallId)
+    return
+  }
+  const success = completion?.status === MothershipStreamV1ToolOutcome.success
+  setTerminalToolCallState(toolCall, {
+    status: success ? MothershipStreamV1ToolOutcome.success : MothershipStreamV1ToolOutcome.error,
+    ...(completion?.data !== undefined ? { output: completion.data } : {}),
+    ...(success ? {} : { error: completion?.message || 'Tool failed' }),
+  })
+  markToolResultSeen(toolCallId)
+}
+
+/**
+ * Emit a synthetic tool_result SSE event to the client after a client-executable
+ * tool completes. The Go backend's actual tool_result is skipped (markToolResultSeen),
+ * so the client would never learn the outcome without this.
+ */
+export async function emitSyntheticToolResult(
+  toolCallId: string,
+  toolName: string,
+  completion: AsyncTerminalCompletionSnapshot | null,
+  options: OrchestratorOptions
+): Promise<void> {
+  const isBackground = completion?.status === ASYNC_TOOL_CONFIRMATION_STATUS.background
+  const success = isBackground || completion?.status === MothershipStreamV1ToolOutcome.success
+  const isCancelled = completion?.status === MothershipStreamV1ToolOutcome.cancelled
+  const completionData = completion?.data
+  const syntheticStatus = isBackground
+    ? MothershipStreamV1ToolOutcome.skipped
+    : completion?.status === MothershipStreamV1ToolOutcome.success ||
+        completion?.status === MothershipStreamV1ToolOutcome.error ||
+        completion?.status === MothershipStreamV1ToolOutcome.cancelled
+      ? completion.status
+      : undefined
+
+  const resultPayload = isCancelled
+    ? isRecordLike(completionData)
+      ? { ...completionData, reason: 'user_cancelled', cancelledByUser: true }
+      : completionData !== undefined
+        ? { output: completionData, reason: 'user_cancelled', cancelledByUser: true }
+        : { reason: 'user_cancelled', cancelledByUser: true }
+    : completionData
+
+  try {
+    await options.onEvent?.({
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        toolCallId,
+        toolName,
+        executor: MothershipStreamV1ToolExecutor.client,
+        mode: MothershipStreamV1ToolMode.async,
+        phase: MothershipStreamV1ToolPhase.result,
+        success,
+        output: resultPayload,
+        ...(syntheticStatus ? { status: syntheticStatus } : {}),
+        ...(!success && completion?.message ? { error: completion.message } : {}),
+      },
+    })
+  } catch (error) {
+    logger.warn('Failed to emit synthetic tool_result', {
+      toolCallId,
+      toolName,
+      error: toError(error).message,
+    })
+  }
+}
+
+export function getToolResultErrorMessage(
+  data: MothershipStreamV1ToolResultPayload | undefined
+): string | undefined {
+  return data?.error
+}
+
+export function inferToolSuccess(data: MothershipStreamV1ToolResultPayload | undefined): {
+  success: boolean
+  hasResultData: boolean
+  hasError: boolean
+} {
+  const errorMessage = getToolResultErrorMessage(data)
+  const hasResultData = data?.output !== undefined
+  const hasError = Boolean(errorMessage)
+  const success = data?.success === true
+  return { success, hasResultData, hasError }
+}
+
+export function ensureTerminalToolCallState(
+  context: StreamingContext,
+  toolCallId: string,
+  toolName: string
+): ToolCallState {
+  const existing = context.toolCalls.get(toolCallId)
+  if (existing) {
+    return existing
+  }
+
+  const toolCall: ToolCallState = {
+    id: toolCallId,
+    name: toolName || 'unknown_tool',
+    status: 'pending',
+    startTime: Date.now(),
+  }
+  context.toolCalls.set(toolCallId, toolCall)
+  addContentBlock(context, { type: 'tool_call', toolCall })
+  return toolCall
+}

apps/sim/lib/copilot/request/http.ts

@@ -0,0 +1,104 @@
+import { safeCompare } from '@sim/security/compare'
+import { generateId } from '@sim/utils/id'
+import type { NextRequest } from 'next/server'
+import { NextResponse } from 'next/server'
+import { getSession } from '@/lib/auth'
+import { ASYNC_TOOL_CONFIRMATION_STATUS } from '@/lib/copilot/async-runs/lifecycle'
+import { env } from '@/lib/core/config/env'
+import { generateRequestId } from '@/lib/core/utils/request'
+
+export const NotificationStatus = {
+  pending: 'pending',
+  ...ASYNC_TOOL_CONFIRMATION_STATUS,
+} as const
+export type NotificationStatus = (typeof NotificationStatus)[keyof typeof NotificationStatus]
+
+export interface CopilotAuthResult {
+  userId: string | null
+  isAuthenticated: boolean
+}
+
+export function createUnauthorizedResponse(): NextResponse {
+  return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+}
+
+/**
+ * Creates a 400 Bad Request response for non-validation errors (business rule
+ * failures, missing entities, semantic mismatches).
+ *
+ * For Zod validation failures, use `validationErrorResponse` from
+ * `@/lib/api/server` instead — it returns the canonical
+ * `{ error, details: ZodIssue[] }` shape that lets clients introspect which
+ * field failed.
+ */
+export function createBadRequestResponse(message: string): NextResponse {
+  return NextResponse.json({ error: message }, { status: 400 })
+}
+
+export function createForbiddenResponse(message: string): NextResponse {
+  return NextResponse.json({ error: message }, { status: 403 })
+}
+
+export function createNotFoundResponse(message: string): NextResponse {
+  return NextResponse.json({ error: message }, { status: 404 })
+}
+
+export function createInternalServerErrorResponse(message: string): NextResponse {
+  return NextResponse.json({ error: message }, { status: 500 })
+}
+
+export function createRequestId(): string {
+  return generateId()
+}
+
+function createShortRequestId(): string {
+  return generateRequestId()
+}
+
+export interface RequestTracker {
+  requestId: string
+  startTime: number
+  getDuration(): number
+}
+
+export function createRequestTracker(short = true): RequestTracker {
+  const requestId = short ? createShortRequestId() : createRequestId()
+  const startTime = Date.now()
+
+  return {
+    requestId,
+    startTime,
+    getDuration(): number {
+      return Date.now() - startTime
+    },
+  }
+}
+
+export async function authenticateCopilotRequestSessionOnly(): Promise<CopilotAuthResult> {
+  const session = await getSession()
+  const userId = session?.user?.id || null
+
+  return {
+    userId,
+    isAuthenticated: userId !== null,
+  }
+}
+
+export function checkInternalApiKey(req: NextRequest) {
+  const apiKey = req.headers.get('x-api-key')
+  const expectedApiKey = env.INTERNAL_API_SECRET
+
+  if (!expectedApiKey) {
+    return { success: false, error: 'Internal API key not configured' }
+  }
+
+  if (!apiKey) {
+    return { success: false, error: 'API key required' }
+  }
+
+  if (!safeCompare(apiKey, expectedApiKey)) {
+    return { success: false, error: 'Invalid API key' }
+  }
+
+  return { success: true }
+}

apps/sim/lib/copilot/request/lifecycle/finalize.ts

@@ -0,0 +1,195 @@
+import { SpanStatusCode, trace } from '@opentelemetry/api'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { updateRunStatus } from '@/lib/copilot/async-runs/repository'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import {
+  type RequestTraceV1Outcome,
+  RequestTraceV1Outcome as RequestTraceV1OutcomeConst,
+} from '@/lib/copilot/generated/request-trace-v1'
+import { CopilotFinalizeOutcome } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import type { StreamWriter } from '@/lib/copilot/request/session'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+
+const logger = createLogger('CopilotStreamFinalize')
+const getTracer = () => trace.getTracer('sim-copilot-finalize', '1.0.0')
+
+// Single finalization path. `outcome` is the caller's resolved verdict
+// so we don't have to re-derive cancel vs error from raw signals.
+export async function finalizeStream(
+  result: OrchestratorResult,
+  publisher: StreamWriter,
+  runId: string,
+  outcome: RequestTraceV1Outcome,
+  requestId: string
+): Promise<void> {
+  const spanOutcome =
+    outcome === RequestTraceV1OutcomeConst.cancelled
+      ? CopilotFinalizeOutcome.Aborted
+      : outcome === RequestTraceV1OutcomeConst.success
+        ? CopilotFinalizeOutcome.Success
+        : CopilotFinalizeOutcome.Error
+  const span = getTracer().startSpan(TraceSpan.CopilotFinalizeStream, {
+    attributes: {
+      [TraceAttr.CopilotFinalizeOutcome]: spanOutcome,
+      [TraceAttr.RunId]: runId,
+      [TraceAttr.RequestId]: requestId,
+      [TraceAttr.CopilotResultToolCalls]: result.toolCalls?.length ?? 0,
+      [TraceAttr.CopilotResultContentBlocks]: result.contentBlocks?.length ?? 0,
+      [TraceAttr.CopilotResultContentLength]: result.content?.length ?? 0,
+      [TraceAttr.CopilotPublisherSawComplete]: publisher.sawComplete,
+      [TraceAttr.CopilotPublisherClientDisconnected]: publisher.clientDisconnected,
+    },
+  })
+  try {
+    if (outcome === RequestTraceV1OutcomeConst.cancelled) {
+      await handleAborted(result, publisher, runId, requestId)
+    } else if (outcome === RequestTraceV1OutcomeConst.error) {
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: result.error || 'orchestration failed',
+      })
+      await handleError(result, publisher, runId, requestId)
+    } else {
+      await handleSuccess(publisher, runId, requestId)
+    }
+    // Successful + cancelled paths fall through as status-unset → set
+    // OK so dashboards don't show "incomplete" for normal terminals.
+    if (outcome !== RequestTraceV1OutcomeConst.error) {
+      span.setStatus({ code: SpanStatusCode.OK })
+    }
+  } catch (error) {
+    span.recordException(toError(error))
+    span.setStatus({ code: SpanStatusCode.ERROR, message: 'finalize threw' })
+    throw error
+  } finally {
+    span.end()
+  }
+}
+
+async function handleAborted(
+  result: OrchestratorResult,
+  publisher: StreamWriter,
+  runId: string,
+  requestId: string
+): Promise<void> {
+  const partialContentLen = result.content?.length ?? 0
+  const toolCallCount = result.toolCalls?.length ?? 0
+  const blockCount = result.contentBlocks?.length ?? 0
+  logger.info(`[${requestId}] Stream aborted by explicit stop`, {
+    partialContentLen,
+    toolCallCount,
+    blockCount,
+  })
+  if (!publisher.sawComplete) {
+    const partialContent = result.content || undefined
+    await publisher.publish({
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.cancelled,
+        ...(partialContent ? { partialContent } : {}),
+        ...(partialContentLen ? { partialContentLen } : {}),
+        ...(toolCallCount ? { toolCallCount } : {}),
+      },
+    })
+  }
+  await publisher.flush()
+  await loggedRunStatusUpdate(runId, MothershipStreamV1CompletionStatus.cancelled, requestId, {
+    completedAt: new Date(),
+  })
+}
+
+async function handleError(
+  result: OrchestratorResult,
+  publisher: StreamWriter,
+  runId: string,
+  requestId: string
+): Promise<void> {
+  const errorMessage =
+    result.error ||
+    result.errors?.[0] ||
+    'An unexpected error occurred while processing the response.'
+
+  // Persist whatever was generated before the failure, exactly like an abort —
+  // a transient provider error (e.g. overloaded) shouldn't discard the partial
+  // assistant output the user already saw streaming.
+  const partialContent = result.content || undefined
+  const partialContentLen = result.content?.length ?? 0
+  const toolCallCount = result.toolCalls?.length ?? 0
+
+  if (publisher.clientDisconnected) {
+    logger.info(`[${requestId}] Stream failed after client disconnect`, { error: errorMessage })
+  }
+  logger.error(`[${requestId}] Orchestration returned failure`, {
+    error: errorMessage,
+    partialContentLen,
+    toolCallCount,
+  })
+
+  // Surface the real error (Go already classifies provider errors like
+  // "overloaded" into a friendly displayMessage). Don't clobber it with a
+  // generic string.
+  await publisher.publish({
+    type: MothershipStreamV1EventType.error,
+    payload: {
+      message: errorMessage,
+      error: errorMessage,
+      displayMessage: errorMessage,
+      data: { displayMessage: errorMessage },
+    },
+  })
+  if (!publisher.sawComplete) {
+    await publisher.publish({
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.error,
+        ...(partialContent ? { partialContent } : {}),
+        ...(partialContentLen ? { partialContentLen } : {}),
+        ...(toolCallCount ? { toolCallCount } : {}),
+      },
+    })
+  }
+  await publisher.flush()
+  await loggedRunStatusUpdate(runId, MothershipStreamV1CompletionStatus.error, requestId, {
+    completedAt: new Date(),
+    error: errorMessage,
+  })
+}
+
+async function handleSuccess(
+  publisher: StreamWriter,
+  runId: string,
+  requestId: string
+): Promise<void> {
+  if (!publisher.sawComplete) {
+    await publisher.publish({
+      type: MothershipStreamV1EventType.complete,
+      payload: { status: MothershipStreamV1CompletionStatus.complete },
+    })
+  }
+  await publisher.flush()
+  await loggedRunStatusUpdate(runId, MothershipStreamV1CompletionStatus.complete, requestId, {
+    completedAt: new Date(),
+  })
+}
+
+async function loggedRunStatusUpdate(
+  runId: string,
+  status: Parameters<typeof updateRunStatus>[1],
+  requestId: string,
+  updates: Parameters<typeof updateRunStatus>[2] = {}
+): Promise<void> {
+  try {
+    await updateRunStatus(runId, status, updates)
+  } catch (error) {
+    logger.warn(`[${requestId}] Failed to update run status to ${status}`, {
+      runId,
+      error: toError(error).message,
+    })
+  }
+}

apps/sim/lib/copilot/request/lifecycle/headless.test.ts

@@ -0,0 +1,171 @@
+/**
+ * @vitest-environment node
+ */
+
+import { propagation, trace } from '@opentelemetry/api'
+import { W3CTraceContextPropagator } from '@opentelemetry/core'
+import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+
+const { runCopilotLifecycle } = vi.hoisted(() => ({
+  runCopilotLifecycle: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/lifecycle/run', () => ({
+  runCopilotLifecycle,
+}))
+
+import { runHeadlessCopilotLifecycle } from './headless'
+
+function createLifecycleResult(overrides?: Partial<OrchestratorResult>): OrchestratorResult {
+  return {
+    success: true,
+    content: 'done',
+    contentBlocks: [],
+    toolCalls: [],
+    chatId: 'chat-1',
+    ...overrides,
+  }
+}
+
+describe('runHeadlessCopilotLifecycle', () => {
+  beforeEach(() => {
+    trace.setGlobalTracerProvider(new BasicTracerProvider())
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator())
+  })
+
+  afterEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('runs the lifecycle and returns its result', async () => {
+    runCopilotLifecycle.mockResolvedValueOnce(
+      createLifecycleResult({
+        usage: { prompt: 10, completion: 5 },
+        cost: { input: 1, output: 2, total: 3 },
+      })
+    )
+
+    const result = await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'req-1',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(result.success).toBe(true)
+    expect(runCopilotLifecycle).toHaveBeenCalledWith(
+      expect.objectContaining({ messageId: 'req-1' }),
+      expect.objectContaining({
+        simRequestId: 'req-1',
+        trace: expect.any(Object),
+        otelContext: expect.any(Object),
+        chatId: 'chat-1',
+      })
+    )
+  })
+
+  it('returns an unsuccessful result from the lifecycle', async () => {
+    runCopilotLifecycle.mockResolvedValueOnce(
+      createLifecycleResult({
+        success: false,
+        error: 'failed',
+      })
+    )
+
+    const result = await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'req-2',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(result.success).toBe(false)
+  })
+
+  it('prefers an explicit simRequestId over the payload messageId', async () => {
+    runCopilotLifecycle.mockResolvedValueOnce(createLifecycleResult())
+
+    await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'message-req-id',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        simRequestId: 'workflow-request-id',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(runCopilotLifecycle).toHaveBeenCalledWith(
+      expect.objectContaining({ messageId: 'message-req-id' }),
+      expect.objectContaining({
+        simRequestId: 'workflow-request-id',
+      })
+    )
+  })
+
+  it('threads a valid OTel context into the lifecycle', async () => {
+    let lifecycleTraceparent = ''
+    runCopilotLifecycle.mockImplementationOnce(async (_payload, options) => {
+      const { traceHeaders } = await import('@/lib/copilot/request/go/propagation')
+      lifecycleTraceparent = traceHeaders({}, options.otelContext).traceparent ?? ''
+      return createLifecycleResult()
+    })
+
+    await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'req-otel',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(lifecycleTraceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
+  })
+
+  it('rethrows when the lifecycle throws', async () => {
+    runCopilotLifecycle.mockRejectedValueOnce(new Error('kaboom'))
+
+    await expect(
+      runHeadlessCopilotLifecycle(
+        {
+          message: 'hello',
+          messageId: 'req-3',
+        },
+        {
+          userId: 'user-1',
+          chatId: 'chat-1',
+          workflowId: 'workflow-1',
+          goRoute: '/api/mothership/execute',
+          interactive: false,
+        }
+      )
+    ).rejects.toThrow('kaboom')
+  })
+})

apps/sim/lib/copilot/request/lifecycle/headless.ts

@@ -0,0 +1,79 @@
+import { createLogger } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
+import type { RequestTraceV1Outcome as RequestTraceOutcome } from '@/lib/copilot/generated/request-trace-v1'
+import {
+  RequestTraceV1Outcome,
+  RequestTraceV1SpanStatus,
+} from '@/lib/copilot/generated/request-trace-v1'
+import { CopilotTransport } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import type { CopilotLifecycleOptions } from '@/lib/copilot/request/lifecycle/run'
+import { runCopilotLifecycle } from '@/lib/copilot/request/lifecycle/run'
+import { withCopilotOtelContext } from '@/lib/copilot/request/otel'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+
+const logger = createLogger('CopilotHeadlessLifecycle')
+
+export async function runHeadlessCopilotLifecycle(
+  requestPayload: Record<string, unknown>,
+  options: CopilotLifecycleOptions
+): Promise<OrchestratorResult> {
+  const simRequestId =
+    typeof options.simRequestId === 'string' && options.simRequestId.length > 0
+      ? options.simRequestId
+      : typeof requestPayload.messageId === 'string' && requestPayload.messageId.length > 0
+        ? requestPayload.messageId
+        : generateId()
+  const trace = new TraceCollector()
+  const requestSpan = trace.startSpan('Headless Sim Agent Request', 'request', {
+    route: options.goRoute,
+    workflowId: options.workflowId,
+    workspaceId: options.workspaceId,
+    chatId: options.chatId,
+  })
+
+  let result: OrchestratorResult | undefined
+  let outcome: RequestTraceOutcome = RequestTraceV1Outcome.error
+
+  return withCopilotOtelContext(
+    {
+      requestId: simRequestId,
+      route: options.goRoute,
+      chatId: options.chatId,
+      workflowId: options.workflowId,
+      executionId: options.executionId,
+      runId: options.runId,
+      transport: CopilotTransport.Headless,
+    },
+    async (otelContext) => {
+      try {
+        result = await runCopilotLifecycle(requestPayload, {
+          ...options,
+          trace,
+          simRequestId,
+          otelContext,
+        })
+        outcome = result.success
+          ? RequestTraceV1Outcome.success
+          : options.abortSignal?.aborted || result.cancelled
+            ? RequestTraceV1Outcome.cancelled
+            : RequestTraceV1Outcome.error
+        return result
+      } catch (error) {
+        outcome = options.abortSignal?.aborted
+          ? RequestTraceV1Outcome.cancelled
+          : RequestTraceV1Outcome.error
+        throw error
+      } finally {
+        trace.endSpan(
+          requestSpan,
+          outcome === RequestTraceV1Outcome.success
+            ? RequestTraceV1SpanStatus.ok
+            : outcome === RequestTraceV1Outcome.cancelled
+              ? RequestTraceV1SpanStatus.cancelled
+              : RequestTraceV1SpanStatus.error
+        )
+      }
+    }
+  )
+}

apps/sim/lib/copilot/request/lifecycle/resume-leg-context.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest'
+import { createStreamingContext } from '@/lib/copilot/request/context/request-context'
+import { makeResumeLegContext, mergeResumeLegOutputs } from '@/lib/copilot/request/lifecycle/run'
+
+// Guards the makeResumeLegContext / mergeResumeLegOutputs contract: the two MUST
+// stay in lockstep (every per-leg-isolated scalar is reset on leg creation and
+// folded back on merge), and the heavy accumulators stay shared by reference so
+// all concurrent legs build one chat. This is the regression the inline comment
+// warns about — without per-leg isolation the orchestrator's pre-fanout content
+// gets multiplied by the leg count on merge.
+describe('resume leg context isolate/merge contract', () => {
+  it('isolates the per-leg scalars while sharing the heavy accumulators by reference', () => {
+    const base = createStreamingContext({
+      accumulatedContent: 'PRE',
+      finalAssistantContent: 'PRE-FINAL',
+      usage: { prompt: 10, completion: 5 },
+      cost: { input: 1, output: 2, total: 3 },
+      errors: ['pre-existing'],
+    })
+
+    const leg = makeResumeLegContext(base)
+
+    // Per-leg scalars reset so a leg accumulates only its OWN output.
+    expect(leg.accumulatedContent).toBe('')
+    expect(leg.finalAssistantContent).toBe('')
+    expect(leg.usage).toBeUndefined()
+    expect(leg.cost).toBeUndefined()
+    expect(leg.errors).toEqual([])
+    expect(leg.streamComplete).toBe(false)
+    expect(leg.awaitingAsyncContinuation).toBeUndefined()
+
+    // A leg's own errors array is a fresh array (not the shared one) so a leg's
+    // retry rollback can't truncate a sibling's errors.
+    expect(leg.errors).not.toBe(base.errors)
+
+    // Heavy accumulators stay shared by reference (one merged chat).
+    expect(leg.contentBlocks).toBe(base.contentBlocks)
+    expect(leg.toolCalls).toBe(base.toolCalls)
+    expect(leg.pendingToolPromises).toBe(base.pendingToolPromises)
+    expect(leg.subAgentContent).toBe(base.subAgentContent)
+  })
+
+  it('folds a leg back exactly once (no double-count of the orchestrator content)', () => {
+    const base = createStreamingContext({ accumulatedContent: 'PRE', errors: ['pre'] })
+
+    const leg = makeResumeLegContext(base)
+    leg.accumulatedContent = 'JOIN'
+    leg.finalAssistantContent = 'JOIN-FINAL'
+    leg.usage = { prompt: 100, completion: 50 }
+    leg.cost = { input: 4, output: 5, total: 9 }
+    leg.errors.push('leg-err')
+
+    mergeResumeLegOutputs(base, leg)
+
+    // PRE seeded once + the leg's own output appended once — not PRE+PRE+JOIN.
+    expect(base.accumulatedContent).toBe('PREJOIN')
+    expect(base.finalAssistantContent).toBe('JOIN-FINAL')
+    expect(base.usage).toEqual({ prompt: 100, completion: 50 })
+    expect(base.cost).toEqual({ input: 4, output: 5, total: 9 })
+    expect(base.errors).toEqual(['pre', 'leg-err'])
+  })
+
+  it('does not multiply pre-fanout content across many legs (N children + one join leg)', () => {
+    const base = createStreamingContext({ accumulatedContent: 'PRE' })
+
+    // Seven child legs that stream subagent content (not main accumulatedContent)
+    // contribute nothing to the join scalars; only the join-carrying leg does.
+    for (let i = 0; i < 7; i++) {
+      const childLeg = makeResumeLegContext(base)
+      mergeResumeLegOutputs(base, childLeg)
+    }
+    const joinLeg = makeResumeLegContext(base)
+    joinLeg.accumulatedContent = 'SUMMARY'
+    joinLeg.usage = { prompt: 1, completion: 1 }
+    mergeResumeLegOutputs(base, joinLeg)
+
+    // Exactly the pre-fanout content + the one join leg's summary — the 7 child
+    // legs must not each re-append 'PRE'.
+    expect(base.accumulatedContent).toBe('PRESUMMARY')
+    expect(base.usage).toEqual({ prompt: 1, completion: 1 })
+  })
+})

apps/sim/lib/copilot/request/lifecycle/run.test.ts

@@ -0,0 +1,690 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types'
+
+const {
+  mockCreateRunSegment,
+  mockForceFailHungToolCall,
+  mockGetEffectiveDecryptedEnv,
+  mockGetMothershipBaseURL,
+  mockGetMothershipSourceEnvHeaders,
+  mockPrepareExecutionContext,
+  mockRunStreamLoop,
+  mockToolWatchdogTimeoutMs,
+  mockUpdateRunStatus,
+} = vi.hoisted(() => ({
+  mockCreateRunSegment: vi.fn(),
+  mockForceFailHungToolCall: vi.fn(),
+  mockGetEffectiveDecryptedEnv: vi.fn(),
+  mockGetMothershipBaseURL: vi.fn(),
+  mockGetMothershipSourceEnvHeaders: vi.fn(),
+  mockPrepareExecutionContext: vi.fn(),
+  mockRunStreamLoop: vi.fn(),
+  mockToolWatchdogTimeoutMs: vi.fn(() => 60_000),
+  mockUpdateRunStatus: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/async-runs/repository', () => ({
+  createRunSegment: mockCreateRunSegment,
+  updateRunStatus: mockUpdateRunStatus,
+}))
+
+vi.mock('@/lib/copilot/request/go/stream', () => {
+  class CopilotBackendError extends Error {
+    status?: number
+
+    constructor(message: string, options?: { status?: number }) {
+      super(message)
+      this.name = 'CopilotBackendError'
+      this.status = options?.status
+    }
+  }
+
+  class BillingLimitError extends Error {
+    userId: string
+
+    constructor(userId: string) {
+      super('Usage limit reached')
+      this.name = 'BillingLimitError'
+      this.userId = userId
+    }
+  }
+
+  return {
+    BillingLimitError,
+    CopilotBackendError,
+    runStreamLoop: mockRunStreamLoop,
+  }
+})
+
+vi.mock('@/lib/copilot/server/agent-url', () => ({
+  getMothershipBaseURL: mockGetMothershipBaseURL,
+  getMothershipSourceEnvHeaders: mockGetMothershipSourceEnvHeaders,
+}))
+
+vi.mock('@/lib/core/config/env', () => ({
+  env: {
+    COPILOT_API_KEY: undefined,
+  },
+  getEnv: vi.fn((key: string) => (key === 'NEXT_PUBLIC_APP_URL' ? 'http://localhost:3000' : '')),
+  isTruthy: vi.fn((value: string | undefined) => value === 'true'),
+  isFalsy: vi.fn((value: string | undefined) => value === 'false'),
+}))
+
+vi.mock('@/lib/environment/utils', () => ({
+  getEffectiveDecryptedEnv: mockGetEffectiveDecryptedEnv,
+}))
+
+vi.mock('@/lib/copilot/tools/handlers/context', () => ({
+  prepareExecutionContext: mockPrepareExecutionContext,
+}))
+
+vi.mock('@/lib/copilot/request/tools/billing', () => ({
+  handleBillingLimitResponse: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/tools/executor', () => ({
+  executeToolAndReport: vi.fn(),
+  forceFailHungToolCall: mockForceFailHungToolCall,
+  toolWatchdogTimeoutMs: mockToolWatchdogTimeoutMs,
+}))
+
+import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1'
+import { CopilotBackendError } from '@/lib/copilot/request/go/stream'
+import { runCopilotLifecycle } from '@/lib/copilot/request/lifecycle/run'
+
+describe('runCopilotLifecycle', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetMothershipBaseURL.mockResolvedValue('http://mothership.test')
+    mockGetMothershipSourceEnvHeaders.mockReturnValue({})
+  })
+
+  it('runs cancelled completion persistence when a stream throws after abort', async () => {
+    const abortController = new AbortController()
+    abortController.abort('stop')
+    const onComplete = vi.fn()
+    const onError = vi.fn()
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'partial answer'
+        context.contentBlocks.push({
+          type: 'text',
+          content: 'partial answer',
+          timestamp: 1,
+        })
+        throw new Error('publisher closed after stop')
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        abortSignal: abortController.signal,
+        executionContext,
+        onComplete,
+        onError,
+      }
+    )
+
+    expect(onError).not.toHaveBeenCalled()
+    expect(onComplete).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+        chatId: 'chat-1',
+        requestId: undefined,
+        error: 'publisher closed after stop',
+        contentBlocks: [
+          expect.objectContaining({
+            type: 'text',
+            content: 'partial answer',
+          }),
+        ],
+      })
+    )
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+        chatId: 'chat-1',
+        error: 'publisher closed after stop',
+      })
+    )
+  })
+
+  it('returns the cancelled result when cancelled completion persistence fails', async () => {
+    const abortController = new AbortController()
+    abortController.abort('stop')
+    const onComplete = vi.fn().mockRejectedValue(new Error('db unavailable'))
+    const onError = vi.fn()
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'partial answer'
+        throw new Error('publisher closed after stop')
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        abortSignal: abortController.signal,
+        executionContext,
+        onComplete,
+        onError,
+      }
+    )
+
+    expect(onError).not.toHaveBeenCalled()
+    expect(onComplete).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+      })
+    )
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+        error: 'publisher closed after stop',
+      })
+    )
+  })
+
+  it('uses the final post-tool assistant content for headless results', async () => {
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'I will check that.Final answer only.'
+        context.finalAssistantContent = 'Final answer only.'
+        context.sawMainToolCall = true
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+        interactive: false,
+      }
+    )
+
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: true,
+        content: 'Final answer only.',
+      })
+    )
+  })
+
+  it('does not fall back to pre-tool narration when headless final content is empty', async () => {
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'I will check that.'
+        context.finalAssistantContent = ''
+        context.sawMainToolCall = true
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+        interactive: false,
+      }
+    )
+
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: true,
+        content: '',
+      })
+    )
+  })
+
+  it('propagates payload userPermission into the generated execution context', async () => {
+    let capturedExecContext: ExecutionContext | undefined
+    mockGetEffectiveDecryptedEnv.mockResolvedValueOnce({})
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        _context: StreamingContext,
+        execContext: ExecutionContext
+      ): Promise<void> => {
+        capturedExecContext = execContext
+      }
+    )
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1', userPermission: 'write' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+      }
+    )
+
+    expect(capturedExecContext).toEqual(
+      expect.objectContaining({
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        userPermission: 'write',
+      })
+    )
+  })
+
+  it('normalizes the initial request body with workspaceId from lifecycle options', async () => {
+    let requestBody: Record<string, unknown> | undefined
+    mockGetEffectiveDecryptedEnv.mockResolvedValueOnce({})
+    mockRunStreamLoop.mockImplementationOnce(
+      async (_fetchUrl: string, fetchOptions: RequestInit): Promise<void> => {
+        requestBody = JSON.parse(String(fetchOptions.body))
+      }
+    )
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+      }
+    )
+
+    expect(requestBody).toEqual(
+      expect.objectContaining({
+        workspaceId: 'ws-1',
+      })
+    )
+  })
+
+  it('uses the lifecycle workspaceId for async tool resume requests', async () => {
+    const requestBodies: Record<string, unknown>[] = []
+    const fetchUrls: string[] = []
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        fetchUrl: string,
+        fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        fetchUrls.push(fetchUrl)
+        requestBodies.push(JSON.parse(String(fetchOptions.body)))
+        context.toolCalls.set('tool-1', {
+          id: 'tool-1',
+          name: 'read',
+          status: MothershipStreamV1ToolOutcome.success,
+          result: { success: true, output: { content: 'file contents' } },
+        })
+        context.awaitingAsyncContinuation = {
+          checkpointId: 'ckpt-1',
+          pendingToolCallIds: ['tool-1'],
+        }
+      }
+    )
+    mockRunStreamLoop.mockImplementationOnce(
+      async (fetchUrl: string, fetchOptions: RequestInit): Promise<void> => {
+        fetchUrls.push(fetchUrl)
+        requestBodies.push(JSON.parse(String(fetchOptions.body)))
+      }
+    )
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        workflowId: 'workflow-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+      }
+    )
+
+    expect(fetchUrls[1]).toBe('http://mothership.test/api/tools/resume')
+    expect(requestBodies[1]).toEqual(
+      expect.objectContaining({
+        checkpointId: 'ckpt-1',
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+      })
+    )
+  })
+
+  it('finalizes as success when a resume fails with a retryable error then the retry succeeds', async () => {
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    // 1) Initial stream pauses on an async tool checkpoint with a resolved
+    //    tool result, so the lifecycle transitions into a resume leg.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.toolCalls.set('tool-1', {
+          id: 'tool-1',
+          name: 'read',
+          status: MothershipStreamV1ToolOutcome.success,
+          result: { success: true, output: { content: 'file contents' } },
+        })
+        context.awaitingAsyncContinuation = {
+          checkpointId: 'ckpt-1',
+          pendingToolCallIds: ['tool-1'],
+        }
+      }
+    )
+
+    // 2) First resume leg dies mid-stream like a transient provider error:
+    //    it records an error AND throws a retryable 5xx.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.errors.push(
+          'Copilot backend stream ended before a terminal event on /api/tools/resume'
+        )
+        throw new CopilotBackendError('backend stream ended before a terminal event', {
+          status: 503,
+        })
+      }
+    )
+
+    // 3) Retry of the same resume leg succeeds cleanly.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'Recovered final answer.'
+        context.finalAssistantContent = 'Recovered final answer.'
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+      }
+    )
+
+    // Three legs ran (initial + failed resume + retried resume), and the
+    // recovered retry must NOT inherit the failed attempt's error.
+    expect(mockRunStreamLoop).toHaveBeenCalledTimes(3)
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: true,
+        cancelled: false,
+        errors: undefined,
+      })
+    )
+  })
+
+  it('marks resume legs willRetryOnStreamError except the final attempt', async () => {
+    const bodies: Record<string, unknown>[] = []
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    // Initial leg pauses on a resolved async tool checkpoint → enters resume.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        bodies.push(JSON.parse(String(fetchOptions.body)))
+        context.toolCalls.set('tool-1', {
+          id: 'tool-1',
+          name: 'read',
+          status: MothershipStreamV1ToolOutcome.success,
+          result: { success: true, output: { content: 'file contents' } },
+        })
+        context.awaitingAsyncContinuation = {
+          checkpointId: 'ckpt-1',
+          pendingToolCallIds: ['tool-1'],
+        }
+      }
+    )
+
+    // Three resume attempts, all failing with a retryable 5xx so the loop
+    // exhausts MAX_RESUME_ATTEMPTS (= 3) and gives up.
+    for (let i = 0; i < 3; i++) {
+      mockRunStreamLoop.mockImplementationOnce(
+        async (
+          _fetchUrl: string,
+          fetchOptions: RequestInit,
+          context: StreamingContext
+        ): Promise<void> => {
+          bodies.push(JSON.parse(String(fetchOptions.body)))
+          context.errors.push('Copilot backend stream ended before a terminal event')
+          throw new CopilotBackendError('backend stream ended before a terminal event', {
+            status: 503,
+          })
+        }
+      )
+    }
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+      }
+    )
+
+    // Initial + 3 resume attempts.
+    expect(mockRunStreamLoop).toHaveBeenCalledTimes(4)
+    // Initial leg is never retried by this loop → no flag.
+    expect(bodies[0].willRetryOnStreamError).toBeUndefined()
+    // Resume attempts 0 and 1 will be retried on a stream error → flagged.
+    expect(bodies[1].willRetryOnStreamError).toBe(true)
+    expect(bodies[2].willRetryOnStreamError).toBe(true)
+    // Final attempt (2) is terminal → not flagged, so Go bills + surfaces it.
+    expect(bodies[3].willRetryOnStreamError).toBeUndefined()
+  })
+
+  it('force-fails a hung tool promise and resumes with an error result instead of wedging', async () => {
+    vi.useFakeTimers()
+    try {
+      const fetchUrls: string[] = []
+      const bodies: Record<string, unknown>[] = []
+      const executionContext: ExecutionContext = {
+        userId: 'user-1',
+        workflowId: '',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        decryptedEnvVars: {},
+      }
+
+      // Mirror the real helper: settle the tool call into a terminal error
+      // state so the resume loop can serialize an error result for it.
+      mockForceFailHungToolCall.mockImplementation(
+        async (toolCallId: string, context: StreamingContext, message: string) => {
+          const tool = context.toolCalls.get(toolCallId)
+          if (!tool) return
+          tool.status = MothershipStreamV1ToolOutcome.error
+          tool.endTime = Date.now()
+          tool.result = { success: false }
+          tool.error = message
+        }
+      )
+
+      // Initial leg checkpoints on an async tool whose promise NEVER settles —
+      // the exact shape of the prod incident (claimed, marked running, hung).
+      mockRunStreamLoop.mockImplementationOnce(
+        async (
+          fetchUrl: string,
+          fetchOptions: RequestInit,
+          context: StreamingContext
+        ): Promise<void> => {
+          fetchUrls.push(fetchUrl)
+          bodies.push(JSON.parse(String(fetchOptions.body)))
+          context.toolCalls.set('tool-hung', {
+            id: 'tool-hung',
+            name: 'read',
+            status: 'executing',
+          })
+          context.pendingToolPromises.set('tool-hung', new Promise(() => {}))
+          context.awaitingAsyncContinuation = {
+            checkpointId: 'ckpt-1',
+            pendingToolCallIds: ['tool-hung'],
+          }
+        }
+      )
+
+      // Resume leg completes normally with the error result delivered.
+      mockRunStreamLoop.mockImplementationOnce(
+        async (
+          fetchUrl: string,
+          fetchOptions: RequestInit,
+          context: StreamingContext
+        ): Promise<void> => {
+          fetchUrls.push(fetchUrl)
+          bodies.push(JSON.parse(String(fetchOptions.body)))
+          context.accumulatedContent = 'The file read failed, but here is what I know.'
+        }
+      )
+
+      const lifecycle = runCopilotLifecycle(
+        { message: 'hello', messageId: 'stream-1' },
+        {
+          userId: 'user-1',
+          workspaceId: 'ws-1',
+          chatId: 'chat-1',
+          executionId: 'exec-1',
+          runId: 'run-1',
+          executionContext,
+        }
+      )
+
+      // Wait budget = watchdog (60s, mocked) + resume grace (30s). Advance past it.
+      await vi.advanceTimersByTimeAsync(91_000)
+      const result = await lifecycle
+
+      expect(mockForceFailHungToolCall).toHaveBeenCalledWith(
+        'tool-hung',
+        expect.anything(),
+        expect.stringContaining('hung')
+      )
+      expect(fetchUrls[1]).toBe('http://mothership.test/api/tools/resume')
+      expect(bodies[1].results).toEqual([
+        expect.objectContaining({
+          callId: 'tool-hung',
+          name: 'read',
+          success: false,
+          data: { error: expect.stringContaining('hung') },
+        }),
+      ])
+      expect(result.success).toBe(true)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})

apps/sim/lib/copilot/request/lifecycle/start.test.ts

@@ -0,0 +1,294 @@
+/**
+ * @vitest-environment node
+ */
+
+import { propagation, trace } from '@opentelemetry/api'
+import { W3CTraceContextPropagator } from '@opentelemetry/core'
+import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+
+const {
+  runCopilotLifecycle,
+  createRunSegment,
+  updateRunStatus,
+  resetBuffer,
+  clearFilePreviewSessions,
+  scheduleBufferCleanup,
+  scheduleFilePreviewSessionCleanup,
+  allocateCursor,
+  appendEvent,
+  cleanupAbortMarker,
+  hasAbortMarker,
+  releasePendingChatStream,
+} = vi.hoisted(() => ({
+  runCopilotLifecycle: vi.fn(),
+  createRunSegment: vi.fn(),
+  updateRunStatus: vi.fn(),
+  resetBuffer: vi.fn(),
+  clearFilePreviewSessions: vi.fn(),
+  scheduleBufferCleanup: vi.fn(),
+  scheduleFilePreviewSessionCleanup: vi.fn(),
+  allocateCursor: vi.fn(),
+  appendEvent: vi.fn(),
+  cleanupAbortMarker: vi.fn(),
+  hasAbortMarker: vi.fn(),
+  releasePendingChatStream: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/lifecycle/run', () => ({
+  runCopilotLifecycle,
+}))
+
+vi.mock('@/lib/copilot/async-runs/repository', () => ({
+  createRunSegment,
+  updateRunStatus,
+}))
+
+let mockPublisherController: ReadableStreamDefaultController | null = null
+
+vi.mock('@/lib/copilot/request/session', () => ({
+  resetBuffer,
+  clearFilePreviewSessions,
+  scheduleBufferCleanup,
+  scheduleFilePreviewSessionCleanup,
+  allocateCursor,
+  appendEvent,
+  cleanupAbortMarker,
+  hasAbortMarker,
+  releasePendingChatStream,
+  registerActiveStream: vi.fn(),
+  unregisterActiveStream: vi.fn(),
+  startAbortPoller: vi.fn().mockReturnValue(setInterval(() => {}, 999999)),
+  isExplicitStopReason: vi.fn().mockReturnValue(false),
+  SSE_RESPONSE_HEADERS: {},
+  StreamWriter: vi.fn().mockImplementation(
+    class {
+      attach = vi.fn().mockImplementation((ctrl: ReadableStreamDefaultController) => {
+        mockPublisherController = ctrl
+      })
+      startKeepalive = vi.fn()
+      stopKeepalive = vi.fn()
+      flush = vi.fn()
+      close = vi.fn().mockImplementation(() => {
+        try {
+          mockPublisherController?.close()
+        } catch {
+          // already closed
+        }
+      })
+      markDisconnected = vi.fn()
+      publish = vi.fn().mockImplementation(async (event: Record<string, unknown>) => {
+        appendEvent(event)
+      })
+      get clientDisconnected() {
+        return false
+      }
+      get sawComplete() {
+        return false
+      }
+    }
+  ),
+}))
+vi.mock('@/lib/copilot/request/session/sse', () => ({
+  SSE_RESPONSE_HEADERS: {},
+}))
+
+vi.mock('@sim/db', () => ({
+  db: {
+    update: vi.fn(() => ({
+      set: vi.fn(() => ({
+        where: vi.fn(),
+      })),
+    })),
+  },
+}))
+
+vi.mock('@/lib/copilot/chat-status', () => ({
+  chatPubSub: null,
+}))
+
+import { createSSEStream } from './start'
+
+async function drainStream(stream: ReadableStream) {
+  const reader = stream.getReader()
+  while (true) {
+    const { done } = await reader.read()
+    if (done) break
+  }
+}
+
+describe('createSSEStream terminal error handling', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    trace.setGlobalTracerProvider(new BasicTracerProvider())
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator())
+    vi.stubGlobal(
+      'fetch',
+      vi.fn().mockResolvedValue(
+        new Response(JSON.stringify({ title: 'Test title' }), {
+          status: 200,
+          headers: {
+            'Content-Type': 'application/json',
+          },
+        })
+      )
+    )
+    resetBuffer.mockResolvedValue(undefined)
+    clearFilePreviewSessions.mockResolvedValue(undefined)
+    scheduleBufferCleanup.mockResolvedValue(undefined)
+    scheduleFilePreviewSessionCleanup.mockResolvedValue(undefined)
+    allocateCursor
+      .mockResolvedValueOnce({ seq: 1, cursor: '1' })
+      .mockResolvedValueOnce({ seq: 2, cursor: '2' })
+      .mockResolvedValueOnce({ seq: 3, cursor: '3' })
+    appendEvent.mockImplementation(async (event: unknown) => event)
+    cleanupAbortMarker.mockResolvedValue(undefined)
+    hasAbortMarker.mockResolvedValue(false)
+    releasePendingChatStream.mockResolvedValue(undefined)
+    createRunSegment.mockResolvedValue(null)
+    updateRunStatus.mockResolvedValue(null)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('writes a terminal error event before close when orchestration returns success=false', async () => {
+    runCopilotLifecycle.mockResolvedValue({
+      success: false,
+      error: 'resume failed',
+      content: '',
+      contentBlocks: [],
+      toolCalls: [],
+    })
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-1',
+      orchestrateOptions: {},
+    })
+
+    await drainStream(stream)
+
+    expect(appendEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.error,
+      })
+    )
+    expect(scheduleBufferCleanup).toHaveBeenCalledWith('stream-1')
+  })
+
+  it('writes the thrown terminal error event before close for replay durability', async () => {
+    runCopilotLifecycle.mockRejectedValue(new Error('kaboom'))
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-1',
+      orchestrateOptions: {},
+    })
+
+    await drainStream(stream)
+
+    expect(appendEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.error,
+      })
+    )
+    expect(scheduleBufferCleanup).toHaveBeenCalledWith('stream-1')
+  })
+
+  it('publishes a cancelled completion (not an error) when the orchestrator reports cancelled without abortSignal aborted', async () => {
+    runCopilotLifecycle.mockResolvedValue({
+      success: false,
+      cancelled: true,
+      content: '',
+      contentBlocks: [],
+      toolCalls: [],
+    })
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-cancelled',
+      orchestrateOptions: {},
+    })
+
+    await drainStream(stream)
+
+    expect(appendEvent).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.error,
+      })
+    )
+    expect(appendEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.complete,
+        payload: expect.objectContaining({
+          status: MothershipStreamV1CompletionStatus.cancelled,
+        }),
+      })
+    )
+  })
+
+  it('passes an OTel context into the streaming lifecycle', async () => {
+    let lifecycleTraceparent = ''
+    runCopilotLifecycle.mockImplementation(async (_payload, options) => {
+      const { traceHeaders } = await import('@/lib/copilot/request/go/propagation')
+      lifecycleTraceparent = traceHeaders({}, options.otelContext).traceparent ?? ''
+      return {
+        success: true,
+        content: 'OK',
+        contentBlocks: [],
+        toolCalls: [],
+      }
+    })
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-otel',
+      orchestrateOptions: {
+        goRoute: '/api/mothership',
+        workflowId: 'workflow-1',
+      },
+    })
+
+    await drainStream(stream)
+
+    expect(lifecycleTraceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
+  })
+})

apps/sim/lib/copilot/request/lifecycle/start.ts

@@ -0,0 +1,531 @@
+import { type Context, context as otelContextApi } from '@opentelemetry/api'
+import { db } from '@sim/db'
+import { copilotChats } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { eq } from 'drizzle-orm'
+import { createRunSegment } from '@/lib/copilot/async-runs/repository'
+import { chatPubSub } from '@/lib/copilot/chat-status'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1SessionKind,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import {
+  RequestTraceV1Outcome,
+  RequestTraceV1SpanStatus,
+} from '@/lib/copilot/generated/request-trace-v1'
+import {
+  CopilotRequestCancelReason,
+  type CopilotRequestCancelReasonValue,
+  CopilotTransport,
+} from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceEvent } from '@/lib/copilot/generated/trace-events-v1'
+import { finalizeStream } from '@/lib/copilot/request/lifecycle/finalize'
+import type { CopilotLifecycleOptions } from '@/lib/copilot/request/lifecycle/run'
+import { runCopilotLifecycle } from '@/lib/copilot/request/lifecycle/run'
+import { type CopilotLifecycleOutcome, startCopilotOtelRoot } from '@/lib/copilot/request/otel'
+import {
+  cleanupAbortMarker,
+  clearFilePreviewSessions,
+  isExplicitStopReason,
+  registerActiveStream,
+  releasePendingChatStream,
+  resetBuffer,
+  StreamWriter,
+  scheduleBufferCleanup,
+  scheduleFilePreviewSessionCleanup,
+  startAbortPoller,
+  unregisterActiveStream,
+} from '@/lib/copilot/request/session'
+import { SSE_RESPONSE_HEADERS } from '@/lib/copilot/request/session/sse'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+import { getMothershipBaseURL, getMothershipSourceEnvHeaders } from '@/lib/copilot/server/agent-url'
+import { env } from '@/lib/core/config/env'
+
+export { SSE_RESPONSE_HEADERS }
+
+const logger = createLogger('CopilotChatStreaming')
+
+type CurrentChatSummary = {
+  title?: string | null
+} | null
+
+export interface StreamingOrchestrationParams {
+  requestPayload: Record<string, unknown>
+  userId: string
+  streamId: string
+  executionId: string
+  runId: string
+  chatId?: string
+  currentChat: CurrentChatSummary
+  isNewChat: boolean
+  message: string
+  titleModel: string
+  titleProvider?: string
+  requestId: string
+  workspaceId?: string
+  orchestrateOptions: Omit<CopilotLifecycleOptions, 'onEvent'>
+  /**
+   * Pre-started root; child spans bind to it and `finish()` fires on
+   * termination. Omit to let the stream start its own root (headless).
+   */
+  otelRoot?: ReturnType<typeof startCopilotOtelRoot>
+}
+
+export function createSSEStream(params: StreamingOrchestrationParams): ReadableStream {
+  const {
+    requestPayload,
+    userId,
+    streamId,
+    executionId,
+    runId,
+    chatId,
+    currentChat,
+    isNewChat,
+    message,
+    titleModel,
+    titleProvider,
+    requestId,
+    workspaceId,
+    orchestrateOptions,
+    otelRoot,
+  } = params
+
+  // Reuse caller's root if provided; otherwise start our own.
+  const activeOtelRoot =
+    otelRoot ??
+    startCopilotOtelRoot({
+      requestId,
+      route: orchestrateOptions.goRoute,
+      chatId,
+      workflowId: orchestrateOptions.workflowId,
+      executionId,
+      runId,
+      streamId,
+      transport: CopilotTransport.Stream,
+    })
+
+  const abortController = new AbortController()
+  registerActiveStream(streamId, abortController)
+
+  const publisher = new StreamWriter({ streamId, chatId, requestId })
+
+  // Classify cancel: signal.reason (explicit-stop set) wins, then
+  // clientDisconnected, else Unknown (latent contract bug — log it).
+  const recordCancelled = (errorMessage?: string): CopilotRequestCancelReasonValue => {
+    const rawReason = abortController.signal.reason
+    let cancelReason: CopilotRequestCancelReasonValue
+    if (isExplicitStopReason(rawReason)) {
+      cancelReason = CopilotRequestCancelReason.ExplicitStop
+    } else if (publisher.clientDisconnected) {
+      cancelReason = CopilotRequestCancelReason.ClientDisconnect
+    } else {
+      cancelReason = CopilotRequestCancelReason.Unknown
+      const serializedReason =
+        rawReason === undefined
+          ? 'undefined'
+          : rawReason instanceof Error
+            ? `${rawReason.name}: ${rawReason.message}`
+            : typeof rawReason === 'string'
+              ? rawReason
+              : (() => {
+                  try {
+                    return JSON.stringify(rawReason)
+                  } catch {
+                    return String(rawReason)
+                  }
+                })()
+      // Contract violation: add the new reason to AbortReason /
+      // isExplicitStopReason or extend the classifier.
+      logger.error(`[${requestId}] Stream cancelled with unknown abort reason`, {
+        streamId,
+        chatId,
+        reason: serializedReason,
+      })
+      activeOtelRoot.span.setAttribute(TraceAttr.CopilotAbortUnknownReason, serializedReason)
+    }
+    activeOtelRoot.span.setAttribute(TraceAttr.CopilotRequestCancelReason, cancelReason)
+    activeOtelRoot.span.addEvent(TraceEvent.RequestCancelled, {
+      [TraceAttr.CopilotRequestCancelReason]: cancelReason,
+      ...(errorMessage ? { [TraceAttr.ErrorMessage]: errorMessage } : {}),
+    })
+    return cancelReason
+  }
+
+  const collector = new TraceCollector()
+
+  return new ReadableStream({
+    async start(controller) {
+      publisher.attach(controller)
+
+      // Re-enter the root OTel context — ALS doesn't survive the
+      // Next handler → ReadableStream.start boundary.
+      await otelContextApi.with(activeOtelRoot.context, async () => {
+        const otelContext = activeOtelRoot.context
+        let rootOutcome: CopilotLifecycleOutcome = RequestTraceV1Outcome.error
+        let rootError: unknown
+        // `cancelReason` must be declared OUTSIDE the outer `try` so
+        // it remains in scope for the outer `finally` that calls
+        // `activeOtelRoot.finish(rootOutcome, rootError, cancelReason)`.
+        // `let` bindings declared inside a `try` block are NOT visible
+        // in the paired `finally`; referencing one there raises a
+        // TDZ ReferenceError, skipping `finish()`, leaving the root
+        // span never-ended, and making Tempo see every child as an
+        // orphan under a phantom parent. (Regression landed 2026-04-21.)
+        let cancelReason: CopilotRequestCancelReasonValue | undefined
+        try {
+          const requestSpan = collector.startSpan('Sim Agent Request', 'request', {
+            streamId,
+            chatId,
+            runId,
+          })
+          let outcome: CopilotLifecycleOutcome = RequestTraceV1Outcome.error
+          let lifecycleResult:
+            | {
+                usage?: { prompt: number; completion: number }
+                cost?: { input: number; output: number; total: number }
+              }
+            | undefined
+
+          await Promise.all([resetBuffer(streamId), clearFilePreviewSessions(streamId)])
+
+          if (chatId) {
+            createRunSegment({
+              id: runId,
+              executionId,
+              chatId,
+              userId,
+              workflowId: (requestPayload.workflowId as string | undefined) || null,
+              workspaceId,
+              streamId,
+              model: (requestPayload.model as string | undefined) || null,
+              provider: (requestPayload.provider as string | undefined) || null,
+              requestContext: { requestId },
+            }).catch((error) => {
+              logger.warn(`[${requestId}] Failed to create copilot run segment`, {
+                error: getErrorMessage(error),
+              })
+            })
+          }
+
+          const abortPoller = startAbortPoller(streamId, abortController, {
+            requestId,
+            chatId,
+          })
+          publisher.startKeepalive()
+
+          if (chatId) {
+            publisher.publish({
+              type: MothershipStreamV1EventType.session,
+              payload: {
+                kind: MothershipStreamV1SessionKind.chat,
+                chatId,
+              },
+            })
+          }
+
+          fireTitleGeneration({
+            chatId,
+            currentChat,
+            isNewChat,
+            userId,
+            message,
+            titleModel,
+            titleProvider,
+            workspaceId,
+            requestId,
+            publisher,
+            otelContext,
+          })
+
+          try {
+            const result = await runCopilotLifecycle(requestPayload, {
+              ...orchestrateOptions,
+              executionId,
+              runId,
+              trace: collector,
+              simRequestId: requestId,
+              otelContext,
+              abortSignal: abortController.signal,
+              onEvent: async (event) => {
+                await publisher.publish(event)
+              },
+              onAbortObserved: (reason) => {
+                if (!abortController.signal.aborted) {
+                  abortController.abort(reason)
+                }
+              },
+            })
+
+            lifecycleResult = result
+            // Outcome classification (priority order):
+            //   1. `result.success` → success. The orchestrator
+            //      reporting "finished cleanly" wins over any later
+            //      signal change. Matters for the narrow race where
+            //      the user clicks Stop a beat after the stream
+            //      completed.
+            //   2. `signal.aborted` (from `abortActiveStream` or the
+            //      Redis-marker poller) OR `clientDisconnected` with
+            //      a non-success result → cancelled. `recordCancelled`
+            //      further refines into explicit_stop / client_disconnect
+            //      / unknown via `signal.reason`.
+            //   3. Otherwise → error.
+            outcome = result.success
+              ? RequestTraceV1Outcome.success
+              : result.cancelled || abortController.signal.aborted || publisher.clientDisconnected
+                ? RequestTraceV1Outcome.cancelled
+                : RequestTraceV1Outcome.error
+            if (outcome === RequestTraceV1Outcome.cancelled) {
+              cancelReason = recordCancelled()
+            }
+            // Pass the resolved outcome — not `signal.aborted` — so
+            // `finalizeStream` classifies the same way we did above.
+            // A client-disconnect-without-controller-abort still needs
+            // to hit `handleAborted` (not `handleError`) so the chat
+            // row gets `cancelled` terminal state instead of `error`.
+            await finalizeStream(result, publisher, runId, outcome, requestId)
+          } catch (error) {
+            // Error-path classification: if the abort signal fired or
+            // the client disconnected, treat the thrown error as a
+            // cancel (same rationale as the try-path above).
+            const wasCancelled = abortController.signal.aborted || publisher.clientDisconnected
+            outcome = wasCancelled ? RequestTraceV1Outcome.cancelled : RequestTraceV1Outcome.error
+            if (outcome === RequestTraceV1Outcome.cancelled) {
+              cancelReason = recordCancelled(getErrorMessage(error))
+            }
+            if (publisher.clientDisconnected) {
+              logger.info(`[${requestId}] Stream errored after client disconnect`, {
+                error: getErrorMessage(error, 'Stream error'),
+              })
+            }
+            // Demote to warn when the throw came from a user-initiated
+            // cancel — it isn't an "unexpected" failure then, and the
+            // error-level log pollutes alerting on normal Stop presses.
+            const logFn = outcome === RequestTraceV1Outcome.cancelled ? logger.warn : logger.error
+            logFn.call(logger, `[${requestId}] Orchestration ended with ${outcome}:`, error)
+
+            const syntheticResult = {
+              success: false as const,
+              content: '',
+              contentBlocks: [],
+              toolCalls: [],
+              error: 'An unexpected error occurred while processing the response.',
+            }
+            await finalizeStream(syntheticResult, publisher, runId, outcome, requestId)
+          } finally {
+            collector.endSpan(
+              requestSpan,
+              outcome === RequestTraceV1Outcome.success
+                ? RequestTraceV1SpanStatus.ok
+                : outcome === RequestTraceV1Outcome.cancelled
+                  ? RequestTraceV1SpanStatus.cancelled
+                  : RequestTraceV1SpanStatus.error
+            )
+
+            clearInterval(abortPoller)
+            try {
+              await publisher.close()
+            } catch (error) {
+              logger.warn(`[${requestId}] Failed to flush stream persistence during close`, {
+                error: getErrorMessage(error),
+              })
+            }
+            unregisterActiveStream(streamId)
+            if (chatId) {
+              await releasePendingChatStream(chatId, streamId)
+            }
+            await scheduleBufferCleanup(streamId)
+            await scheduleFilePreviewSessionCleanup(streamId)
+            await cleanupAbortMarker(streamId)
+
+            rootOutcome = outcome
+            if (lifecycleResult?.usage) {
+              activeOtelRoot.span.setAttributes({
+                [TraceAttr.GenAiUsageInputTokens]: lifecycleResult.usage.prompt ?? 0,
+                [TraceAttr.GenAiUsageOutputTokens]: lifecycleResult.usage.completion ?? 0,
+              })
+            }
+            if (lifecycleResult?.cost) {
+              activeOtelRoot.span.setAttributes({
+                [TraceAttr.BillingCostInputUsd]: lifecycleResult.cost.input ?? 0,
+                [TraceAttr.BillingCostOutputUsd]: lifecycleResult.cost.output ?? 0,
+                [TraceAttr.BillingCostTotalUsd]: lifecycleResult.cost.total ?? 0,
+              })
+            }
+          }
+        } catch (error) {
+          rootOutcome = RequestTraceV1Outcome.error
+          rootError = error
+          throw error
+        } finally {
+          // `finish` is idempotent, so it's safe whether the POST
+          // handler started the root (and may also call finish on an
+          // error path before the stream ran) or we did. The cancel
+          // reason (if any) determines whether `cancelled` is an
+          // expected outcome (explicit_stop → status OK) or a real
+          // error (client_disconnect / unknown → status ERROR).
+          //
+          // Belt-and-suspenders: if `finish()` itself throws (e.g. an
+          // argument in the TDZ, a bad attribute, a regression in
+          // status-setting), fall back to `span.end()` directly. A
+          // root that never ends leaves every child orphaned in Tempo
+          // under a phantom parent; force-ending it keeps the trace
+          // shape intact even when the pretty-finalize path is
+          // broken. The error is logged so Loki greps surface the
+          // regression instead of it silently costing us trace
+          // fidelity for hours.
+          try {
+            activeOtelRoot.finish(rootOutcome, rootError, cancelReason)
+          } catch (finishError) {
+            logger.error(`[${requestId}] activeOtelRoot.finish threw; force-ending root span`, {
+              error: getErrorMessage(finishError),
+            })
+            try {
+              activeOtelRoot.span.end()
+            } catch {
+              // Already ended or an OTel internal failure — nothing
+              // more we can do. The export pipe has already had its
+              // chance; swallow to avoid masking the original error
+              // path.
+            }
+          }
+        }
+      })
+    },
+    cancel() {
+      // The browser's SSE reader closed. Flip `clientDisconnected` so
+      // in-flight `publisher.publish` calls silently no-op (prevents
+      // enqueueing on a closed controller).
+      //
+      // Browser disconnect is NOT an abort — firing the controller
+      // here retroactively reclassifies in-flight successful streams
+      // as aborted and skips assistant persistence. Let the
+      // orchestrator drain naturally; publish no-ops post-disconnect.
+      // Explicit Stop still fires the controller via /chat/abort.
+      publisher.markDisconnected()
+    },
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Title generation (fire-and-forget side effect)
+// ---------------------------------------------------------------------------
+
+function fireTitleGeneration(params: {
+  chatId?: string
+  currentChat: CurrentChatSummary
+  isNewChat: boolean
+  userId?: string
+  message: string
+  titleModel: string
+  titleProvider?: string
+  workspaceId?: string
+  requestId: string
+  publisher: StreamWriter
+  otelContext?: Context
+}): void {
+  const {
+    chatId,
+    currentChat,
+    isNewChat,
+    userId,
+    message,
+    titleModel,
+    titleProvider,
+    workspaceId,
+    requestId,
+    publisher,
+    otelContext,
+  } = params
+  if (!chatId || currentChat?.title || !isNewChat) return
+
+  requestChatTitle({
+    message,
+    model: titleModel,
+    provider: titleProvider,
+    userId,
+    workspaceId,
+    otelContext,
+  })
+    .then(async (title) => {
+      if (!title) return
+      await db.update(copilotChats).set({ title }).where(eq(copilotChats.id, chatId))
+      await publisher.publish({
+        type: MothershipStreamV1EventType.session,
+        payload: { kind: MothershipStreamV1SessionKind.title, title },
+      })
+      if (workspaceId) {
+        chatPubSub?.publishStatusChanged({
+          workspaceId,
+          chatId,
+          type: 'renamed',
+        })
+      }
+    })
+    .catch((error) => {
+      logger.error(`[${requestId}] Title generation failed:`, error)
+    })
+}
+
+// ---------------------------------------------------------------------------
+// Chat title helper
+// ---------------------------------------------------------------------------
+
+export async function requestChatTitle(params: {
+  message: string
+  model: string
+  provider?: string
+  userId?: string
+  workspaceId?: string
+  otelContext?: Context
+}): Promise<string | null> {
+  const { message, model, provider, userId, workspaceId, otelContext } = params
+  if (!message || !model) return null
+
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+  }
+  if (env.COPILOT_API_KEY) {
+    headers['x-api-key'] = env.COPILOT_API_KEY
+  }
+  Object.assign(headers, getMothershipSourceEnvHeaders())
+
+  try {
+    const { fetchGo } = await import('@/lib/copilot/request/go/fetch')
+    const mothershipBaseURL = await getMothershipBaseURL({ userId })
+    const response = await fetchGo(`${mothershipBaseURL}/api/generate-chat-title`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        message,
+        model,
+        ...(provider ? { provider } : {}),
+        ...(workspaceId ? { workspaceId } : {}),
+        ...(userId ? { userId } : {}),
+      }),
+      otelContext,
+      spanName: 'sim → go /api/generate-chat-title',
+      operation: 'generate_chat_title',
+      attributes: {
+        [TraceAttr.GenAiRequestModel]: model,
+        ...(provider ? { [TraceAttr.GenAiSystem]: provider } : {}),
+      },
+    })
+
+    const payload = await response.json().catch(() => ({}))
+    if (!response.ok) {
+      logger.warn('Failed to generate chat title via copilot backend', {
+        status: response.status,
+        error: payload,
+      })
+      return null
+    }
+
+    const title = typeof payload?.title === 'string' ? payload.title.trim() : ''
+    return title || null
+  } catch (error) {
+    logger.error('Error generating chat title:', error)
+    return null
+  }
+}

apps/sim/lib/copilot/request/metrics.ts

@@ -0,0 +1,101 @@
+// Sim server-side copilot metrics (U17). Sim's MeterProvider is wired in
+// instrumentation-node.ts (OTLP → Mimir, 60s) but had no copilot instruments;
+// this module is its first consumer. We emit the SAME metric names + label keys
+// + histogram bucket boundaries as the Go side (copilot internal/telemetry +
+// contracts/metrics_v1.go) so the Go∪Sim union is queryable as one series set
+// — e.g. `copilot.tool.duration` split by `tool.executor` (go|client|sim).
+//
+// Bounded cardinality only: tool.name is capped to the shared tool catalog
+// (else "other"); vfs phase / file-read outcome are bounded sets. NEVER a
+// user/chat/request id (those explode Prometheus series).
+import { type Counter, type Histogram, metrics } from '@opentelemetry/api'
+import { Metric } from '@/lib/copilot/generated/metrics-v1'
+import { TOOL_CATALOG } from '@/lib/copilot/generated/tool-catalog-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+
+// MUST match Go's copilot/internal/telemetry/metrics.go LatencyBucketsMs
+// exactly — a histogram_quantile(sum by (le) …) over the Go∪Sim union is only
+// valid with identical boundaries. If you change one side, change the other.
+const LATENCY_BUCKETS_MS = [
+  50, 100, 250, 500, 1000, 2000, 5000, 10000, 20000, 30000, 60000, 120000, 300000,
+]
+
+// File sizes span KB→tens of MB; a bytes-appropriate bucket set (not latency).
+const BYTE_BUCKETS = [1024, 8192, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456]
+
+interface CopilotMeterInstruments {
+  toolDuration: Histogram
+  toolCalls: Counter
+  vfsMaterializeDuration: Histogram
+  fileReadDuration: Histogram
+  fileReadBytes: Histogram
+}
+
+let cached: CopilotMeterInstruments | undefined
+
+// Lazy init: Turbopack/Next can evaluate this module before the NodeSDK
+// installs the real MeterProvider, so resolve instruments on first use (a
+// no-op meter before then simply drops records — same pattern as getCopilotTracer).
+function instruments(): CopilotMeterInstruments {
+  if (cached) return cached
+  const meter = metrics.getMeter('sim-copilot')
+  cached = {
+    toolDuration: meter.createHistogram(Metric.CopilotToolDuration, {
+      unit: 'ms',
+      advice: { explicitBucketBoundaries: LATENCY_BUCKETS_MS },
+    }),
+    toolCalls: meter.createCounter(Metric.CopilotToolCalls),
+    vfsMaterializeDuration: meter.createHistogram(Metric.CopilotVfsMaterializeDuration, {
+      unit: 'ms',
+      advice: { explicitBucketBoundaries: LATENCY_BUCKETS_MS },
+    }),
+    fileReadDuration: meter.createHistogram(Metric.CopilotFileReadDuration, {
+      unit: 'ms',
+      advice: { explicitBucketBoundaries: LATENCY_BUCKETS_MS },
+    }),
+    fileReadBytes: meter.createHistogram(Metric.CopilotFileReadSize, {
+      unit: 'By',
+      advice: { explicitBucketBoundaries: BYTE_BUCKETS },
+    }),
+  }
+  return cached
+}
+
+// Caps tool.name to the shared catalog (matches Go's cappedToolName): a
+// catalog tool keeps its name, everything else (user MCP/custom/unknown)
+// collapses to "other" so series count stays finite.
+function cappedToolName(name: string): string {
+  return TOOL_CATALOG[name] ? name : 'other'
+}
+
+// recordSimToolMetric emits copilot.tool.calls (+1) and copilot.tool.duration
+// for one server-side Sim tool dispatch (executor=sim). outcome is the bounded
+// tool outcome (success/error/…). Pure telemetry.
+export function recordSimToolMetric(name: string, outcome: string, durationMs: number): void {
+  const { toolDuration, toolCalls } = instruments()
+  const attrs = {
+    [TraceAttr.ToolName]: cappedToolName(name),
+    [TraceAttr.ToolExecutor]: 'sim',
+    [TraceAttr.ToolOutcome]: outcome,
+  }
+  toolCalls.add(1, attrs)
+  if (durationMs >= 0) toolDuration.record(durationMs, attrs)
+}
+
+// recordVfsMaterialize records VFS materialization time. Call once per phase
+// with that phase's duration and once with phase="total" for the whole op, so
+// the dashboard can show total + per-phase. phase must be a bounded value.
+export function recordVfsMaterialize(phase: string, durationMs: number): void {
+  if (durationMs < 0) return
+  instruments().vfsMaterializeDuration.record(durationMs, {
+    [TraceAttr.CopilotVfsPhase]: phase,
+  })
+}
+
+// recordFileRead records server-side file-read duration + size by outcome.
+export function recordFileRead(outcome: string, durationMs: number, bytes: number): void {
+  const { fileReadDuration, fileReadBytes } = instruments()
+  const attrs = { [TraceAttr.CopilotVfsReadOutcome]: outcome }
+  if (durationMs >= 0) fileReadDuration.record(durationMs, attrs)
+  if (bytes >= 0) fileReadBytes.record(bytes, attrs)
+}

apps/sim/lib/copilot/request/otel.ts

@@ -0,0 +1,593 @@
+import { randomBytes } from 'crypto'
+import {
+  type Context,
+  context,
+  ROOT_CONTEXT,
+  type Span,
+  type SpanContext,
+  SpanKind,
+  SpanStatusCode,
+  TraceFlags,
+  trace,
+} from '@opentelemetry/api'
+import { describeError, toError } from '@sim/utils/errors'
+import { RequestTraceV1Outcome } from '@/lib/copilot/generated/request-trace-v1'
+import {
+  CopilotBranchKind,
+  CopilotRequestCancelReason,
+  type CopilotRequestCancelReasonValue,
+  CopilotSurface,
+  CopilotTransport,
+} from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { contextFromRequestHeaders } from '@/lib/copilot/request/go/propagation'
+import { isExplicitStopReason } from '@/lib/copilot/request/session/abort-reason'
+
+// OTel GenAI content-capture env var (spec:
+// https://opentelemetry.io/docs/specs/semconv/gen-ai/). Mirrored on
+// the Go side so a single var controls both halves.
+const GENAI_CAPTURE_ENV = 'OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT'
+
+// OTLP backends commonly reject attrs over 64 KiB; cap proactively.
+const GENAI_MESSAGE_ATTR_MAX_BYTES = 60 * 1024
+
+function isGenAIMessageCaptureEnabled(): boolean {
+  const raw = (process.env[GENAI_CAPTURE_ENV] || '').toLowerCase().trim()
+  return raw === 'true' || raw === '1' || raw === 'yes'
+}
+
+// True iff `err` represents the user explicitly clicking Stop — the
+// only cancellation we treat as expected (non-error).
+//
+// Policy across the codebase: an explicit user stop leaves span
+// status UNSET; every other cancellation (client tab close,
+// network drop, internal timeout, uncategorized abort) escalates
+// to `status=error` so it shows up on error dashboards. This is
+// the Sim mirror of `requestctx.IsExplicitUserStop` on the Go
+// side; keep the two semantically aligned.
+//
+// Detection modes:
+//
+//   - Plain-string reject value: `controller.abort('user_stop:...')`
+//     rejects fetch() with the reason STRING directly. Matches
+//     `isExplicitStopReason()` exactly (UserStop / RedisPoller).
+//   - DOMException / Error object: `controller.abort()` with no arg
+//     (or older runtimes) rejects with an AbortError whose `.cause`
+//     or `.message` may carry the reason. We inspect both.
+//
+// Anything that doesn't resolve to an explicit-stop reason (plain
+// AbortError with no identifiable cause, timeout-flavored aborts,
+// arbitrary Error instances) returns false and gets `status=error`.
+export function isExplicitUserStopError(err: unknown): boolean {
+  if (err == null) return false
+  if (typeof err === 'string') return isExplicitStopReason(err)
+  if (typeof err === 'object') {
+    const e = err as { cause?: unknown; message?: unknown }
+    if (isExplicitStopReason(e.cause)) return true
+    if (typeof e.message === 'string' && isExplicitStopReason(e.message)) return true
+  }
+  return false
+}
+
+/**
+ * True iff an HTTP response status code represents a real server-side
+ * problem (5xx) or a user-visible condition we want to alert on
+ * (402 Payment Required, 409 Conflict, 429 Too Many Requests).
+ *
+ * Everything else — in particular the 4xx flood from bot probes and
+ * expected auth/validation rejections — stays UNSET on the span so
+ * dashboards don't treat normal rejections as errors.
+ *
+ * Mirrored on the Go side in
+ * `copilot/internal/http/middleware/telemetry.go`. Keep the two in
+ * sync if you change the actionable set.
+ */
+export function isActionableErrorStatus(code: number): boolean {
+  if (code >= 500) return true
+  return code === 402 || code === 409 || code === 429
+}
+
+// Record exception + set ERROR unless the error is an explicit user
+// stop (see `isExplicitUserStopError`). Every other cancellation —
+// client disconnect, internal timeout, uncategorized AbortError —
+// becomes a real error that the dashboards will surface.
+export function markSpanForError(span: Span, error: unknown): void {
+  const asError = toError(error)
+  span.recordException(asError)
+  const described = describeError(error)
+  if (described.code) {
+    span.setAttribute(TraceAttr.ErrorCode, described.code)
+  }
+  if (!isExplicitUserStopError(error)) {
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: described.causeChain ? described.causeChain.join(' <- ') : asError.message,
+    })
+  }
+}
+
+// OTel GenAI message shape (kept minimal). Mirror changes on the Go side.
+interface GenAIAgentPart {
+  type: 'text' | 'tool_call' | 'tool_call_response'
+  content?: string
+  id?: string
+  name?: string
+  arguments?: Record<string, unknown>
+  response?: string
+}
+
+interface GenAIAgentMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool'
+  parts: GenAIAgentPart[]
+}
+
+function marshalAgentMessages(messages: GenAIAgentMessage[]): string | undefined {
+  if (messages.length === 0) return undefined
+  const json = JSON.stringify(messages)
+  if (json.length <= GENAI_MESSAGE_ATTR_MAX_BYTES) return json
+  // Simple tail-preserving truncation: drop from the front until we
+  // fit. Matches the Go side's behavior. The last message is
+  // usually the most diagnostic for span-level outcome.
+  let remaining = messages.slice()
+  while (remaining.length > 1) {
+    remaining = remaining.slice(1)
+    const candidate = JSON.stringify(remaining)
+    if (candidate.length <= GENAI_MESSAGE_ATTR_MAX_BYTES) return candidate
+  }
+  // Single message still over cap — truncate the text part in place
+  // with a marker so the partial content is still readable.
+  const only = remaining[0]
+  for (const part of only.parts) {
+    if (part.type === 'text' && part.content) {
+      const headroom = GENAI_MESSAGE_ATTR_MAX_BYTES - 1024
+      if (part.content.length > headroom) {
+        part.content = `${part.content.slice(0, headroom)}\n\n[truncated: capture cap ${GENAI_MESSAGE_ATTR_MAX_BYTES} bytes]`
+      }
+    }
+  }
+  const final = JSON.stringify([only])
+  return final.length <= GENAI_MESSAGE_ATTR_MAX_BYTES ? final : undefined
+}
+
+interface CopilotAgentInputMessages {
+  userMessage?: string
+  systemPrompt?: string
+}
+
+interface CopilotAgentOutputMessages {
+  assistantText?: string
+  toolCalls?: Array<{
+    id: string
+    name: string
+    arguments?: Record<string, unknown>
+  }>
+}
+
+function setAgentInputMessages(span: Span, input: CopilotAgentInputMessages): void {
+  if (!isGenAIMessageCaptureEnabled()) return
+  const messages: GenAIAgentMessage[] = []
+  if (input.systemPrompt) {
+    messages.push({
+      role: 'system',
+      parts: [{ type: 'text', content: input.systemPrompt }],
+    })
+  }
+  if (input.userMessage) {
+    messages.push({
+      role: 'user',
+      parts: [{ type: 'text', content: input.userMessage }],
+    })
+  }
+  const serialized = marshalAgentMessages(messages)
+  if (serialized) {
+    span.setAttribute(TraceAttr.GenAiInputMessages, serialized)
+  }
+}
+
+function setAgentOutputMessages(span: Span, output: CopilotAgentOutputMessages): void {
+  if (!isGenAIMessageCaptureEnabled()) return
+  const parts: GenAIAgentPart[] = []
+  if (output.assistantText) {
+    parts.push({ type: 'text', content: output.assistantText })
+  }
+  for (const tc of output.toolCalls ?? []) {
+    parts.push({
+      type: 'tool_call',
+      id: tc.id,
+      name: tc.name,
+      ...(tc.arguments ? { arguments: tc.arguments } : {}),
+    })
+  }
+  if (parts.length === 0) return
+  const serialized = marshalAgentMessages([{ role: 'assistant', parts }])
+  if (serialized) {
+    span.setAttribute(TraceAttr.GenAiOutputMessages, serialized)
+  }
+}
+
+export type CopilotLifecycleOutcome =
+  (typeof RequestTraceV1Outcome)[keyof typeof RequestTraceV1Outcome]
+
+// Lazy tracer — Next 16/Turbopack can evaluate modules before NodeSDK
+// installs the real TracerProvider; resolving per call avoids a
+// cached NoOpTracer silently disabling OTel.
+export function getCopilotTracer() {
+  return trace.getTracer('sim-ai-platform', '1.0.0')
+}
+
+function getTracer() {
+  return getCopilotTracer()
+}
+
+// Wrap an inbound handler that Go called into so its span parents
+// under the Go-side trace (via `traceparent`).
+export async function withIncomingGoSpan<T>(
+  headers: Headers,
+  spanName: string,
+  attributes: Record<string, string | number | boolean> | undefined,
+  fn: (span: Span) => Promise<T>
+): Promise<T> {
+  const parentContext = contextFromRequestHeaders(headers)
+  const tracer = getTracer()
+  return tracer.startActiveSpan(
+    spanName,
+    { kind: SpanKind.SERVER, attributes },
+    parentContext,
+    async (span) => {
+      try {
+        const result = await fn(span)
+        span.setStatus({ code: SpanStatusCode.OK })
+        return result
+      } catch (error) {
+        markSpanForError(span, error)
+        throw error
+      } finally {
+        span.end()
+      }
+    }
+  )
+}
+
+// Wrap a copilot-lifecycle op in an OTel span. Pass `parentContext`
+// explicitly when AsyncLocalStorage-tracked context can be dropped
+// across multiple awaits (otherwise the child falls back to a framework
+// span that the sampler drops).
+export async function withCopilotSpan<T>(
+  spanName: string,
+  attributes: Record<string, string | number | boolean> | undefined,
+  fn: (span: Span) => Promise<T>,
+  parentContext?: Context
+): Promise<T> {
+  const tracer = getTracer()
+  const runBody = async (span: Span) => {
+    try {
+      const result = await fn(span)
+      span.setStatus({ code: SpanStatusCode.OK })
+      return result
+    } catch (error) {
+      markSpanForError(span, error)
+      throw error
+    } finally {
+      span.end()
+    }
+  }
+  if (parentContext) {
+    return tracer.startActiveSpan(spanName, { attributes }, parentContext, runBody)
+  }
+  return tracer.startActiveSpan(spanName, { attributes }, runBody)
+}
+
+// External OTel `tool.execute` span for Sim-side tool work (the Go
+// side's `tool.execute` is just the enqueue, stays ~0ms).
+export async function withCopilotToolSpan<T>(
+  input: {
+    toolName: string
+    toolCallId: string
+    runId?: string
+    chatId?: string
+    argsBytes?: number
+    argsPreview?: string
+  },
+  fn: (span: Span) => Promise<T>
+): Promise<T> {
+  const tracer = getTracer()
+  return tracer.startActiveSpan(
+    `tool.execute ${input.toolName}`,
+    {
+      attributes: {
+        [TraceAttr.ToolName]: input.toolName,
+        [TraceAttr.ToolCallId]: input.toolCallId,
+        [TraceAttr.ToolExecutor]: 'sim',
+        ...(input.runId ? { [TraceAttr.RunId]: input.runId } : {}),
+        ...(input.chatId ? { [TraceAttr.ChatId]: input.chatId } : {}),
+        ...(typeof input.argsBytes === 'number'
+          ? { [TraceAttr.ToolArgsBytes]: input.argsBytes }
+          : {}),
+        // argsPreview can leak pasted credentials in tool args; gate
+        // behind the GenAI content-capture env var.
+        ...(input.argsPreview && isGenAIMessageCaptureEnabled()
+          ? { [TraceAttr.ToolArgsPreview]: input.argsPreview }
+          : {}),
+      },
+    },
+    async (span) => {
+      try {
+        const result = await fn(span)
+        span.setStatus({ code: SpanStatusCode.OK })
+        return result
+      } catch (error) {
+        markSpanForError(span, error)
+        throw error
+      } finally {
+        span.end()
+      }
+    }
+  )
+}
+
+function isValidSpanContext(spanContext: SpanContext): boolean {
+  return (
+    /^[0-9a-f]{32}$/.test(spanContext.traceId) &&
+    spanContext.traceId !== '00000000000000000000000000000000' &&
+    /^[0-9a-f]{16}$/.test(spanContext.spanId) &&
+    spanContext.spanId !== '0000000000000000'
+  )
+}
+
+function createFallbackSpanContext(): SpanContext {
+  return {
+    traceId: randomBytes(16).toString('hex'),
+    spanId: randomBytes(8).toString('hex'),
+    traceFlags: TraceFlags.SAMPLED,
+  }
+}
+
+interface CopilotOtelScope {
+  // Leave unset on the chat POST — startCopilotOtelRoot will derive
+  // from the root span's OTel trace ID (same value Grafana uses).
+  // Set explicitly on paths that need a non-trace-derived ID (headless,
+  // resume taking an ID from persisted state).
+  requestId?: string
+  route?: string
+  chatId?: string
+  workflowId?: string
+  executionId?: string
+  runId?: string
+  streamId?: string
+  transport: 'headless' | 'stream'
+  userMessagePreview?: string
+}
+
+// Dashboard-column width; long enough for triage disambiguation.
+const USER_MESSAGE_PREVIEW_MAX_CHARS = 500
+function buildAgentSpanAttributes(
+  scope: CopilotOtelScope & { requestId: string }
+): Record<string, string | number | boolean> {
+  // Gated behind the same env var as full GenAI message capture — a
+  // 500-char preview is still user prompt content.
+  const preview = isGenAIMessageCaptureEnabled()
+    ? truncateUserMessagePreview(scope.userMessagePreview)
+    : undefined
+  return {
+    [TraceAttr.GenAiAgentName]: 'mothership',
+    [TraceAttr.GenAiAgentId]:
+      scope.transport === CopilotTransport.Stream ? 'mothership-stream' : 'mothership-headless',
+    [TraceAttr.GenAiOperationName]:
+      scope.transport === CopilotTransport.Stream ? 'chat' : 'invoke_agent',
+    [TraceAttr.RequestId]: scope.requestId,
+    [TraceAttr.SimRequestId]: scope.requestId,
+    [TraceAttr.CopilotRoute]: scope.route ?? '',
+    [TraceAttr.CopilotTransport]: scope.transport,
+    ...(scope.chatId ? { [TraceAttr.ChatId]: scope.chatId } : {}),
+    ...(scope.workflowId ? { [TraceAttr.WorkflowId]: scope.workflowId } : {}),
+    ...(scope.executionId ? { [TraceAttr.CopilotExecutionId]: scope.executionId } : {}),
+    ...(scope.runId ? { [TraceAttr.RunId]: scope.runId } : {}),
+    ...(scope.streamId ? { [TraceAttr.StreamId]: scope.streamId } : {}),
+    ...(preview ? { [TraceAttr.CopilotUserMessagePreview]: preview } : {}),
+  }
+}
+
+function truncateUserMessagePreview(raw: unknown): string | undefined {
+  if (typeof raw !== 'string') return undefined
+  const collapsed = raw.replace(/\s+/g, ' ').trim()
+  if (!collapsed) return undefined
+  if (collapsed.length <= USER_MESSAGE_PREVIEW_MAX_CHARS) return collapsed
+  return `${collapsed.slice(0, USER_MESSAGE_PREVIEW_MAX_CHARS - 1)}…`
+}
+
+// Request-shape metadata known only after branch resolution. Stamped
+// on the root span for dashboard filtering.
+interface CopilotOtelRequestShape {
+  branchKind?: 'workflow' | 'workspace'
+  mode?: string
+  model?: string
+  provider?: string
+  createNewChat?: boolean
+  prefetch?: boolean
+  fileAttachmentsCount?: number
+  resourceAttachmentsCount?: number
+  contextsCount?: number
+  commandsCount?: number
+  pendingStreamWaitMs?: number
+  interruptedPriorStream?: boolean
+}
+
+interface CopilotOtelRoot {
+  span: Span
+  context: Context
+  /**
+   * Finalize the root span. `cancelReason`, when provided, decides
+   * whether a `cancelled` outcome leaves span status UNSET (for
+   * explicit user stops — our single non-error cancel class) or
+   * escalates to ERROR (client disconnect, unknown, etc.). Omit it
+   * for non-cancellation outcomes.
+   */
+  finish: (
+    outcome?: CopilotLifecycleOutcome,
+    error?: unknown,
+    cancelReason?: CopilotRequestCancelReasonValue
+  ) => void
+  setInputMessages: (input: CopilotAgentInputMessages) => void
+  setOutputMessages: (output: CopilotAgentOutputMessages) => void
+  setRequestShape: (shape: CopilotOtelRequestShape) => void
+}
+
+export function startCopilotOtelRoot(
+  scope: CopilotOtelScope
+): CopilotOtelRoot & { requestId: string } {
+  // TRUE root — don't inherit from Next's HTTP handler span (the
+  // sampler drops those; we'd orphan the whole mothership tree).
+  const parentContext = ROOT_CONTEXT
+  // Start with a placeholder `requestId`, then overwrite using the
+  // span's actual trace ID so the UI copy-button value pastes
+  // directly into Grafana.
+  const span = getTracer().startSpan(
+    TraceSpan.GenAiAgentExecute,
+    { attributes: buildAgentSpanAttributes({ ...scope, requestId: '' }) },
+    parentContext
+  )
+  const carrierSpan = isValidSpanContext(span.spanContext())
+    ? span
+    : trace.wrapSpanContext(createFallbackSpanContext())
+  const spanContext = carrierSpan.spanContext()
+  const requestId =
+    scope.requestId ??
+    (spanContext.traceId && spanContext.traceId.length === 32 ? spanContext.traceId : '')
+  span.setAttribute(TraceAttr.RequestId, requestId)
+  span.setAttribute(TraceAttr.SimRequestId, requestId)
+  const rootContext = trace.setSpan(parentContext, carrierSpan)
+
+  let finished = false
+  const finish: CopilotOtelRoot['finish'] = (outcome, error, cancelReason) => {
+    if (finished) return
+    finished = true
+    const resolvedOutcome = outcome ?? RequestTraceV1Outcome.success
+    span.setAttribute(TraceAttr.CopilotRequestOutcome, resolvedOutcome)
+    // Policy: `explicit_stop` is the ONLY cancellation we treat as
+    // expected (status unset → dashboards see it as OK). Everything
+    // else — client_disconnect, unknown reason, bug-case cancels —
+    // escalates to ERROR so it shows up on error panels.
+    const isExplicitStop = cancelReason === CopilotRequestCancelReason.ExplicitStop
+    if (error) {
+      markSpanForError(span, error)
+      if (isExplicitStop || isExplicitUserStopError(error)) {
+        span.setStatus({ code: SpanStatusCode.OK })
+      }
+    } else if (resolvedOutcome === RequestTraceV1Outcome.success) {
+      span.setStatus({ code: SpanStatusCode.OK })
+    } else if (resolvedOutcome === RequestTraceV1Outcome.cancelled) {
+      if (isExplicitStop) {
+        span.setStatus({ code: SpanStatusCode.OK })
+      } else {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: `cancelled: ${cancelReason ?? 'unknown'}`,
+        })
+      }
+    }
+    span.end()
+  }
+
+  return {
+    span,
+    context: rootContext,
+    requestId,
+    finish,
+    setInputMessages: (input) => setAgentInputMessages(span, input),
+    setOutputMessages: (output) => setAgentOutputMessages(span, output),
+    setRequestShape: (shape) => applyRequestShape(span, shape),
+  }
+}
+
+// Pending-stream-lock wait above this = inferred send-to-interrupt.
+const INTERRUPT_WAIT_MS_THRESHOLD = 50
+
+function applyRequestShape(span: Span, shape: CopilotOtelRequestShape): void {
+  if (shape.branchKind) {
+    span.setAttribute(TraceAttr.CopilotBranchKind, shape.branchKind)
+    span.setAttribute(
+      TraceAttr.CopilotSurface,
+      shape.branchKind === CopilotBranchKind.Workflow
+        ? CopilotSurface.Copilot
+        : CopilotSurface.Mothership
+    )
+  }
+  if (shape.mode) span.setAttribute(TraceAttr.CopilotMode, shape.mode)
+  if (shape.model) span.setAttribute(TraceAttr.GenAiRequestModel, shape.model)
+  if (shape.provider) span.setAttribute(TraceAttr.GenAiSystem, shape.provider)
+  if (typeof shape.createNewChat === 'boolean') {
+    span.setAttribute(TraceAttr.CopilotChatIsNew, shape.createNewChat)
+  }
+  if (typeof shape.prefetch === 'boolean') {
+    span.setAttribute(TraceAttr.CopilotPrefetch, shape.prefetch)
+  }
+  if (typeof shape.fileAttachmentsCount === 'number') {
+    span.setAttribute(TraceAttr.CopilotFileAttachmentsCount, shape.fileAttachmentsCount)
+  }
+  if (typeof shape.resourceAttachmentsCount === 'number') {
+    span.setAttribute(TraceAttr.CopilotResourceAttachmentsCount, shape.resourceAttachmentsCount)
+  }
+  if (typeof shape.contextsCount === 'number') {
+    span.setAttribute(TraceAttr.CopilotContextsCount, shape.contextsCount)
+  }
+  if (typeof shape.commandsCount === 'number') {
+    span.setAttribute(TraceAttr.CopilotCommandsCount, shape.commandsCount)
+  }
+  if (typeof shape.pendingStreamWaitMs === 'number') {
+    span.setAttribute(TraceAttr.CopilotPendingStreamWaitMs, shape.pendingStreamWaitMs)
+    const interrupted =
+      typeof shape.interruptedPriorStream === 'boolean'
+        ? shape.interruptedPriorStream
+        : shape.pendingStreamWaitMs > INTERRUPT_WAIT_MS_THRESHOLD
+    span.setAttribute(TraceAttr.CopilotInterruptedPriorStream, interrupted)
+  } else if (typeof shape.interruptedPriorStream === 'boolean') {
+    span.setAttribute(TraceAttr.CopilotInterruptedPriorStream, shape.interruptedPriorStream)
+  }
+}
+
+export async function withCopilotOtelContext<T>(
+  scope: CopilotOtelScope,
+  fn: (otelContext: Context) => Promise<T>
+): Promise<T> {
+  const parentContext = context.active()
+  // Same trace-id-derives-requestId dance as startCopilotOtelRoot — see
+  // that function for the rationale. Stamp a placeholder, read the real
+  // trace ID off the span, then overwrite.
+  const span = getTracer().startSpan(
+    TraceSpan.GenAiAgentExecute,
+    { attributes: buildAgentSpanAttributes({ ...scope, requestId: scope.requestId ?? '' }) },
+    parentContext
+  )
+  const carrierSpan = isValidSpanContext(span.spanContext())
+    ? span
+    : trace.wrapSpanContext(createFallbackSpanContext())
+  const spanContext = carrierSpan.spanContext()
+  const resolvedRequestId =
+    scope.requestId ??
+    (spanContext.traceId && spanContext.traceId.length === 32 ? spanContext.traceId : '')
+  if (resolvedRequestId) {
+    span.setAttribute(TraceAttr.RequestId, resolvedRequestId)
+    span.setAttribute(TraceAttr.SimRequestId, resolvedRequestId)
+  }
+  const otelContext = trace.setSpan(parentContext, carrierSpan)
+  let terminalStatusSet = false
+
+  try {
+    const result = await context.with(otelContext, () => fn(otelContext))
+    span.setStatus({ code: SpanStatusCode.OK })
+    terminalStatusSet = true
+    return result
+  } catch (error) {
+    markSpanForError(span, error)
+    terminalStatusSet = true
+    throw error
+  } finally {
+    if (!terminalStatusSet) {
+      // Extremely defensive: should be unreachable, but avoids leaking
+      // an unset span status if some future refactor breaks both arms.
+      span.setStatus({ code: SpanStatusCode.OK })
+    }
+    span.end()
+  }
+}

apps/sim/lib/copilot/request/session/abort-reason.ts

@@ -0,0 +1,52 @@
+/**
+ * Abort-reason vocabulary for Sim-originated cancellations.
+ *
+ * This is deliberately a zero-dependency module (no OTel, no logger,
+ * no DB) so it can be imported from both the telemetry layer
+ * (`request/otel.ts`) and the abort layer (`request/session/abort.ts`)
+ * without creating a circular dependency. The longer prose lives in
+ * `abort.ts`; anything here is the raw classification vocabulary
+ * consumed by span-status / finalizer code.
+ */
+
+/**
+ * Reason strings passed to `AbortController.abort(reason)` for every
+ * Sim-originated cancel path.
+ */
+export const AbortReason = {
+  /** Same-process stop: browser→Sim→abortActiveStream. */
+  UserStop: 'user_stop:abortActiveStream',
+  /**
+   * Cross-process stop: the Sim node that holds the SSE didn't
+   * receive the Stop HTTP call, but it polled the Redis abort marker
+   * that the node that DID receive it wrote, and aborts on the poll.
+   */
+  RedisPoller: 'redis_abort_marker:poller',
+  /**
+   * Cross-process stop: same root cause as `RedisPoller`, but observed
+   * by `runStreamLoop` at body close (the Go body ended before the
+   * 250ms poller's next tick) rather than by the polling timer.
+   */
+  MarkerObservedAtBodyClose: 'redis_abort_marker:body_close',
+  /** Internal timeout on the outbound explicit-abort fetch to Go. */
+  ExplicitAbortFetchTimeout: 'timeout:go_explicit_abort_fetch',
+} as const
+
+export type AbortReasonValue = (typeof AbortReason)[keyof typeof AbortReason]
+
+/**
+ * True iff `reason` indicates the user explicitly triggered the abort
+ * (as opposed to an implicit client disconnect or server timeout).
+ * Treated as a small closed vocabulary — any string not in
+ * `AbortReason` is presumed non-explicit. This is the canonical
+ * "should I treat this cancellation as expected?" predicate: span
+ * status-setters consult it to suppress ERROR only for user-initiated
+ * stops, mirroring `requestctx.IsExplicitUserStop` on the Go side.
+ */
+export function isExplicitStopReason(reason: unknown): boolean {
+  return (
+    reason === AbortReason.UserStop ||
+    reason === AbortReason.RedisPoller ||
+    reason === AbortReason.MarkerObservedAtBodyClose
+  )
+}

apps/sim/lib/copilot/request/session/abort.test.ts

@@ -0,0 +1,256 @@
+/**
+ * @vitest-environment node
+ */
+
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockHasAbortMarker, mockClearAbortMarker, mockWriteAbortMarker } = vi.hoisted(() => ({
+  mockHasAbortMarker: vi.fn().mockResolvedValue(false),
+  mockClearAbortMarker: vi.fn().mockResolvedValue(undefined),
+  mockWriteAbortMarker: vi.fn().mockResolvedValue(undefined),
+}))
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+vi.mock('@/lib/copilot/request/session/buffer', () => ({
+  hasAbortMarker: mockHasAbortMarker,
+  clearAbortMarker: mockClearAbortMarker,
+  writeAbortMarker: mockWriteAbortMarker,
+}))
+vi.mock('@/lib/copilot/request/otel', () => ({
+  withCopilotSpan: (_span: unknown, _attrs: unknown, fn: (span: unknown) => unknown) =>
+    fn({ setAttribute: vi.fn() }),
+}))
+
+import {
+  acquirePendingChatStream,
+  getChatStreamLockOwners,
+  releasePendingChatStream,
+  startAbortPoller,
+} from '@/lib/copilot/request/session/abort'
+
+describe('startAbortPoller heartbeat', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.useFakeTimers()
+    mockHasAbortMarker.mockResolvedValue(false)
+    redisConfigMockFns.mockExtendLock.mockResolvedValue(true)
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('extends the chat stream lock approximately every heartbeat interval', async () => {
+    const controller = new AbortController()
+    const streamId = 'stream-heartbeat-1'
+    const chatId = 'chat-heartbeat-1'
+
+    const interval = startAbortPoller(streamId, controller, { chatId })
+
+    try {
+      await vi.advanceTimersByTimeAsync(15_000)
+      expect(redisConfigMockFns.mockExtendLock).not.toHaveBeenCalled()
+
+      await vi.advanceTimersByTimeAsync(6_000)
+
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(1)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenLastCalledWith(
+        `copilot:chat-stream-lock:${chatId}`,
+        streamId,
+        60
+      )
+
+      await vi.advanceTimersByTimeAsync(20_000)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(2)
+
+      await vi.advanceTimersByTimeAsync(20_000)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(3)
+    } finally {
+      clearInterval(interval)
+    }
+  })
+
+  it('does not extend the lock when no chatId is passed (backward compat)', async () => {
+    const controller = new AbortController()
+    const interval = startAbortPoller('stream-no-chat', controller, {})
+
+    try {
+      await vi.advanceTimersByTimeAsync(90_000)
+      expect(redisConfigMockFns.mockExtendLock).not.toHaveBeenCalled()
+    } finally {
+      clearInterval(interval)
+    }
+  })
+
+  it('retries on the next tick when extendLock throws (no 20s backoff)', async () => {
+    const controller = new AbortController()
+    const streamId = 'stream-retry'
+    const chatId = 'chat-retry'
+
+    redisConfigMockFns.mockExtendLock.mockRejectedValueOnce(new Error('redis down'))
+
+    const interval = startAbortPoller(streamId, controller, { chatId })
+
+    try {
+      await vi.advanceTimersByTimeAsync(20_000)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(1)
+
+      await vi.advanceTimersByTimeAsync(1_000)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(2)
+    } finally {
+      clearInterval(interval)
+    }
+  })
+
+  it('aborts the controller before clearing the marker so the marker is never observable as cleared while the signal is still unaborted', async () => {
+    const controller = new AbortController()
+    const streamId = 'stream-order-1'
+
+    let signalAbortedWhenMarkerCleared: boolean | null = null
+    mockClearAbortMarker.mockImplementationOnce(async () => {
+      signalAbortedWhenMarkerCleared = controller.signal.aborted
+    })
+    mockHasAbortMarker.mockResolvedValueOnce(true)
+
+    const interval = startAbortPoller(streamId, controller, {})
+
+    try {
+      await vi.advanceTimersByTimeAsync(300)
+
+      expect(mockClearAbortMarker).toHaveBeenCalledWith(streamId)
+      expect(signalAbortedWhenMarkerCleared).toBe(true)
+      expect(controller.signal.aborted).toBe(true)
+    } finally {
+      clearInterval(interval)
+    }
+  })
+
+  it('does not clear the marker when the signal is already aborted (no double abort)', async () => {
+    const controller = new AbortController()
+    controller.abort('preexisting')
+    const streamId = 'stream-order-2'
+
+    mockHasAbortMarker.mockResolvedValueOnce(true)
+
+    const interval = startAbortPoller(streamId, controller, {})
+
+    try {
+      await vi.advanceTimersByTimeAsync(300)
+
+      expect(mockClearAbortMarker).not.toHaveBeenCalled()
+    } finally {
+      clearInterval(interval)
+    }
+  })
+
+  it('stops heartbeating after ownership is lost', async () => {
+    const controller = new AbortController()
+    const streamId = 'stream-lost'
+    const chatId = 'chat-lost'
+
+    redisConfigMockFns.mockExtendLock.mockResolvedValueOnce(false)
+
+    const interval = startAbortPoller(streamId, controller, { chatId })
+
+    try {
+      await vi.advanceTimersByTimeAsync(21_000)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(1)
+
+      await vi.advanceTimersByTimeAsync(60_000)
+      expect(redisConfigMockFns.mockExtendLock).toHaveBeenCalledTimes(1)
+    } finally {
+      clearInterval(interval)
+    }
+  })
+})
+
+describe('getChatStreamLockOwners', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue(null)
+  })
+
+  it('returns a verified empty owner map when no chat ids are provided', async () => {
+    const result = await getChatStreamLockOwners([])
+    expect(result.status).toBe('verified')
+    expect(result.ownersByChatId.size).toBe(0)
+  })
+
+  it('returns Redis lock owners keyed by chat id', async () => {
+    const mget = vi.fn().mockResolvedValue(['stream-1', null, 'stream-3'])
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never)
+
+    const result = await getChatStreamLockOwners(['chat-1', 'chat-2', 'chat-3'])
+
+    expect(mget).toHaveBeenCalledWith([
+      'copilot:chat-stream-lock:chat-1',
+      'copilot:chat-stream-lock:chat-2',
+      'copilot:chat-stream-lock:chat-3',
+    ])
+    expect(result.status).toBe('verified')
+    expect(result.ownersByChatId).toEqual(
+      new Map([
+        ['chat-1', 'stream-1'],
+        ['chat-3', 'stream-3'],
+      ])
+    )
+  })
+
+  it('returns a verified empty map when every lock has expired in Redis', async () => {
+    const mget = vi.fn().mockResolvedValue([null, null])
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never)
+
+    const result = await getChatStreamLockOwners(['chat-stuck-1', 'chat-stuck-2'])
+
+    expect(result.status).toBe('verified')
+    expect(result.ownersByChatId.size).toBe(0)
+  })
+
+  it('trusts verified Redis null over a process-local pending stream', async () => {
+    const mget = vi.fn().mockResolvedValue([null])
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never)
+    await acquirePendingChatStream('chat-local', 'stream-local')
+
+    try {
+      const result = await getChatStreamLockOwners(['chat-local'])
+
+      expect(result.status).toBe('verified')
+      expect(result.ownersByChatId.size).toBe(0)
+    } finally {
+      await releasePendingChatStream('chat-local', 'stream-local')
+    }
+  })
+
+  it('returns unknown status when Redis is unavailable', async () => {
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue(null)
+
+    const result = await getChatStreamLockOwners(['chat-1', 'chat-2'])
+
+    expect(result.status).toBe('unknown')
+    expect(result.ownersByChatId.size).toBe(0)
+  })
+
+  it('preserves local pending stream owners when Redis is unavailable', async () => {
+    await acquirePendingChatStream('chat-local', 'stream-local')
+
+    try {
+      const result = await getChatStreamLockOwners(['chat-local', 'chat-remote'])
+
+      expect(result.status).toBe('unknown')
+      expect(result.ownersByChatId).toEqual(new Map([['chat-local', 'stream-local']]))
+    } finally {
+      await releasePendingChatStream('chat-local', 'stream-local')
+    }
+  })
+
+  it('returns unknown status without throwing when mget rejects', async () => {
+    const mget = vi.fn().mockRejectedValue(new Error('redis down'))
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never)
+
+    const result = await getChatStreamLockOwners(['chat-1', 'chat-2'])
+
+    expect(result.status).toBe('unknown')
+    expect(result.ownersByChatId.size).toBe(0)
+  })
+})

apps/sim/lib/copilot/request/session/abort.ts

@@ -0,0 +1,398 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { AbortBackend } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { withCopilotSpan } from '@/lib/copilot/request/otel'
+import { acquireLock, extendLock, getRedisClient, releaseLock } from '@/lib/core/config/redis'
+import { AbortReason } from './abort-reason'
+import { clearAbortMarker, hasAbortMarker, writeAbortMarker } from './buffer'
+
+const logger = createLogger('SessionAbort')
+
+const activeStreams = new Map<string, AbortController>()
+const pendingChatStreams = new Map<
+  string,
+  { promise: Promise<void>; resolve: () => void; streamId: string }
+>()
+
+const DEFAULT_ABORT_POLL_MS = 250
+
+/**
+ * TTL for the per-chat stream lock. Kept short so that if the Sim pod
+ * holding the lock dies (SIGKILL, OOM, a SIGTERM drain that doesn't
+ * reach the release path), the lock self-heals inside a minute rather
+ * than stranding the chat for hours. A live stream keeps the lock alive
+ * via `CHAT_STREAM_LOCK_HEARTBEAT_INTERVAL_MS` heartbeats.
+ */
+const CHAT_STREAM_LOCK_TTL_SECONDS = 60
+
+/**
+ * Heartbeat cadence for extending the per-chat stream lock. Set to a
+ * third of the TTL so one missed beat still leaves room for recovery
+ * before the lock expires under a still-live stream.
+ */
+const CHAT_STREAM_LOCK_HEARTBEAT_INTERVAL_MS = 20_000
+
+export interface ChatStreamLockOwnersResult {
+  status: 'verified' | 'unknown'
+  ownersByChatId: Map<string, string>
+}
+
+function registerPendingChatStream(chatId: string, streamId: string): void {
+  let resolve!: () => void
+  const promise = new Promise<void>((r) => {
+    resolve = r
+  })
+  pendingChatStreams.set(chatId, { promise, resolve, streamId })
+}
+
+function resolvePendingChatStream(chatId: string, streamId: string): void {
+  const entry = pendingChatStreams.get(chatId)
+  if (entry && entry.streamId === streamId) {
+    entry.resolve()
+    pendingChatStreams.delete(chatId)
+  }
+}
+
+function getChatStreamLockKey(chatId: string): string {
+  return `copilot:chat-stream-lock:${chatId}`
+}
+
+export function registerActiveStream(streamId: string, controller: AbortController): void {
+  activeStreams.set(streamId, controller)
+}
+
+export function unregisterActiveStream(streamId: string): void {
+  activeStreams.delete(streamId)
+}
+
+export async function waitForPendingChatStream(
+  chatId: string,
+  timeoutMs = 5_000,
+  expectedStreamId?: string
+): Promise<boolean> {
+  const redis = getRedisClient()
+  const deadline = Date.now() + timeoutMs
+
+  for (;;) {
+    const entry = pendingChatStreams.get(chatId)
+    const localPending = !!entry && (!expectedStreamId || entry.streamId === expectedStreamId)
+
+    if (redis) {
+      try {
+        const ownerStreamId = await redis.get(getChatStreamLockKey(chatId))
+        const lockReleased =
+          !ownerStreamId || (expectedStreamId !== undefined && ownerStreamId !== expectedStreamId)
+        if (!localPending && lockReleased) {
+          return true
+        }
+      } catch (error) {
+        logger.warn('Failed to inspect chat stream lock while waiting', {
+          chatId,
+          expectedStreamId,
+          error: toError(error).message,
+        })
+      }
+    } else if (!localPending) {
+      return true
+    }
+
+    if (Date.now() >= deadline) {
+      return false
+    }
+    await sleep(200)
+  }
+}
+
+export async function getPendingChatStreamId(chatId: string): Promise<string | null> {
+  const localEntry = pendingChatStreams.get(chatId)
+  if (localEntry?.streamId) {
+    return localEntry.streamId
+  }
+
+  const redis = getRedisClient()
+  if (!redis) {
+    return null
+  }
+
+  try {
+    return (await redis.get(getChatStreamLockKey(chatId))) || null
+  } catch (error) {
+    logger.warn('Failed to load chat stream lock owner', {
+      chatId,
+      error: toError(error).message,
+    })
+    return null
+  }
+}
+
+/**
+ * Loads canonical stream lock owners for chat IDs.
+ *
+ * `status: 'verified'` means Redis was queried successfully, so a missing
+ * owner is authoritative. `status: 'unknown'` means only the process-local
+ * pending map is known, which is not enough to declare remote streams inactive
+ * in a multi-pod deployment.
+ */
+export async function getChatStreamLockOwners(
+  chatIds: string[]
+): Promise<ChatStreamLockOwnersResult> {
+  const localOwnersByChatId = new Map<string, string>()
+  if (chatIds.length === 0) {
+    return { status: 'verified', ownersByChatId: localOwnersByChatId }
+  }
+
+  for (const chatId of chatIds) {
+    const entry = pendingChatStreams.get(chatId)
+    if (entry?.streamId) localOwnersByChatId.set(chatId, entry.streamId)
+  }
+
+  const redis = getRedisClient()
+  if (!redis) {
+    return { status: 'unknown', ownersByChatId: localOwnersByChatId }
+  }
+
+  try {
+    const keys = chatIds.map(getChatStreamLockKey)
+    const values = await redis.mget(keys)
+    const redisOwnersByChatId = new Map<string, string>()
+    for (let i = 0; i < chatIds.length; i++) {
+      const owner = values[i]
+      if (owner) redisOwnersByChatId.set(chatIds[i], owner)
+    }
+    return { status: 'verified', ownersByChatId: redisOwnersByChatId }
+  } catch (error) {
+    logger.warn('Failed to load chat stream lock owners (batch)', {
+      count: chatIds.length,
+      error: toError(error).message,
+    })
+    return { status: 'unknown', ownersByChatId: localOwnersByChatId }
+  }
+}
+
+export async function releasePendingChatStream(chatId: string, streamId: string): Promise<void> {
+  try {
+    await releaseLock(getChatStreamLockKey(chatId), streamId)
+  } catch (error) {
+    logger.warn('Failed to release chat stream lock', {
+      chatId,
+      streamId,
+      error: toError(error).message,
+    })
+  } finally {
+    resolvePendingChatStream(chatId, streamId)
+  }
+}
+
+export async function acquirePendingChatStream(
+  chatId: string,
+  streamId: string,
+  timeoutMs = 5_000
+): Promise<boolean> {
+  // Span records wall time spent waiting for the per-chat stream lock.
+  // Typical case: sub-10ms uncontested acquire. Worst case: up to
+  // `timeoutMs` spent polling while a prior stream finishes. Previously
+  // this time looked like "unexplained gap before llm.stream".
+  return withCopilotSpan(
+    TraceSpan.CopilotChatAcquirePendingStreamLock,
+    {
+      [TraceAttr.ChatId]: chatId,
+      [TraceAttr.StreamId]: streamId,
+      [TraceAttr.LockTimeoutMs]: timeoutMs,
+    },
+    async (span) => {
+      const redis = getRedisClient()
+      span.setAttribute(TraceAttr.LockBackend, redis ? AbortBackend.Redis : AbortBackend.InProcess)
+      if (redis) {
+        const deadline = Date.now() + timeoutMs
+        for (;;) {
+          try {
+            const acquired = await acquireLock(
+              getChatStreamLockKey(chatId),
+              streamId,
+              CHAT_STREAM_LOCK_TTL_SECONDS
+            )
+            if (acquired) {
+              registerPendingChatStream(chatId, streamId)
+              span.setAttribute(TraceAttr.LockAcquired, true)
+              return true
+            }
+            if (!pendingChatStreams.has(chatId)) {
+              const ownerStreamId = await redis.get(getChatStreamLockKey(chatId))
+              if (ownerStreamId) {
+                const settled = await waitForPendingChatStream(chatId, 0, ownerStreamId)
+                if (settled) {
+                  continue
+                }
+              }
+            }
+          } catch (error) {
+            logger.warn('Failed to acquire chat stream lock', {
+              chatId,
+              streamId,
+              error: toError(error).message,
+            })
+          }
+
+          if (Date.now() >= deadline) {
+            span.setAttribute(TraceAttr.LockAcquired, false)
+            span.setAttribute(TraceAttr.LockTimedOut, true)
+            return false
+          }
+          await sleep(200)
+        }
+      }
+
+      for (;;) {
+        const existing = pendingChatStreams.get(chatId)
+        if (!existing) {
+          registerPendingChatStream(chatId, streamId)
+          span.setAttribute(TraceAttr.LockAcquired, true)
+          return true
+        }
+
+        const settled = await Promise.race([
+          existing.promise.then(() => true),
+          new Promise<boolean>((resolve) => setTimeout(() => resolve(false), timeoutMs)),
+        ])
+        if (!settled) {
+          span.setAttribute(TraceAttr.LockAcquired, false)
+          span.setAttribute(TraceAttr.LockTimedOut, true)
+          return false
+        }
+      }
+    }
+  )
+}
+
+/**
+ * Returns `true` if it aborted an in-process controller,
+ * `false` if it only wrote the marker (no local controller found).
+ *
+ * Spanned because the two operations inside can stall independently
+ * — Redis latency on `writeAbortMarker` was previously invisible, and
+ * the "no local controller" branch (happens when the stream handler
+ * is on a different Sim box than the one receiving /chat/abort) is
+ * a subtle but important outcome to distinguish from "aborted a live
+ * controller" in dashboards.
+ */
+export async function abortActiveStream(streamId: string): Promise<boolean> {
+  return withCopilotSpan(
+    TraceSpan.CopilotChatAbortActiveStream,
+    { [TraceAttr.StreamId]: streamId },
+    async (span) => {
+      await writeAbortMarker(streamId)
+      span.setAttribute(TraceAttr.CopilotAbortMarkerWritten, true)
+      const controller = activeStreams.get(streamId)
+      if (!controller) {
+        span.setAttribute(TraceAttr.CopilotAbortControllerFired, false)
+        return false
+      }
+      controller.abort(AbortReason.UserStop)
+      activeStreams.delete(streamId)
+      span.setAttribute(TraceAttr.CopilotAbortControllerFired, true)
+      return true
+    }
+  )
+}
+
+export type { AbortReasonValue } from './abort-reason'
+/**
+ * `AbortReason` vocabulary and the `isExplicitStopReason` classifier
+ * live in a sibling zero-dependency module so the telemetry layer
+ * (`request/otel.ts`) can import them without creating a circular
+ * import back through `session/abort.ts`'s OTel-wrapped helpers.
+ *
+ * Context on why the distinction matters: when the user clicks Stop,
+ * we fire `abortController.abort(AbortReason.UserStop)` from
+ * `abortActiveStream()`. That causes Sim's SSE writer to close,
+ * which in turn makes the BROWSER's SSE reader see the stream end
+ * — which fires the browser-side fetch AbortController and
+ * propagates back to Sim as `publisher.markDisconnected()`. So on
+ * an explicit Stop you observe BOTH "explicit reason" AND
+ * "client disconnected" — the discriminator is the reason string,
+ * not the client flag.
+ *
+ * For any NEW abort path, add its reason in `./abort-reason.ts` and
+ * update `isExplicitStopReason` if it should be classified as a user
+ * stop.
+ */
+export { AbortReason, isExplicitStopReason } from './abort-reason'
+
+const pollingStreams = new Set<string>()
+
+export function startAbortPoller(
+  streamId: string,
+  abortController: AbortController,
+  options?: { pollMs?: number; requestId?: string; chatId?: string }
+): ReturnType<typeof setInterval> {
+  const pollMs = options?.pollMs ?? DEFAULT_ABORT_POLL_MS
+  const requestId = options?.requestId
+  const chatId = options?.chatId
+
+  let lastHeartbeatAt = Date.now()
+  let heartbeatOwnershipLost = false
+
+  return setInterval(() => {
+    if (pollingStreams.has(streamId)) return
+    pollingStreams.add(streamId)
+
+    void (async () => {
+      try {
+        const shouldAbort = await hasAbortMarker(streamId)
+        if (shouldAbort && !abortController.signal.aborted) {
+          abortController.abort(AbortReason.RedisPoller)
+          await clearAbortMarker(streamId)
+        }
+      } catch (error) {
+        logger.warn('Failed to poll stream abort marker', {
+          streamId,
+          ...(requestId ? { requestId } : {}),
+          error: toError(error).message,
+        })
+      } finally {
+        pollingStreams.delete(streamId)
+      }
+
+      if (!chatId || heartbeatOwnershipLost) return
+      if (Date.now() - lastHeartbeatAt < CHAT_STREAM_LOCK_HEARTBEAT_INTERVAL_MS) return
+
+      try {
+        const owned = await extendLock(
+          getChatStreamLockKey(chatId),
+          streamId,
+          CHAT_STREAM_LOCK_TTL_SECONDS
+        )
+        lastHeartbeatAt = Date.now()
+        if (!owned) {
+          heartbeatOwnershipLost = true
+          logger.warn('Lost ownership of chat stream lock — stopping heartbeat', {
+            chatId,
+            streamId,
+            ...(requestId ? { requestId } : {}),
+          })
+        }
+      } catch (error) {
+        logger.warn('Failed to extend chat stream lock TTL', {
+          chatId,
+          streamId,
+          ...(requestId ? { requestId } : {}),
+          error: toError(error).message,
+        })
+      }
+    })()
+  }, pollMs)
+}
+
+export async function cleanupAbortMarker(streamId: string): Promise<void> {
+  try {
+    await clearAbortMarker(streamId)
+  } catch (error) {
+    logger.warn('Failed to clear stream abort marker during cleanup', {
+      streamId,
+      error: toError(error).message,
+    })
+  }
+}

apps/sim/lib/copilot/request/session/buffer.test.ts

@@ -0,0 +1,250 @@
+/**
+ * @vitest-environment node
+ */
+
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1TextChannel,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { createEvent } from '@/lib/copilot/request/session/event'
+
+type StoredEnvelope = {
+  score: number
+  value: string
+}
+
+const createRedisStub = () => {
+  const counters = new Map<string, number>()
+  const values = new Map<string, string>()
+  const sortedSets = new Map<string, StoredEnvelope[]>()
+
+  const api = {
+    incr: vi.fn().mockImplementation((key: string) => {
+      const next = (counters.get(key) ?? 0) + 1
+      counters.set(key, next)
+      return next
+    }),
+    expire: vi.fn().mockResolvedValue(1),
+    del: vi.fn().mockImplementation((...keys: string[]) => {
+      for (const key of keys) {
+        values.delete(key)
+        sortedSets.delete(key)
+        counters.delete(key)
+      }
+      return Promise.resolve(keys.length)
+    }),
+    zadd: vi.fn().mockImplementation((key: string, score: number, value: string) => {
+      const entries = sortedSets.get(key) ?? []
+      entries.push({ score, value })
+      sortedSets.set(key, entries)
+      return Promise.resolve(1)
+    }),
+    zremrangebyrank: vi.fn().mockImplementation((key: string, start: number, stop: number) => {
+      const entries = [...(sortedSets.get(key) ?? [])].sort((a, b) => a.score - b.score)
+      const normalizedStart = start < 0 ? Math.max(entries.length + start, 0) : start
+      const normalizedStop = stop < 0 ? entries.length + stop : stop
+      const next = entries.filter(
+        (_entry, index) => index < normalizedStart || index > normalizedStop
+      )
+      sortedSets.set(key, next)
+      return Promise.resolve(1)
+    }),
+    zrangebyscore: vi.fn().mockImplementation((key: string, min: number, max: string) => {
+      const upperBound = max === '+inf' ? Number.POSITIVE_INFINITY : Number(max)
+      const entries = [...(sortedSets.get(key) ?? [])]
+        .filter((entry) => entry.score >= min && entry.score <= upperBound)
+        .sort((a, b) => a.score - b.score)
+        .map((entry) => entry.value)
+      return Promise.resolve(entries)
+    }),
+    set: vi.fn().mockImplementation((key: string, value: string) => {
+      values.set(key, value)
+      return Promise.resolve('OK')
+    }),
+    get: vi.fn().mockImplementation((key: string) => Promise.resolve(values.get(key) ?? null)),
+    pipeline: vi.fn().mockImplementation(() => {
+      const operations: Array<() => Promise<unknown>> = []
+      const pipeline = {
+        zadd: (...args: [string, number, string]) => {
+          operations.push(() => api.zadd(...args))
+          return pipeline
+        },
+        expire: (...args: [string, number]) => {
+          operations.push(() => api.expire(...args))
+          return pipeline
+        },
+        set: (...args: [string, string, 'EX', number]) => {
+          operations.push(() => api.set(args[0], args[1]))
+          return pipeline
+        },
+        zremrangebyrank: (...args: [string, number, number]) => {
+          operations.push(() => api.zremrangebyrank(...args))
+          return pipeline
+        },
+        exec: vi.fn().mockImplementation(async () => {
+          const results: Array<[null, unknown]> = []
+          for (const operation of operations) {
+            results.push([null, await operation()])
+          }
+          return results
+        }),
+      }
+      return pipeline
+    }),
+  }
+
+  return api
+}
+
+let mockRedis: ReturnType<typeof createRedisStub>
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+
+import {
+  allocateCursor,
+  appendEvent,
+  clearBuffer,
+  readEvents,
+  scheduleBufferCleanup,
+} from '@/lib/copilot/request/session/buffer'
+
+describe('mothership-stream-outbox', () => {
+  beforeEach(() => {
+    mockRedis = createRedisStub()
+    vi.clearAllMocks()
+    redisConfigMockFns.mockGetRedisClient.mockImplementation(() => mockRedis)
+  })
+
+  it('replays envelopes after a given cursor', async () => {
+    const firstCursor = await allocateCursor('stream-1')
+    const secondCursor = await allocateCursor('stream-1')
+
+    await appendEvent(
+      createEvent({
+        streamId: 'stream-1',
+        cursor: firstCursor.cursor,
+        seq: firstCursor.seq,
+        requestId: 'req-1',
+        type: MothershipStreamV1EventType.text,
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'hello' },
+      })
+    )
+    await appendEvent(
+      createEvent({
+        streamId: 'stream-1',
+        cursor: secondCursor.cursor,
+        seq: secondCursor.seq,
+        requestId: 'req-1',
+        type: MothershipStreamV1EventType.text,
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'world' },
+      })
+    )
+
+    const allEvents = await readEvents('stream-1', '0')
+    expect(allEvents.map((entry) => entry.payload.text)).toEqual(['hello', 'world'])
+
+    const replayed = await readEvents('stream-1', '1')
+    expect(replayed.map((entry) => entry.payload.text)).toEqual(['world'])
+  })
+
+  it('trims active stream history to eventLimit on every append', async () => {
+    const cursor = await allocateCursor('stream-1')
+
+    await appendEvent(
+      createEvent({
+        streamId: 'stream-1',
+        cursor: cursor.cursor,
+        seq: cursor.seq,
+        requestId: 'req-1',
+        type: MothershipStreamV1EventType.text,
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'hello' },
+      })
+    )
+
+    expect(mockRedis.zremrangebyrank).toHaveBeenCalledWith(
+      'mothership_stream:stream-1:events',
+      0,
+      -100_001
+    )
+  })
+
+  it('clears persisted stream state during teardown cleanup', async () => {
+    const cursor = await allocateCursor('stream-1')
+
+    await appendEvent(
+      createEvent({
+        streamId: 'stream-1',
+        cursor: cursor.cursor,
+        seq: cursor.seq,
+        requestId: 'req-1',
+        type: MothershipStreamV1EventType.text,
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'hello' },
+      })
+    )
+
+    expect((await readEvents('stream-1', '0')).length).toBe(1)
+
+    await clearBuffer('stream-1')
+
+    expect(await readEvents('stream-1', '0')).toEqual([])
+  })
+
+  it('shortens completed stream retention without deleting replay data immediately', async () => {
+    const cursor = await allocateCursor('stream-1')
+
+    await appendEvent(
+      createEvent({
+        streamId: 'stream-1',
+        cursor: cursor.cursor,
+        seq: cursor.seq,
+        requestId: 'req-1',
+        type: MothershipStreamV1EventType.text,
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'hello' },
+      })
+    )
+
+    await scheduleBufferCleanup('stream-1', 30)
+
+    expect(mockRedis.expire).toHaveBeenCalledWith('mothership_stream:stream-1:events', 30)
+    expect(mockRedis.expire).toHaveBeenCalledWith('mothership_stream:stream-1:seq', 30)
+    expect(mockRedis.expire).toHaveBeenCalledWith('mothership_stream:stream-1:abort', 30)
+    expect((await readEvents('stream-1', '0')).map((entry) => entry.payload.text)).toEqual([
+      'hello',
+    ])
+  })
+
+  it('skips corrupt replay entries that fail stream validation', async () => {
+    const cursor = await allocateCursor('stream-1')
+
+    await appendEvent(
+      createEvent({
+        streamId: 'stream-1',
+        cursor: cursor.cursor,
+        seq: cursor.seq,
+        requestId: 'req-1',
+        type: MothershipStreamV1EventType.text,
+        payload: { channel: MothershipStreamV1TextChannel.assistant, text: 'hello' },
+      })
+    )
+
+    await mockRedis.zadd(
+      'mothership_stream:stream-1:events',
+      cursor.seq + 1,
+      JSON.stringify({
+        v: 1,
+        type: 'tool',
+        seq: cursor.seq + 1,
+        ts: '2026-04-11T00:00:00.000Z',
+        stream: { streamId: 'stream-1' },
+        payload: { toolCallId: 'broken-tool' },
+      })
+    )
+
+    const replayed = await readEvents('stream-1', '0')
+
+    expect(replayed).toHaveLength(1)
+    expect(replayed[0]?.payload.text).toBe('hello')
+  })
+})

apps/sim/lib/copilot/request/session/buffer.ts

@@ -0,0 +1,249 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { env, envNumber } from '@/lib/core/config/env'
+import { getRedisClient } from '@/lib/core/config/redis'
+import {
+  type PersistedStreamEventEnvelope,
+  parsePersistedStreamEventEnvelopeJson,
+} from './contract'
+
+const logger = createLogger('SessionBuffer')
+
+const STREAM_OUTBOX_PREFIX = 'mothership_stream:'
+const DEFAULT_TTL_SECONDS = 60 * 60
+const DEFAULT_COMPLETED_TTL_SECONDS = 5 * 60
+const DEFAULT_EVENT_LIMIT = 100_000
+const RETRY_DELAYS_MS = [0, 50, 150] as const
+
+type RedisOperationMetadata = {
+  operation: string
+  streamId: string
+}
+
+function getEventsKey(streamId: string) {
+  return `${STREAM_OUTBOX_PREFIX}${streamId}:events`
+}
+
+function getSeqKey(streamId: string) {
+  return `${STREAM_OUTBOX_PREFIX}${streamId}:seq`
+}
+
+function getAbortKey(streamId: string) {
+  return `${STREAM_OUTBOX_PREFIX}${streamId}:abort`
+}
+
+export type StreamConfig = {
+  ttlSeconds: number
+  eventLimit: number
+}
+
+export function getStreamConfig(): StreamConfig {
+  return {
+    ttlSeconds: envNumber(env.COPILOT_STREAM_TTL_SECONDS, DEFAULT_TTL_SECONDS, { min: 1 }),
+    eventLimit: envNumber(env.COPILOT_STREAM_EVENT_LIMIT, DEFAULT_EVENT_LIMIT, { min: 1 }),
+  }
+}
+
+async function withRedisRetry<T>(
+  metadata: RedisOperationMetadata,
+  operation: (redis: NonNullable<ReturnType<typeof getRedisClient>>) => Promise<T>
+): Promise<T> {
+  const redis = getRedisClient()
+  if (!redis) {
+    throw new Error('Redis is required for mothership stream durability')
+  }
+
+  let lastError: unknown
+
+  for (let attempt = 0; attempt < RETRY_DELAYS_MS.length; attempt++) {
+    const delay = RETRY_DELAYS_MS[attempt]
+    if (delay > 0) {
+      await sleep(delay)
+    }
+
+    try {
+      return await operation(redis)
+    } catch (error) {
+      lastError = error
+      logger.warn('Redis stream operation failed', {
+        operation: metadata.operation,
+        streamId: metadata.streamId,
+        attempt: attempt + 1,
+        error: toError(error).message,
+      })
+    }
+  }
+
+  throw lastError instanceof Error
+    ? lastError
+    : new Error(`${metadata.operation} failed for stream ${metadata.streamId}`)
+}
+
+export async function allocateCursor(streamId: string): Promise<{
+  seq: number
+  cursor: string
+}> {
+  const config = getStreamConfig()
+  const seq = await withRedisRetry({ operation: 'allocate_cursor', streamId }, async (redis) => {
+    const nextValue = await redis.incr(getSeqKey(streamId))
+    await redis.expire(getSeqKey(streamId), config.ttlSeconds)
+    return typeof nextValue === 'number' ? nextValue : Number(nextValue)
+  })
+
+  return { seq, cursor: String(seq) }
+}
+
+export async function resetBuffer(streamId: string): Promise<void> {
+  await clearBuffer(streamId, 'reset_outbox')
+}
+
+export async function clearBuffer(streamId: string, operation = 'clear_outbox'): Promise<void> {
+  await withRedisRetry({ operation, streamId }, async (redis) => {
+    await redis.del(getEventsKey(streamId), getSeqKey(streamId), getAbortKey(streamId))
+  })
+}
+
+export async function scheduleBufferCleanup(
+  streamId: string,
+  ttlSeconds = DEFAULT_COMPLETED_TTL_SECONDS
+): Promise<void> {
+  try {
+    await withRedisRetry({ operation: 'schedule_outbox_cleanup', streamId }, async (redis) => {
+      const pipeline = redis.pipeline()
+      pipeline.expire(getEventsKey(streamId), ttlSeconds)
+      pipeline.expire(getSeqKey(streamId), ttlSeconds)
+      pipeline.expire(getAbortKey(streamId), ttlSeconds)
+      await pipeline.exec()
+    })
+  } catch (error) {
+    logger.warn('Failed to shorten stream buffer TTL during cleanup', {
+      streamId,
+      ttlSeconds,
+      error: toError(error).message,
+    })
+  }
+}
+
+export async function appendEvents(
+  envelopes: PersistedStreamEventEnvelope[]
+): Promise<PersistedStreamEventEnvelope[]> {
+  if (envelopes.length === 0) {
+    return envelopes
+  }
+
+  const streamId = envelopes[0].stream.streamId
+  const config = getStreamConfig()
+
+  await withRedisRetry({ operation: 'append_event', streamId }, async (redis) => {
+    const key = getEventsKey(streamId)
+    const seqKey = getSeqKey(streamId)
+    const pipeline = redis.pipeline()
+    const zaddArgs: Array<number | string> = []
+    for (const envelope of envelopes) {
+      zaddArgs.push(envelope.seq, JSON.stringify(envelope))
+    }
+    pipeline.zadd(key, ...(zaddArgs as [number, string, ...Array<number | string>]))
+    pipeline.zremrangebyrank(key, 0, -config.eventLimit - 1)
+    pipeline.expire(key, config.ttlSeconds)
+    pipeline.set(seqKey, String(envelopes[envelopes.length - 1].seq), 'EX', config.ttlSeconds)
+    await pipeline.exec()
+  })
+
+  return envelopes
+}
+
+export async function appendEvent(
+  envelope: PersistedStreamEventEnvelope
+): Promise<PersistedStreamEventEnvelope> {
+  await appendEvents([envelope])
+  return envelope
+}
+
+export class InvalidCursorError extends Error {
+  constructor(
+    public readonly streamId: string,
+    public readonly cursor: string
+  ) {
+    super(`Invalid non-numeric cursor "${cursor}" for stream ${streamId}`)
+    this.name = 'InvalidCursorError'
+  }
+}
+
+export async function readEvents(
+  streamId: string,
+  afterCursor: string
+): Promise<PersistedStreamEventEnvelope[]> {
+  const afterSeq = Number(afterCursor || '0')
+  if (!Number.isFinite(afterSeq)) {
+    throw new InvalidCursorError(streamId, afterCursor)
+  }
+  const minScore = afterSeq + 1
+
+  const rawEntries = await withRedisRetry({ operation: 'read_events', streamId }, async (redis) => {
+    return redis.zrangebyscore(getEventsKey(streamId), minScore, '+inf')
+  })
+
+  const envelopes: PersistedStreamEventEnvelope[] = []
+  for (const entry of rawEntries) {
+    const parsed = parsePersistedStreamEventEnvelopeJson(entry)
+    if (!parsed.ok) {
+      logger.warn('Skipping corrupt outbox entry', {
+        streamId,
+        reason: parsed.reason,
+        message: parsed.message,
+        errors: parsed.errors,
+      })
+      continue
+    }
+    envelopes.push(parsed.event)
+  }
+  return envelopes
+}
+
+export async function getOldestSeq(streamId: string): Promise<number | null> {
+  return withRedisRetry({ operation: 'get_oldest_seq', streamId }, async (redis) => {
+    const entries = await redis.zrangebyscore(getEventsKey(streamId), '-inf', '+inf', 'LIMIT', 0, 1)
+    if (!entries || entries.length === 0) {
+      return null
+    }
+    try {
+      const parsed = JSON.parse(entries[0]) as { seq?: number }
+      return typeof parsed.seq === 'number' ? parsed.seq : null
+    } catch {
+      logger.warn('Failed to parse oldest outbox entry', { streamId })
+      return null
+    }
+  })
+}
+
+export async function getLatestSeq(streamId: string): Promise<number | null> {
+  return withRedisRetry({ operation: 'get_latest_seq', streamId }, async (redis) => {
+    const currentSeq = await redis.get(getSeqKey(streamId))
+    if (currentSeq === null) {
+      return null
+    }
+    const parsed = Number(currentSeq)
+    return Number.isFinite(parsed) ? parsed : null
+  })
+}
+
+export async function writeAbortMarker(streamId: string): Promise<void> {
+  const ttlSeconds = getStreamConfig().ttlSeconds
+  await withRedisRetry({ operation: 'write_abort_marker', streamId }, async (redis) => {
+    await redis.set(getAbortKey(streamId), '1', 'EX', ttlSeconds)
+  })
+}
+
+export async function hasAbortMarker(streamId: string): Promise<boolean> {
+  return withRedisRetry({ operation: 'read_abort_marker', streamId }, async (redis) => {
+    const marker = await redis.get(getAbortKey(streamId))
+    return marker === '1'
+  })
+}
+
+export async function clearAbortMarker(streamId: string): Promise<void> {
+  await withRedisRetry({ operation: 'clear_abort_marker', streamId }, async (redis) => {
+    await redis.del(getAbortKey(streamId))
+  })
+}

apps/sim/lib/copilot/request/session/contract.test.ts

@@ -0,0 +1,216 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import {
+  isContractStreamEventEnvelope,
+  isSyntheticFilePreviewEventEnvelope,
+  parsePersistedStreamEventEnvelope,
+  parsePersistedStreamEventEnvelopeJson,
+} from './contract'
+
+const BASE_ENVELOPE = {
+  v: 1 as const,
+  seq: 1,
+  ts: '2026-04-11T00:00:00.000Z',
+  stream: {
+    streamId: 'stream-1',
+    cursor: '1',
+  },
+  trace: {
+    requestId: 'req-1',
+  },
+}
+
+describe('stream session contract parser', () => {
+  it('accepts contract text events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      trace: {
+        ...BASE_ENVELOPE.trace,
+        goTraceId: 'go-trace-1',
+      },
+      type: 'text' as const,
+      payload: {
+        channel: 'assistant' as const,
+        text: 'hello',
+      },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+
+    const parsed = parsePersistedStreamEventEnvelope(event)
+    expect(parsed).toEqual({
+      ok: true,
+      event,
+    })
+  })
+
+  it('accepts contract session chat events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'session' as const,
+      payload: { kind: 'chat' as const, chatId: 'chat-1' },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts contract complete events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'complete' as const,
+      payload: { status: 'complete' as const },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts contract error events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'error' as const,
+      payload: { message: 'something went wrong' },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts contract tool call events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'tool' as const,
+      payload: {
+        toolCallId: 'tc-1',
+        toolName: 'read',
+        phase: 'call' as const,
+        executor: 'sim' as const,
+        mode: 'sync' as const,
+      },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts contract span events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'span' as const,
+      payload: {
+        kind: 'subagent' as const,
+        event: 'start' as const,
+        agent: 'file',
+      },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts contract resource events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'resource' as const,
+      payload: {
+        op: 'upsert' as const,
+        resource: { id: 'r-1', type: 'file', title: 'test.md' },
+      },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts contract run events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'run' as const,
+      payload: { kind: 'compaction_start' as const },
+    }
+
+    expect(isContractStreamEventEnvelope(event)).toBe(true)
+    expect(parsePersistedStreamEventEnvelope(event).ok).toBe(true)
+  })
+
+  it('accepts synthetic file preview events', () => {
+    const event = {
+      ...BASE_ENVELOPE,
+      type: 'tool' as const,
+      payload: {
+        toolCallId: 'preview-1',
+        toolName: 'workspace_file' as const,
+        previewPhase: 'file_preview_content' as const,
+        content: 'draft body',
+        contentMode: 'snapshot' as const,
+        previewVersion: 2,
+        fileName: 'draft.md',
+      },
+    }
+
+    expect(isSyntheticFilePreviewEventEnvelope(event)).toBe(true)
+
+    const parsed = parsePersistedStreamEventEnvelope(event)
+    expect(parsed).toEqual({
+      ok: true,
+      event,
+    })
+  })
+
+  it('rejects invalid tool events with structured validation errors', () => {
+    const parsed = parsePersistedStreamEventEnvelope({
+      ...BASE_ENVELOPE,
+      type: 'tool',
+      payload: {
+        toolCallId: 'tool-1',
+        toolName: 'read',
+      },
+    })
+
+    expect(parsed.ok).toBe(false)
+    if (parsed.ok) {
+      throw new Error('expected invalid result')
+    }
+    expect(parsed.reason).toBe('invalid_stream_event')
+  })
+
+  it('rejects unknown event types', () => {
+    const parsed = parsePersistedStreamEventEnvelope({
+      ...BASE_ENVELOPE,
+      type: 'unknown_type',
+      payload: {},
+    })
+
+    expect(parsed.ok).toBe(false)
+    if (parsed.ok) {
+      throw new Error('expected invalid result')
+    }
+    expect(parsed.reason).toBe('invalid_stream_event')
+    expect(parsed.errors).toContain('unknown type="unknown_type"')
+  })
+
+  it('rejects non-object values', () => {
+    const parsed = parsePersistedStreamEventEnvelope('not an object')
+
+    expect(parsed.ok).toBe(false)
+    if (parsed.ok) {
+      throw new Error('expected invalid result')
+    }
+    expect(parsed.reason).toBe('invalid_stream_event')
+    expect(parsed.errors).toContain('value is not an object')
+  })
+
+  it('reports invalid JSON separately from schema failures', () => {
+    const parsed = parsePersistedStreamEventEnvelopeJson('{')
+
+    expect(parsed.ok).toBe(false)
+    if (parsed.ok) {
+      throw new Error('expected invalid json result')
+    }
+    expect(parsed.reason).toBe('invalid_json')
+  })
+})

apps/sim/lib/copilot/request/session/contract.ts

@@ -0,0 +1,475 @@
+import { getErrorMessage } from '@sim/utils/errors'
+import { isRecordLike } from '@sim/utils/object'
+import type {
+  MothershipStreamV1EventEnvelope,
+  MothershipStreamV1StreamRef,
+  MothershipStreamV1StreamScope,
+  MothershipStreamV1Trace,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1ResourceOp,
+  MothershipStreamV1RunKind,
+  MothershipStreamV1SessionKind,
+  MothershipStreamV1SpanPayloadKind,
+  MothershipStreamV1TextChannel,
+  MothershipStreamV1ToolPhase,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import type { FilePreviewTargetKind } from './file-preview-session-contract'
+
+type JsonRecord = Record<string, unknown>
+
+const FILE_PREVIEW_PHASE = {
+  start: 'file_preview_start',
+  target: 'file_preview_target',
+  editMeta: 'file_preview_edit_meta',
+  content: 'file_preview_content',
+  complete: 'file_preview_complete',
+} as const
+
+type EnvelopeToStreamEvent<T> = T extends {
+  type: infer TType
+  payload: infer TPayload
+  scope?: infer TScope
+}
+  ? { type: TType; payload: TPayload; scope?: Exclude<TScope, undefined> }
+  : never
+
+export type SyntheticFilePreviewPhase = (typeof FILE_PREVIEW_PHASE)[keyof typeof FILE_PREVIEW_PHASE]
+
+export interface SyntheticFilePreviewTarget {
+  kind: FilePreviewTargetKind
+  fileId?: string
+  fileName?: string
+}
+
+export interface SyntheticFilePreviewStartPayload {
+  previewPhase: typeof FILE_PREVIEW_PHASE.start
+  toolCallId: string
+  toolName: 'workspace_file'
+}
+
+export interface SyntheticFilePreviewTargetPayload {
+  operation?: string
+  previewPhase: typeof FILE_PREVIEW_PHASE.target
+  target: SyntheticFilePreviewTarget
+  title?: string
+  toolCallId: string
+  toolName: 'workspace_file'
+}
+
+export interface SyntheticFilePreviewEditMetaPayload {
+  edit: JsonRecord
+  previewPhase: typeof FILE_PREVIEW_PHASE.editMeta
+  toolCallId: string
+  toolName: 'workspace_file'
+}
+
+export interface SyntheticFilePreviewContentPayload {
+  content: string
+  contentMode: 'delta' | 'snapshot'
+  edit?: JsonRecord
+  fileId?: string
+  fileName: string
+  operation?: string
+  previewPhase: typeof FILE_PREVIEW_PHASE.content
+  previewVersion: number
+  targetKind?: string
+  toolCallId: string
+  toolName: 'workspace_file'
+}
+
+export interface SyntheticFilePreviewCompletePayload {
+  fileId?: string
+  output?: unknown
+  previewPhase: typeof FILE_PREVIEW_PHASE.complete
+  previewVersion?: number
+  toolCallId: string
+  toolName: 'workspace_file'
+}
+
+export type SyntheticFilePreviewPayload =
+  | SyntheticFilePreviewStartPayload
+  | SyntheticFilePreviewTargetPayload
+  | SyntheticFilePreviewEditMetaPayload
+  | SyntheticFilePreviewContentPayload
+  | SyntheticFilePreviewCompletePayload
+
+export interface SyntheticFilePreviewEventEnvelope {
+  payload: SyntheticFilePreviewPayload
+  scope?: MothershipStreamV1StreamScope
+  seq: number
+  stream: MothershipStreamV1StreamRef
+  trace?: MothershipStreamV1Trace
+  ts: string
+  type: 'tool'
+  v: 1
+}
+
+export type PersistedStreamEventEnvelope =
+  | MothershipStreamV1EventEnvelope
+  | SyntheticFilePreviewEventEnvelope
+
+export type ContractStreamEvent = EnvelopeToStreamEvent<MothershipStreamV1EventEnvelope>
+export type SyntheticStreamEvent = EnvelopeToStreamEvent<SyntheticFilePreviewEventEnvelope>
+export type SessionStreamEvent = ContractStreamEvent | SyntheticStreamEvent
+export type StreamEvent = SessionStreamEvent
+export type ToolCallStreamEvent = Extract<
+  ContractStreamEvent,
+  { type: 'tool'; payload: { phase: 'call' } }
+>
+export type ToolArgsDeltaStreamEvent = Extract<
+  ContractStreamEvent,
+  { type: 'tool'; payload: { phase: 'args_delta' } }
+>
+export type ToolResultStreamEvent = Extract<
+  ContractStreamEvent,
+  { type: 'tool'; payload: { phase: 'result' } }
+>
+export type SubagentSpanStreamEvent = Extract<
+  ContractStreamEvent,
+  { type: 'span'; payload: { kind: 'subagent' } }
+>
+
+export interface ParseStreamEventEnvelopeSuccess {
+  ok: true
+  event: PersistedStreamEventEnvelope
+}
+
+export interface ParseStreamEventEnvelopeFailure {
+  errors?: string[]
+  message: string
+  ok: false
+  reason: 'invalid_json' | 'invalid_stream_event'
+}
+
+export type ParseStreamEventEnvelopeResult =
+  | ParseStreamEventEnvelopeSuccess
+  | ParseStreamEventEnvelopeFailure
+
+// ---------------------------------------------------------------------------
+// Structural helpers (CSP-safe – no codegen / eval / new Function)
+// ---------------------------------------------------------------------------
+
+function isOptionalString(value: unknown): value is string | undefined {
+  return value === undefined || typeof value === 'string'
+}
+
+function isOptionalFiniteNumber(value: unknown): value is number | undefined {
+  return value === undefined || (typeof value === 'number' && Number.isFinite(value))
+}
+
+function isStreamRef(value: unknown): value is MothershipStreamV1StreamRef {
+  return (
+    isRecordLike(value) &&
+    typeof value.streamId === 'string' &&
+    value.streamId.length > 0 &&
+    isOptionalString(value.chatId) &&
+    isOptionalString(value.cursor)
+  )
+}
+
+function isTrace(value: unknown): value is MothershipStreamV1Trace {
+  return (
+    isRecordLike(value) &&
+    typeof value.requestId === 'string' &&
+    isOptionalString(value.goTraceId) &&
+    isOptionalString(value.spanId)
+  )
+}
+
+function isStreamScope(value: unknown): value is MothershipStreamV1StreamScope {
+  return (
+    isRecordLike(value) &&
+    value.lane === 'subagent' &&
+    isOptionalString(value.agentId) &&
+    isOptionalString(value.parentToolCallId)
+  )
+}
+
+// ---------------------------------------------------------------------------
+// Contract envelope validator (replaces Ajv runtime compilation)
+//
+// Validates the envelope shell (v, seq, ts, stream, trace?, scope?) and that
+// `type` is one of the known event types with a non-null payload object.
+// Per-payload-variant validation is intentionally lightweight: the server
+// already performs strict schema validation; the client only needs enough
+// structural checking to safely dispatch inside the switch statement.
+// ---------------------------------------------------------------------------
+
+const KNOWN_EVENT_TYPES: ReadonlySet<string> = new Set(Object.values(MothershipStreamV1EventType))
+
+function isValidEnvelopeShell(value: unknown): value is JsonRecord & {
+  v: 1
+  seq: number
+  ts: string
+  stream: MothershipStreamV1StreamRef
+  type: string
+  payload: JsonRecord
+} {
+  if (!isRecordLike(value)) return false
+  if (value.v !== 1) return false
+  if (typeof value.seq !== 'number' || !Number.isFinite(value.seq)) return false
+  if (typeof value.ts !== 'string') return false
+  if (!isStreamRef(value.stream)) return false
+  if (value.trace !== undefined && !isTrace(value.trace)) return false
+  if (value.scope !== undefined && !isStreamScope(value.scope)) return false
+  if (typeof value.type !== 'string' || !KNOWN_EVENT_TYPES.has(value.type)) return false
+  if (!isRecordLike(value.payload)) return false
+  return true
+}
+
+function isValidSessionPayload(payload: JsonRecord): boolean {
+  const kind = payload.kind
+  if (typeof kind !== 'string') return false
+  switch (kind) {
+    case MothershipStreamV1SessionKind.start:
+      return true
+    case MothershipStreamV1SessionKind.chat:
+      return typeof payload.chatId === 'string'
+    case MothershipStreamV1SessionKind.title:
+      return typeof payload.title === 'string'
+    case MothershipStreamV1SessionKind.trace:
+      return typeof payload.requestId === 'string'
+    default:
+      return false
+  }
+}
+
+function isValidTextPayload(payload: JsonRecord): boolean {
+  return (
+    (payload.channel === MothershipStreamV1TextChannel.assistant ||
+      payload.channel === MothershipStreamV1TextChannel.thinking) &&
+    typeof payload.text === 'string'
+  )
+}
+
+function isValidToolPayload(payload: JsonRecord): boolean {
+  if (typeof payload.toolCallId !== 'string') return false
+  if (typeof payload.toolName !== 'string') return false
+  const phase = payload.phase
+  return (
+    phase === MothershipStreamV1ToolPhase.call ||
+    phase === MothershipStreamV1ToolPhase.args_delta ||
+    phase === MothershipStreamV1ToolPhase.result
+  )
+}
+
+function isValidSpanPayload(payload: JsonRecord): boolean {
+  const kind = payload.kind
+  return (
+    kind === MothershipStreamV1SpanPayloadKind.subagent ||
+    kind === MothershipStreamV1SpanPayloadKind.structured_result ||
+    kind === MothershipStreamV1SpanPayloadKind.subagent_result
+  )
+}
+
+function isValidResourcePayload(payload: JsonRecord): boolean {
+  return (
+    (payload.op === MothershipStreamV1ResourceOp.upsert ||
+      payload.op === MothershipStreamV1ResourceOp.remove) &&
+    isRecordLike(payload.resource) &&
+    typeof (payload.resource as JsonRecord).id === 'string' &&
+    typeof (payload.resource as JsonRecord).type === 'string'
+  )
+}
+
+function isValidRunPayload(payload: JsonRecord): boolean {
+  const kind = payload.kind
+  return (
+    kind === MothershipStreamV1RunKind.checkpoint_pause ||
+    kind === MothershipStreamV1RunKind.resumed ||
+    kind === MothershipStreamV1RunKind.compaction_start ||
+    kind === MothershipStreamV1RunKind.compaction_done
+  )
+}
+
+function isValidErrorPayload(payload: JsonRecord): boolean {
+  return typeof payload.message === 'string' || typeof payload.error === 'string'
+}
+
+function isValidCompletePayload(payload: JsonRecord): boolean {
+  return typeof payload.status === 'string'
+}
+
+function isContractEnvelope(value: unknown): value is MothershipStreamV1EventEnvelope {
+  if (!isValidEnvelopeShell(value)) return false
+  const payload = value.payload as JsonRecord
+  switch (value.type) {
+    case MothershipStreamV1EventType.session:
+      return isValidSessionPayload(payload)
+    case MothershipStreamV1EventType.text:
+      return isValidTextPayload(payload)
+    case MothershipStreamV1EventType.tool:
+      return isValidToolPayload(payload)
+    case MothershipStreamV1EventType.span:
+      return isValidSpanPayload(payload)
+    case MothershipStreamV1EventType.resource:
+      return isValidResourcePayload(payload)
+    case MothershipStreamV1EventType.run:
+      return isValidRunPayload(payload)
+    case MothershipStreamV1EventType.error:
+      return isValidErrorPayload(payload)
+    case MothershipStreamV1EventType.complete:
+      return isValidCompletePayload(payload)
+    default:
+      return false
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Synthetic file-preview envelope validators
+// ---------------------------------------------------------------------------
+
+function isSyntheticEnvelopeBase(value: unknown): value is Omit<
+  SyntheticFilePreviewEventEnvelope,
+  'payload'
+> & {
+  payload?: unknown
+} {
+  return (
+    isRecordLike(value) &&
+    value.v === 1 &&
+    value.type === 'tool' &&
+    typeof value.seq === 'number' &&
+    Number.isFinite(value.seq) &&
+    typeof value.ts === 'string' &&
+    isStreamRef(value.stream) &&
+    (value.trace === undefined || isTrace(value.trace)) &&
+    (value.scope === undefined || isStreamScope(value.scope))
+  )
+}
+
+function isSyntheticFilePreviewTarget(value: unknown): value is SyntheticFilePreviewTarget {
+  return (
+    isRecordLike(value) &&
+    (value.kind === 'new_file' || value.kind === 'file_id') &&
+    isOptionalString(value.fileId) &&
+    isOptionalString(value.fileName)
+  )
+}
+
+function isSyntheticFilePreviewPayload(value: unknown): value is SyntheticFilePreviewPayload {
+  if (!isRecordLike(value)) {
+    return false
+  }
+
+  if (typeof value.toolCallId !== 'string' || value.toolName !== 'workspace_file') {
+    return false
+  }
+
+  switch (value.previewPhase) {
+    case FILE_PREVIEW_PHASE.start:
+      return true
+    case FILE_PREVIEW_PHASE.target:
+      return (
+        isSyntheticFilePreviewTarget(value.target) &&
+        isOptionalString(value.operation) &&
+        isOptionalString(value.title)
+      )
+    case FILE_PREVIEW_PHASE.editMeta:
+      return isRecordLike(value.edit)
+    case FILE_PREVIEW_PHASE.content:
+      return (
+        typeof value.content === 'string' &&
+        (value.contentMode === 'delta' || value.contentMode === 'snapshot') &&
+        typeof value.previewVersion === 'number' &&
+        Number.isFinite(value.previewVersion) &&
+        typeof value.fileName === 'string' &&
+        isOptionalString(value.fileId) &&
+        isOptionalString(value.targetKind) &&
+        isOptionalString(value.operation) &&
+        (value.edit === undefined || isRecordLike(value.edit))
+      )
+    case FILE_PREVIEW_PHASE.complete:
+      return isOptionalString(value.fileId) && isOptionalFiniteNumber(value.previewVersion)
+    default:
+      return false
+  }
+}
+
+export function isSyntheticFilePreviewEventEnvelope(
+  value: unknown
+): value is SyntheticFilePreviewEventEnvelope {
+  return isSyntheticEnvelopeBase(value) && isSyntheticFilePreviewPayload(value.payload)
+}
+
+// ---------------------------------------------------------------------------
+// Stream event type guards
+// ---------------------------------------------------------------------------
+
+export function isToolCallStreamEvent(event: SessionStreamEvent): event is ToolCallStreamEvent {
+  return event.type === 'tool' && isRecordLike(event.payload) && event.payload.phase === 'call'
+}
+
+export function isToolArgsDeltaStreamEvent(
+  event: SessionStreamEvent
+): event is ToolArgsDeltaStreamEvent {
+  return (
+    event.type === 'tool' && isRecordLike(event.payload) && event.payload.phase === 'args_delta'
+  )
+}
+
+export function isToolResultStreamEvent(event: SessionStreamEvent): event is ToolResultStreamEvent {
+  return event.type === 'tool' && isRecordLike(event.payload) && event.payload.phase === 'result'
+}
+
+export function isSubagentSpanStreamEvent(
+  event: SessionStreamEvent
+): event is SubagentSpanStreamEvent {
+  return event.type === 'span' && isRecordLike(event.payload) && event.payload.kind === 'subagent'
+}
+
+// ---------------------------------------------------------------------------
+// Public contract validators & parsers
+// ---------------------------------------------------------------------------
+
+export function isContractStreamEventEnvelope(
+  value: unknown
+): value is MothershipStreamV1EventEnvelope {
+  return isContractEnvelope(value)
+}
+
+export function parsePersistedStreamEventEnvelope(value: unknown): ParseStreamEventEnvelopeResult {
+  if (isContractEnvelope(value)) {
+    return { ok: true, event: value }
+  }
+
+  if (isSyntheticFilePreviewEventEnvelope(value)) {
+    return { ok: true, event: value }
+  }
+
+  const hints: string[] = []
+  if (!isRecordLike(value)) {
+    hints.push('value is not an object')
+  } else {
+    if (value.v !== 1) hints.push(`unexpected v=${JSON.stringify(value.v)}`)
+    if (typeof value.type !== 'string') hints.push('missing type')
+    else if (!KNOWN_EVENT_TYPES.has(value.type)) hints.push(`unknown type="${value.type}"`)
+    if (!isRecordLike(value.payload)) hints.push('missing or invalid payload')
+  }
+
+  return {
+    ok: false,
+    reason: 'invalid_stream_event',
+    message: 'A stream event failed validation.',
+    ...(hints.length > 0 ? { errors: hints } : {}),
+  }
+}
+
+export function parsePersistedStreamEventEnvelopeJson(raw: string): ParseStreamEventEnvelopeResult {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw)
+  } catch (error) {
+    const rawMessage = getErrorMessage(error, 'Invalid JSON')
+    return {
+      ok: false,
+      reason: 'invalid_json',
+      message: 'Received invalid JSON while parsing a stream event.',
+      ...(rawMessage ? { errors: [rawMessage] } : {}),
+    }
+  }
+
+  return parsePersistedStreamEventEnvelope(parsed)
+}

apps/sim/lib/copilot/request/session/event.test.ts

@@ -0,0 +1,63 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1TextChannel,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { parsePersistedStreamEventEnvelope } from '@/lib/copilot/request/session'
+import { createEvent, eventToStreamEvent } from '@/lib/copilot/request/session/event'
+
+describe('createEvent', () => {
+  it('creates contract envelopes that pass validation', () => {
+    const envelope = createEvent({
+      streamId: 'stream-1',
+      cursor: '1',
+      seq: 1,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.text,
+      payload: {
+        channel: MothershipStreamV1TextChannel.assistant,
+        text: 'hello',
+      },
+    })
+
+    const parsed = parsePersistedStreamEventEnvelope(envelope)
+    expect(parsed.ok).toBe(true)
+    expect(envelope.type).toBe(MothershipStreamV1EventType.text)
+    expect(envelope.payload).toEqual({
+      channel: MothershipStreamV1TextChannel.assistant,
+      text: 'hello',
+    })
+  })
+
+  it('creates synthetic preview envelopes that round-trip to stream events', () => {
+    const envelope = createEvent({
+      streamId: 'stream-1',
+      cursor: '2',
+      seq: 2,
+      requestId: 'req-1',
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        previewPhase: 'file_preview_start',
+        toolCallId: 'preview-1',
+        toolName: 'workspace_file',
+      },
+    })
+
+    const parsed = parsePersistedStreamEventEnvelope(envelope)
+    expect(parsed.ok).toBe(true)
+
+    const streamEvent = eventToStreamEvent(envelope)
+    expect(streamEvent).toEqual({
+      type: MothershipStreamV1EventType.tool,
+      payload: {
+        previewPhase: 'file_preview_start',
+        toolCallId: 'preview-1',
+        toolName: 'workspace_file',
+      },
+    })
+  })
+})

apps/sim/lib/copilot/request/session/event.ts

@@ -0,0 +1,74 @@
+import {
+  type PersistedStreamEventEnvelope,
+  parsePersistedStreamEventEnvelope,
+  type SessionStreamEvent,
+} from './contract'
+
+type CreateEventBase = {
+  chatId?: string
+  cursor: string
+  requestId: string
+  seq: number
+  streamId: string
+  ts?: string
+}
+
+type CreateEventVariant<TEvent extends SessionStreamEvent> = CreateEventBase &
+  Pick<TEvent, 'type' | 'payload' | 'scope'>
+
+export type CreateEventInput = SessionStreamEvent extends infer TEvent
+  ? TEvent extends SessionStreamEvent
+    ? CreateEventVariant<TEvent>
+    : never
+  : never
+
+type CreateEventResult<TInput extends CreateEventInput> = Extract<
+  PersistedStreamEventEnvelope,
+  { type: TInput['type']; payload: TInput['payload'] }
+>
+
+type StreamEventFromEnvelope<TEnvelope extends PersistedStreamEventEnvelope> = Extract<
+  SessionStreamEvent,
+  { type: TEnvelope['type']; payload: TEnvelope['payload'] }
+>
+
+export const TOOL_CALL_STATUS = {
+  generating: 'generating',
+} as const
+
+export function createEvent<TInput extends CreateEventInput>(
+  input: TInput
+): CreateEventResult<TInput> {
+  const { streamId, chatId, cursor, seq, requestId, type, payload, scope, ts } = input
+
+  return {
+    v: 1,
+    type,
+    seq,
+    ts: ts ?? new Date().toISOString(),
+    stream: {
+      streamId,
+      ...(chatId ? { chatId } : {}),
+      cursor,
+    },
+    trace: {
+      requestId,
+    },
+    ...(scope ? { scope } : {}),
+    payload,
+  } as CreateEventResult<TInput>
+}
+
+export function isEventRecord(value: unknown): value is PersistedStreamEventEnvelope {
+  return parsePersistedStreamEventEnvelope(value).ok
+}
+
+export function eventToStreamEvent<TEnvelope extends PersistedStreamEventEnvelope>(
+  envelope: TEnvelope
+): StreamEventFromEnvelope<TEnvelope> {
+  return {
+    type: envelope.type,
+    payload: envelope.payload,
+    ...(envelope.scope ? { scope: envelope.scope } : {}),
+  } as StreamEventFromEnvelope<TEnvelope>
+}

apps/sim/lib/copilot/request/session/explicit-abort.ts

@@ -0,0 +1,68 @@
+import type { Context } from '@opentelemetry/api'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { fetchGo } from '@/lib/copilot/request/go/fetch'
+import { AbortReason } from '@/lib/copilot/request/session/abort'
+import { getMothershipBaseURL, getMothershipSourceEnvHeaders } from '@/lib/copilot/server/agent-url'
+import { env } from '@/lib/core/config/env'
+
+export const DEFAULT_EXPLICIT_ABORT_TIMEOUT_MS = 3000
+
+export async function requestExplicitStreamAbort(params: {
+  streamId: string
+  userId: string
+  chatId?: string
+  workspaceId?: string
+  timeoutMs?: number
+  otelContext?: Context
+}): Promise<void> {
+  const {
+    streamId,
+    userId,
+    chatId,
+    workspaceId,
+    timeoutMs = DEFAULT_EXPLICIT_ABORT_TIMEOUT_MS,
+    otelContext,
+  } = params
+
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+  }
+  if (env.COPILOT_API_KEY) {
+    headers['x-api-key'] = env.COPILOT_API_KEY
+  }
+  Object.assign(headers, getMothershipSourceEnvHeaders())
+
+  const controller = new AbortController()
+  const timeout = setTimeout(
+    () => controller.abort(AbortReason.ExplicitAbortFetchTimeout),
+    timeoutMs
+  )
+
+  try {
+    const mothershipBaseURL = await getMothershipBaseURL({ userId })
+    const response = await fetchGo(`${mothershipBaseURL}/api/streams/explicit-abort`, {
+      method: 'POST',
+      headers,
+      signal: controller.signal,
+      body: JSON.stringify({
+        messageId: streamId,
+        userId,
+        ...(chatId ? { chatId } : {}),
+        ...(workspaceId ? { workspaceId } : {}),
+      }),
+      otelContext,
+      spanName: 'sim → go /api/streams/explicit-abort',
+      operation: 'explicit_abort',
+      attributes: {
+        [TraceAttr.StreamId]: streamId,
+        ...(chatId ? { [TraceAttr.ChatId]: chatId } : {}),
+      },
+    })
+
+    if (!response.ok) {
+      throw new Error(`Explicit abort marker request failed: ${response.status}`)
+    }
+  } finally {
+    clearTimeout(timeout)
+  }
+}

apps/sim/lib/copilot/request/session/file-preview-session-contract.ts

@@ -0,0 +1,53 @@
+export const FILE_PREVIEW_SESSION_SCHEMA_VERSION = 1 as const
+
+export type FilePreviewTargetKind = 'new_file' | 'file_id'
+export type FilePreviewStatus = 'pending' | 'streaming' | 'complete'
+export type FilePreviewContentMode = 'delta' | 'snapshot'
+
+export interface FilePreviewSession {
+  schemaVersion: typeof FILE_PREVIEW_SESSION_SCHEMA_VERSION
+  id: string
+  streamId: string
+  toolCallId: string
+  status: FilePreviewStatus
+  fileName: string
+  fileId?: string
+  targetKind?: FilePreviewTargetKind
+  operation?: string
+  edit?: Record<string, unknown>
+  baseContent?: string
+  previewText: string
+  previewVersion: number
+  updatedAt: string
+  completedAt?: string
+}
+
+export function isFilePreviewSession(value: unknown): value is FilePreviewSession {
+  if (!value || typeof value !== 'object') {
+    return false
+  }
+
+  const record = value as Record<string, unknown>
+  return (
+    record.schemaVersion === FILE_PREVIEW_SESSION_SCHEMA_VERSION &&
+    typeof record.id === 'string' &&
+    typeof record.streamId === 'string' &&
+    typeof record.toolCallId === 'string' &&
+    typeof record.status === 'string' &&
+    typeof record.fileName === 'string' &&
+    (record.baseContent === undefined || typeof record.baseContent === 'string') &&
+    typeof record.previewText === 'string' &&
+    typeof record.previewVersion === 'number' &&
+    typeof record.updatedAt === 'string'
+  )
+}
+
+export function sortFilePreviewSessions(sessions: FilePreviewSession[]): FilePreviewSession[] {
+  return [...sessions].sort((a, b) => {
+    const updatedAtCompare = a.updatedAt.localeCompare(b.updatedAt)
+    if (updatedAtCompare !== 0) {
+      return updatedAtCompare
+    }
+    return a.id.localeCompare(b.id)
+  })
+}

apps/sim/lib/copilot/request/session/file-preview-session.test.ts

@@ -0,0 +1,42 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  createFilePreviewSession,
+  sortFilePreviewSessions,
+} from '@/lib/copilot/request/session/file-preview-session'
+
+describe('file preview session helpers', () => {
+  it('preserves baseContent when creating a preview session', () => {
+    const session = createFilePreviewSession({
+      streamId: 'stream-1',
+      toolCallId: 'preview-1',
+      fileName: 'draft.md',
+      baseContent: 'existing content',
+    })
+
+    expect(session.baseContent).toBe('existing content')
+  })
+
+  it('sorts preview sessions by updatedAt across tool call ids', () => {
+    const sessions = sortFilePreviewSessions([
+      createFilePreviewSession({
+        streamId: 'stream-1',
+        toolCallId: 'preview-2',
+        fileName: 'b.md',
+        previewVersion: 10,
+        updatedAt: '2026-04-10T00:00:02.000Z',
+      }),
+      createFilePreviewSession({
+        streamId: 'stream-1',
+        toolCallId: 'preview-1',
+        fileName: 'a.md',
+        previewVersion: 1,
+        updatedAt: '2026-04-10T00:00:01.000Z',
+      }),
+    ])
+
+    expect(sessions.map((session) => session.id)).toEqual(['preview-1', 'preview-2'])
+  })
+})

apps/sim/lib/copilot/request/session/file-preview-session.ts

@@ -0,0 +1,174 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { getRedisClient } from '@/lib/core/config/redis'
+import { getStreamConfig } from './buffer'
+import {
+  FILE_PREVIEW_SESSION_SCHEMA_VERSION,
+  type FilePreviewSession,
+  type FilePreviewStatus,
+  type FilePreviewTargetKind,
+  isFilePreviewSession,
+  sortFilePreviewSessions,
+} from './file-preview-session-contract'
+
+const logger = createLogger('FilePreviewSessionStore')
+
+const STREAM_OUTBOX_PREFIX = 'mothership_stream:'
+const DEFAULT_COMPLETED_TTL_SECONDS = 5 * 60
+const RETRY_DELAYS_MS = [0, 50, 150] as const
+
+export type {
+  FilePreviewSession,
+  FilePreviewStatus,
+  FilePreviewTargetKind,
+} from './file-preview-session-contract'
+export { sortFilePreviewSessions } from './file-preview-session-contract'
+
+function getPreviewSessionsKey(streamId: string): string {
+  return `${STREAM_OUTBOX_PREFIX}${streamId}:preview_sessions`
+}
+
+type RedisOperationMetadata = {
+  operation: string
+  streamId: string
+}
+
+async function withRedisRetry<T>(
+  metadata: RedisOperationMetadata,
+  operation: (redis: NonNullable<ReturnType<typeof getRedisClient>>) => Promise<T>
+): Promise<T> {
+  const redis = getRedisClient()
+  if (!redis) {
+    throw new Error('Redis is required for mothership preview durability')
+  }
+
+  let lastError: unknown
+
+  for (let attempt = 0; attempt < RETRY_DELAYS_MS.length; attempt++) {
+    const delay = RETRY_DELAYS_MS[attempt]
+    if (delay > 0) {
+      await sleep(delay)
+    }
+
+    try {
+      return await operation(redis)
+    } catch (error) {
+      lastError = error
+      logger.warn('Redis preview session operation failed', {
+        operation: metadata.operation,
+        streamId: metadata.streamId,
+        attempt: attempt + 1,
+        error: toError(error).message,
+      })
+    }
+  }
+
+  throw lastError instanceof Error
+    ? lastError
+    : new Error(`${metadata.operation} failed for stream ${metadata.streamId}`)
+}
+
+export function createFilePreviewSession(input: {
+  streamId: string
+  toolCallId: string
+  fileName?: string
+  fileId?: string
+  targetKind?: FilePreviewTargetKind
+  operation?: string
+  edit?: Record<string, unknown>
+  baseContent?: string
+  previewText?: string
+  previewVersion?: number
+  status?: FilePreviewStatus
+  updatedAt?: string
+  completedAt?: string
+}): FilePreviewSession {
+  return {
+    schemaVersion: FILE_PREVIEW_SESSION_SCHEMA_VERSION,
+    id: input.toolCallId,
+    streamId: input.streamId,
+    toolCallId: input.toolCallId,
+    status: input.status ?? 'pending',
+    fileName: input.fileName ?? '',
+    ...(input.fileId ? { fileId: input.fileId } : {}),
+    ...(input.targetKind ? { targetKind: input.targetKind } : {}),
+    ...(input.operation ? { operation: input.operation } : {}),
+    ...(input.edit ? { edit: input.edit } : {}),
+    ...(typeof input.baseContent === 'string' ? { baseContent: input.baseContent } : {}),
+    previewText: input.previewText ?? '',
+    previewVersion: input.previewVersion ?? 0,
+    updatedAt: input.updatedAt ?? new Date().toISOString(),
+    ...(input.completedAt ? { completedAt: input.completedAt } : {}),
+  }
+}
+
+export async function upsertFilePreviewSession(
+  session: FilePreviewSession
+): Promise<FilePreviewSession> {
+  const config = getStreamConfig()
+  await withRedisRetry(
+    { operation: 'upsert_preview_session', streamId: session.streamId },
+    async (redis) => {
+      const key = getPreviewSessionsKey(session.streamId)
+      const pipeline = redis.pipeline()
+      pipeline.hset(key, session.id, JSON.stringify(session))
+      pipeline.expire(key, config.ttlSeconds)
+      await pipeline.exec()
+    }
+  )
+  return session
+}
+
+export async function readFilePreviewSessions(streamId: string): Promise<FilePreviewSession[]> {
+  const raw = await withRedisRetry(
+    { operation: 'read_preview_sessions', streamId },
+    async (redis) => redis.hgetall(getPreviewSessionsKey(streamId))
+  )
+
+  const sessions: FilePreviewSession[] = []
+  const values = Object.values(raw ?? {})
+  for (const entry of values) {
+    try {
+      const parsed = JSON.parse(entry) as unknown
+      if (!isFilePreviewSession(parsed)) {
+        logger.warn('Skipping invalid file preview session entry', { streamId })
+        continue
+      }
+      sessions.push(parsed)
+    } catch (error) {
+      logger.warn('Failed to parse file preview session entry', {
+        streamId,
+        error: toError(error).message,
+      })
+    }
+  }
+
+  return sortFilePreviewSessions(sessions)
+}
+
+export async function clearFilePreviewSessions(streamId: string): Promise<void> {
+  await withRedisRetry({ operation: 'clear_preview_sessions', streamId }, async (redis) => {
+    await redis.del(getPreviewSessionsKey(streamId))
+  })
+}
+
+export async function scheduleFilePreviewSessionCleanup(
+  streamId: string,
+  ttlSeconds = DEFAULT_COMPLETED_TTL_SECONDS
+): Promise<void> {
+  try {
+    await withRedisRetry(
+      { operation: 'schedule_preview_session_cleanup', streamId },
+      async (redis) => {
+        await redis.expire(getPreviewSessionsKey(streamId), ttlSeconds)
+      }
+    )
+  } catch (error) {
+    logger.warn('Failed to shorten preview session retention', {
+      streamId,
+      ttlSeconds,
+      error: toError(error).message,
+    })
+  }
+}

apps/sim/lib/copilot/request/session/index.ts

@@ -0,0 +1,76 @@
+export type { ChatStreamLockOwnersResult } from './abort'
+export {
+  AbortReason,
+  type AbortReasonValue,
+  abortActiveStream,
+  acquirePendingChatStream,
+  cleanupAbortMarker,
+  getChatStreamLockOwners,
+  getPendingChatStreamId,
+  isExplicitStopReason,
+  registerActiveStream,
+  releasePendingChatStream,
+  startAbortPoller,
+  unregisterActiveStream,
+  waitForPendingChatStream,
+} from './abort'
+export {
+  allocateCursor,
+  appendEvent,
+  appendEvents,
+  clearAbortMarker,
+  clearBuffer,
+  getLatestSeq,
+  getOldestSeq,
+  hasAbortMarker,
+  InvalidCursorError,
+  readEvents,
+  resetBuffer,
+  scheduleBufferCleanup,
+  writeAbortMarker,
+} from './buffer'
+export type {
+  ContractStreamEvent,
+  PersistedStreamEventEnvelope,
+  SessionStreamEvent,
+  StreamEvent,
+  SubagentSpanStreamEvent,
+  SyntheticFilePreviewEventEnvelope,
+  SyntheticFilePreviewPayload,
+  SyntheticStreamEvent,
+  ToolArgsDeltaStreamEvent,
+  ToolCallStreamEvent,
+  ToolResultStreamEvent,
+} from './contract'
+export {
+  isContractStreamEventEnvelope,
+  isSubagentSpanStreamEvent,
+  isSyntheticFilePreviewEventEnvelope,
+  isToolArgsDeltaStreamEvent,
+  isToolCallStreamEvent,
+  isToolResultStreamEvent,
+  parsePersistedStreamEventEnvelope,
+  parsePersistedStreamEventEnvelopeJson,
+} from './contract'
+export { createEvent, eventToStreamEvent, isEventRecord, TOOL_CALL_STATUS } from './event'
+export {
+  clearFilePreviewSessions,
+  createFilePreviewSession,
+  readFilePreviewSessions,
+  scheduleFilePreviewSessionCleanup,
+  upsertFilePreviewSession,
+} from './file-preview-session'
+export type {
+  FilePreviewContentMode,
+  FilePreviewSession,
+  FilePreviewStatus,
+  FilePreviewTargetKind,
+} from './file-preview-session-contract'
+export {
+  FILE_PREVIEW_SESSION_SCHEMA_VERSION,
+  isFilePreviewSession,
+} from './file-preview-session-contract'
+export { checkForReplayGap, type ReplayGapResult } from './recovery'
+export { encodeSSEComment, encodeSSEEnvelope, SSE_RESPONSE_HEADERS } from './sse'
+export type { StreamBatchEvent } from './types'
+export { StreamWriter, type StreamWriterOptions } from './writer'

apps/sim/lib/copilot/request/session/recovery.test.ts

@@ -0,0 +1,38 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it, vi } from 'vitest'
+
+const { getLatestSeq, getOldestSeq, readEvents } = vi.hoisted(() => ({
+  getLatestSeq: vi.fn(),
+  getOldestSeq: vi.fn(),
+  readEvents: vi.fn(),
+}))
+
+vi.mock('./buffer', () => ({
+  getLatestSeq,
+  getOldestSeq,
+  readEvents,
+}))
+
+import { checkForReplayGap } from './recovery'
+
+describe('checkForReplayGap', () => {
+  it('uses the latest buffered request id when run metadata is missing it', async () => {
+    getOldestSeq.mockResolvedValue(10)
+    getLatestSeq.mockResolvedValue(12)
+    readEvents.mockResolvedValue([
+      {
+        trace: { requestId: 'req-live-123' },
+      },
+    ])
+
+    const result = await checkForReplayGap('stream-1', '1')
+
+    expect(readEvents).toHaveBeenCalledWith('stream-1', '11')
+    expect(result?.gapDetected).toBe(true)
+    expect(result?.envelopes[0].trace.requestId).toBe('req-live-123')
+    expect(result?.envelopes[1].trace.requestId).toBe('req-live-123')
+  })
+})

apps/sim/lib/copilot/request/session/recovery.ts

@@ -0,0 +1,124 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import { CopilotRecoveryOutcome } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import { withCopilotSpan } from '@/lib/copilot/request/otel'
+import { getLatestSeq, getOldestSeq, readEvents } from './buffer'
+import { createEvent } from './event'
+
+const logger = createLogger('SessionRecovery')
+
+export interface ReplayGapResult {
+  gapDetected: true
+  envelopes: ReturnType<typeof createEvent>[]
+}
+
+export async function checkForReplayGap(
+  streamId: string,
+  afterCursor: string,
+  requestId?: string
+): Promise<ReplayGapResult | null> {
+  const requestedAfterSeq = Number(afterCursor || '0')
+  if (requestedAfterSeq <= 0) {
+    // Fast path: no cursor → nothing to check. Skip the span to avoid
+    // emitting zero-work spans on every stream connect.
+    return null
+  }
+
+  return withCopilotSpan(
+    TraceSpan.CopilotRecoveryCheckReplayGap,
+    {
+      [TraceAttr.StreamId]: streamId,
+      [TraceAttr.CopilotRecoveryRequestedAfterSeq]: requestedAfterSeq,
+      ...(requestId ? { [TraceAttr.RequestId]: requestId } : {}),
+    },
+    async (span) => {
+      const oldestSeq = await getOldestSeq(streamId)
+      const latestSeq = await getLatestSeq(streamId)
+      span.setAttributes({
+        [TraceAttr.CopilotRecoveryOldestSeq]: oldestSeq ?? -1,
+        [TraceAttr.CopilotRecoveryLatestSeq]: latestSeq ?? -1,
+      })
+
+      if (
+        latestSeq !== null &&
+        latestSeq > 0 &&
+        oldestSeq !== null &&
+        requestedAfterSeq < oldestSeq - 1
+      ) {
+        const resolvedRequestId = await resolveReplayGapRequestId(streamId, latestSeq, requestId)
+        logger.warn('Replay gap detected: requested cursor is below oldest available event', {
+          streamId,
+          requestedAfterSeq,
+          oldestAvailableSeq: oldestSeq,
+          latestSeq,
+        })
+        span.setAttribute(TraceAttr.CopilotRecoveryOutcome, CopilotRecoveryOutcome.GapDetected)
+
+        const gapEnvelope = createEvent({
+          streamId,
+          cursor: String(latestSeq + 1),
+          seq: latestSeq + 1,
+          requestId: resolvedRequestId,
+          type: MothershipStreamV1EventType.error,
+          payload: {
+            message: 'Replay history is no longer available. Some events may have been lost.',
+            code: 'replay_gap',
+            data: {
+              oldestAvailableSeq: oldestSeq,
+              requestedAfterSeq,
+            },
+          },
+        })
+
+        const terminalEnvelope = createEvent({
+          streamId,
+          cursor: String(latestSeq + 2),
+          seq: latestSeq + 2,
+          requestId: resolvedRequestId,
+          type: MothershipStreamV1EventType.complete,
+          payload: {
+            status: MothershipStreamV1CompletionStatus.error,
+            reason: 'replay_gap',
+          },
+        })
+
+        return {
+          gapDetected: true,
+          envelopes: [gapEnvelope, terminalEnvelope],
+        }
+      }
+
+      span.setAttribute(TraceAttr.CopilotRecoveryOutcome, CopilotRecoveryOutcome.InRange)
+      return null
+    }
+  )
+}
+
+async function resolveReplayGapRequestId(
+  streamId: string,
+  latestSeq: number,
+  requestId?: string
+): Promise<string> {
+  if (typeof requestId === 'string' && requestId.length > 0) {
+    return requestId
+  }
+
+  try {
+    const latestEvents = await readEvents(streamId, String(Math.max(latestSeq - 1, 0)))
+    const latestRequestId = latestEvents[0]?.trace?.requestId
+    return typeof latestRequestId === 'string' ? latestRequestId : ''
+  } catch (error) {
+    logger.warn('Failed to resolve request ID for replay gap', {
+      streamId,
+      latestSeq,
+      error: getErrorMessage(error),
+    })
+    return ''
+  }
+}

apps/sim/lib/copilot/request/session/sse.ts

@@ -0,0 +1,16 @@
+import { SSE_HEADERS } from '@/lib/core/utils/sse'
+
+const encoder = new TextEncoder()
+
+export function encodeSSEEnvelope(envelope: unknown): Uint8Array {
+  return encoder.encode(`data: ${JSON.stringify(envelope)}\n\n`)
+}
+
+export function encodeSSEComment(comment: string): Uint8Array {
+  return encoder.encode(`: ${comment}\n\n`)
+}
+
+export const SSE_RESPONSE_HEADERS = {
+  ...SSE_HEADERS,
+  'Content-Encoding': 'none',
+} as const