chore: import upstream snapshot with attribution

apps/sim/lib/copilot/request/lifecycle/finalize.ts

@@ -0,0 +1,195 @@
+import { SpanStatusCode, trace } from '@opentelemetry/api'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { updateRunStatus } from '@/lib/copilot/async-runs/repository'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import {
+  type RequestTraceV1Outcome,
+  RequestTraceV1Outcome as RequestTraceV1OutcomeConst,
+} from '@/lib/copilot/generated/request-trace-v1'
+import { CopilotFinalizeOutcome } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
+import type { StreamWriter } from '@/lib/copilot/request/session'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+
+const logger = createLogger('CopilotStreamFinalize')
+const getTracer = () => trace.getTracer('sim-copilot-finalize', '1.0.0')
+
+// Single finalization path. `outcome` is the caller's resolved verdict
+// so we don't have to re-derive cancel vs error from raw signals.
+export async function finalizeStream(
+  result: OrchestratorResult,
+  publisher: StreamWriter,
+  runId: string,
+  outcome: RequestTraceV1Outcome,
+  requestId: string
+): Promise<void> {
+  const spanOutcome =
+    outcome === RequestTraceV1OutcomeConst.cancelled
+      ? CopilotFinalizeOutcome.Aborted
+      : outcome === RequestTraceV1OutcomeConst.success
+        ? CopilotFinalizeOutcome.Success
+        : CopilotFinalizeOutcome.Error
+  const span = getTracer().startSpan(TraceSpan.CopilotFinalizeStream, {
+    attributes: {
+      [TraceAttr.CopilotFinalizeOutcome]: spanOutcome,
+      [TraceAttr.RunId]: runId,
+      [TraceAttr.RequestId]: requestId,
+      [TraceAttr.CopilotResultToolCalls]: result.toolCalls?.length ?? 0,
+      [TraceAttr.CopilotResultContentBlocks]: result.contentBlocks?.length ?? 0,
+      [TraceAttr.CopilotResultContentLength]: result.content?.length ?? 0,
+      [TraceAttr.CopilotPublisherSawComplete]: publisher.sawComplete,
+      [TraceAttr.CopilotPublisherClientDisconnected]: publisher.clientDisconnected,
+    },
+  })
+  try {
+    if (outcome === RequestTraceV1OutcomeConst.cancelled) {
+      await handleAborted(result, publisher, runId, requestId)
+    } else if (outcome === RequestTraceV1OutcomeConst.error) {
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: result.error || 'orchestration failed',
+      })
+      await handleError(result, publisher, runId, requestId)
+    } else {
+      await handleSuccess(publisher, runId, requestId)
+    }
+    // Successful + cancelled paths fall through as status-unset → set
+    // OK so dashboards don't show "incomplete" for normal terminals.
+    if (outcome !== RequestTraceV1OutcomeConst.error) {
+      span.setStatus({ code: SpanStatusCode.OK })
+    }
+  } catch (error) {
+    span.recordException(toError(error))
+    span.setStatus({ code: SpanStatusCode.ERROR, message: 'finalize threw' })
+    throw error
+  } finally {
+    span.end()
+  }
+}
+
+async function handleAborted(
+  result: OrchestratorResult,
+  publisher: StreamWriter,
+  runId: string,
+  requestId: string
+): Promise<void> {
+  const partialContentLen = result.content?.length ?? 0
+  const toolCallCount = result.toolCalls?.length ?? 0
+  const blockCount = result.contentBlocks?.length ?? 0
+  logger.info(`[${requestId}] Stream aborted by explicit stop`, {
+    partialContentLen,
+    toolCallCount,
+    blockCount,
+  })
+  if (!publisher.sawComplete) {
+    const partialContent = result.content || undefined
+    await publisher.publish({
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.cancelled,
+        ...(partialContent ? { partialContent } : {}),
+        ...(partialContentLen ? { partialContentLen } : {}),
+        ...(toolCallCount ? { toolCallCount } : {}),
+      },
+    })
+  }
+  await publisher.flush()
+  await loggedRunStatusUpdate(runId, MothershipStreamV1CompletionStatus.cancelled, requestId, {
+    completedAt: new Date(),
+  })
+}
+
+async function handleError(
+  result: OrchestratorResult,
+  publisher: StreamWriter,
+  runId: string,
+  requestId: string
+): Promise<void> {
+  const errorMessage =
+    result.error ||
+    result.errors?.[0] ||
+    'An unexpected error occurred while processing the response.'
+
+  // Persist whatever was generated before the failure, exactly like an abort —
+  // a transient provider error (e.g. overloaded) shouldn't discard the partial
+  // assistant output the user already saw streaming.
+  const partialContent = result.content || undefined
+  const partialContentLen = result.content?.length ?? 0
+  const toolCallCount = result.toolCalls?.length ?? 0
+
+  if (publisher.clientDisconnected) {
+    logger.info(`[${requestId}] Stream failed after client disconnect`, { error: errorMessage })
+  }
+  logger.error(`[${requestId}] Orchestration returned failure`, {
+    error: errorMessage,
+    partialContentLen,
+    toolCallCount,
+  })
+
+  // Surface the real error (Go already classifies provider errors like
+  // "overloaded" into a friendly displayMessage). Don't clobber it with a
+  // generic string.
+  await publisher.publish({
+    type: MothershipStreamV1EventType.error,
+    payload: {
+      message: errorMessage,
+      error: errorMessage,
+      displayMessage: errorMessage,
+      data: { displayMessage: errorMessage },
+    },
+  })
+  if (!publisher.sawComplete) {
+    await publisher.publish({
+      type: MothershipStreamV1EventType.complete,
+      payload: {
+        status: MothershipStreamV1CompletionStatus.error,
+        ...(partialContent ? { partialContent } : {}),
+        ...(partialContentLen ? { partialContentLen } : {}),
+        ...(toolCallCount ? { toolCallCount } : {}),
+      },
+    })
+  }
+  await publisher.flush()
+  await loggedRunStatusUpdate(runId, MothershipStreamV1CompletionStatus.error, requestId, {
+    completedAt: new Date(),
+    error: errorMessage,
+  })
+}
+
+async function handleSuccess(
+  publisher: StreamWriter,
+  runId: string,
+  requestId: string
+): Promise<void> {
+  if (!publisher.sawComplete) {
+    await publisher.publish({
+      type: MothershipStreamV1EventType.complete,
+      payload: { status: MothershipStreamV1CompletionStatus.complete },
+    })
+  }
+  await publisher.flush()
+  await loggedRunStatusUpdate(runId, MothershipStreamV1CompletionStatus.complete, requestId, {
+    completedAt: new Date(),
+  })
+}
+
+async function loggedRunStatusUpdate(
+  runId: string,
+  status: Parameters<typeof updateRunStatus>[1],
+  requestId: string,
+  updates: Parameters<typeof updateRunStatus>[2] = {}
+): Promise<void> {
+  try {
+    await updateRunStatus(runId, status, updates)
+  } catch (error) {
+    logger.warn(`[${requestId}] Failed to update run status to ${status}`, {
+      runId,
+      error: toError(error).message,
+    })
+  }
+}

apps/sim/lib/copilot/request/lifecycle/headless.test.ts

@@ -0,0 +1,171 @@
+/**
+ * @vitest-environment node
+ */
+
+import { propagation, trace } from '@opentelemetry/api'
+import { W3CTraceContextPropagator } from '@opentelemetry/core'
+import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+
+const { runCopilotLifecycle } = vi.hoisted(() => ({
+  runCopilotLifecycle: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/lifecycle/run', () => ({
+  runCopilotLifecycle,
+}))
+
+import { runHeadlessCopilotLifecycle } from './headless'
+
+function createLifecycleResult(overrides?: Partial<OrchestratorResult>): OrchestratorResult {
+  return {
+    success: true,
+    content: 'done',
+    contentBlocks: [],
+    toolCalls: [],
+    chatId: 'chat-1',
+    ...overrides,
+  }
+}
+
+describe('runHeadlessCopilotLifecycle', () => {
+  beforeEach(() => {
+    trace.setGlobalTracerProvider(new BasicTracerProvider())
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator())
+  })
+
+  afterEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('runs the lifecycle and returns its result', async () => {
+    runCopilotLifecycle.mockResolvedValueOnce(
+      createLifecycleResult({
+        usage: { prompt: 10, completion: 5 },
+        cost: { input: 1, output: 2, total: 3 },
+      })
+    )
+
+    const result = await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'req-1',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(result.success).toBe(true)
+    expect(runCopilotLifecycle).toHaveBeenCalledWith(
+      expect.objectContaining({ messageId: 'req-1' }),
+      expect.objectContaining({
+        simRequestId: 'req-1',
+        trace: expect.any(Object),
+        otelContext: expect.any(Object),
+        chatId: 'chat-1',
+      })
+    )
+  })
+
+  it('returns an unsuccessful result from the lifecycle', async () => {
+    runCopilotLifecycle.mockResolvedValueOnce(
+      createLifecycleResult({
+        success: false,
+        error: 'failed',
+      })
+    )
+
+    const result = await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'req-2',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(result.success).toBe(false)
+  })
+
+  it('prefers an explicit simRequestId over the payload messageId', async () => {
+    runCopilotLifecycle.mockResolvedValueOnce(createLifecycleResult())
+
+    await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'message-req-id',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        simRequestId: 'workflow-request-id',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(runCopilotLifecycle).toHaveBeenCalledWith(
+      expect.objectContaining({ messageId: 'message-req-id' }),
+      expect.objectContaining({
+        simRequestId: 'workflow-request-id',
+      })
+    )
+  })
+
+  it('threads a valid OTel context into the lifecycle', async () => {
+    let lifecycleTraceparent = ''
+    runCopilotLifecycle.mockImplementationOnce(async (_payload, options) => {
+      const { traceHeaders } = await import('@/lib/copilot/request/go/propagation')
+      lifecycleTraceparent = traceHeaders({}, options.otelContext).traceparent ?? ''
+      return createLifecycleResult()
+    })
+
+    await runHeadlessCopilotLifecycle(
+      {
+        message: 'hello',
+        messageId: 'req-otel',
+      },
+      {
+        userId: 'user-1',
+        chatId: 'chat-1',
+        workflowId: 'workflow-1',
+        goRoute: '/api/mothership/execute',
+        interactive: false,
+      }
+    )
+
+    expect(lifecycleTraceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
+  })
+
+  it('rethrows when the lifecycle throws', async () => {
+    runCopilotLifecycle.mockRejectedValueOnce(new Error('kaboom'))
+
+    await expect(
+      runHeadlessCopilotLifecycle(
+        {
+          message: 'hello',
+          messageId: 'req-3',
+        },
+        {
+          userId: 'user-1',
+          chatId: 'chat-1',
+          workflowId: 'workflow-1',
+          goRoute: '/api/mothership/execute',
+          interactive: false,
+        }
+      )
+    ).rejects.toThrow('kaboom')
+  })
+})

apps/sim/lib/copilot/request/lifecycle/headless.ts

@@ -0,0 +1,79 @@
+import { createLogger } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
+import type { RequestTraceV1Outcome as RequestTraceOutcome } from '@/lib/copilot/generated/request-trace-v1'
+import {
+  RequestTraceV1Outcome,
+  RequestTraceV1SpanStatus,
+} from '@/lib/copilot/generated/request-trace-v1'
+import { CopilotTransport } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import type { CopilotLifecycleOptions } from '@/lib/copilot/request/lifecycle/run'
+import { runCopilotLifecycle } from '@/lib/copilot/request/lifecycle/run'
+import { withCopilotOtelContext } from '@/lib/copilot/request/otel'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+import type { OrchestratorResult } from '@/lib/copilot/request/types'
+
+const logger = createLogger('CopilotHeadlessLifecycle')
+
+export async function runHeadlessCopilotLifecycle(
+  requestPayload: Record<string, unknown>,
+  options: CopilotLifecycleOptions
+): Promise<OrchestratorResult> {
+  const simRequestId =
+    typeof options.simRequestId === 'string' && options.simRequestId.length > 0
+      ? options.simRequestId
+      : typeof requestPayload.messageId === 'string' && requestPayload.messageId.length > 0
+        ? requestPayload.messageId
+        : generateId()
+  const trace = new TraceCollector()
+  const requestSpan = trace.startSpan('Headless Sim Agent Request', 'request', {
+    route: options.goRoute,
+    workflowId: options.workflowId,
+    workspaceId: options.workspaceId,
+    chatId: options.chatId,
+  })
+
+  let result: OrchestratorResult | undefined
+  let outcome: RequestTraceOutcome = RequestTraceV1Outcome.error
+
+  return withCopilotOtelContext(
+    {
+      requestId: simRequestId,
+      route: options.goRoute,
+      chatId: options.chatId,
+      workflowId: options.workflowId,
+      executionId: options.executionId,
+      runId: options.runId,
+      transport: CopilotTransport.Headless,
+    },
+    async (otelContext) => {
+      try {
+        result = await runCopilotLifecycle(requestPayload, {
+          ...options,
+          trace,
+          simRequestId,
+          otelContext,
+        })
+        outcome = result.success
+          ? RequestTraceV1Outcome.success
+          : options.abortSignal?.aborted || result.cancelled
+            ? RequestTraceV1Outcome.cancelled
+            : RequestTraceV1Outcome.error
+        return result
+      } catch (error) {
+        outcome = options.abortSignal?.aborted
+          ? RequestTraceV1Outcome.cancelled
+          : RequestTraceV1Outcome.error
+        throw error
+      } finally {
+        trace.endSpan(
+          requestSpan,
+          outcome === RequestTraceV1Outcome.success
+            ? RequestTraceV1SpanStatus.ok
+            : outcome === RequestTraceV1Outcome.cancelled
+              ? RequestTraceV1SpanStatus.cancelled
+              : RequestTraceV1SpanStatus.error
+        )
+      }
+    }
+  )
+}

apps/sim/lib/copilot/request/lifecycle/resume-leg-context.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest'
+import { createStreamingContext } from '@/lib/copilot/request/context/request-context'
+import { makeResumeLegContext, mergeResumeLegOutputs } from '@/lib/copilot/request/lifecycle/run'
+
+// Guards the makeResumeLegContext / mergeResumeLegOutputs contract: the two MUST
+// stay in lockstep (every per-leg-isolated scalar is reset on leg creation and
+// folded back on merge), and the heavy accumulators stay shared by reference so
+// all concurrent legs build one chat. This is the regression the inline comment
+// warns about — without per-leg isolation the orchestrator's pre-fanout content
+// gets multiplied by the leg count on merge.
+describe('resume leg context isolate/merge contract', () => {
+  it('isolates the per-leg scalars while sharing the heavy accumulators by reference', () => {
+    const base = createStreamingContext({
+      accumulatedContent: 'PRE',
+      finalAssistantContent: 'PRE-FINAL',
+      usage: { prompt: 10, completion: 5 },
+      cost: { input: 1, output: 2, total: 3 },
+      errors: ['pre-existing'],
+    })
+
+    const leg = makeResumeLegContext(base)
+
+    // Per-leg scalars reset so a leg accumulates only its OWN output.
+    expect(leg.accumulatedContent).toBe('')
+    expect(leg.finalAssistantContent).toBe('')
+    expect(leg.usage).toBeUndefined()
+    expect(leg.cost).toBeUndefined()
+    expect(leg.errors).toEqual([])
+    expect(leg.streamComplete).toBe(false)
+    expect(leg.awaitingAsyncContinuation).toBeUndefined()
+
+    // A leg's own errors array is a fresh array (not the shared one) so a leg's
+    // retry rollback can't truncate a sibling's errors.
+    expect(leg.errors).not.toBe(base.errors)
+
+    // Heavy accumulators stay shared by reference (one merged chat).
+    expect(leg.contentBlocks).toBe(base.contentBlocks)
+    expect(leg.toolCalls).toBe(base.toolCalls)
+    expect(leg.pendingToolPromises).toBe(base.pendingToolPromises)
+    expect(leg.subAgentContent).toBe(base.subAgentContent)
+  })
+
+  it('folds a leg back exactly once (no double-count of the orchestrator content)', () => {
+    const base = createStreamingContext({ accumulatedContent: 'PRE', errors: ['pre'] })
+
+    const leg = makeResumeLegContext(base)
+    leg.accumulatedContent = 'JOIN'
+    leg.finalAssistantContent = 'JOIN-FINAL'
+    leg.usage = { prompt: 100, completion: 50 }
+    leg.cost = { input: 4, output: 5, total: 9 }
+    leg.errors.push('leg-err')
+
+    mergeResumeLegOutputs(base, leg)
+
+    // PRE seeded once + the leg's own output appended once — not PRE+PRE+JOIN.
+    expect(base.accumulatedContent).toBe('PREJOIN')
+    expect(base.finalAssistantContent).toBe('JOIN-FINAL')
+    expect(base.usage).toEqual({ prompt: 100, completion: 50 })
+    expect(base.cost).toEqual({ input: 4, output: 5, total: 9 })
+    expect(base.errors).toEqual(['pre', 'leg-err'])
+  })
+
+  it('does not multiply pre-fanout content across many legs (N children + one join leg)', () => {
+    const base = createStreamingContext({ accumulatedContent: 'PRE' })
+
+    // Seven child legs that stream subagent content (not main accumulatedContent)
+    // contribute nothing to the join scalars; only the join-carrying leg does.
+    for (let i = 0; i < 7; i++) {
+      const childLeg = makeResumeLegContext(base)
+      mergeResumeLegOutputs(base, childLeg)
+    }
+    const joinLeg = makeResumeLegContext(base)
+    joinLeg.accumulatedContent = 'SUMMARY'
+    joinLeg.usage = { prompt: 1, completion: 1 }
+    mergeResumeLegOutputs(base, joinLeg)
+
+    // Exactly the pre-fanout content + the one join leg's summary — the 7 child
+    // legs must not each re-append 'PRE'.
+    expect(base.accumulatedContent).toBe('PRESUMMARY')
+    expect(base.usage).toEqual({ prompt: 1, completion: 1 })
+  })
+})

apps/sim/lib/copilot/request/lifecycle/run.test.ts

@@ -0,0 +1,690 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types'
+
+const {
+  mockCreateRunSegment,
+  mockForceFailHungToolCall,
+  mockGetEffectiveDecryptedEnv,
+  mockGetMothershipBaseURL,
+  mockGetMothershipSourceEnvHeaders,
+  mockPrepareExecutionContext,
+  mockRunStreamLoop,
+  mockToolWatchdogTimeoutMs,
+  mockUpdateRunStatus,
+} = vi.hoisted(() => ({
+  mockCreateRunSegment: vi.fn(),
+  mockForceFailHungToolCall: vi.fn(),
+  mockGetEffectiveDecryptedEnv: vi.fn(),
+  mockGetMothershipBaseURL: vi.fn(),
+  mockGetMothershipSourceEnvHeaders: vi.fn(),
+  mockPrepareExecutionContext: vi.fn(),
+  mockRunStreamLoop: vi.fn(),
+  mockToolWatchdogTimeoutMs: vi.fn(() => 60_000),
+  mockUpdateRunStatus: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/async-runs/repository', () => ({
+  createRunSegment: mockCreateRunSegment,
+  updateRunStatus: mockUpdateRunStatus,
+}))
+
+vi.mock('@/lib/copilot/request/go/stream', () => {
+  class CopilotBackendError extends Error {
+    status?: number
+
+    constructor(message: string, options?: { status?: number }) {
+      super(message)
+      this.name = 'CopilotBackendError'
+      this.status = options?.status
+    }
+  }
+
+  class BillingLimitError extends Error {
+    userId: string
+
+    constructor(userId: string) {
+      super('Usage limit reached')
+      this.name = 'BillingLimitError'
+      this.userId = userId
+    }
+  }
+
+  return {
+    BillingLimitError,
+    CopilotBackendError,
+    runStreamLoop: mockRunStreamLoop,
+  }
+})
+
+vi.mock('@/lib/copilot/server/agent-url', () => ({
+  getMothershipBaseURL: mockGetMothershipBaseURL,
+  getMothershipSourceEnvHeaders: mockGetMothershipSourceEnvHeaders,
+}))
+
+vi.mock('@/lib/core/config/env', () => ({
+  env: {
+    COPILOT_API_KEY: undefined,
+  },
+  getEnv: vi.fn((key: string) => (key === 'NEXT_PUBLIC_APP_URL' ? 'http://localhost:3000' : '')),
+  isTruthy: vi.fn((value: string | undefined) => value === 'true'),
+  isFalsy: vi.fn((value: string | undefined) => value === 'false'),
+}))
+
+vi.mock('@/lib/environment/utils', () => ({
+  getEffectiveDecryptedEnv: mockGetEffectiveDecryptedEnv,
+}))
+
+vi.mock('@/lib/copilot/tools/handlers/context', () => ({
+  prepareExecutionContext: mockPrepareExecutionContext,
+}))
+
+vi.mock('@/lib/copilot/request/tools/billing', () => ({
+  handleBillingLimitResponse: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/tools/executor', () => ({
+  executeToolAndReport: vi.fn(),
+  forceFailHungToolCall: mockForceFailHungToolCall,
+  toolWatchdogTimeoutMs: mockToolWatchdogTimeoutMs,
+}))
+
+import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1'
+import { CopilotBackendError } from '@/lib/copilot/request/go/stream'
+import { runCopilotLifecycle } from '@/lib/copilot/request/lifecycle/run'
+
+describe('runCopilotLifecycle', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetMothershipBaseURL.mockResolvedValue('http://mothership.test')
+    mockGetMothershipSourceEnvHeaders.mockReturnValue({})
+  })
+
+  it('runs cancelled completion persistence when a stream throws after abort', async () => {
+    const abortController = new AbortController()
+    abortController.abort('stop')
+    const onComplete = vi.fn()
+    const onError = vi.fn()
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'partial answer'
+        context.contentBlocks.push({
+          type: 'text',
+          content: 'partial answer',
+          timestamp: 1,
+        })
+        throw new Error('publisher closed after stop')
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        abortSignal: abortController.signal,
+        executionContext,
+        onComplete,
+        onError,
+      }
+    )
+
+    expect(onError).not.toHaveBeenCalled()
+    expect(onComplete).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+        chatId: 'chat-1',
+        requestId: undefined,
+        error: 'publisher closed after stop',
+        contentBlocks: [
+          expect.objectContaining({
+            type: 'text',
+            content: 'partial answer',
+          }),
+        ],
+      })
+    )
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+        chatId: 'chat-1',
+        error: 'publisher closed after stop',
+      })
+    )
+  })
+
+  it('returns the cancelled result when cancelled completion persistence fails', async () => {
+    const abortController = new AbortController()
+    abortController.abort('stop')
+    const onComplete = vi.fn().mockRejectedValue(new Error('db unavailable'))
+    const onError = vi.fn()
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'partial answer'
+        throw new Error('publisher closed after stop')
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        abortSignal: abortController.signal,
+        executionContext,
+        onComplete,
+        onError,
+      }
+    )
+
+    expect(onError).not.toHaveBeenCalled()
+    expect(onComplete).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+      })
+    )
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: false,
+        cancelled: true,
+        content: 'partial answer',
+        error: 'publisher closed after stop',
+      })
+    )
+  })
+
+  it('uses the final post-tool assistant content for headless results', async () => {
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'I will check that.Final answer only.'
+        context.finalAssistantContent = 'Final answer only.'
+        context.sawMainToolCall = true
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+        interactive: false,
+      }
+    )
+
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: true,
+        content: 'Final answer only.',
+      })
+    )
+  })
+
+  it('does not fall back to pre-tool narration when headless final content is empty', async () => {
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'I will check that.'
+        context.finalAssistantContent = ''
+        context.sawMainToolCall = true
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+        interactive: false,
+      }
+    )
+
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: true,
+        content: '',
+      })
+    )
+  })
+
+  it('propagates payload userPermission into the generated execution context', async () => {
+    let capturedExecContext: ExecutionContext | undefined
+    mockGetEffectiveDecryptedEnv.mockResolvedValueOnce({})
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        _context: StreamingContext,
+        execContext: ExecutionContext
+      ): Promise<void> => {
+        capturedExecContext = execContext
+      }
+    )
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1', userPermission: 'write' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+      }
+    )
+
+    expect(capturedExecContext).toEqual(
+      expect.objectContaining({
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        userPermission: 'write',
+      })
+    )
+  })
+
+  it('normalizes the initial request body with workspaceId from lifecycle options', async () => {
+    let requestBody: Record<string, unknown> | undefined
+    mockGetEffectiveDecryptedEnv.mockResolvedValueOnce({})
+    mockRunStreamLoop.mockImplementationOnce(
+      async (_fetchUrl: string, fetchOptions: RequestInit): Promise<void> => {
+        requestBody = JSON.parse(String(fetchOptions.body))
+      }
+    )
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+      }
+    )
+
+    expect(requestBody).toEqual(
+      expect.objectContaining({
+        workspaceId: 'ws-1',
+      })
+    )
+  })
+
+  it('uses the lifecycle workspaceId for async tool resume requests', async () => {
+    const requestBodies: Record<string, unknown>[] = []
+    const fetchUrls: string[] = []
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        fetchUrl: string,
+        fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        fetchUrls.push(fetchUrl)
+        requestBodies.push(JSON.parse(String(fetchOptions.body)))
+        context.toolCalls.set('tool-1', {
+          id: 'tool-1',
+          name: 'read',
+          status: MothershipStreamV1ToolOutcome.success,
+          result: { success: true, output: { content: 'file contents' } },
+        })
+        context.awaitingAsyncContinuation = {
+          checkpointId: 'ckpt-1',
+          pendingToolCallIds: ['tool-1'],
+        }
+      }
+    )
+    mockRunStreamLoop.mockImplementationOnce(
+      async (fetchUrl: string, fetchOptions: RequestInit): Promise<void> => {
+        fetchUrls.push(fetchUrl)
+        requestBodies.push(JSON.parse(String(fetchOptions.body)))
+      }
+    )
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        workflowId: 'workflow-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+      }
+    )
+
+    expect(fetchUrls[1]).toBe('http://mothership.test/api/tools/resume')
+    expect(requestBodies[1]).toEqual(
+      expect.objectContaining({
+        checkpointId: 'ckpt-1',
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+      })
+    )
+  })
+
+  it('finalizes as success when a resume fails with a retryable error then the retry succeeds', async () => {
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    // 1) Initial stream pauses on an async tool checkpoint with a resolved
+    //    tool result, so the lifecycle transitions into a resume leg.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.toolCalls.set('tool-1', {
+          id: 'tool-1',
+          name: 'read',
+          status: MothershipStreamV1ToolOutcome.success,
+          result: { success: true, output: { content: 'file contents' } },
+        })
+        context.awaitingAsyncContinuation = {
+          checkpointId: 'ckpt-1',
+          pendingToolCallIds: ['tool-1'],
+        }
+      }
+    )
+
+    // 2) First resume leg dies mid-stream like a transient provider error:
+    //    it records an error AND throws a retryable 5xx.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.errors.push(
+          'Copilot backend stream ended before a terminal event on /api/tools/resume'
+        )
+        throw new CopilotBackendError('backend stream ended before a terminal event', {
+          status: 503,
+        })
+      }
+    )
+
+    // 3) Retry of the same resume leg succeeds cleanly.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        _fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        context.accumulatedContent = 'Recovered final answer.'
+        context.finalAssistantContent = 'Recovered final answer.'
+      }
+    )
+
+    const result = await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+      }
+    )
+
+    // Three legs ran (initial + failed resume + retried resume), and the
+    // recovered retry must NOT inherit the failed attempt's error.
+    expect(mockRunStreamLoop).toHaveBeenCalledTimes(3)
+    expect(result).toEqual(
+      expect.objectContaining({
+        success: true,
+        cancelled: false,
+        errors: undefined,
+      })
+    )
+  })
+
+  it('marks resume legs willRetryOnStreamError except the final attempt', async () => {
+    const bodies: Record<string, unknown>[] = []
+    const executionContext: ExecutionContext = {
+      userId: 'user-1',
+      workflowId: '',
+      workspaceId: 'ws-1',
+      chatId: 'chat-1',
+      decryptedEnvVars: {},
+    }
+
+    // Initial leg pauses on a resolved async tool checkpoint → enters resume.
+    mockRunStreamLoop.mockImplementationOnce(
+      async (
+        _fetchUrl: string,
+        fetchOptions: RequestInit,
+        context: StreamingContext
+      ): Promise<void> => {
+        bodies.push(JSON.parse(String(fetchOptions.body)))
+        context.toolCalls.set('tool-1', {
+          id: 'tool-1',
+          name: 'read',
+          status: MothershipStreamV1ToolOutcome.success,
+          result: { success: true, output: { content: 'file contents' } },
+        })
+        context.awaitingAsyncContinuation = {
+          checkpointId: 'ckpt-1',
+          pendingToolCallIds: ['tool-1'],
+        }
+      }
+    )
+
+    // Three resume attempts, all failing with a retryable 5xx so the loop
+    // exhausts MAX_RESUME_ATTEMPTS (= 3) and gives up.
+    for (let i = 0; i < 3; i++) {
+      mockRunStreamLoop.mockImplementationOnce(
+        async (
+          _fetchUrl: string,
+          fetchOptions: RequestInit,
+          context: StreamingContext
+        ): Promise<void> => {
+          bodies.push(JSON.parse(String(fetchOptions.body)))
+          context.errors.push('Copilot backend stream ended before a terminal event')
+          throw new CopilotBackendError('backend stream ended before a terminal event', {
+            status: 503,
+          })
+        }
+      )
+    }
+
+    await runCopilotLifecycle(
+      { message: 'hello', messageId: 'stream-1' },
+      {
+        userId: 'user-1',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        executionId: 'exec-1',
+        runId: 'run-1',
+        executionContext,
+      }
+    )
+
+    // Initial + 3 resume attempts.
+    expect(mockRunStreamLoop).toHaveBeenCalledTimes(4)
+    // Initial leg is never retried by this loop → no flag.
+    expect(bodies[0].willRetryOnStreamError).toBeUndefined()
+    // Resume attempts 0 and 1 will be retried on a stream error → flagged.
+    expect(bodies[1].willRetryOnStreamError).toBe(true)
+    expect(bodies[2].willRetryOnStreamError).toBe(true)
+    // Final attempt (2) is terminal → not flagged, so Go bills + surfaces it.
+    expect(bodies[3].willRetryOnStreamError).toBeUndefined()
+  })
+
+  it('force-fails a hung tool promise and resumes with an error result instead of wedging', async () => {
+    vi.useFakeTimers()
+    try {
+      const fetchUrls: string[] = []
+      const bodies: Record<string, unknown>[] = []
+      const executionContext: ExecutionContext = {
+        userId: 'user-1',
+        workflowId: '',
+        workspaceId: 'ws-1',
+        chatId: 'chat-1',
+        decryptedEnvVars: {},
+      }
+
+      // Mirror the real helper: settle the tool call into a terminal error
+      // state so the resume loop can serialize an error result for it.
+      mockForceFailHungToolCall.mockImplementation(
+        async (toolCallId: string, context: StreamingContext, message: string) => {
+          const tool = context.toolCalls.get(toolCallId)
+          if (!tool) return
+          tool.status = MothershipStreamV1ToolOutcome.error
+          tool.endTime = Date.now()
+          tool.result = { success: false }
+          tool.error = message
+        }
+      )
+
+      // Initial leg checkpoints on an async tool whose promise NEVER settles —
+      // the exact shape of the prod incident (claimed, marked running, hung).
+      mockRunStreamLoop.mockImplementationOnce(
+        async (
+          fetchUrl: string,
+          fetchOptions: RequestInit,
+          context: StreamingContext
+        ): Promise<void> => {
+          fetchUrls.push(fetchUrl)
+          bodies.push(JSON.parse(String(fetchOptions.body)))
+          context.toolCalls.set('tool-hung', {
+            id: 'tool-hung',
+            name: 'read',
+            status: 'executing',
+          })
+          context.pendingToolPromises.set('tool-hung', new Promise(() => {}))
+          context.awaitingAsyncContinuation = {
+            checkpointId: 'ckpt-1',
+            pendingToolCallIds: ['tool-hung'],
+          }
+        }
+      )
+
+      // Resume leg completes normally with the error result delivered.
+      mockRunStreamLoop.mockImplementationOnce(
+        async (
+          fetchUrl: string,
+          fetchOptions: RequestInit,
+          context: StreamingContext
+        ): Promise<void> => {
+          fetchUrls.push(fetchUrl)
+          bodies.push(JSON.parse(String(fetchOptions.body)))
+          context.accumulatedContent = 'The file read failed, but here is what I know.'
+        }
+      )
+
+      const lifecycle = runCopilotLifecycle(
+        { message: 'hello', messageId: 'stream-1' },
+        {
+          userId: 'user-1',
+          workspaceId: 'ws-1',
+          chatId: 'chat-1',
+          executionId: 'exec-1',
+          runId: 'run-1',
+          executionContext,
+        }
+      )
+
+      // Wait budget = watchdog (60s, mocked) + resume grace (30s). Advance past it.
+      await vi.advanceTimersByTimeAsync(91_000)
+      const result = await lifecycle
+
+      expect(mockForceFailHungToolCall).toHaveBeenCalledWith(
+        'tool-hung',
+        expect.anything(),
+        expect.stringContaining('hung')
+      )
+      expect(fetchUrls[1]).toBe('http://mothership.test/api/tools/resume')
+      expect(bodies[1].results).toEqual([
+        expect.objectContaining({
+          callId: 'tool-hung',
+          name: 'read',
+          success: false,
+          data: { error: expect.stringContaining('hung') },
+        }),
+      ])
+      expect(result.success).toBe(true)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})

apps/sim/lib/copilot/request/lifecycle/start.test.ts

@@ -0,0 +1,294 @@
+/**
+ * @vitest-environment node
+ */
+
+import { propagation, trace } from '@opentelemetry/api'
+import { W3CTraceContextPropagator } from '@opentelemetry/core'
+import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  MothershipStreamV1CompletionStatus,
+  MothershipStreamV1EventType,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+
+const {
+  runCopilotLifecycle,
+  createRunSegment,
+  updateRunStatus,
+  resetBuffer,
+  clearFilePreviewSessions,
+  scheduleBufferCleanup,
+  scheduleFilePreviewSessionCleanup,
+  allocateCursor,
+  appendEvent,
+  cleanupAbortMarker,
+  hasAbortMarker,
+  releasePendingChatStream,
+} = vi.hoisted(() => ({
+  runCopilotLifecycle: vi.fn(),
+  createRunSegment: vi.fn(),
+  updateRunStatus: vi.fn(),
+  resetBuffer: vi.fn(),
+  clearFilePreviewSessions: vi.fn(),
+  scheduleBufferCleanup: vi.fn(),
+  scheduleFilePreviewSessionCleanup: vi.fn(),
+  allocateCursor: vi.fn(),
+  appendEvent: vi.fn(),
+  cleanupAbortMarker: vi.fn(),
+  hasAbortMarker: vi.fn(),
+  releasePendingChatStream: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/lifecycle/run', () => ({
+  runCopilotLifecycle,
+}))
+
+vi.mock('@/lib/copilot/async-runs/repository', () => ({
+  createRunSegment,
+  updateRunStatus,
+}))
+
+let mockPublisherController: ReadableStreamDefaultController | null = null
+
+vi.mock('@/lib/copilot/request/session', () => ({
+  resetBuffer,
+  clearFilePreviewSessions,
+  scheduleBufferCleanup,
+  scheduleFilePreviewSessionCleanup,
+  allocateCursor,
+  appendEvent,
+  cleanupAbortMarker,
+  hasAbortMarker,
+  releasePendingChatStream,
+  registerActiveStream: vi.fn(),
+  unregisterActiveStream: vi.fn(),
+  startAbortPoller: vi.fn().mockReturnValue(setInterval(() => {}, 999999)),
+  isExplicitStopReason: vi.fn().mockReturnValue(false),
+  SSE_RESPONSE_HEADERS: {},
+  StreamWriter: vi.fn().mockImplementation(
+    class {
+      attach = vi.fn().mockImplementation((ctrl: ReadableStreamDefaultController) => {
+        mockPublisherController = ctrl
+      })
+      startKeepalive = vi.fn()
+      stopKeepalive = vi.fn()
+      flush = vi.fn()
+      close = vi.fn().mockImplementation(() => {
+        try {
+          mockPublisherController?.close()
+        } catch {
+          // already closed
+        }
+      })
+      markDisconnected = vi.fn()
+      publish = vi.fn().mockImplementation(async (event: Record<string, unknown>) => {
+        appendEvent(event)
+      })
+      get clientDisconnected() {
+        return false
+      }
+      get sawComplete() {
+        return false
+      }
+    }
+  ),
+}))
+vi.mock('@/lib/copilot/request/session/sse', () => ({
+  SSE_RESPONSE_HEADERS: {},
+}))
+
+vi.mock('@sim/db', () => ({
+  db: {
+    update: vi.fn(() => ({
+      set: vi.fn(() => ({
+        where: vi.fn(),
+      })),
+    })),
+  },
+}))
+
+vi.mock('@/lib/copilot/chat-status', () => ({
+  chatPubSub: null,
+}))
+
+import { createSSEStream } from './start'
+
+async function drainStream(stream: ReadableStream) {
+  const reader = stream.getReader()
+  while (true) {
+    const { done } = await reader.read()
+    if (done) break
+  }
+}
+
+describe('createSSEStream terminal error handling', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    trace.setGlobalTracerProvider(new BasicTracerProvider())
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator())
+    vi.stubGlobal(
+      'fetch',
+      vi.fn().mockResolvedValue(
+        new Response(JSON.stringify({ title: 'Test title' }), {
+          status: 200,
+          headers: {
+            'Content-Type': 'application/json',
+          },
+        })
+      )
+    )
+    resetBuffer.mockResolvedValue(undefined)
+    clearFilePreviewSessions.mockResolvedValue(undefined)
+    scheduleBufferCleanup.mockResolvedValue(undefined)
+    scheduleFilePreviewSessionCleanup.mockResolvedValue(undefined)
+    allocateCursor
+      .mockResolvedValueOnce({ seq: 1, cursor: '1' })
+      .mockResolvedValueOnce({ seq: 2, cursor: '2' })
+      .mockResolvedValueOnce({ seq: 3, cursor: '3' })
+    appendEvent.mockImplementation(async (event: unknown) => event)
+    cleanupAbortMarker.mockResolvedValue(undefined)
+    hasAbortMarker.mockResolvedValue(false)
+    releasePendingChatStream.mockResolvedValue(undefined)
+    createRunSegment.mockResolvedValue(null)
+    updateRunStatus.mockResolvedValue(null)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('writes a terminal error event before close when orchestration returns success=false', async () => {
+    runCopilotLifecycle.mockResolvedValue({
+      success: false,
+      error: 'resume failed',
+      content: '',
+      contentBlocks: [],
+      toolCalls: [],
+    })
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-1',
+      orchestrateOptions: {},
+    })
+
+    await drainStream(stream)
+
+    expect(appendEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.error,
+      })
+    )
+    expect(scheduleBufferCleanup).toHaveBeenCalledWith('stream-1')
+  })
+
+  it('writes the thrown terminal error event before close for replay durability', async () => {
+    runCopilotLifecycle.mockRejectedValue(new Error('kaboom'))
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-1',
+      orchestrateOptions: {},
+    })
+
+    await drainStream(stream)
+
+    expect(appendEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.error,
+      })
+    )
+    expect(scheduleBufferCleanup).toHaveBeenCalledWith('stream-1')
+  })
+
+  it('publishes a cancelled completion (not an error) when the orchestrator reports cancelled without abortSignal aborted', async () => {
+    runCopilotLifecycle.mockResolvedValue({
+      success: false,
+      cancelled: true,
+      content: '',
+      contentBlocks: [],
+      toolCalls: [],
+    })
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-cancelled',
+      orchestrateOptions: {},
+    })
+
+    await drainStream(stream)
+
+    expect(appendEvent).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.error,
+      })
+    )
+    expect(appendEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: MothershipStreamV1EventType.complete,
+        payload: expect.objectContaining({
+          status: MothershipStreamV1CompletionStatus.cancelled,
+        }),
+      })
+    )
+  })
+
+  it('passes an OTel context into the streaming lifecycle', async () => {
+    let lifecycleTraceparent = ''
+    runCopilotLifecycle.mockImplementation(async (_payload, options) => {
+      const { traceHeaders } = await import('@/lib/copilot/request/go/propagation')
+      lifecycleTraceparent = traceHeaders({}, options.otelContext).traceparent ?? ''
+      return {
+        success: true,
+        content: 'OK',
+        contentBlocks: [],
+        toolCalls: [],
+      }
+    })
+
+    const stream = createSSEStream({
+      requestPayload: { message: 'hello' },
+      userId: 'user-1',
+      streamId: 'stream-1',
+      executionId: 'exec-1',
+      runId: 'run-1',
+      currentChat: null,
+      isNewChat: false,
+      message: 'hello',
+      titleModel: 'gpt-5.4',
+      requestId: 'req-otel',
+      orchestrateOptions: {
+        goRoute: '/api/mothership',
+        workflowId: 'workflow-1',
+      },
+    })
+
+    await drainStream(stream)
+
+    expect(lifecycleTraceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
+  })
+})

apps/sim/lib/copilot/request/lifecycle/start.ts

@@ -0,0 +1,531 @@
+import { type Context, context as otelContextApi } from '@opentelemetry/api'
+import { db } from '@sim/db'
+import { copilotChats } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { eq } from 'drizzle-orm'
+import { createRunSegment } from '@/lib/copilot/async-runs/repository'
+import { chatPubSub } from '@/lib/copilot/chat-status'
+import {
+  MothershipStreamV1EventType,
+  MothershipStreamV1SessionKind,
+} from '@/lib/copilot/generated/mothership-stream-v1'
+import {
+  RequestTraceV1Outcome,
+  RequestTraceV1SpanStatus,
+} from '@/lib/copilot/generated/request-trace-v1'
+import {
+  CopilotRequestCancelReason,
+  type CopilotRequestCancelReasonValue,
+  CopilotTransport,
+} from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceEvent } from '@/lib/copilot/generated/trace-events-v1'
+import { finalizeStream } from '@/lib/copilot/request/lifecycle/finalize'
+import type { CopilotLifecycleOptions } from '@/lib/copilot/request/lifecycle/run'
+import { runCopilotLifecycle } from '@/lib/copilot/request/lifecycle/run'
+import { type CopilotLifecycleOutcome, startCopilotOtelRoot } from '@/lib/copilot/request/otel'
+import {
+  cleanupAbortMarker,
+  clearFilePreviewSessions,
+  isExplicitStopReason,
+  registerActiveStream,
+  releasePendingChatStream,
+  resetBuffer,
+  StreamWriter,
+  scheduleBufferCleanup,
+  scheduleFilePreviewSessionCleanup,
+  startAbortPoller,
+  unregisterActiveStream,
+} from '@/lib/copilot/request/session'
+import { SSE_RESPONSE_HEADERS } from '@/lib/copilot/request/session/sse'
+import { TraceCollector } from '@/lib/copilot/request/trace'
+import { getMothershipBaseURL, getMothershipSourceEnvHeaders } from '@/lib/copilot/server/agent-url'
+import { env } from '@/lib/core/config/env'
+
+export { SSE_RESPONSE_HEADERS }
+
+const logger = createLogger('CopilotChatStreaming')
+
+type CurrentChatSummary = {
+  title?: string | null
+} | null
+
+export interface StreamingOrchestrationParams {
+  requestPayload: Record<string, unknown>
+  userId: string
+  streamId: string
+  executionId: string
+  runId: string
+  chatId?: string
+  currentChat: CurrentChatSummary
+  isNewChat: boolean
+  message: string
+  titleModel: string
+  titleProvider?: string
+  requestId: string
+  workspaceId?: string
+  orchestrateOptions: Omit<CopilotLifecycleOptions, 'onEvent'>
+  /**
+   * Pre-started root; child spans bind to it and `finish()` fires on
+   * termination. Omit to let the stream start its own root (headless).
+   */
+  otelRoot?: ReturnType<typeof startCopilotOtelRoot>
+}
+
+export function createSSEStream(params: StreamingOrchestrationParams): ReadableStream {
+  const {
+    requestPayload,
+    userId,
+    streamId,
+    executionId,
+    runId,
+    chatId,
+    currentChat,
+    isNewChat,
+    message,
+    titleModel,
+    titleProvider,
+    requestId,
+    workspaceId,
+    orchestrateOptions,
+    otelRoot,
+  } = params
+
+  // Reuse caller's root if provided; otherwise start our own.
+  const activeOtelRoot =
+    otelRoot ??
+    startCopilotOtelRoot({
+      requestId,
+      route: orchestrateOptions.goRoute,
+      chatId,
+      workflowId: orchestrateOptions.workflowId,
+      executionId,
+      runId,
+      streamId,
+      transport: CopilotTransport.Stream,
+    })
+
+  const abortController = new AbortController()
+  registerActiveStream(streamId, abortController)
+
+  const publisher = new StreamWriter({ streamId, chatId, requestId })
+
+  // Classify cancel: signal.reason (explicit-stop set) wins, then
+  // clientDisconnected, else Unknown (latent contract bug — log it).
+  const recordCancelled = (errorMessage?: string): CopilotRequestCancelReasonValue => {
+    const rawReason = abortController.signal.reason
+    let cancelReason: CopilotRequestCancelReasonValue
+    if (isExplicitStopReason(rawReason)) {
+      cancelReason = CopilotRequestCancelReason.ExplicitStop
+    } else if (publisher.clientDisconnected) {
+      cancelReason = CopilotRequestCancelReason.ClientDisconnect
+    } else {
+      cancelReason = CopilotRequestCancelReason.Unknown
+      const serializedReason =
+        rawReason === undefined
+          ? 'undefined'
+          : rawReason instanceof Error
+            ? `${rawReason.name}: ${rawReason.message}`
+            : typeof rawReason === 'string'
+              ? rawReason
+              : (() => {
+                  try {
+                    return JSON.stringify(rawReason)
+                  } catch {
+                    return String(rawReason)
+                  }
+                })()
+      // Contract violation: add the new reason to AbortReason /
+      // isExplicitStopReason or extend the classifier.
+      logger.error(`[${requestId}] Stream cancelled with unknown abort reason`, {
+        streamId,
+        chatId,
+        reason: serializedReason,
+      })
+      activeOtelRoot.span.setAttribute(TraceAttr.CopilotAbortUnknownReason, serializedReason)
+    }
+    activeOtelRoot.span.setAttribute(TraceAttr.CopilotRequestCancelReason, cancelReason)
+    activeOtelRoot.span.addEvent(TraceEvent.RequestCancelled, {
+      [TraceAttr.CopilotRequestCancelReason]: cancelReason,
+      ...(errorMessage ? { [TraceAttr.ErrorMessage]: errorMessage } : {}),
+    })
+    return cancelReason
+  }
+
+  const collector = new TraceCollector()
+
+  return new ReadableStream({
+    async start(controller) {
+      publisher.attach(controller)
+
+      // Re-enter the root OTel context — ALS doesn't survive the
+      // Next handler → ReadableStream.start boundary.
+      await otelContextApi.with(activeOtelRoot.context, async () => {
+        const otelContext = activeOtelRoot.context
+        let rootOutcome: CopilotLifecycleOutcome = RequestTraceV1Outcome.error
+        let rootError: unknown
+        // `cancelReason` must be declared OUTSIDE the outer `try` so
+        // it remains in scope for the outer `finally` that calls
+        // `activeOtelRoot.finish(rootOutcome, rootError, cancelReason)`.
+        // `let` bindings declared inside a `try` block are NOT visible
+        // in the paired `finally`; referencing one there raises a
+        // TDZ ReferenceError, skipping `finish()`, leaving the root
+        // span never-ended, and making Tempo see every child as an
+        // orphan under a phantom parent. (Regression landed 2026-04-21.)
+        let cancelReason: CopilotRequestCancelReasonValue | undefined
+        try {
+          const requestSpan = collector.startSpan('Sim Agent Request', 'request', {
+            streamId,
+            chatId,
+            runId,
+          })
+          let outcome: CopilotLifecycleOutcome = RequestTraceV1Outcome.error
+          let lifecycleResult:
+            | {
+                usage?: { prompt: number; completion: number }
+                cost?: { input: number; output: number; total: number }
+              }
+            | undefined
+
+          await Promise.all([resetBuffer(streamId), clearFilePreviewSessions(streamId)])
+
+          if (chatId) {
+            createRunSegment({
+              id: runId,
+              executionId,
+              chatId,
+              userId,
+              workflowId: (requestPayload.workflowId as string | undefined) || null,
+              workspaceId,
+              streamId,
+              model: (requestPayload.model as string | undefined) || null,
+              provider: (requestPayload.provider as string | undefined) || null,
+              requestContext: { requestId },
+            }).catch((error) => {
+              logger.warn(`[${requestId}] Failed to create copilot run segment`, {
+                error: getErrorMessage(error),
+              })
+            })
+          }
+
+          const abortPoller = startAbortPoller(streamId, abortController, {
+            requestId,
+            chatId,
+          })
+          publisher.startKeepalive()
+
+          if (chatId) {
+            publisher.publish({
+              type: MothershipStreamV1EventType.session,
+              payload: {
+                kind: MothershipStreamV1SessionKind.chat,
+                chatId,
+              },
+            })
+          }
+
+          fireTitleGeneration({
+            chatId,
+            currentChat,
+            isNewChat,
+            userId,
+            message,
+            titleModel,
+            titleProvider,
+            workspaceId,
+            requestId,
+            publisher,
+            otelContext,
+          })
+
+          try {
+            const result = await runCopilotLifecycle(requestPayload, {
+              ...orchestrateOptions,
+              executionId,
+              runId,
+              trace: collector,
+              simRequestId: requestId,
+              otelContext,
+              abortSignal: abortController.signal,
+              onEvent: async (event) => {
+                await publisher.publish(event)
+              },
+              onAbortObserved: (reason) => {
+                if (!abortController.signal.aborted) {
+                  abortController.abort(reason)
+                }
+              },
+            })
+
+            lifecycleResult = result
+            // Outcome classification (priority order):
+            //   1. `result.success` → success. The orchestrator
+            //      reporting "finished cleanly" wins over any later
+            //      signal change. Matters for the narrow race where
+            //      the user clicks Stop a beat after the stream
+            //      completed.
+            //   2. `signal.aborted` (from `abortActiveStream` or the
+            //      Redis-marker poller) OR `clientDisconnected` with
+            //      a non-success result → cancelled. `recordCancelled`
+            //      further refines into explicit_stop / client_disconnect
+            //      / unknown via `signal.reason`.
+            //   3. Otherwise → error.
+            outcome = result.success
+              ? RequestTraceV1Outcome.success
+              : result.cancelled || abortController.signal.aborted || publisher.clientDisconnected
+                ? RequestTraceV1Outcome.cancelled
+                : RequestTraceV1Outcome.error
+            if (outcome === RequestTraceV1Outcome.cancelled) {
+              cancelReason = recordCancelled()
+            }
+            // Pass the resolved outcome — not `signal.aborted` — so
+            // `finalizeStream` classifies the same way we did above.
+            // A client-disconnect-without-controller-abort still needs
+            // to hit `handleAborted` (not `handleError`) so the chat
+            // row gets `cancelled` terminal state instead of `error`.
+            await finalizeStream(result, publisher, runId, outcome, requestId)
+          } catch (error) {
+            // Error-path classification: if the abort signal fired or
+            // the client disconnected, treat the thrown error as a
+            // cancel (same rationale as the try-path above).
+            const wasCancelled = abortController.signal.aborted || publisher.clientDisconnected
+            outcome = wasCancelled ? RequestTraceV1Outcome.cancelled : RequestTraceV1Outcome.error
+            if (outcome === RequestTraceV1Outcome.cancelled) {
+              cancelReason = recordCancelled(getErrorMessage(error))
+            }
+            if (publisher.clientDisconnected) {
+              logger.info(`[${requestId}] Stream errored after client disconnect`, {
+                error: getErrorMessage(error, 'Stream error'),
+              })
+            }
+            // Demote to warn when the throw came from a user-initiated
+            // cancel — it isn't an "unexpected" failure then, and the
+            // error-level log pollutes alerting on normal Stop presses.
+            const logFn = outcome === RequestTraceV1Outcome.cancelled ? logger.warn : logger.error
+            logFn.call(logger, `[${requestId}] Orchestration ended with ${outcome}:`, error)
+
+            const syntheticResult = {
+              success: false as const,
+              content: '',
+              contentBlocks: [],
+              toolCalls: [],
+              error: 'An unexpected error occurred while processing the response.',
+            }
+            await finalizeStream(syntheticResult, publisher, runId, outcome, requestId)
+          } finally {
+            collector.endSpan(
+              requestSpan,
+              outcome === RequestTraceV1Outcome.success
+                ? RequestTraceV1SpanStatus.ok
+                : outcome === RequestTraceV1Outcome.cancelled
+                  ? RequestTraceV1SpanStatus.cancelled
+                  : RequestTraceV1SpanStatus.error
+            )
+
+            clearInterval(abortPoller)
+            try {
+              await publisher.close()
+            } catch (error) {
+              logger.warn(`[${requestId}] Failed to flush stream persistence during close`, {
+                error: getErrorMessage(error),
+              })
+            }
+            unregisterActiveStream(streamId)
+            if (chatId) {
+              await releasePendingChatStream(chatId, streamId)
+            }
+            await scheduleBufferCleanup(streamId)
+            await scheduleFilePreviewSessionCleanup(streamId)
+            await cleanupAbortMarker(streamId)
+
+            rootOutcome = outcome
+            if (lifecycleResult?.usage) {
+              activeOtelRoot.span.setAttributes({
+                [TraceAttr.GenAiUsageInputTokens]: lifecycleResult.usage.prompt ?? 0,
+                [TraceAttr.GenAiUsageOutputTokens]: lifecycleResult.usage.completion ?? 0,
+              })
+            }
+            if (lifecycleResult?.cost) {
+              activeOtelRoot.span.setAttributes({
+                [TraceAttr.BillingCostInputUsd]: lifecycleResult.cost.input ?? 0,
+                [TraceAttr.BillingCostOutputUsd]: lifecycleResult.cost.output ?? 0,
+                [TraceAttr.BillingCostTotalUsd]: lifecycleResult.cost.total ?? 0,
+              })
+            }
+          }
+        } catch (error) {
+          rootOutcome = RequestTraceV1Outcome.error
+          rootError = error
+          throw error
+        } finally {
+          // `finish` is idempotent, so it's safe whether the POST
+          // handler started the root (and may also call finish on an
+          // error path before the stream ran) or we did. The cancel
+          // reason (if any) determines whether `cancelled` is an
+          // expected outcome (explicit_stop → status OK) or a real
+          // error (client_disconnect / unknown → status ERROR).
+          //
+          // Belt-and-suspenders: if `finish()` itself throws (e.g. an
+          // argument in the TDZ, a bad attribute, a regression in
+          // status-setting), fall back to `span.end()` directly. A
+          // root that never ends leaves every child orphaned in Tempo
+          // under a phantom parent; force-ending it keeps the trace
+          // shape intact even when the pretty-finalize path is
+          // broken. The error is logged so Loki greps surface the
+          // regression instead of it silently costing us trace
+          // fidelity for hours.
+          try {
+            activeOtelRoot.finish(rootOutcome, rootError, cancelReason)
+          } catch (finishError) {
+            logger.error(`[${requestId}] activeOtelRoot.finish threw; force-ending root span`, {
+              error: getErrorMessage(finishError),
+            })
+            try {
+              activeOtelRoot.span.end()
+            } catch {
+              // Already ended or an OTel internal failure — nothing
+              // more we can do. The export pipe has already had its
+              // chance; swallow to avoid masking the original error
+              // path.
+            }
+          }
+        }
+      })
+    },
+    cancel() {
+      // The browser's SSE reader closed. Flip `clientDisconnected` so
+      // in-flight `publisher.publish` calls silently no-op (prevents
+      // enqueueing on a closed controller).
+      //
+      // Browser disconnect is NOT an abort — firing the controller
+      // here retroactively reclassifies in-flight successful streams
+      // as aborted and skips assistant persistence. Let the
+      // orchestrator drain naturally; publish no-ops post-disconnect.
+      // Explicit Stop still fires the controller via /chat/abort.
+      publisher.markDisconnected()
+    },
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Title generation (fire-and-forget side effect)
+// ---------------------------------------------------------------------------
+
+function fireTitleGeneration(params: {
+  chatId?: string
+  currentChat: CurrentChatSummary
+  isNewChat: boolean
+  userId?: string
+  message: string
+  titleModel: string
+  titleProvider?: string
+  workspaceId?: string
+  requestId: string
+  publisher: StreamWriter
+  otelContext?: Context
+}): void {
+  const {
+    chatId,
+    currentChat,
+    isNewChat,
+    userId,
+    message,
+    titleModel,
+    titleProvider,
+    workspaceId,
+    requestId,
+    publisher,
+    otelContext,
+  } = params
+  if (!chatId || currentChat?.title || !isNewChat) return
+
+  requestChatTitle({
+    message,
+    model: titleModel,
+    provider: titleProvider,
+    userId,
+    workspaceId,
+    otelContext,
+  })
+    .then(async (title) => {
+      if (!title) return
+      await db.update(copilotChats).set({ title }).where(eq(copilotChats.id, chatId))
+      await publisher.publish({
+        type: MothershipStreamV1EventType.session,
+        payload: { kind: MothershipStreamV1SessionKind.title, title },
+      })
+      if (workspaceId) {
+        chatPubSub?.publishStatusChanged({
+          workspaceId,
+          chatId,
+          type: 'renamed',
+        })
+      }
+    })
+    .catch((error) => {
+      logger.error(`[${requestId}] Title generation failed:`, error)
+    })
+}
+
+// ---------------------------------------------------------------------------
+// Chat title helper
+// ---------------------------------------------------------------------------
+
+export async function requestChatTitle(params: {
+  message: string
+  model: string
+  provider?: string
+  userId?: string
+  workspaceId?: string
+  otelContext?: Context
+}): Promise<string | null> {
+  const { message, model, provider, userId, workspaceId, otelContext } = params
+  if (!message || !model) return null
+
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+  }
+  if (env.COPILOT_API_KEY) {
+    headers['x-api-key'] = env.COPILOT_API_KEY
+  }
+  Object.assign(headers, getMothershipSourceEnvHeaders())
+
+  try {
+    const { fetchGo } = await import('@/lib/copilot/request/go/fetch')
+    const mothershipBaseURL = await getMothershipBaseURL({ userId })
+    const response = await fetchGo(`${mothershipBaseURL}/api/generate-chat-title`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        message,
+        model,
+        ...(provider ? { provider } : {}),
+        ...(workspaceId ? { workspaceId } : {}),
+        ...(userId ? { userId } : {}),
+      }),
+      otelContext,
+      spanName: 'sim → go /api/generate-chat-title',
+      operation: 'generate_chat_title',
+      attributes: {
+        [TraceAttr.GenAiRequestModel]: model,
+        ...(provider ? { [TraceAttr.GenAiSystem]: provider } : {}),
+      },
+    })
+
+    const payload = await response.json().catch(() => ({}))
+    if (!response.ok) {
+      logger.warn('Failed to generate chat title via copilot backend', {
+        status: response.status,
+        error: payload,
+      })
+      return null
+    }
+
+    const title = typeof payload?.title === 'string' ? payload.title.trim() : ''
+    return title || null
+  } catch (error) {
+    logger.error('Error generating chat title:', error)
+    return null
+  }
+}