chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,79 @@
import { trace } from '@opentelemetry/api'
import {
BasicTracerProvider,
InMemorySpanExporter,
SimpleSpanProcessor,
} from '@opentelemetry/sdk-trace-base'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { fetchGo } from '@/lib/copilot/request/go/fetch'
describe('fetchGo', () => {
const exporter = new InMemorySpanExporter()
const provider = new BasicTracerProvider({
spanProcessors: [new SimpleSpanProcessor(exporter)],
})
beforeEach(() => {
exporter.reset()
trace.setGlobalTracerProvider(provider)
vi.restoreAllMocks()
})
it('emits a client span with http.* attrs and injects traceparent', async () => {
const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {
const headers = init.headers as Record<string, string>
expect(headers.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
return new Response('ok', {
status: 200,
headers: { 'content-length': '2' },
})
})
vi.stubGlobal('fetch', fetchMock)
const res = await fetchGo('https://backend.example.com/api/copilot', {
method: 'POST',
body: 'payload',
operation: 'stream',
attributes: { 'copilot.leg': 'sim_to_go' },
})
expect(res.status).toBe(200)
const spans = exporter.getFinishedSpans()
expect(spans).toHaveLength(1)
const attrs = spans[0].attributes
expect(spans[0].name).toBe('sim → go /api/copilot')
expect(attrs['http.method']).toBe('POST')
expect(attrs['http.url']).toBe('https://backend.example.com/api/copilot')
expect(attrs['http.target']).toBe('/api/copilot')
expect(attrs['http.status_code']).toBe(200)
expect(attrs['copilot.operation']).toBe('stream')
expect(attrs['copilot.leg']).toBe('sim_to_go')
expect(typeof attrs['http.response.headers_ms']).toBe('number')
})
it('marks span as error on non-2xx response', async () => {
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('nope', { status: 500 })))
const res = await fetchGo('https://backend.example.com/api/tools/resume', {
method: 'POST',
})
expect(res.status).toBe(500)
const spans = exporter.getFinishedSpans()
expect(spans).toHaveLength(1)
expect(spans[0].status.code).toBe(2)
})
it('records exceptions when fetch throws', async () => {
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network boom')))
await expect(
fetchGo('https://backend.example.com/api/traces', { method: 'POST' })
).rejects.toThrow('network boom')
const spans = exporter.getFinishedSpans()
expect(spans).toHaveLength(1)
expect(spans[0].status.code).toBe(2)
expect(spans[0].events.some((e) => e.name === 'exception')).toBe(true)
})
})
+112
View File
@@ -0,0 +1,112 @@
import { type Context, context, SpanStatusCode, trace } from '@opentelemetry/api'
import { CopilotLeg } from '@/lib/copilot/generated/trace-attribute-values-v1'
import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
import { traceHeaders } from '@/lib/copilot/request/go/propagation'
import { isActionableErrorStatus, markSpanForError } from '@/lib/copilot/request/otel'
// Lazy tracer resolution: module-level `trace.getTracer()` can be evaluated
// before `instrumentation-node.ts` installs the TracerProvider under
// Next.js 16 + Turbopack dev, freezing a NoOp tracer and silently dropping
// every outbound Sim → Go span. Resolving per-call avoids the race.
const getTracer = () => trace.getTracer('sim-copilot-http', '1.0.0')
interface OutboundFetchOptions extends RequestInit {
otelContext?: Context
spanName?: string
operation?: string
attributes?: Record<string, string | number | boolean>
}
/**
* Perform an outbound Sim → Go fetch wrapped in an OTel child span so each
* call shows up as a distinct segment in Jaeger, and propagates the W3C
* traceparent so the Go-side span joins the same trace.
*
* The span captures generic attributes (method, status, duration, response
* size, error code) so any future latency investigation — not just images or
* Bedrock — has uniform metadata to work with.
*/
export async function fetchGo(url: string, options: OutboundFetchOptions = {}): Promise<Response> {
const {
otelContext,
spanName,
operation,
attributes,
headers: providedHeaders,
...init
} = options
const parsed = safeParseUrl(url)
const pathname = parsed?.pathname ?? url
const method = (init.method ?? 'GET').toUpperCase()
const parentContext = otelContext ?? context.active()
const span = getTracer().startSpan(
spanName ?? `sim → go ${pathname}`,
{
attributes: {
[TraceAttr.HttpMethod]: method,
[TraceAttr.HttpUrl]: url,
[TraceAttr.HttpTarget]: pathname,
[TraceAttr.NetPeerName]: parsed?.host ?? '',
[TraceAttr.CopilotLeg]: CopilotLeg.SimToGo,
...(operation ? { [TraceAttr.CopilotOperation]: operation } : {}),
...(attributes ?? {}),
},
},
parentContext
)
const activeContext = trace.setSpan(parentContext, span)
const propagatedHeaders = traceHeaders({}, activeContext)
const mergedHeaders = {
...(providedHeaders as Record<string, string> | undefined),
...propagatedHeaders,
}
const start = performance.now()
try {
const response = await context.with(activeContext, () =>
fetch(url, {
...init,
method,
headers: mergedHeaders,
})
)
const elapsedMs = performance.now() - start
const contentLength = Number(response.headers.get('content-length') ?? 0)
span.setAttribute(TraceAttr.HttpStatusCode, response.status)
span.setAttribute(TraceAttr.HttpResponseHeadersMs, Math.round(elapsedMs))
if (contentLength > 0) {
span.setAttribute(TraceAttr.HttpResponseContentLength, contentLength)
}
// Only mark ERROR for actionable status codes. 4xx that represent
// normal auth/validation rejections (400/401/403/404/405/422/etc.)
// stay UNSET so error dashboards don't drown in expected rejection
// paths. See `isActionableErrorStatus` in Go's telemetry middleware
// for the mirror rule (5xx + 402/409/429).
if (isActionableErrorStatus(response.status)) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: `HTTP ${response.status}`,
})
} else {
span.setStatus({ code: SpanStatusCode.OK })
}
return response
} catch (error) {
span.setAttribute(TraceAttr.HttpResponseHeadersMs, Math.round(performance.now() - start))
markSpanForError(span, error)
throw error
} finally {
span.end()
}
}
function safeParseUrl(url: string): URL | null {
try {
return new URL(url)
} catch {
return null
}
}
@@ -0,0 +1,761 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { MothershipStreamV1EventType } from '@/lib/copilot/generated/mothership-stream-v1'
import {
createFilePreviewSession,
type FilePreviewContentMode,
type FilePreviewSession,
type FilePreviewTargetKind,
isToolArgsDeltaStreamEvent,
isToolCallStreamEvent,
isToolResultStreamEvent,
type SyntheticFilePreviewPayload,
upsertFilePreviewSession,
} from '@/lib/copilot/request/session'
import type {
ActiveFileIntent,
ExecutionContext,
OrchestratorOptions,
StreamEvent,
StreamingContext,
} from '@/lib/copilot/request/types'
import { peekFileIntent } from '@/lib/copilot/tools/server/files/file-intent-store'
import {
buildFilePreviewText,
loadWorkspaceFileTextForPreview,
} from '@/lib/copilot/tools/server/files/file-preview'
import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
const logger = createLogger('CopilotFilePreviewAdapter')
type JsonRecord = Record<string, unknown>
type FileIntent = ActiveFileIntent
type EditContentStreamState = {
raw: string
lastContentSnapshot?: string
}
type FilePreviewStreamState = {
session: FilePreviewSession
lastEmittedPreviewText: string
lastSnapshotAt: number
}
type ParsedWorkspaceFileArgs = {
operation: string
target: FileIntent['target']
title?: string
contentType?: string
edit?: JsonRecord
}
const PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS = 80
const DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS = 1000
function asJsonRecord(value: unknown): JsonRecord | undefined {
return value && typeof value === 'object' && !Array.isArray(value)
? (value as JsonRecord)
: undefined
}
function toPreviewTargetKind(kind: string | undefined): FilePreviewTargetKind | undefined {
return kind === 'new_file' || kind === 'file_id' ? kind : undefined
}
async function resolvePreviewTarget(args: {
workspaceId?: string
target: FileIntent['target']
}): Promise<FileIntent['target']> {
if (args.target.kind !== 'path' || !args.workspaceId || !args.target.path) {
return args.target
}
const file = await resolveWorkspaceFileReference(args.workspaceId, args.target.path)
if (!file) {
return args.target
}
return {
kind: 'file_id',
fileId: file.id,
fileName: args.target.fileName ?? file.name,
path: args.target.path,
}
}
function parseWorkspaceFileArgs(value: unknown): ParsedWorkspaceFileArgs | undefined {
const args = asJsonRecord(value)
if (!args) {
return undefined
}
const operation = typeof args.operation === 'string' ? args.operation : undefined
const target = asJsonRecord(args.target)
const targetKind = typeof target?.kind === 'string' ? target.kind : undefined
if (!operation || !target || !targetKind) {
return undefined
}
const fileId = typeof target.fileId === 'string' ? target.fileId : undefined
const fileName = typeof target.fileName === 'string' ? target.fileName : undefined
const path = typeof target.path === 'string' ? target.path : undefined
const title = typeof args.title === 'string' ? args.title : undefined
const contentType = typeof args.contentType === 'string' ? args.contentType : undefined
const edit = asJsonRecord(args.edit)
return {
operation,
target: {
kind: targetKind,
...(fileId ? { fileId } : {}),
...(fileName ? { fileName } : {}),
...(path ? { path } : {}),
},
...(title ? { title } : {}),
...(contentType ? { contentType } : {}),
...(edit ? { edit } : {}),
}
}
function extractWorkspaceFileResult(output: unknown): { fileId?: string; fileName?: string } {
const candidates: JsonRecord[] = []
const root = asJsonRecord(output)
if (root) {
candidates.push(root)
const rootData = asJsonRecord(root.data)
if (rootData) candidates.push(rootData)
const rootOutput = asJsonRecord(root.output)
if (rootOutput) {
candidates.push(rootOutput)
const outputData = asJsonRecord(rootOutput.data)
if (outputData) candidates.push(outputData)
}
}
for (const candidate of candidates) {
const fileId =
typeof candidate.id === 'string'
? candidate.id
: typeof candidate.fileId === 'string'
? candidate.fileId
: undefined
if (!fileId) continue
const fileName =
typeof candidate.name === 'string'
? candidate.name
: typeof candidate.fileName === 'string'
? candidate.fileName
: undefined
return { fileId, fileName }
}
return {}
}
export function decodeJsonStringPrefix(input: string): string {
let output = ''
for (let i = 0; i < input.length; i++) {
const ch = input[i]
if (ch !== '\\') {
output += ch
continue
}
const next = input[i + 1]
if (!next) break
if (next === 'n') {
output += '\n'
i++
continue
}
if (next === 't') {
output += '\t'
i++
continue
}
if (next === 'r') {
output += '\r'
i++
continue
}
if (next === '"') {
output += '"'
i++
continue
}
if (next === '\\') {
output += '\\'
i++
continue
}
if (next === '/') {
output += '/'
i++
continue
}
if (next === 'b') {
output += '\b'
i++
continue
}
if (next === 'f') {
output += '\f'
i++
continue
}
if (next === 'u') {
const hex = input.slice(i + 2, i + 6)
if (hex.length < 4 || !/^[0-9a-fA-F]{4}$/.test(hex)) break
output += String.fromCharCode(Number.parseInt(hex, 16))
i += 5
continue
}
break
}
return output
}
export function extractEditContent(raw: string): string {
const marker = '"content":'
const idx = raw.indexOf(marker)
if (idx === -1) return ''
const rest = raw.slice(idx + marker.length).trimStart()
if (!rest.startsWith('"')) return rest
let end = -1
for (let i = 1; i < rest.length; i++) {
if (rest[i] === '\\') {
i++
continue
}
if (rest[i] === '"') {
end = i
break
}
}
const inner = end === -1 ? rest.slice(1) : rest.slice(1, end)
return decodeJsonStringPrefix(inner)
}
function isContentOperation(
operation: string | undefined
): operation is 'append' | 'update' | 'patch' {
return operation === 'append' || operation === 'update' || operation === 'patch'
}
function isDocFormat(fileName: string | undefined): boolean {
return /\.(pptx|docx|pdf)$/i.test(fileName ?? '')
}
function buildPreviewSessionFromIntent(
streamId: string,
intent: FileIntent,
current?: FilePreviewSession
): FilePreviewSession {
const targetKind = toPreviewTargetKind(intent.target.kind)
return createFilePreviewSession({
streamId,
toolCallId: intent.toolCallId,
fileName: intent.target.fileName ?? current?.fileName,
...(intent.target.fileId ? { fileId: intent.target.fileId } : {}),
...(targetKind ? { targetKind } : {}),
operation: intent.operation,
...(intent.edit ? { edit: intent.edit } : {}),
...(typeof current?.baseContent === 'string' ? { baseContent: current.baseContent } : {}),
previewText: current?.previewText ?? '',
previewVersion: current?.previewVersion ?? 0,
status: current?.status ?? 'pending',
completedAt: current?.completedAt,
})
}
async function persistFilePreviewSession(session: FilePreviewSession): Promise<void> {
try {
await upsertFilePreviewSession(session)
} catch (error) {
logger.warn('Failed to persist file preview session', {
streamId: session.streamId,
toolCallId: session.toolCallId,
previewVersion: session.previewVersion,
error: toError(error).message,
})
}
}
export function buildPreviewContentUpdate(
previousText: string,
nextText: string,
lastSnapshotAt: number,
now: number,
operation: string | undefined
): { content: string; contentMode: FilePreviewContentMode; lastSnapshotAt: number } {
const shouldForceSnapshot =
previousText.length === 0 ||
!nextText.startsWith(previousText) ||
operation === 'patch' ||
operation === 'append' ||
now - lastSnapshotAt >= DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS
if (shouldForceSnapshot) {
return {
content: nextText,
contentMode: 'snapshot',
lastSnapshotAt: now,
}
}
return {
content: nextText.slice(previousText.length),
contentMode: 'delta',
lastSnapshotAt,
}
}
export interface FilePreviewAdapterState {
editContentState: Map<string, EditContentStreamState>
filePreviewState: Map<string, FilePreviewStreamState>
}
export function createFilePreviewAdapterState(): FilePreviewAdapterState {
return {
editContentState: new Map<string, EditContentStreamState>(),
filePreviewState: new Map<string, FilePreviewStreamState>(),
}
}
async function emitPreviewEvent(
streamEvent: StreamEvent,
options: Pick<OrchestratorOptions, 'onEvent'>,
payload: SyntheticFilePreviewPayload
): Promise<void> {
await options.onEvent?.({
type: MothershipStreamV1EventType.tool,
payload,
...(streamEvent.scope ? { scope: streamEvent.scope } : {}),
})
}
export async function processFilePreviewStreamEvent(input: {
streamId: string
streamEvent: StreamEvent
context: StreamingContext
execContext: ExecutionContext
options: Pick<OrchestratorOptions, 'onEvent'>
state: FilePreviewAdapterState
}): Promise<void> {
const { streamId, streamEvent, context, execContext, options, state } = input
const { editContentState, filePreviewState } = state
// Scope the in-flight intent to the invoking file subagent's channel (its
// outer tool_use id) so two file agents streaming concurrently never read or
// overwrite each other's intent. workspace_file and edit_content from the same
// file agent share this channel id, so they pair up; siblings stay isolated.
const channelId = streamEvent.scope?.parentToolCallId ?? ''
const getIntent = (): FileIntent | null => context.activeFileIntents.get(channelId) ?? null
const setIntent = (intent: FileIntent): void => {
context.activeFileIntents.set(channelId, intent)
}
const clearIntent = (): void => {
context.activeFileIntents.delete(channelId)
}
if (isToolCallStreamEvent(streamEvent) && streamEvent.payload.toolName === 'workspace_file') {
const toolCallId = streamEvent.payload.toolCallId
const parsedArgs = parseWorkspaceFileArgs(streamEvent.payload.arguments)
if (toolCallId && parsedArgs) {
const { operation, title, contentType, edit } = parsedArgs
const target = await resolvePreviewTarget({
workspaceId: execContext.workspaceId,
target: parsedArgs.target,
})
const previewTargetKind = toPreviewTargetKind(target.kind)
const { fileId, fileName } = target
const isContentOp = isContentOperation(operation)
// Per-channel: a re-declared workspace_file just overwrites THIS channel's
// slot. No cross-message intent clearing — that would wipe a concurrent
// sibling file agent's pending intent.
const intent: FileIntent = {
toolCallId,
operation,
target,
...(title ? { title } : {}),
...(contentType ? { contentType } : {}),
...(edit ? { edit } : {}),
}
setIntent(intent)
if (isContentOp && previewTargetKind) {
let previewBaseContent: string | undefined
if (
execContext.workspaceId &&
fileId &&
(operation === 'append' || operation === 'patch')
) {
previewBaseContent = await loadWorkspaceFileTextForPreview(
execContext.workspaceId,
fileId
)
}
let session = buildPreviewSessionFromIntent(streamId, intent)
if (previewBaseContent !== undefined) {
session = { ...session, baseContent: previewBaseContent }
}
filePreviewState.set(toolCallId, {
session,
lastEmittedPreviewText: '',
lastSnapshotAt: 0,
})
await persistFilePreviewSession(session)
await emitPreviewEvent(streamEvent, options, {
toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_start',
})
await emitPreviewEvent(streamEvent, options, {
toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_target',
operation,
target: {
kind: previewTargetKind,
...(fileId ? { fileId } : {}),
...(fileName ? { fileName } : {}),
},
...(title ? { title } : {}),
})
if (edit) {
await emitPreviewEvent(streamEvent, options, {
toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_edit_meta',
edit,
})
}
}
}
}
const workspaceResultIntent = getIntent()
if (
isToolResultStreamEvent(streamEvent) &&
streamEvent.payload.toolName === 'workspace_file' &&
workspaceResultIntent &&
isContentOperation(workspaceResultIntent.operation)
) {
const result = extractWorkspaceFileResult(streamEvent.payload.output)
if (result.fileId && workspaceResultIntent.target.kind === 'path') {
const intent: FileIntent = {
...workspaceResultIntent,
target: {
kind: 'file_id',
fileId: result.fileId,
fileName: result.fileName ?? workspaceResultIntent.target.fileName,
path: workspaceResultIntent.target.path,
},
}
setIntent(intent)
let previewBaseContent: string | undefined
if (
execContext.workspaceId &&
(intent.operation === 'append' || intent.operation === 'patch')
) {
previewBaseContent = await loadWorkspaceFileTextForPreview(
execContext.workspaceId,
result.fileId
)
}
let session = buildPreviewSessionFromIntent(streamId, intent)
if (previewBaseContent !== undefined) {
session = { ...session, baseContent: previewBaseContent }
}
filePreviewState.set(intent.toolCallId, {
session,
lastEmittedPreviewText: '',
lastSnapshotAt: 0,
})
await persistFilePreviewSession(session)
await emitPreviewEvent(streamEvent, options, {
toolCallId: intent.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_start',
})
await emitPreviewEvent(streamEvent, options, {
toolCallId: intent.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_target',
operation: intent.operation,
target: {
kind: 'file_id',
fileId: result.fileId,
...(result.fileName ? { fileName: result.fileName } : {}),
},
...(intent.title ? { title: intent.title } : {}),
})
if (intent.edit) {
await emitPreviewEvent(streamEvent, options, {
toolCallId: intent.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_edit_meta',
edit: intent.edit,
})
}
}
}
const patchDeleteIntent = getIntent()
if (
isToolResultStreamEvent(streamEvent) &&
streamEvent.payload.toolName === 'workspace_file' &&
patchDeleteIntent &&
isContentOperation(patchDeleteIntent.operation) &&
patchDeleteIntent.operation === 'patch' &&
patchDeleteIntent.edit?.strategy === 'anchored' &&
patchDeleteIntent.edit?.mode === 'delete_between' &&
execContext.workspaceId &&
patchDeleteIntent.target.fileId &&
!isDocFormat(patchDeleteIntent.target.fileName)
) {
const currentPreview = filePreviewState.get(patchDeleteIntent.toolCallId)
const previewText = buildFilePreviewText({
operation: 'patch',
streamedContent: '',
existingContent: currentPreview?.session.baseContent,
edit: currentPreview?.session.edit,
})
if (previewText !== undefined) {
const baseSession = buildPreviewSessionFromIntent(
streamId,
patchDeleteIntent,
currentPreview?.session
)
const nextSession: FilePreviewSession = {
...baseSession,
status: 'streaming',
previewText,
previewVersion: (currentPreview?.session.previewVersion ?? 0) + 1,
updatedAt: new Date().toISOString(),
}
filePreviewState.set(patchDeleteIntent.toolCallId, {
session: nextSession,
lastEmittedPreviewText: previewText,
lastSnapshotAt: Date.now(),
})
await persistFilePreviewSession(nextSession)
await emitPreviewEvent(streamEvent, options, {
toolCallId: nextSession.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_content',
content: previewText,
contentMode: 'snapshot',
previewVersion: nextSession.previewVersion,
fileName: nextSession.fileName,
...(nextSession.fileId ? { fileId: nextSession.fileId } : {}),
...(nextSession.targetKind ? { targetKind: nextSession.targetKind } : {}),
...(nextSession.operation ? { operation: nextSession.operation } : {}),
...(nextSession.edit ? { edit: nextSession.edit } : {}),
})
}
}
if (isToolArgsDeltaStreamEvent(streamEvent) && streamEvent.payload.toolName === 'edit_content') {
const toolCallId = streamEvent.payload.toolCallId
const delta = streamEvent.payload.argumentsDelta
const stateForTool = editContentState.get(toolCallId) ?? { raw: '' }
stateForTool.raw += delta
const editIntent = getIntent()
if (editIntent) {
const streamedContent = extractEditContent(stateForTool.raw)
if (streamedContent !== (stateForTool.lastContentSnapshot ?? '')) {
stateForTool.lastContentSnapshot = streamedContent
let currentPreview = filePreviewState.get(editIntent.toolCallId) ?? {
session: buildPreviewSessionFromIntent(streamId, editIntent),
lastEmittedPreviewText: '',
lastSnapshotAt: 0,
}
if (
currentPreview.session.baseContent === undefined &&
(editIntent.operation === 'append' || editIntent.operation === 'patch') &&
execContext.workspaceId &&
editIntent.target.fileId
) {
const intentBase = await peekFileIntent(
execContext.workspaceId,
editIntent.target.fileId,
{
chatId: execContext.chatId,
messageId: execContext.messageId,
channelId,
}
)
if (typeof intentBase?.existingContent === 'string') {
const seededSession: FilePreviewSession = {
...currentPreview.session,
baseContent: intentBase.existingContent,
...(intentBase.edit ? { edit: intentBase.edit } : {}),
}
currentPreview = {
...currentPreview,
session: seededSession,
}
filePreviewState.set(editIntent.toolCallId, currentPreview)
await persistFilePreviewSession(seededSession)
}
}
const previewText = isContentOperation(editIntent.operation)
? buildFilePreviewText({
operation: editIntent.operation,
streamedContent,
existingContent: currentPreview.session.baseContent,
edit: currentPreview.session.edit,
})
: undefined
if (previewText !== undefined) {
const baseSession = buildPreviewSessionFromIntent(
streamId,
editIntent,
currentPreview.session
)
const now = Date.now()
const nextSession: FilePreviewSession = {
...baseSession,
status: 'streaming',
previewText,
previewVersion: (currentPreview.session.previewVersion ?? 0) + 1,
updatedAt: new Date(now).toISOString(),
}
await persistFilePreviewSession(nextSession)
if (
nextSession.operation === 'patch' &&
now - currentPreview.lastSnapshotAt < PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS
) {
filePreviewState.set(editIntent.toolCallId, {
session: nextSession,
lastEmittedPreviewText: currentPreview.lastEmittedPreviewText,
lastSnapshotAt: currentPreview.lastSnapshotAt,
})
} else {
const previewUpdate = buildPreviewContentUpdate(
currentPreview.lastEmittedPreviewText,
nextSession.previewText,
currentPreview.lastSnapshotAt,
now,
nextSession.operation
)
filePreviewState.set(editIntent.toolCallId, {
session: nextSession,
lastEmittedPreviewText: nextSession.previewText,
lastSnapshotAt: previewUpdate.lastSnapshotAt,
})
await emitPreviewEvent(streamEvent, options, {
toolCallId: nextSession.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_content',
content: previewUpdate.content,
contentMode: previewUpdate.contentMode,
previewVersion: nextSession.previewVersion,
fileName: nextSession.fileName,
...(nextSession.fileId ? { fileId: nextSession.fileId } : {}),
...(nextSession.targetKind ? { targetKind: nextSession.targetKind } : {}),
...(nextSession.operation ? { operation: nextSession.operation } : {}),
...(nextSession.edit ? { edit: nextSession.edit } : {}),
})
}
} else {
filePreviewState.set(editIntent.toolCallId, {
session: currentPreview.session,
lastEmittedPreviewText: currentPreview.lastEmittedPreviewText,
lastSnapshotAt: currentPreview.lastSnapshotAt,
})
}
}
}
editContentState.set(toolCallId, stateForTool)
}
if (isToolCallStreamEvent(streamEvent) && streamEvent.payload.toolName === 'edit_content') {
const toolCallId = streamEvent.payload.toolCallId
if (toolCallId) {
editContentState.delete(toolCallId)
}
}
const editResultIntent = getIntent()
if (
isToolResultStreamEvent(streamEvent) &&
streamEvent.payload.toolName === 'edit_content' &&
editResultIntent
) {
const currentPreview = filePreviewState.get(editResultIntent.toolCallId)
const completedAt = new Date().toISOString()
if (
currentPreview &&
currentPreview.lastEmittedPreviewText !== currentPreview.session.previewText &&
currentPreview.session.previewText.length > 0
) {
filePreviewState.set(editResultIntent.toolCallId, {
session: currentPreview.session,
lastEmittedPreviewText: currentPreview.session.previewText,
lastSnapshotAt: Date.now(),
})
await emitPreviewEvent(streamEvent, options, {
toolCallId: currentPreview.session.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_content',
content: currentPreview.session.previewText,
contentMode: 'snapshot',
previewVersion: currentPreview.session.previewVersion,
fileName: currentPreview.session.fileName,
...(currentPreview.session.fileId ? { fileId: currentPreview.session.fileId } : {}),
...(currentPreview.session.targetKind
? { targetKind: currentPreview.session.targetKind }
: {}),
...(currentPreview.session.operation
? { operation: currentPreview.session.operation }
: {}),
...(currentPreview.session.edit ? { edit: currentPreview.session.edit } : {}),
})
}
if (currentPreview) {
const completedSession: FilePreviewSession = {
...currentPreview.session,
status: 'complete',
updatedAt: completedAt,
completedAt,
}
filePreviewState.set(editResultIntent.toolCallId, {
session: completedSession,
lastEmittedPreviewText: completedSession.previewText,
lastSnapshotAt: Date.now(),
})
await persistFilePreviewSession(completedSession)
}
await emitPreviewEvent(streamEvent, options, {
toolCallId: editResultIntent.toolCallId,
toolName: 'workspace_file',
previewPhase: 'file_preview_complete',
fileId: editResultIntent.target.fileId,
output: streamEvent.payload.output,
...(currentPreview ? { previewVersion: currentPreview.session.previewVersion } : {}),
})
clearIntent()
}
}
+129
View File
@@ -0,0 +1,129 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
const logger = createLogger('CopilotSseParser')
export class FatalSseEventError extends Error {}
function createParseFailure(message: string, preview: string): FatalSseEventError {
logger.error(message, { preview })
return new FatalSseEventError(message)
}
function normalizeSseLine(line: string): string {
return line.endsWith('\r') ? line.slice(0, -1) : line
}
/**
* Processes an SSE stream by calling onEvent for each parsed event.
*
* @param onEvent Called per parsed event. Return true to stop processing.
*/
export async function processSSEStream(
reader: ReadableStreamDefaultReader<Uint8Array>,
decoder: TextDecoder,
abortSignal: AbortSignal | undefined,
onEvent: (event: unknown) => boolean | undefined | Promise<boolean | undefined>
): Promise<void> {
let buffer = ''
try {
try {
while (true) {
if (abortSignal?.aborted) {
logger.info('SSE stream aborted by signal')
break
}
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
const lines = buffer.split('\n')
buffer = lines.pop() || ''
let stopped = false
for (const line of lines) {
const normalizedLine = normalizeSseLine(line)
if (abortSignal?.aborted) {
logger.info('SSE stream aborted mid-chunk (between events)')
return
}
if (!normalizedLine.trim()) continue
if (!normalizedLine.startsWith('data: ')) continue
const jsonStr = normalizedLine.slice(6)
if (jsonStr === '[DONE]') continue
let parsed: unknown
try {
parsed = JSON.parse(jsonStr)
} catch (error) {
const preview = jsonStr.slice(0, 200)
const detail = toError(error).message
throw createParseFailure(`Failed to parse SSE event JSON: ${detail}`, preview)
}
try {
if (await onEvent(parsed)) {
stopped = true
break
}
} catch (error) {
if (error instanceof FatalSseEventError) {
throw error
}
logger.warn('Failed to handle SSE event', {
preview: jsonStr.slice(0, 200),
error: toError(error).message,
})
}
}
if (stopped) break
}
} catch (error) {
const aborted =
abortSignal?.aborted || (error instanceof DOMException && error.name === 'AbortError')
if (aborted) {
logger.info('SSE stream read aborted')
return
}
throw error
}
const normalizedBuffer = normalizeSseLine(buffer)
if (normalizedBuffer.trim() && normalizedBuffer.startsWith('data: ')) {
const jsonStr = normalizedBuffer.slice(6)
if (jsonStr === '[DONE]') {
return
}
let parsed: unknown
try {
parsed = JSON.parse(jsonStr)
} catch (error) {
const preview = normalizedBuffer.slice(0, 200)
const detail = toError(error).message
throw createParseFailure(`Failed to parse final SSE buffer JSON: ${detail}`, preview)
}
try {
await onEvent(parsed)
} catch (error) {
if (error instanceof FatalSseEventError) {
throw error
}
logger.warn('Failed to handle final SSE event', {
preview: normalizedBuffer.slice(0, 200),
error: toError(error).message,
})
}
}
} finally {
try {
reader.releaseLock()
} catch {
logger.warn('Failed to release SSE reader lock')
}
}
}
@@ -0,0 +1,57 @@
import { type Context, context } from '@opentelemetry/api'
import { W3CTraceContextPropagator } from '@opentelemetry/core'
const propagator = new W3CTraceContextPropagator()
const headerSetter = {
set(carrier: Record<string, string>, key: string, value: string) {
carrier[key] = value
},
}
const headerGetter = {
keys(carrier: Headers): string[] {
const out: string[] = []
carrier.forEach((_, key) => {
out.push(key)
})
return out
},
get(carrier: Headers, key: string): string | undefined {
return carrier.get(key) ?? undefined
},
}
/**
* Injects W3C trace context (traceparent, tracestate) into outbound HTTP
* headers so Go-side spans join the same OTel trace tree as the calling
* Sim span.
*
* Usage: spread the result into your fetch headers:
* fetch(url, { headers: { ...myHeaders, ...traceHeaders() } })
*/
export function traceHeaders(
carrier?: Record<string, string>,
otelContext?: Context
): Record<string, string> {
const headers: Record<string, string> = carrier ?? {}
propagator.inject(otelContext ?? context.active(), headers, headerSetter)
return headers
}
/**
* Extracts W3C trace context from incoming request headers (traceparent /
* tracestate) and returns an OTel Context seeded with the upstream span.
*
* Use this at the top of inbound Sim route handlers that Go calls into
* (e.g. /api/billing/update-cost, /api/copilot/api-keys/validate) so the
* Sim-side span becomes a proper child of the Go-side client span in the
* same trace — closing the round trip in Jaeger.
*
* When no traceparent is present (e.g. calls from a browser or a client
* that hasn't been instrumented), this returns `context.active()`
* unchanged, and any span started under it becomes a new root — the same
* behavior as before this helper existed.
*/
export function contextFromRequestHeaders(headers: Headers): Context {
return propagator.extract(context.active(), headers, headerGetter)
}
@@ -0,0 +1,869 @@
/**
* @vitest-environment node
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
MothershipStreamV1CompletionStatus,
MothershipStreamV1EventType,
MothershipStreamV1ToolExecutor,
MothershipStreamV1ToolMode,
MothershipStreamV1ToolOutcome,
MothershipStreamV1ToolPhase,
} from '@/lib/copilot/generated/mothership-stream-v1'
vi.mock('@/lib/copilot/request/session', async () => {
const actual = await vi.importActual<typeof import('@/lib/copilot/request/session')>(
'@/lib/copilot/request/session'
)
return {
...actual,
hasAbortMarker: vi.fn().mockResolvedValue(false),
upsertFilePreviewSession: vi.fn(async (session) => session),
}
})
const resolveWorkspaceFileReferenceMock = vi.hoisted(() => vi.fn())
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
resolveWorkspaceFileReference: resolveWorkspaceFileReferenceMock,
}))
vi.mock('@/lib/copilot/tools/server/files/file-preview', async () => {
const actual = await vi.importActual<
typeof import('@/lib/copilot/tools/server/files/file-preview')
>('@/lib/copilot/tools/server/files/file-preview')
return {
...actual,
loadWorkspaceFileTextForPreview: vi.fn().mockResolvedValue(''),
}
})
import {
buildPreviewContentUpdate,
decodeJsonStringPrefix,
extractEditContent,
runStreamLoop,
} from '@/lib/copilot/request/go/stream'
import { AbortReason, createEvent, hasAbortMarker } from '@/lib/copilot/request/session'
import { RequestTraceV1Outcome, TraceCollector } from '@/lib/copilot/request/trace'
import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types'
function createSseResponse(events: unknown[]): Response {
const payload = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join('')
return new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(payload))
controller.close()
},
}),
{
status: 200,
headers: {
'Content-Type': 'text/event-stream',
},
}
)
}
function createRawSseResponse(payload: string): Response {
return new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(payload))
controller.close()
},
}),
{
status: 200,
headers: {
'Content-Type': 'text/event-stream',
},
}
)
}
function createStreamingContext(): StreamingContext {
return {
messageId: 'msg-1',
accumulatedContent: '',
finalAssistantContent: '',
sawMainToolCall: false,
contentBlocks: [],
toolCalls: new Map(),
pendingToolPromises: new Map(),
currentThinkingBlock: null,
subagentThinkingBlocks: new Map(),
isInThinkingBlock: false,
subAgentContent: {},
subAgentToolCalls: {},
pendingContent: '',
streamComplete: false,
wasAborted: false,
errors: [],
activeFileIntents: new Map(),
trace: new TraceCollector(),
}
}
describe('copilot go stream helpers', () => {
beforeEach(() => {
vi.stubGlobal('fetch', vi.fn())
resolveWorkspaceFileReferenceMock.mockReset()
resolveWorkspaceFileReferenceMock.mockResolvedValue(null)
})
afterEach(() => {
vi.unstubAllGlobals()
})
it('decodes complete escapes and stops at incomplete unicode escapes', () => {
expect(decodeJsonStringPrefix('hello\\nworld')).toBe('hello\nworld')
expect(decodeJsonStringPrefix('emoji \\u263A')).toBe('emoji ☺')
expect(decodeJsonStringPrefix('partial \\u26')).toBe('partial ')
})
it('extracts the streamed edit_content prefix from partial JSON', () => {
expect(extractEditContent('{"content":"hello\\nwor')).toBe('hello\nwor')
expect(extractEditContent('{"content":"tab\\tvalue"}')).toBe('tab\tvalue')
})
it('emits full snapshots for append (sidebar viewer uses replace mode; no delta merge)', () => {
expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'append')).toEqual({
content: 'hello world',
contentMode: 'snapshot',
lastSnapshotAt: 200,
})
})
it('emits deltas for update when the preview extends the previous text', () => {
expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'update')).toEqual({
content: ' world',
contentMode: 'delta',
lastSnapshotAt: 100,
})
})
it('falls back to snapshots for patches and divergent content', () => {
expect(buildPreviewContentUpdate('hello', 'goodbye', 100, 200, 'update')).toEqual({
content: 'goodbye',
contentMode: 'snapshot',
lastSnapshotAt: 200,
})
expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'patch')).toEqual({
content: 'hello world',
contentMode: 'snapshot',
lastSnapshotAt: 200,
})
})
it('hydrates path-based workspace_file edits into file preview events before edit_content streams', async () => {
resolveWorkspaceFileReferenceMock.mockResolvedValue({
id: 'file-1',
name: 'notes.md',
})
const workspaceFileCall = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'workspace-file-path-1',
toolName: 'workspace_file',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.call,
arguments: {
operation: 'update',
target: { kind: 'path', path: 'files/notes.md' },
title: 'Update notes',
},
},
})
const workspaceFileResult = createEvent({
streamId: 'stream-1',
cursor: '2',
seq: 2,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'workspace-file-path-1',
toolName: 'workspace_file',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.result,
success: true,
output: {
success: true,
data: { id: 'file-1', name: 'notes.md', operation: 'update' },
},
},
})
const editContentDelta = createEvent({
streamId: 'stream-1',
cursor: '3',
seq: 3,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'edit-content-path-1',
toolName: 'edit_content',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.args_delta,
argumentsDelta: '{"content":"hello world',
},
})
const editContentResult = createEvent({
streamId: 'stream-1',
cursor: '4',
seq: 4,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'edit-content-path-1',
toolName: 'edit_content',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.result,
success: true,
output: {
success: true,
data: { id: 'file-1', name: 'notes.md' },
},
},
})
const complete = createEvent({
streamId: 'stream-1',
cursor: '5',
seq: 5,
requestId: 'req-1',
type: MothershipStreamV1EventType.complete,
payload: {
status: MothershipStreamV1CompletionStatus.complete,
},
})
vi.mocked(fetch).mockResolvedValueOnce(
createSseResponse([
workspaceFileCall,
workspaceFileResult,
editContentDelta,
editContentResult,
complete,
])
)
const onEvent = vi.fn()
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
messageId: 'msg-1',
}
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
onEvent,
timeout: 1000,
})
const previewEvents = onEvent.mock.calls
.map(([event]) => event)
.filter(
(event) =>
event.type === MothershipStreamV1EventType.tool && 'previewPhase' in event.payload
)
expect(previewEvents.map((event) => event.payload.previewPhase)).toEqual([
'file_preview_start',
'file_preview_target',
'file_preview_content',
'file_preview_complete',
])
expect(previewEvents[1].payload).toMatchObject({
previewPhase: 'file_preview_target',
target: { kind: 'file_id', fileId: 'file-1', fileName: 'notes.md' },
})
expect(previewEvents[2].payload).toMatchObject({
previewPhase: 'file_preview_content',
fileId: 'file-1',
targetKind: 'file_id',
content: 'hello world',
})
expect(previewEvents[3].payload).toMatchObject({
previewPhase: 'file_preview_complete',
fileId: 'file-1',
})
expect(resolveWorkspaceFileReferenceMock).toHaveBeenCalledWith('workspace-1', 'files/notes.md')
})
it('resolves workflow alias paths to the backing file before streaming previews', async () => {
resolveWorkspaceFileReferenceMock.mockResolvedValue({
id: 'changelog-file-1',
name: 'workflow-1.md',
})
const workspaceFileCall = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'workspace-file-alias-1',
toolName: 'workspace_file',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.call,
arguments: {
operation: 'append',
target: { kind: 'path', path: 'workflows/My%20Workflow/changelog.md' },
title: 'Update changelog',
},
},
})
const editContentDelta = createEvent({
streamId: 'stream-1',
cursor: '2',
seq: 2,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'edit-content-alias-1',
toolName: 'edit_content',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.args_delta,
argumentsDelta: '{"content":"\\n- Added a workflow step',
},
})
const editContentResult = createEvent({
streamId: 'stream-1',
cursor: '3',
seq: 3,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'edit-content-alias-1',
toolName: 'edit_content',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.result,
success: true,
output: {
success: true,
data: { id: 'changelog-file-1', name: 'workflow-1.md' },
},
},
})
const complete = createEvent({
streamId: 'stream-1',
cursor: '4',
seq: 4,
requestId: 'req-1',
type: MothershipStreamV1EventType.complete,
payload: {
status: MothershipStreamV1CompletionStatus.complete,
},
})
vi.mocked(fetch).mockResolvedValueOnce(
createSseResponse([workspaceFileCall, editContentDelta, editContentResult, complete])
)
const onEvent = vi.fn()
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
messageId: 'msg-1',
}
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
onEvent,
timeout: 1000,
})
const previewEvents = onEvent.mock.calls
.map(([event]) => event)
.filter(
(event) =>
event.type === MothershipStreamV1EventType.tool && 'previewPhase' in event.payload
)
expect(previewEvents.map((event) => event.payload.previewPhase)).toEqual([
'file_preview_start',
'file_preview_target',
'file_preview_content',
'file_preview_complete',
])
expect(previewEvents[1].payload).toMatchObject({
previewPhase: 'file_preview_target',
target: { kind: 'file_id', fileId: 'changelog-file-1', fileName: 'workflow-1.md' },
})
expect(previewEvents[2].payload).toMatchObject({
previewPhase: 'file_preview_content',
fileId: 'changelog-file-1',
targetKind: 'file_id',
content: '\n- Added a workflow step',
})
expect(previewEvents[3].payload).toMatchObject({
previewPhase: 'file_preview_complete',
fileId: 'changelog-file-1',
})
expect(resolveWorkspaceFileReferenceMock).toHaveBeenCalledWith(
'workspace-1',
'workflows/My%20Workflow/changelog.md'
)
})
it('drops duplicate tool_result events before forwarding them', async () => {
const toolResult = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.tool,
payload: {
toolCallId: 'tool-result-dedupe',
toolName: 'search_online',
executor: MothershipStreamV1ToolExecutor.sim,
mode: MothershipStreamV1ToolMode.async,
phase: MothershipStreamV1ToolPhase.result,
success: true,
output: { value: 'ok' },
},
})
const complete = createEvent({
streamId: 'stream-1',
cursor: '2',
seq: 2,
requestId: 'req-1',
type: MothershipStreamV1EventType.complete,
payload: {
status: MothershipStreamV1CompletionStatus.complete,
},
})
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([toolResult, toolResult, complete]))
const onEvent = vi.fn()
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
onEvent,
timeout: 1000,
})
expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
MothershipStreamV1EventType.tool,
MothershipStreamV1EventType.complete,
])
expect(onEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: MothershipStreamV1EventType.tool,
payload: expect.objectContaining({
toolCallId: 'tool-result-dedupe',
phase: MothershipStreamV1ToolPhase.result,
}),
})
)
expect(context.toolCalls.get('tool-result-dedupe')).toEqual(
expect.objectContaining({
id: 'tool-result-dedupe',
name: 'search_online',
status: MothershipStreamV1ToolOutcome.success,
result: { success: true, output: { value: 'ok' } },
})
)
})
it('does not retry transient backend statuses because stream requests are not idempotent', async () => {
vi.mocked(fetch).mockResolvedValueOnce(new Response('bad gateway', { status: 502 }))
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toMatchObject({
name: 'CopilotBackendError',
status: 502,
body: 'bad gateway',
})
expect(fetch).toHaveBeenCalledTimes(1)
})
it('does not retry non-transient backend statuses before the SSE stream opens', async () => {
vi.mocked(fetch).mockResolvedValueOnce(new Response('limit reached', { status: 402 }))
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toThrow('Usage limit reached')
expect(fetch).toHaveBeenCalledTimes(1)
})
it('does not retry network errors because Go may already be executing the request', async () => {
vi.mocked(fetch).mockRejectedValueOnce(new TypeError('fetch failed'))
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toThrow('fetch failed')
expect(fetch).toHaveBeenCalledTimes(1)
})
it('fails closed when the shared stream ends before a terminal event', async () => {
const textEvent = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.text,
payload: {
channel: 'assistant',
text: 'partial response',
},
})
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toThrow('Copilot backend stream ended before a terminal event')
expect(
context.errors.some((message) =>
message.includes('Copilot backend stream ended before a terminal event')
)
).toBe(true)
})
it('reclassifies as aborted when the body closes without terminal but the abort marker is set', async () => {
const textEvent = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.text,
payload: {
channel: 'assistant',
text: 'partial response',
},
})
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
expect(hasAbortMarker).toHaveBeenCalledWith(context.messageId)
expect(context.wasAborted).toBe(true)
expect(
context.errors.some((message) =>
message.includes('Copilot backend stream ended before a terminal event')
)
).toBe(false)
})
it('invokes onAbortObserved with MarkerObservedAtBodyClose when reclassifying via the abort marker', async () => {
const textEvent = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.text,
payload: {
channel: 'assistant',
text: 'partial response',
},
})
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
const onAbortObserved = vi.fn()
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
onAbortObserved,
})
expect(onAbortObserved).toHaveBeenCalledTimes(1)
expect(onAbortObserved).toHaveBeenCalledWith(AbortReason.MarkerObservedAtBodyClose)
expect(context.wasAborted).toBe(true)
})
it('does not invoke onAbortObserved when no abort marker is present at body close', async () => {
const textEvent = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.text,
payload: {
channel: 'assistant',
text: 'partial response',
},
})
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
vi.mocked(hasAbortMarker).mockResolvedValueOnce(false)
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
const onAbortObserved = vi.fn()
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
onAbortObserved,
})
).rejects.toThrow('Copilot backend stream ended before a terminal event')
expect(onAbortObserved).not.toHaveBeenCalled()
})
it('still fails closed when the body closes without terminal and the abort marker check throws', async () => {
const textEvent = createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.text,
payload: {
channel: 'assistant',
text: 'partial response',
},
})
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
vi.mocked(hasAbortMarker).mockRejectedValueOnce(new Error('redis unavailable'))
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toThrow('Copilot backend stream ended before a terminal event')
expect(context.wasAborted).toBe(false)
})
it('fails closed when the shared stream receives an invalid event', async () => {
vi.mocked(fetch).mockResolvedValueOnce(
createSseResponse([
{
v: 1,
type: MothershipStreamV1EventType.tool,
seq: 1,
ts: '2026-01-01T00:00:00.000Z',
stream: { streamId: 'stream-1', cursor: '1' },
payload: {
phase: MothershipStreamV1ToolPhase.result,
},
},
])
)
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toThrow('Received invalid stream event on shared path')
expect(
context.errors.some((message) =>
message.includes('Received invalid stream event on shared path')
)
).toBe(true)
})
it('fails closed when the shared stream receives malformed JSON', async () => {
vi.mocked(fetch).mockResolvedValueOnce(
createRawSseResponse('data: {"v":1,"type":"text","payload":\n\n')
)
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await expect(
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
).rejects.toThrow('Failed to parse SSE event JSON')
expect(
context.errors.some((message) => message.includes('Failed to parse SSE event JSON'))
).toBe(true)
})
it('records a split canonical request id and go trace id from the stream envelope', async () => {
vi.mocked(fetch).mockResolvedValueOnce(
createSseResponse([
{
v: 1,
type: MothershipStreamV1EventType.text,
seq: 1,
ts: '2026-01-01T00:00:00.000Z',
stream: { streamId: 'stream-1', cursor: '1' },
trace: {
requestId: 'sim-request-1',
goTraceId: 'go-trace-1',
},
payload: {
channel: 'assistant',
text: 'hello',
},
},
createEvent({
streamId: 'stream-1',
cursor: '2',
seq: 2,
requestId: 'sim-request-1',
type: MothershipStreamV1EventType.complete,
payload: {
status: MothershipStreamV1CompletionStatus.complete,
},
}),
])
)
const context = createStreamingContext()
context.requestId = 'sim-request-1'
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
expect(context.requestId).toBe('sim-request-1')
expect(
context.trace.build({
outcome: RequestTraceV1Outcome.success,
simRequestId: 'sim-request-1',
}).goTraceId
).toBe('go-trace-1')
})
it('records span identity on the subagent block from the scope', async () => {
vi.mocked(fetch).mockResolvedValueOnce(
createSseResponse([
createEvent({
streamId: 'stream-1',
cursor: '1',
seq: 1,
requestId: 'req-1',
type: MothershipStreamV1EventType.span,
scope: {
lane: 'subagent',
agentId: 'deploy',
parentToolCallId: 'tc-deploy-inner',
spanId: 'S2',
parentSpanId: 'S1',
},
payload: {
kind: 'subagent',
event: 'start',
agent: 'deploy',
data: { tool_call_id: 'tc-deploy-inner', nested: true },
},
}),
createEvent({
streamId: 'stream-1',
cursor: '2',
seq: 2,
requestId: 'req-1',
type: MothershipStreamV1EventType.complete,
payload: { status: MothershipStreamV1CompletionStatus.complete },
}),
])
)
const context = createStreamingContext()
const execContext: ExecutionContext = {
userId: 'user-1',
workflowId: 'workflow-1',
}
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
timeout: 1000,
})
const subagentBlock = context.contentBlocks.find((block) => block.type === 'subagent')
expect(subagentBlock).toBeDefined()
expect(subagentBlock?.spanId).toBe('S2')
expect(subagentBlock?.parentSpanId).toBe('S1')
expect(subagentBlock?.parentToolCallId).toBe('tc-deploy-inner')
})
})
+708
View File
@@ -0,0 +1,708 @@
import { type Context, SpanStatusCode } from '@opentelemetry/api'
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { ORCHESTRATION_TIMEOUT_MS } from '@/lib/copilot/constants'
import {
MothershipStreamV1EventType,
MothershipStreamV1SpanLifecycleEvent,
} from '@/lib/copilot/generated/mothership-stream-v1'
import { CopilotSseCloseReason } from '@/lib/copilot/generated/trace-attribute-values-v1'
import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
import { TraceEvent } from '@/lib/copilot/generated/trace-events-v1'
import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
import { fetchGo } from '@/lib/copilot/request/go/fetch'
import {
buildPreviewContentUpdate,
createFilePreviewAdapterState,
decodeJsonStringPrefix,
extractEditContent,
processFilePreviewStreamEvent,
} from '@/lib/copilot/request/go/file-preview-adapter'
import { FatalSseEventError, processSSEStream } from '@/lib/copilot/request/go/parser'
import {
handleSubagentRouting,
prePersistClientExecutableToolCall,
sseHandlers,
subAgentHandlers,
} from '@/lib/copilot/request/handlers'
import {
flushSubagentThinkingBlock,
flushThinkingBlock,
} from '@/lib/copilot/request/handlers/types'
import { getCopilotTracer } from '@/lib/copilot/request/otel'
import {
AbortReason,
eventToStreamEvent,
hasAbortMarker,
isSubagentSpanStreamEvent,
parsePersistedStreamEventEnvelope,
} from '@/lib/copilot/request/session'
import { shouldSkipToolCallEvent, shouldSkipToolResultEvent } from '@/lib/copilot/request/sse-utils'
import type {
ExecutionContext,
OrchestratorOptions,
StreamEvent,
StreamingContext,
} from '@/lib/copilot/request/types'
const logger = createLogger('CopilotGoStream')
export { buildPreviewContentUpdate, decodeJsonStringPrefix, extractEditContent }
type JsonRecord = Record<string, unknown>
type SubagentSpanData = {
pending?: boolean
toolCallId?: string
}
function asJsonRecord(value: unknown): JsonRecord | undefined {
return value && typeof value === 'object' && !Array.isArray(value)
? (value as JsonRecord)
: undefined
}
function parseSubagentSpanData(value: unknown): SubagentSpanData | undefined {
const data = asJsonRecord(value)
if (!data) {
return undefined
}
const toolCallId = typeof data.tool_call_id === 'string' ? data.tool_call_id : undefined
const pending = typeof data.pending === 'boolean' ? data.pending : undefined
return {
...(toolCallId ? { toolCallId } : {}),
...(pending !== undefined ? { pending } : {}),
}
}
export class CopilotBackendError extends Error {
status?: number
body?: string
constructor(message: string, options?: { status?: number; body?: string }) {
super(message)
this.name = 'CopilotBackendError'
this.status = options?.status
this.body = options?.body
}
}
export class BillingLimitError extends Error {
constructor(public readonly userId: string) {
super('Usage limit reached')
this.name = 'BillingLimitError'
}
}
/**
* Options for the shared stream processing loop.
*/
export interface StreamLoopOptions extends OrchestratorOptions {
/**
* Called for each normalized event BEFORE standard handler dispatch.
* Return true to skip the default handler for this event.
*/
onBeforeDispatch?: (event: StreamEvent, context: StreamingContext) => boolean | undefined
/**
* Called when the Go backend's trace ID (go_trace_id) is first received via SSE.
*/
onGoTraceId?: (goTraceId: string) => void
otelContext?: Context
}
/**
* Run the SSE stream processing loop against the Go backend.
*
* Handles: fetch -> parse -> normalize -> dedupe -> subagent routing -> handler dispatch.
* Callers provide the fetch URL/options and can intercept events via onBeforeDispatch.
* Feature-specific normalization runs through dedicated adapters before the raw event is forwarded.
*/
export async function runStreamLoop(
fetchUrl: string,
fetchOptions: RequestInit,
context: StreamingContext,
execContext: ExecutionContext,
options: StreamLoopOptions
): Promise<void> {
const { timeout = ORCHESTRATION_TIMEOUT_MS, abortSignal } = options
const filePreviewAdapterState = createFilePreviewAdapterState()
const pathname = new URL(fetchUrl).pathname
const requestBodyBytes = estimateBodyBytes(fetchOptions.body)
const fetchSpan = context.trace.startSpan(`HTTP Request → ${pathname}`, 'sim.http.fetch', {
url: fetchUrl,
method: fetchOptions.method ?? 'GET',
requestBodyBytes,
})
const fetchStart = performance.now()
let response: Response
try {
response = await fetchGo(fetchUrl, {
...fetchOptions,
signal: abortSignal,
otelContext: options.otelContext,
spanName: `sim → go ${pathname}`,
operation: 'stream',
attributes: {
[TraceAttr.CopilotStream]: true,
...(requestBodyBytes ? { [TraceAttr.HttpRequestContentLength]: requestBodyBytes } : {}),
},
})
} catch (error) {
fetchSpan.attributes = {
...(fetchSpan.attributes ?? {}),
headersMs: Math.round(performance.now() - fetchStart),
}
context.trace.endSpan(fetchSpan, abortSignal?.aborted ? 'cancelled' : 'error')
throw error
}
const headersElapsedMs = Math.round(performance.now() - fetchStart)
fetchSpan.attributes = {
...(fetchSpan.attributes ?? {}),
status: response.status,
headersMs: headersElapsedMs,
}
if (!response.ok) {
context.trace.endSpan(fetchSpan, 'error')
const errorText = await response.text().catch(() => '')
if (response.status === 402) {
throw new BillingLimitError(execContext.userId)
}
throw new CopilotBackendError(
`Copilot backend error (${response.status}): ${errorText || response.statusText}`,
{ status: response.status, body: errorText || response.statusText }
)
}
if (!response.body) {
context.trace.endSpan(fetchSpan, 'error')
throw new CopilotBackendError('Copilot backend response missing body')
}
context.trace.endSpan(fetchSpan)
const bodySpan = context.trace.startSpan(`SSE Body → ${pathname}`, 'sim.http.stream_body', {
url: fetchUrl,
method: fetchOptions.method ?? 'GET',
})
// Aggregate counters populated inline by the reader wrapper + onEvent
// dispatcher below and flushed to both the legacy TraceCollector span
// and the OTel read-loop span when the loop terminates. Kept as plain
// JS variables (not span attrs) so incrementing them is free — we
// only pay OTel cost once at span End().
//
// Idle-gap tracking is split two ways so we can tell apart
// upstream-silent from we-were-busy:
//
// - `longestInboundGapMs`: biggest time between consecutive
// `reader.read()` calls returning bytes. Upper bound on
// "Go silent". Actually also includes Node waiting for main
// thread free, so see dispatchMs below.
// - `longestDispatchMs`: biggest time any single event handler
// took between "event received" and "returned control". Upper
// bound on "Sim was CPU-bound on a handler". If this is high
// AND inbound gap is high at the same time, it's Sim. If only
// inbound gap is high, it's upstream.
// - `totalDispatchMs`: sum of all handler times. Helps gauge
// whether handlers in aggregate ate a meaningful fraction of
// the read loop.
const counters = {
bytes: 0,
chunks: 0,
events: 0,
eventsByType: {
session: 0,
text: 0,
tool: 0,
span: 0,
resource: 0,
run: 0,
error: 0,
complete: 0,
} as Record<MothershipStreamV1EventType, number>,
firstEventMs: undefined as number | undefined,
lastChunkMs: performance.now(),
longestInboundGapMs: 0,
longestDispatchMs: 0,
totalDispatchMs: 0,
}
const bodyStart = performance.now()
let endedOn: string = CopilotSseCloseReason.Terminal
// Wrap the body's reader so we can track per-chunk bytes and the gap
// between chunks. `processSSEStream` consumes this reader exactly as
// it would the raw one — no API changes there.
const IDLE_GAP_EVENT_THRESHOLD_MS = 10000
const rawReader = response.body.getReader()
const reader: ReadableStreamDefaultReader<Uint8Array> = {
async read() {
const result = await rawReader.read()
if (!result.done && result.value) {
const now = performance.now()
const gap = now - counters.lastChunkMs
if (gap > counters.longestInboundGapMs) counters.longestInboundGapMs = gap
counters.lastChunkMs = now
counters.chunks += 1
counters.bytes += result.value.byteLength
}
return result
},
cancel: (reason) => rawReader.cancel(reason),
releaseLock: () => rawReader.releaseLock(),
get closed() {
return rawReader.closed
},
}
const decoder = new TextDecoder()
const timeoutId = setTimeout(() => {
context.errors.push('Request timed out')
context.streamComplete = true
endedOn = CopilotSseCloseReason.Timeout
reader.cancel().catch(() => {})
}, timeout)
try {
await processSSEStream(reader, decoder, abortSignal, async (raw) => {
// Track how long THIS handler invocation takes so we can tell
// apart "Go was silent" from "we were CPU-bound on a handler".
// `longestInboundGapMs` includes handler time (the next reader.read
// doesn't run until the previous handler returns), so dispatch
// time is the correction needed to isolate upstream silence.
const dispatchStart = performance.now()
try {
if (counters.events === 0) {
counters.firstEventMs = Math.round(performance.now() - bodyStart)
}
counters.events += 1
if (abortSignal?.aborted) {
context.wasAborted = true
return true
}
const parsedEvent = parsePersistedStreamEventEnvelope(raw)
if (!parsedEvent.ok) {
const detail = [parsedEvent.message, ...(parsedEvent.errors ?? [])]
.filter(Boolean)
.join('; ')
const failureMessage = `Received invalid stream event on shared path: ${detail}`
context.errors.push(failureMessage)
logger.error('Received invalid stream event on shared path', {
reason: parsedEvent.reason,
message: parsedEvent.message,
errors: parsedEvent.errors,
})
throw new FatalSseEventError(failureMessage)
}
const envelope = parsedEvent.event
const streamEvent = eventToStreamEvent(envelope)
if (envelope.trace?.requestId) {
const goTraceId = envelope.trace.goTraceId || envelope.trace.requestId
context.trace.setGoTraceId(goTraceId)
options.onGoTraceId?.(goTraceId)
}
// Per-type counters for the copilot.sse.read_loop span. Bound set
// (8 types) so this can never blow up into high cardinality.
if (streamEvent.type in counters.eventsByType) {
counters.eventsByType[streamEvent.type as MothershipStreamV1EventType] += 1
}
// Surface the full error payload the moment it arrives on the wire. This
// is the single chokepoint every error event passes through (main AND
// subagent lanes), before subagent routing — which has no `error`
// handler — would otherwise swallow it. The client only renders
// `message`/`displayMessage`, so log `code`/`provider`/`data` (the raw
// upstream provider error) here to explain a client-side "Stream error".
if (streamEvent.type === MothershipStreamV1EventType.error) {
const errorPayload = streamEvent.payload
logger.error('Received error event from Go copilot stream', {
path: pathname,
lane: streamEvent.scope?.lane ?? 'main',
parentToolCallId: streamEvent.scope?.parentToolCallId,
agentId: streamEvent.scope?.agentId,
code: errorPayload.code,
provider: errorPayload.provider,
message: errorPayload.message,
error: errorPayload.error,
displayMessage: errorPayload.displayMessage,
data: errorPayload.data,
requestId: context.requestId,
messageId: context.messageId,
})
}
if (shouldSkipToolCallEvent(streamEvent) || shouldSkipToolResultEvent(streamEvent)) {
return
}
await processFilePreviewStreamEvent({
streamId: envelope.stream.streamId,
streamEvent,
context,
execContext,
options,
state: filePreviewAdapterState,
})
await prePersistClientExecutableToolCall(streamEvent, context)
try {
await options.onEvent?.(streamEvent)
} catch (error) {
logger.warn('Failed to forward stream event', {
type: streamEvent.type,
error: getErrorMessage(error),
})
}
// Yield a macrotask so Node.js flushes the HTTP response buffer to
// the browser. Microtask yields (await Promise.resolve()) are not
// enough — the I/O layer needs a full event loop tick to write.
await new Promise<void>((resolve) => setImmediate(resolve))
if (options.onBeforeDispatch?.(streamEvent, context)) {
return context.streamComplete || undefined
}
if (isSubagentSpanStreamEvent(streamEvent)) {
const spanData = parseSubagentSpanData(streamEvent.payload.data)
const toolCallId = streamEvent.scope?.parentToolCallId || spanData?.toolCallId
// Deterministic nesting identity. spanId / parentSpanId are the
// primary keys; the toolCallId-keyed stack below is the legacy
// fallback for streams that predate span identity.
const spanId = streamEvent.scope?.spanId
const parentSpanId = streamEvent.scope?.parentSpanId
const subagentName = streamEvent.payload.agent
const spanEvt = streamEvent.payload.event
const isPendingPause = spanData?.pending === true
// A subagent lifecycle boundary breaks the main thinking stream.
// Flush any open thinking block into contentBlocks BEFORE we push
// the `subagent` marker, or the persisted order ends up
// [subagent, thinking] and the UI renders the subagent group
// above a thinking block that actually happened first.
flushSubagentThinkingBlock(context)
flushThinkingBlock(context)
if (spanEvt === MothershipStreamV1SpanLifecycleEvent.start) {
if (toolCallId) {
context.subAgentContent[toolCallId] ??= ''
context.subAgentToolCalls[toolCallId] ??= []
}
if (toolCallId && subagentName) {
const openParents = (context.openSubagentParents ??= new Set<string>())
if (!openParents.has(toolCallId)) {
openParents.add(toolCallId)
context.contentBlocks.push({
type: 'subagent',
content: subagentName,
parentToolCallId: toolCallId,
...(spanId ? { spanId } : {}),
...(parentSpanId ? { parentSpanId } : {}),
timestamp: Date.now(),
})
}
} else {
logger.warn('subagent start missing toolCallId or agent name', {
hasToolCallId: Boolean(toolCallId),
hasSubagentName: Boolean(subagentName),
})
}
return
}
if (spanEvt === MothershipStreamV1SpanLifecycleEvent.end) {
if (isPendingPause) {
return
}
if (!toolCallId) {
logger.warn('subagent end missing toolCallId')
}
if (toolCallId) {
for (let i = context.contentBlocks.length - 1; i >= 0; i--) {
const b = context.contentBlocks[i]
if (
b.type === 'subagent' &&
b.endedAt === undefined &&
b.parentToolCallId === toolCallId
) {
b.endedAt = Date.now()
break
}
}
context.openSubagentParents?.delete(toolCallId)
}
return
}
}
// Subagent-lane events are routed ONLY by their own scope. A valid one
// (has parentToolCallId) goes to the subagent handler; a malformed one
// (missing parentToolCallId — Go always stamps it, so this is defensive)
// is DROPPED rather than falling through to the main handler, which would
// merge foreign subagent text/tools into the durable main assistant
// message and mis-attribute it.
if (streamEvent.scope?.lane === 'subagent') {
if (handleSubagentRouting(streamEvent, context)) {
const handler = subAgentHandlers[streamEvent.type]
if (handler) {
await handler(streamEvent, context, execContext, options)
}
}
return context.streamComplete || undefined
}
const handler = sseHandlers[streamEvent.type]
if (handler) {
await handler(streamEvent, context, execContext, options)
}
return context.streamComplete || undefined
} finally {
const dispatchMs = performance.now() - dispatchStart
counters.totalDispatchMs += dispatchMs
if (dispatchMs > counters.longestDispatchMs) counters.longestDispatchMs = dispatchMs
}
})
if (!context.streamComplete && !abortSignal?.aborted && !context.wasAborted) {
let abortRequested = false
try {
abortRequested = await hasAbortMarker(context.messageId)
} catch (error) {
logger.warn('Failed to read abort marker at body close', {
streamId: context.messageId,
error: getErrorMessage(error),
})
}
if (abortRequested) {
options.onAbortObserved?.(AbortReason.MarkerObservedAtBodyClose)
context.wasAborted = true
endedOn = CopilotSseCloseReason.Aborted
} else {
const streamPath = new URL(fetchUrl).pathname
const message = `Copilot backend stream ended before a terminal event on ${streamPath}`
context.errors.push(message)
logger.error('Copilot backend stream ended before a terminal event', {
path: streamPath,
requestId: context.requestId,
messageId: context.messageId,
})
endedOn = CopilotSseCloseReason.ClosedNoTerminal
throw new CopilotBackendError(message, { status: 503 })
}
}
} catch (error) {
if (error instanceof FatalSseEventError && !context.errors.includes(error.message)) {
context.errors.push(error.message)
}
if (endedOn === CopilotSseCloseReason.Terminal) {
endedOn =
error instanceof CopilotBackendError
? CopilotSseCloseReason.BackendError
: error instanceof BillingLimitError
? CopilotSseCloseReason.BillingLimit
: CopilotSseCloseReason.Error
}
throw error
} finally {
if (abortSignal?.aborted) {
context.wasAborted = true
await reader.cancel().catch(() => {})
if (endedOn === CopilotSseCloseReason.Terminal) {
endedOn = CopilotSseCloseReason.Aborted
}
}
// An abort or error can tear down the loop mid-thinking. Flush any
// open thinking blocks so partial-persistence on /chat/stop sees
// them in contentBlocks with endedAt stamped, instead of silently
// dropping the in-flight reasoning.
flushSubagentThinkingBlock(context)
flushThinkingBlock(context)
clearTimeout(timeoutId)
// Legacy TraceCollector span (consumed by the in-memory trace
// collector, kept for backwards compatibility with existing
// tooling). The real OTel span is stamped below.
const bodyDurationMs = Math.round(performance.now() - bodyStart)
bodySpan.attributes = {
...(bodySpan.attributes ?? {}),
eventsReceived: counters.events,
firstEventMs: counters.firstEventMs,
endedOn,
durationMs: bodyDurationMs,
}
context.trace.endSpan(
bodySpan,
endedOn === CopilotSseCloseReason.Terminal
? 'ok'
: endedOn === CopilotSseCloseReason.Aborted
? 'cancelled'
: 'error'
)
// Real OTel span for Tempo/Grafana. Stamped aggregate-only so
// there is no per-chunk OTel cost — one span per read loop with
// integer counters, plus a bounded set of events.
//
// `expectedTerminal` = "the caller considered this leg the FINAL
// leg and genuinely expected a terminal event on the wire." We
// derive it from `context.streamComplete` MINUS the tool-pause
// case: when the server emits a `run.checkpoint_pause`, its
// handler also sets `streamComplete=true` to stop the read loop
// cleanly, but no `complete` SSE event is ever sent in that
// case — that's the tool-pause protocol, not a missing terminal.
// `awaitingAsyncContinuation` is set by the same handler, so
// its presence distinguishes "tool pause, no terminal expected"
// from "caller thought stream was done but server never said so"
// (= the real disappeared-response bug class).
const expectedTerminal = context.streamComplete && !context.awaitingAsyncContinuation
stampSseReadLoopSpan(bodyStart, counters, endedOn, fetchUrl, pathname, {
idleGapEventThresholdMs: IDLE_GAP_EVENT_THRESHOLD_MS,
expectedTerminal,
})
}
}
function estimateBodyBytes(body: BodyInit | null | undefined): number {
if (!body) {
return 0
}
if (typeof body === 'string') {
return body.length
}
if (body instanceof ArrayBuffer) {
return body.byteLength
}
if (ArrayBuffer.isView(body)) {
return body.byteLength
}
return 0
}
type SseReadLoopCounters = {
bytes: number
chunks: number
events: number
eventsByType: Record<MothershipStreamV1EventType, number>
firstEventMs: number | undefined
longestInboundGapMs: number
longestDispatchMs: number
totalDispatchMs: number
}
/**
* Ship a one-shot `copilot.sse.read_loop` OTel span with the aggregate
* counters collected during the read loop. Uses `startTime` so the
* span's duration reflects the actual loop wall clock even though we
* only talk to OTel once at the end.
*
* Deliberately synchronous, no per-chunk span calls: total OTel cost
* per read loop is fixed (~10 attrs + up to 3 events), independent of
* chunk count.
*/
function stampSseReadLoopSpan(
startPerfMs: number,
counters: SseReadLoopCounters,
closeReason: string,
fetchUrl: string,
pathname: string,
opts: { idleGapEventThresholdMs: number; expectedTerminal: boolean }
): void {
// Translate performance.now() values into wall-clock Date values so
// the span's timestamps land in real time (OTel accepts both, but we
// need to pair startTime with a matching "now" for .end()).
const nowPerf = performance.now()
const nowWall = Date.now()
const startWall = nowWall - (nowPerf - startPerfMs)
const terminalEventSeen = counters.eventsByType.complete > 0 || counters.eventsByType.error > 0
// `terminal_event_missing` is the single-attribute dashboard signal
// for the "disappeared response" bug class: the caller considered
// this leg to be the final one (`context.streamComplete === true`)
// but no terminal `complete` or `error` event arrived on the wire.
// Tool-pause legs have expectedTerminal=false and never trip this, so
// dashboards can filter on `{ .copilot.sse.terminal_event_missing = true }`
// without false positives.
const terminalEventMissing = opts.expectedTerminal && !terminalEventSeen
const tracer = getCopilotTracer()
const span = tracer.startSpan(TraceSpan.CopilotSseReadLoop, {
startTime: startWall,
attributes: {
[TraceAttr.HttpUrl]: fetchUrl,
[TraceAttr.HttpPath]: pathname,
[TraceAttr.CopilotSseBytesReceived]: counters.bytes,
[TraceAttr.CopilotSseChunksReceived]: counters.chunks,
[TraceAttr.CopilotSseEventsReceived]: counters.events,
[TraceAttr.CopilotSseEventsSession]: counters.eventsByType.session,
[TraceAttr.CopilotSseEventsText]: counters.eventsByType.text,
[TraceAttr.CopilotSseEventsTool]: counters.eventsByType.tool,
[TraceAttr.CopilotSseEventsSpan]: counters.eventsByType.span,
[TraceAttr.CopilotSseEventsResource]: counters.eventsByType.resource,
[TraceAttr.CopilotSseEventsRun]: counters.eventsByType.run,
[TraceAttr.CopilotSseEventsError]: counters.eventsByType.error,
[TraceAttr.CopilotSseEventsComplete]: counters.eventsByType.complete,
[TraceAttr.CopilotSseLongestInboundGapMs]: Math.round(counters.longestInboundGapMs),
[TraceAttr.CopilotSseLongestDispatchMs]: Math.round(counters.longestDispatchMs),
[TraceAttr.CopilotSseTotalDispatchMs]: Math.round(counters.totalDispatchMs),
[TraceAttr.CopilotSseCloseReason]: closeReason,
[TraceAttr.CopilotSseExpectedTerminal]: opts.expectedTerminal,
[TraceAttr.CopilotSseTerminalEventSeen]: terminalEventSeen,
[TraceAttr.CopilotSseTerminalEventMissing]: terminalEventMissing,
},
})
if (counters.firstEventMs !== undefined) {
span.setAttribute(TraceAttr.CopilotSseFirstEventMs, counters.firstEventMs)
// Anchor the event to the moment the first SSE event was actually
// received (startWall + firstEventMs), not `now`, so a trace
// waterfall shows the diamond at the TTFT point — not at span end.
span.addEvent(
TraceEvent.CopilotSseFirstEvent,
{ [TraceAttr.CopilotSseFirstEventMs]: counters.firstEventMs },
startWall + counters.firstEventMs
)
}
// Fire the idle-gap event when the INBOUND gap (time between TCP
// reads returning bytes) exceeds the threshold. This is the
// "upstream was silent or Sim was CPU-bound" signal; dispatch time
// on its own doesn't warrant an event because it's within our
// control and visible on a dedicated attribute.
if (counters.longestInboundGapMs >= opts.idleGapEventThresholdMs) {
span.addEvent(TraceEvent.CopilotSseIdleGapExceeded, {
[TraceAttr.CopilotSseLongestInboundGapMs]: Math.round(counters.longestInboundGapMs),
[TraceAttr.CopilotSseLongestDispatchMs]: Math.round(counters.longestDispatchMs),
})
}
if (terminalEventSeen) {
span.addEvent(TraceEvent.CopilotSseTerminalEventReceived)
}
// Span status: only mark ERROR for real failures. User aborts and
// clean terminals stay UNSET so dashboards filtering `status=error`
// don't light up for normal cancellations. Tool-pause legs (caller
// didn't set streamComplete) are NOT errors even though they have
// no complete event.
if (terminalEventMissing) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: 'SSE read loop finished without terminal event (caller expected one)',
})
} else if (
closeReason !== CopilotSseCloseReason.Terminal &&
closeReason !== CopilotSseCloseReason.Aborted
) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: `SSE read loop ended with reason: ${closeReason}`,
})
}
span.end(nowWall)
}