chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
import { trace } from '@opentelemetry/api'
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
InMemorySpanExporter,
|
||||
SimpleSpanProcessor,
|
||||
} from '@opentelemetry/sdk-trace-base'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { fetchGo } from '@/lib/copilot/request/go/fetch'
|
||||
|
||||
describe('fetchGo', () => {
|
||||
const exporter = new InMemorySpanExporter()
|
||||
const provider = new BasicTracerProvider({
|
||||
spanProcessors: [new SimpleSpanProcessor(exporter)],
|
||||
})
|
||||
|
||||
beforeEach(() => {
|
||||
exporter.reset()
|
||||
trace.setGlobalTracerProvider(provider)
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('emits a client span with http.* attrs and injects traceparent', async () => {
|
||||
const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {
|
||||
const headers = init.headers as Record<string, string>
|
||||
expect(headers.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[0-9a-f]$/)
|
||||
return new Response('ok', {
|
||||
status: 200,
|
||||
headers: { 'content-length': '2' },
|
||||
})
|
||||
})
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
const res = await fetchGo('https://backend.example.com/api/copilot', {
|
||||
method: 'POST',
|
||||
body: 'payload',
|
||||
operation: 'stream',
|
||||
attributes: { 'copilot.leg': 'sim_to_go' },
|
||||
})
|
||||
expect(res.status).toBe(200)
|
||||
|
||||
const spans = exporter.getFinishedSpans()
|
||||
expect(spans).toHaveLength(1)
|
||||
const attrs = spans[0].attributes
|
||||
expect(spans[0].name).toBe('sim → go /api/copilot')
|
||||
expect(attrs['http.method']).toBe('POST')
|
||||
expect(attrs['http.url']).toBe('https://backend.example.com/api/copilot')
|
||||
expect(attrs['http.target']).toBe('/api/copilot')
|
||||
expect(attrs['http.status_code']).toBe(200)
|
||||
expect(attrs['copilot.operation']).toBe('stream')
|
||||
expect(attrs['copilot.leg']).toBe('sim_to_go')
|
||||
expect(typeof attrs['http.response.headers_ms']).toBe('number')
|
||||
})
|
||||
|
||||
it('marks span as error on non-2xx response', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('nope', { status: 500 })))
|
||||
|
||||
const res = await fetchGo('https://backend.example.com/api/tools/resume', {
|
||||
method: 'POST',
|
||||
})
|
||||
expect(res.status).toBe(500)
|
||||
|
||||
const spans = exporter.getFinishedSpans()
|
||||
expect(spans).toHaveLength(1)
|
||||
expect(spans[0].status.code).toBe(2)
|
||||
})
|
||||
|
||||
it('records exceptions when fetch throws', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network boom')))
|
||||
|
||||
await expect(
|
||||
fetchGo('https://backend.example.com/api/traces', { method: 'POST' })
|
||||
).rejects.toThrow('network boom')
|
||||
|
||||
const spans = exporter.getFinishedSpans()
|
||||
expect(spans).toHaveLength(1)
|
||||
expect(spans[0].status.code).toBe(2)
|
||||
expect(spans[0].events.some((e) => e.name === 'exception')).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,112 @@
|
||||
import { type Context, context, SpanStatusCode, trace } from '@opentelemetry/api'
|
||||
import { CopilotLeg } from '@/lib/copilot/generated/trace-attribute-values-v1'
|
||||
import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
|
||||
import { traceHeaders } from '@/lib/copilot/request/go/propagation'
|
||||
import { isActionableErrorStatus, markSpanForError } from '@/lib/copilot/request/otel'
|
||||
|
||||
// Lazy tracer resolution: module-level `trace.getTracer()` can be evaluated
|
||||
// before `instrumentation-node.ts` installs the TracerProvider under
|
||||
// Next.js 16 + Turbopack dev, freezing a NoOp tracer and silently dropping
|
||||
// every outbound Sim → Go span. Resolving per-call avoids the race.
|
||||
const getTracer = () => trace.getTracer('sim-copilot-http', '1.0.0')
|
||||
|
||||
interface OutboundFetchOptions extends RequestInit {
|
||||
otelContext?: Context
|
||||
spanName?: string
|
||||
operation?: string
|
||||
attributes?: Record<string, string | number | boolean>
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform an outbound Sim → Go fetch wrapped in an OTel child span so each
|
||||
* call shows up as a distinct segment in Jaeger, and propagates the W3C
|
||||
* traceparent so the Go-side span joins the same trace.
|
||||
*
|
||||
* The span captures generic attributes (method, status, duration, response
|
||||
* size, error code) so any future latency investigation — not just images or
|
||||
* Bedrock — has uniform metadata to work with.
|
||||
*/
|
||||
export async function fetchGo(url: string, options: OutboundFetchOptions = {}): Promise<Response> {
|
||||
const {
|
||||
otelContext,
|
||||
spanName,
|
||||
operation,
|
||||
attributes,
|
||||
headers: providedHeaders,
|
||||
...init
|
||||
} = options
|
||||
|
||||
const parsed = safeParseUrl(url)
|
||||
const pathname = parsed?.pathname ?? url
|
||||
const method = (init.method ?? 'GET').toUpperCase()
|
||||
const parentContext = otelContext ?? context.active()
|
||||
|
||||
const span = getTracer().startSpan(
|
||||
spanName ?? `sim → go ${pathname}`,
|
||||
{
|
||||
attributes: {
|
||||
[TraceAttr.HttpMethod]: method,
|
||||
[TraceAttr.HttpUrl]: url,
|
||||
[TraceAttr.HttpTarget]: pathname,
|
||||
[TraceAttr.NetPeerName]: parsed?.host ?? '',
|
||||
[TraceAttr.CopilotLeg]: CopilotLeg.SimToGo,
|
||||
...(operation ? { [TraceAttr.CopilotOperation]: operation } : {}),
|
||||
...(attributes ?? {}),
|
||||
},
|
||||
},
|
||||
parentContext
|
||||
)
|
||||
|
||||
const activeContext = trace.setSpan(parentContext, span)
|
||||
const propagatedHeaders = traceHeaders({}, activeContext)
|
||||
const mergedHeaders = {
|
||||
...(providedHeaders as Record<string, string> | undefined),
|
||||
...propagatedHeaders,
|
||||
}
|
||||
|
||||
const start = performance.now()
|
||||
try {
|
||||
const response = await context.with(activeContext, () =>
|
||||
fetch(url, {
|
||||
...init,
|
||||
method,
|
||||
headers: mergedHeaders,
|
||||
})
|
||||
)
|
||||
const elapsedMs = performance.now() - start
|
||||
const contentLength = Number(response.headers.get('content-length') ?? 0)
|
||||
span.setAttribute(TraceAttr.HttpStatusCode, response.status)
|
||||
span.setAttribute(TraceAttr.HttpResponseHeadersMs, Math.round(elapsedMs))
|
||||
if (contentLength > 0) {
|
||||
span.setAttribute(TraceAttr.HttpResponseContentLength, contentLength)
|
||||
}
|
||||
// Only mark ERROR for actionable status codes. 4xx that represent
|
||||
// normal auth/validation rejections (400/401/403/404/405/422/etc.)
|
||||
// stay UNSET so error dashboards don't drown in expected rejection
|
||||
// paths. See `isActionableErrorStatus` in Go's telemetry middleware
|
||||
// for the mirror rule (5xx + 402/409/429).
|
||||
if (isActionableErrorStatus(response.status)) {
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: `HTTP ${response.status}`,
|
||||
})
|
||||
} else {
|
||||
span.setStatus({ code: SpanStatusCode.OK })
|
||||
}
|
||||
return response
|
||||
} catch (error) {
|
||||
span.setAttribute(TraceAttr.HttpResponseHeadersMs, Math.round(performance.now() - start))
|
||||
markSpanForError(span, error)
|
||||
throw error
|
||||
} finally {
|
||||
span.end()
|
||||
}
|
||||
}
|
||||
|
||||
function safeParseUrl(url: string): URL | null {
|
||||
try {
|
||||
return new URL(url)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,761 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { MothershipStreamV1EventType } from '@/lib/copilot/generated/mothership-stream-v1'
|
||||
import {
|
||||
createFilePreviewSession,
|
||||
type FilePreviewContentMode,
|
||||
type FilePreviewSession,
|
||||
type FilePreviewTargetKind,
|
||||
isToolArgsDeltaStreamEvent,
|
||||
isToolCallStreamEvent,
|
||||
isToolResultStreamEvent,
|
||||
type SyntheticFilePreviewPayload,
|
||||
upsertFilePreviewSession,
|
||||
} from '@/lib/copilot/request/session'
|
||||
import type {
|
||||
ActiveFileIntent,
|
||||
ExecutionContext,
|
||||
OrchestratorOptions,
|
||||
StreamEvent,
|
||||
StreamingContext,
|
||||
} from '@/lib/copilot/request/types'
|
||||
import { peekFileIntent } from '@/lib/copilot/tools/server/files/file-intent-store'
|
||||
import {
|
||||
buildFilePreviewText,
|
||||
loadWorkspaceFileTextForPreview,
|
||||
} from '@/lib/copilot/tools/server/files/file-preview'
|
||||
import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
|
||||
const logger = createLogger('CopilotFilePreviewAdapter')
|
||||
|
||||
type JsonRecord = Record<string, unknown>
|
||||
type FileIntent = ActiveFileIntent
|
||||
|
||||
type EditContentStreamState = {
|
||||
raw: string
|
||||
lastContentSnapshot?: string
|
||||
}
|
||||
|
||||
type FilePreviewStreamState = {
|
||||
session: FilePreviewSession
|
||||
lastEmittedPreviewText: string
|
||||
lastSnapshotAt: number
|
||||
}
|
||||
|
||||
type ParsedWorkspaceFileArgs = {
|
||||
operation: string
|
||||
target: FileIntent['target']
|
||||
title?: string
|
||||
contentType?: string
|
||||
edit?: JsonRecord
|
||||
}
|
||||
|
||||
const PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS = 80
|
||||
const DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS = 1000
|
||||
|
||||
function asJsonRecord(value: unknown): JsonRecord | undefined {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as JsonRecord)
|
||||
: undefined
|
||||
}
|
||||
|
||||
function toPreviewTargetKind(kind: string | undefined): FilePreviewTargetKind | undefined {
|
||||
return kind === 'new_file' || kind === 'file_id' ? kind : undefined
|
||||
}
|
||||
|
||||
async function resolvePreviewTarget(args: {
|
||||
workspaceId?: string
|
||||
target: FileIntent['target']
|
||||
}): Promise<FileIntent['target']> {
|
||||
if (args.target.kind !== 'path' || !args.workspaceId || !args.target.path) {
|
||||
return args.target
|
||||
}
|
||||
|
||||
const file = await resolveWorkspaceFileReference(args.workspaceId, args.target.path)
|
||||
if (!file) {
|
||||
return args.target
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'file_id',
|
||||
fileId: file.id,
|
||||
fileName: args.target.fileName ?? file.name,
|
||||
path: args.target.path,
|
||||
}
|
||||
}
|
||||
|
||||
function parseWorkspaceFileArgs(value: unknown): ParsedWorkspaceFileArgs | undefined {
|
||||
const args = asJsonRecord(value)
|
||||
if (!args) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const operation = typeof args.operation === 'string' ? args.operation : undefined
|
||||
const target = asJsonRecord(args.target)
|
||||
const targetKind = typeof target?.kind === 'string' ? target.kind : undefined
|
||||
if (!operation || !target || !targetKind) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const fileId = typeof target.fileId === 'string' ? target.fileId : undefined
|
||||
const fileName = typeof target.fileName === 'string' ? target.fileName : undefined
|
||||
const path = typeof target.path === 'string' ? target.path : undefined
|
||||
const title = typeof args.title === 'string' ? args.title : undefined
|
||||
const contentType = typeof args.contentType === 'string' ? args.contentType : undefined
|
||||
const edit = asJsonRecord(args.edit)
|
||||
|
||||
return {
|
||||
operation,
|
||||
target: {
|
||||
kind: targetKind,
|
||||
...(fileId ? { fileId } : {}),
|
||||
...(fileName ? { fileName } : {}),
|
||||
...(path ? { path } : {}),
|
||||
},
|
||||
...(title ? { title } : {}),
|
||||
...(contentType ? { contentType } : {}),
|
||||
...(edit ? { edit } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
function extractWorkspaceFileResult(output: unknown): { fileId?: string; fileName?: string } {
|
||||
const candidates: JsonRecord[] = []
|
||||
const root = asJsonRecord(output)
|
||||
if (root) {
|
||||
candidates.push(root)
|
||||
const rootData = asJsonRecord(root.data)
|
||||
if (rootData) candidates.push(rootData)
|
||||
const rootOutput = asJsonRecord(root.output)
|
||||
if (rootOutput) {
|
||||
candidates.push(rootOutput)
|
||||
const outputData = asJsonRecord(rootOutput.data)
|
||||
if (outputData) candidates.push(outputData)
|
||||
}
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const fileId =
|
||||
typeof candidate.id === 'string'
|
||||
? candidate.id
|
||||
: typeof candidate.fileId === 'string'
|
||||
? candidate.fileId
|
||||
: undefined
|
||||
if (!fileId) continue
|
||||
|
||||
const fileName =
|
||||
typeof candidate.name === 'string'
|
||||
? candidate.name
|
||||
: typeof candidate.fileName === 'string'
|
||||
? candidate.fileName
|
||||
: undefined
|
||||
return { fileId, fileName }
|
||||
}
|
||||
return {}
|
||||
}
|
||||
|
||||
export function decodeJsonStringPrefix(input: string): string {
|
||||
let output = ''
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const ch = input[i]
|
||||
if (ch !== '\\') {
|
||||
output += ch
|
||||
continue
|
||||
}
|
||||
const next = input[i + 1]
|
||||
if (!next) break
|
||||
if (next === 'n') {
|
||||
output += '\n'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === 't') {
|
||||
output += '\t'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === 'r') {
|
||||
output += '\r'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === '"') {
|
||||
output += '"'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === '\\') {
|
||||
output += '\\'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === '/') {
|
||||
output += '/'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === 'b') {
|
||||
output += '\b'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === 'f') {
|
||||
output += '\f'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (next === 'u') {
|
||||
const hex = input.slice(i + 2, i + 6)
|
||||
if (hex.length < 4 || !/^[0-9a-fA-F]{4}$/.test(hex)) break
|
||||
output += String.fromCharCode(Number.parseInt(hex, 16))
|
||||
i += 5
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
export function extractEditContent(raw: string): string {
|
||||
const marker = '"content":'
|
||||
const idx = raw.indexOf(marker)
|
||||
if (idx === -1) return ''
|
||||
const rest = raw.slice(idx + marker.length).trimStart()
|
||||
if (!rest.startsWith('"')) return rest
|
||||
let end = -1
|
||||
for (let i = 1; i < rest.length; i++) {
|
||||
if (rest[i] === '\\') {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (rest[i] === '"') {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
}
|
||||
const inner = end === -1 ? rest.slice(1) : rest.slice(1, end)
|
||||
return decodeJsonStringPrefix(inner)
|
||||
}
|
||||
|
||||
function isContentOperation(
|
||||
operation: string | undefined
|
||||
): operation is 'append' | 'update' | 'patch' {
|
||||
return operation === 'append' || operation === 'update' || operation === 'patch'
|
||||
}
|
||||
|
||||
function isDocFormat(fileName: string | undefined): boolean {
|
||||
return /\.(pptx|docx|pdf)$/i.test(fileName ?? '')
|
||||
}
|
||||
|
||||
function buildPreviewSessionFromIntent(
|
||||
streamId: string,
|
||||
intent: FileIntent,
|
||||
current?: FilePreviewSession
|
||||
): FilePreviewSession {
|
||||
const targetKind = toPreviewTargetKind(intent.target.kind)
|
||||
|
||||
return createFilePreviewSession({
|
||||
streamId,
|
||||
toolCallId: intent.toolCallId,
|
||||
fileName: intent.target.fileName ?? current?.fileName,
|
||||
...(intent.target.fileId ? { fileId: intent.target.fileId } : {}),
|
||||
...(targetKind ? { targetKind } : {}),
|
||||
operation: intent.operation,
|
||||
...(intent.edit ? { edit: intent.edit } : {}),
|
||||
...(typeof current?.baseContent === 'string' ? { baseContent: current.baseContent } : {}),
|
||||
previewText: current?.previewText ?? '',
|
||||
previewVersion: current?.previewVersion ?? 0,
|
||||
status: current?.status ?? 'pending',
|
||||
completedAt: current?.completedAt,
|
||||
})
|
||||
}
|
||||
|
||||
async function persistFilePreviewSession(session: FilePreviewSession): Promise<void> {
|
||||
try {
|
||||
await upsertFilePreviewSession(session)
|
||||
} catch (error) {
|
||||
logger.warn('Failed to persist file preview session', {
|
||||
streamId: session.streamId,
|
||||
toolCallId: session.toolCallId,
|
||||
previewVersion: session.previewVersion,
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export function buildPreviewContentUpdate(
|
||||
previousText: string,
|
||||
nextText: string,
|
||||
lastSnapshotAt: number,
|
||||
now: number,
|
||||
operation: string | undefined
|
||||
): { content: string; contentMode: FilePreviewContentMode; lastSnapshotAt: number } {
|
||||
const shouldForceSnapshot =
|
||||
previousText.length === 0 ||
|
||||
!nextText.startsWith(previousText) ||
|
||||
operation === 'patch' ||
|
||||
operation === 'append' ||
|
||||
now - lastSnapshotAt >= DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS
|
||||
|
||||
if (shouldForceSnapshot) {
|
||||
return {
|
||||
content: nextText,
|
||||
contentMode: 'snapshot',
|
||||
lastSnapshotAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content: nextText.slice(previousText.length),
|
||||
contentMode: 'delta',
|
||||
lastSnapshotAt,
|
||||
}
|
||||
}
|
||||
|
||||
export interface FilePreviewAdapterState {
|
||||
editContentState: Map<string, EditContentStreamState>
|
||||
filePreviewState: Map<string, FilePreviewStreamState>
|
||||
}
|
||||
|
||||
export function createFilePreviewAdapterState(): FilePreviewAdapterState {
|
||||
return {
|
||||
editContentState: new Map<string, EditContentStreamState>(),
|
||||
filePreviewState: new Map<string, FilePreviewStreamState>(),
|
||||
}
|
||||
}
|
||||
|
||||
async function emitPreviewEvent(
|
||||
streamEvent: StreamEvent,
|
||||
options: Pick<OrchestratorOptions, 'onEvent'>,
|
||||
payload: SyntheticFilePreviewPayload
|
||||
): Promise<void> {
|
||||
await options.onEvent?.({
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload,
|
||||
...(streamEvent.scope ? { scope: streamEvent.scope } : {}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function processFilePreviewStreamEvent(input: {
|
||||
streamId: string
|
||||
streamEvent: StreamEvent
|
||||
context: StreamingContext
|
||||
execContext: ExecutionContext
|
||||
options: Pick<OrchestratorOptions, 'onEvent'>
|
||||
state: FilePreviewAdapterState
|
||||
}): Promise<void> {
|
||||
const { streamId, streamEvent, context, execContext, options, state } = input
|
||||
const { editContentState, filePreviewState } = state
|
||||
|
||||
// Scope the in-flight intent to the invoking file subagent's channel (its
|
||||
// outer tool_use id) so two file agents streaming concurrently never read or
|
||||
// overwrite each other's intent. workspace_file and edit_content from the same
|
||||
// file agent share this channel id, so they pair up; siblings stay isolated.
|
||||
const channelId = streamEvent.scope?.parentToolCallId ?? ''
|
||||
const getIntent = (): FileIntent | null => context.activeFileIntents.get(channelId) ?? null
|
||||
const setIntent = (intent: FileIntent): void => {
|
||||
context.activeFileIntents.set(channelId, intent)
|
||||
}
|
||||
const clearIntent = (): void => {
|
||||
context.activeFileIntents.delete(channelId)
|
||||
}
|
||||
|
||||
if (isToolCallStreamEvent(streamEvent) && streamEvent.payload.toolName === 'workspace_file') {
|
||||
const toolCallId = streamEvent.payload.toolCallId
|
||||
const parsedArgs = parseWorkspaceFileArgs(streamEvent.payload.arguments)
|
||||
if (toolCallId && parsedArgs) {
|
||||
const { operation, title, contentType, edit } = parsedArgs
|
||||
const target = await resolvePreviewTarget({
|
||||
workspaceId: execContext.workspaceId,
|
||||
target: parsedArgs.target,
|
||||
})
|
||||
const previewTargetKind = toPreviewTargetKind(target.kind)
|
||||
const { fileId, fileName } = target
|
||||
|
||||
const isContentOp = isContentOperation(operation)
|
||||
// Per-channel: a re-declared workspace_file just overwrites THIS channel's
|
||||
// slot. No cross-message intent clearing — that would wipe a concurrent
|
||||
// sibling file agent's pending intent.
|
||||
const intent: FileIntent = {
|
||||
toolCallId,
|
||||
operation,
|
||||
target,
|
||||
...(title ? { title } : {}),
|
||||
...(contentType ? { contentType } : {}),
|
||||
...(edit ? { edit } : {}),
|
||||
}
|
||||
setIntent(intent)
|
||||
|
||||
if (isContentOp && previewTargetKind) {
|
||||
let previewBaseContent: string | undefined
|
||||
if (
|
||||
execContext.workspaceId &&
|
||||
fileId &&
|
||||
(operation === 'append' || operation === 'patch')
|
||||
) {
|
||||
previewBaseContent = await loadWorkspaceFileTextForPreview(
|
||||
execContext.workspaceId,
|
||||
fileId
|
||||
)
|
||||
}
|
||||
|
||||
let session = buildPreviewSessionFromIntent(streamId, intent)
|
||||
if (previewBaseContent !== undefined) {
|
||||
session = { ...session, baseContent: previewBaseContent }
|
||||
}
|
||||
filePreviewState.set(toolCallId, {
|
||||
session,
|
||||
lastEmittedPreviewText: '',
|
||||
lastSnapshotAt: 0,
|
||||
})
|
||||
await persistFilePreviewSession(session)
|
||||
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_start',
|
||||
})
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_target',
|
||||
operation,
|
||||
target: {
|
||||
kind: previewTargetKind,
|
||||
...(fileId ? { fileId } : {}),
|
||||
...(fileName ? { fileName } : {}),
|
||||
},
|
||||
...(title ? { title } : {}),
|
||||
})
|
||||
if (edit) {
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_edit_meta',
|
||||
edit,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workspaceResultIntent = getIntent()
|
||||
if (
|
||||
isToolResultStreamEvent(streamEvent) &&
|
||||
streamEvent.payload.toolName === 'workspace_file' &&
|
||||
workspaceResultIntent &&
|
||||
isContentOperation(workspaceResultIntent.operation)
|
||||
) {
|
||||
const result = extractWorkspaceFileResult(streamEvent.payload.output)
|
||||
if (result.fileId && workspaceResultIntent.target.kind === 'path') {
|
||||
const intent: FileIntent = {
|
||||
...workspaceResultIntent,
|
||||
target: {
|
||||
kind: 'file_id',
|
||||
fileId: result.fileId,
|
||||
fileName: result.fileName ?? workspaceResultIntent.target.fileName,
|
||||
path: workspaceResultIntent.target.path,
|
||||
},
|
||||
}
|
||||
setIntent(intent)
|
||||
|
||||
let previewBaseContent: string | undefined
|
||||
if (
|
||||
execContext.workspaceId &&
|
||||
(intent.operation === 'append' || intent.operation === 'patch')
|
||||
) {
|
||||
previewBaseContent = await loadWorkspaceFileTextForPreview(
|
||||
execContext.workspaceId,
|
||||
result.fileId
|
||||
)
|
||||
}
|
||||
|
||||
let session = buildPreviewSessionFromIntent(streamId, intent)
|
||||
if (previewBaseContent !== undefined) {
|
||||
session = { ...session, baseContent: previewBaseContent }
|
||||
}
|
||||
filePreviewState.set(intent.toolCallId, {
|
||||
session,
|
||||
lastEmittedPreviewText: '',
|
||||
lastSnapshotAt: 0,
|
||||
})
|
||||
await persistFilePreviewSession(session)
|
||||
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: intent.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_start',
|
||||
})
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: intent.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_target',
|
||||
operation: intent.operation,
|
||||
target: {
|
||||
kind: 'file_id',
|
||||
fileId: result.fileId,
|
||||
...(result.fileName ? { fileName: result.fileName } : {}),
|
||||
},
|
||||
...(intent.title ? { title: intent.title } : {}),
|
||||
})
|
||||
if (intent.edit) {
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: intent.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_edit_meta',
|
||||
edit: intent.edit,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const patchDeleteIntent = getIntent()
|
||||
if (
|
||||
isToolResultStreamEvent(streamEvent) &&
|
||||
streamEvent.payload.toolName === 'workspace_file' &&
|
||||
patchDeleteIntent &&
|
||||
isContentOperation(patchDeleteIntent.operation) &&
|
||||
patchDeleteIntent.operation === 'patch' &&
|
||||
patchDeleteIntent.edit?.strategy === 'anchored' &&
|
||||
patchDeleteIntent.edit?.mode === 'delete_between' &&
|
||||
execContext.workspaceId &&
|
||||
patchDeleteIntent.target.fileId &&
|
||||
!isDocFormat(patchDeleteIntent.target.fileName)
|
||||
) {
|
||||
const currentPreview = filePreviewState.get(patchDeleteIntent.toolCallId)
|
||||
const previewText = buildFilePreviewText({
|
||||
operation: 'patch',
|
||||
streamedContent: '',
|
||||
existingContent: currentPreview?.session.baseContent,
|
||||
edit: currentPreview?.session.edit,
|
||||
})
|
||||
|
||||
if (previewText !== undefined) {
|
||||
const baseSession = buildPreviewSessionFromIntent(
|
||||
streamId,
|
||||
patchDeleteIntent,
|
||||
currentPreview?.session
|
||||
)
|
||||
const nextSession: FilePreviewSession = {
|
||||
...baseSession,
|
||||
status: 'streaming',
|
||||
previewText,
|
||||
previewVersion: (currentPreview?.session.previewVersion ?? 0) + 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
filePreviewState.set(patchDeleteIntent.toolCallId, {
|
||||
session: nextSession,
|
||||
lastEmittedPreviewText: previewText,
|
||||
lastSnapshotAt: Date.now(),
|
||||
})
|
||||
await persistFilePreviewSession(nextSession)
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: nextSession.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_content',
|
||||
content: previewText,
|
||||
contentMode: 'snapshot',
|
||||
previewVersion: nextSession.previewVersion,
|
||||
fileName: nextSession.fileName,
|
||||
...(nextSession.fileId ? { fileId: nextSession.fileId } : {}),
|
||||
...(nextSession.targetKind ? { targetKind: nextSession.targetKind } : {}),
|
||||
...(nextSession.operation ? { operation: nextSession.operation } : {}),
|
||||
...(nextSession.edit ? { edit: nextSession.edit } : {}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (isToolArgsDeltaStreamEvent(streamEvent) && streamEvent.payload.toolName === 'edit_content') {
|
||||
const toolCallId = streamEvent.payload.toolCallId
|
||||
const delta = streamEvent.payload.argumentsDelta
|
||||
const stateForTool = editContentState.get(toolCallId) ?? { raw: '' }
|
||||
stateForTool.raw += delta
|
||||
|
||||
const editIntent = getIntent()
|
||||
if (editIntent) {
|
||||
const streamedContent = extractEditContent(stateForTool.raw)
|
||||
if (streamedContent !== (stateForTool.lastContentSnapshot ?? '')) {
|
||||
stateForTool.lastContentSnapshot = streamedContent
|
||||
let currentPreview = filePreviewState.get(editIntent.toolCallId) ?? {
|
||||
session: buildPreviewSessionFromIntent(streamId, editIntent),
|
||||
lastEmittedPreviewText: '',
|
||||
lastSnapshotAt: 0,
|
||||
}
|
||||
|
||||
if (
|
||||
currentPreview.session.baseContent === undefined &&
|
||||
(editIntent.operation === 'append' || editIntent.operation === 'patch') &&
|
||||
execContext.workspaceId &&
|
||||
editIntent.target.fileId
|
||||
) {
|
||||
const intentBase = await peekFileIntent(
|
||||
execContext.workspaceId,
|
||||
editIntent.target.fileId,
|
||||
{
|
||||
chatId: execContext.chatId,
|
||||
messageId: execContext.messageId,
|
||||
channelId,
|
||||
}
|
||||
)
|
||||
if (typeof intentBase?.existingContent === 'string') {
|
||||
const seededSession: FilePreviewSession = {
|
||||
...currentPreview.session,
|
||||
baseContent: intentBase.existingContent,
|
||||
...(intentBase.edit ? { edit: intentBase.edit } : {}),
|
||||
}
|
||||
currentPreview = {
|
||||
...currentPreview,
|
||||
session: seededSession,
|
||||
}
|
||||
filePreviewState.set(editIntent.toolCallId, currentPreview)
|
||||
await persistFilePreviewSession(seededSession)
|
||||
}
|
||||
}
|
||||
|
||||
const previewText = isContentOperation(editIntent.operation)
|
||||
? buildFilePreviewText({
|
||||
operation: editIntent.operation,
|
||||
streamedContent,
|
||||
existingContent: currentPreview.session.baseContent,
|
||||
edit: currentPreview.session.edit,
|
||||
})
|
||||
: undefined
|
||||
|
||||
if (previewText !== undefined) {
|
||||
const baseSession = buildPreviewSessionFromIntent(
|
||||
streamId,
|
||||
editIntent,
|
||||
currentPreview.session
|
||||
)
|
||||
const now = Date.now()
|
||||
const nextSession: FilePreviewSession = {
|
||||
...baseSession,
|
||||
status: 'streaming',
|
||||
previewText,
|
||||
previewVersion: (currentPreview.session.previewVersion ?? 0) + 1,
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
}
|
||||
|
||||
await persistFilePreviewSession(nextSession)
|
||||
|
||||
if (
|
||||
nextSession.operation === 'patch' &&
|
||||
now - currentPreview.lastSnapshotAt < PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS
|
||||
) {
|
||||
filePreviewState.set(editIntent.toolCallId, {
|
||||
session: nextSession,
|
||||
lastEmittedPreviewText: currentPreview.lastEmittedPreviewText,
|
||||
lastSnapshotAt: currentPreview.lastSnapshotAt,
|
||||
})
|
||||
} else {
|
||||
const previewUpdate = buildPreviewContentUpdate(
|
||||
currentPreview.lastEmittedPreviewText,
|
||||
nextSession.previewText,
|
||||
currentPreview.lastSnapshotAt,
|
||||
now,
|
||||
nextSession.operation
|
||||
)
|
||||
|
||||
filePreviewState.set(editIntent.toolCallId, {
|
||||
session: nextSession,
|
||||
lastEmittedPreviewText: nextSession.previewText,
|
||||
lastSnapshotAt: previewUpdate.lastSnapshotAt,
|
||||
})
|
||||
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: nextSession.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_content',
|
||||
content: previewUpdate.content,
|
||||
contentMode: previewUpdate.contentMode,
|
||||
previewVersion: nextSession.previewVersion,
|
||||
fileName: nextSession.fileName,
|
||||
...(nextSession.fileId ? { fileId: nextSession.fileId } : {}),
|
||||
...(nextSession.targetKind ? { targetKind: nextSession.targetKind } : {}),
|
||||
...(nextSession.operation ? { operation: nextSession.operation } : {}),
|
||||
...(nextSession.edit ? { edit: nextSession.edit } : {}),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
filePreviewState.set(editIntent.toolCallId, {
|
||||
session: currentPreview.session,
|
||||
lastEmittedPreviewText: currentPreview.lastEmittedPreviewText,
|
||||
lastSnapshotAt: currentPreview.lastSnapshotAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
editContentState.set(toolCallId, stateForTool)
|
||||
}
|
||||
|
||||
if (isToolCallStreamEvent(streamEvent) && streamEvent.payload.toolName === 'edit_content') {
|
||||
const toolCallId = streamEvent.payload.toolCallId
|
||||
if (toolCallId) {
|
||||
editContentState.delete(toolCallId)
|
||||
}
|
||||
}
|
||||
|
||||
const editResultIntent = getIntent()
|
||||
if (
|
||||
isToolResultStreamEvent(streamEvent) &&
|
||||
streamEvent.payload.toolName === 'edit_content' &&
|
||||
editResultIntent
|
||||
) {
|
||||
const currentPreview = filePreviewState.get(editResultIntent.toolCallId)
|
||||
const completedAt = new Date().toISOString()
|
||||
|
||||
if (
|
||||
currentPreview &&
|
||||
currentPreview.lastEmittedPreviewText !== currentPreview.session.previewText &&
|
||||
currentPreview.session.previewText.length > 0
|
||||
) {
|
||||
filePreviewState.set(editResultIntent.toolCallId, {
|
||||
session: currentPreview.session,
|
||||
lastEmittedPreviewText: currentPreview.session.previewText,
|
||||
lastSnapshotAt: Date.now(),
|
||||
})
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: currentPreview.session.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_content',
|
||||
content: currentPreview.session.previewText,
|
||||
contentMode: 'snapshot',
|
||||
previewVersion: currentPreview.session.previewVersion,
|
||||
fileName: currentPreview.session.fileName,
|
||||
...(currentPreview.session.fileId ? { fileId: currentPreview.session.fileId } : {}),
|
||||
...(currentPreview.session.targetKind
|
||||
? { targetKind: currentPreview.session.targetKind }
|
||||
: {}),
|
||||
...(currentPreview.session.operation
|
||||
? { operation: currentPreview.session.operation }
|
||||
: {}),
|
||||
...(currentPreview.session.edit ? { edit: currentPreview.session.edit } : {}),
|
||||
})
|
||||
}
|
||||
|
||||
if (currentPreview) {
|
||||
const completedSession: FilePreviewSession = {
|
||||
...currentPreview.session,
|
||||
status: 'complete',
|
||||
updatedAt: completedAt,
|
||||
completedAt,
|
||||
}
|
||||
filePreviewState.set(editResultIntent.toolCallId, {
|
||||
session: completedSession,
|
||||
lastEmittedPreviewText: completedSession.previewText,
|
||||
lastSnapshotAt: Date.now(),
|
||||
})
|
||||
await persistFilePreviewSession(completedSession)
|
||||
}
|
||||
|
||||
await emitPreviewEvent(streamEvent, options, {
|
||||
toolCallId: editResultIntent.toolCallId,
|
||||
toolName: 'workspace_file',
|
||||
previewPhase: 'file_preview_complete',
|
||||
fileId: editResultIntent.target.fileId,
|
||||
output: streamEvent.payload.output,
|
||||
...(currentPreview ? { previewVersion: currentPreview.session.previewVersion } : {}),
|
||||
})
|
||||
clearIntent()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
|
||||
const logger = createLogger('CopilotSseParser')
|
||||
|
||||
export class FatalSseEventError extends Error {}
|
||||
|
||||
function createParseFailure(message: string, preview: string): FatalSseEventError {
|
||||
logger.error(message, { preview })
|
||||
return new FatalSseEventError(message)
|
||||
}
|
||||
|
||||
function normalizeSseLine(line: string): string {
|
||||
return line.endsWith('\r') ? line.slice(0, -1) : line
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes an SSE stream by calling onEvent for each parsed event.
|
||||
*
|
||||
* @param onEvent Called per parsed event. Return true to stop processing.
|
||||
*/
|
||||
export async function processSSEStream(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
decoder: TextDecoder,
|
||||
abortSignal: AbortSignal | undefined,
|
||||
onEvent: (event: unknown) => boolean | undefined | Promise<boolean | undefined>
|
||||
): Promise<void> {
|
||||
let buffer = ''
|
||||
|
||||
try {
|
||||
try {
|
||||
while (true) {
|
||||
if (abortSignal?.aborted) {
|
||||
logger.info('SSE stream aborted by signal')
|
||||
break
|
||||
}
|
||||
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
let stopped = false
|
||||
for (const line of lines) {
|
||||
const normalizedLine = normalizeSseLine(line)
|
||||
if (abortSignal?.aborted) {
|
||||
logger.info('SSE stream aborted mid-chunk (between events)')
|
||||
return
|
||||
}
|
||||
if (!normalizedLine.trim()) continue
|
||||
if (!normalizedLine.startsWith('data: ')) continue
|
||||
|
||||
const jsonStr = normalizedLine.slice(6)
|
||||
if (jsonStr === '[DONE]') continue
|
||||
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(jsonStr)
|
||||
} catch (error) {
|
||||
const preview = jsonStr.slice(0, 200)
|
||||
const detail = toError(error).message
|
||||
throw createParseFailure(`Failed to parse SSE event JSON: ${detail}`, preview)
|
||||
}
|
||||
|
||||
try {
|
||||
if (await onEvent(parsed)) {
|
||||
stopped = true
|
||||
break
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof FatalSseEventError) {
|
||||
throw error
|
||||
}
|
||||
logger.warn('Failed to handle SSE event', {
|
||||
preview: jsonStr.slice(0, 200),
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
}
|
||||
if (stopped) break
|
||||
}
|
||||
} catch (error) {
|
||||
const aborted =
|
||||
abortSignal?.aborted || (error instanceof DOMException && error.name === 'AbortError')
|
||||
if (aborted) {
|
||||
logger.info('SSE stream read aborted')
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const normalizedBuffer = normalizeSseLine(buffer)
|
||||
if (normalizedBuffer.trim() && normalizedBuffer.startsWith('data: ')) {
|
||||
const jsonStr = normalizedBuffer.slice(6)
|
||||
if (jsonStr === '[DONE]') {
|
||||
return
|
||||
}
|
||||
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(jsonStr)
|
||||
} catch (error) {
|
||||
const preview = normalizedBuffer.slice(0, 200)
|
||||
const detail = toError(error).message
|
||||
throw createParseFailure(`Failed to parse final SSE buffer JSON: ${detail}`, preview)
|
||||
}
|
||||
|
||||
try {
|
||||
await onEvent(parsed)
|
||||
} catch (error) {
|
||||
if (error instanceof FatalSseEventError) {
|
||||
throw error
|
||||
}
|
||||
logger.warn('Failed to handle final SSE event', {
|
||||
preview: normalizedBuffer.slice(0, 200),
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
reader.releaseLock()
|
||||
} catch {
|
||||
logger.warn('Failed to release SSE reader lock')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { type Context, context } from '@opentelemetry/api'
|
||||
import { W3CTraceContextPropagator } from '@opentelemetry/core'
|
||||
|
||||
const propagator = new W3CTraceContextPropagator()
|
||||
const headerSetter = {
|
||||
set(carrier: Record<string, string>, key: string, value: string) {
|
||||
carrier[key] = value
|
||||
},
|
||||
}
|
||||
|
||||
const headerGetter = {
|
||||
keys(carrier: Headers): string[] {
|
||||
const out: string[] = []
|
||||
carrier.forEach((_, key) => {
|
||||
out.push(key)
|
||||
})
|
||||
return out
|
||||
},
|
||||
get(carrier: Headers, key: string): string | undefined {
|
||||
return carrier.get(key) ?? undefined
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Injects W3C trace context (traceparent, tracestate) into outbound HTTP
|
||||
* headers so Go-side spans join the same OTel trace tree as the calling
|
||||
* Sim span.
|
||||
*
|
||||
* Usage: spread the result into your fetch headers:
|
||||
* fetch(url, { headers: { ...myHeaders, ...traceHeaders() } })
|
||||
*/
|
||||
export function traceHeaders(
|
||||
carrier?: Record<string, string>,
|
||||
otelContext?: Context
|
||||
): Record<string, string> {
|
||||
const headers: Record<string, string> = carrier ?? {}
|
||||
propagator.inject(otelContext ?? context.active(), headers, headerSetter)
|
||||
return headers
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts W3C trace context from incoming request headers (traceparent /
|
||||
* tracestate) and returns an OTel Context seeded with the upstream span.
|
||||
*
|
||||
* Use this at the top of inbound Sim route handlers that Go calls into
|
||||
* (e.g. /api/billing/update-cost, /api/copilot/api-keys/validate) so the
|
||||
* Sim-side span becomes a proper child of the Go-side client span in the
|
||||
* same trace — closing the round trip in Jaeger.
|
||||
*
|
||||
* When no traceparent is present (e.g. calls from a browser or a client
|
||||
* that hasn't been instrumented), this returns `context.active()`
|
||||
* unchanged, and any span started under it becomes a new root — the same
|
||||
* behavior as before this helper existed.
|
||||
*/
|
||||
export function contextFromRequestHeaders(headers: Headers): Context {
|
||||
return propagator.extract(context.active(), headers, headerGetter)
|
||||
}
|
||||
@@ -0,0 +1,869 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
MothershipStreamV1CompletionStatus,
|
||||
MothershipStreamV1EventType,
|
||||
MothershipStreamV1ToolExecutor,
|
||||
MothershipStreamV1ToolMode,
|
||||
MothershipStreamV1ToolOutcome,
|
||||
MothershipStreamV1ToolPhase,
|
||||
} from '@/lib/copilot/generated/mothership-stream-v1'
|
||||
|
||||
vi.mock('@/lib/copilot/request/session', async () => {
|
||||
const actual = await vi.importActual<typeof import('@/lib/copilot/request/session')>(
|
||||
'@/lib/copilot/request/session'
|
||||
)
|
||||
return {
|
||||
...actual,
|
||||
hasAbortMarker: vi.fn().mockResolvedValue(false),
|
||||
upsertFilePreviewSession: vi.fn(async (session) => session),
|
||||
}
|
||||
})
|
||||
|
||||
const resolveWorkspaceFileReferenceMock = vi.hoisted(() => vi.fn())
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
|
||||
resolveWorkspaceFileReference: resolveWorkspaceFileReferenceMock,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/copilot/tools/server/files/file-preview', async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import('@/lib/copilot/tools/server/files/file-preview')
|
||||
>('@/lib/copilot/tools/server/files/file-preview')
|
||||
return {
|
||||
...actual,
|
||||
loadWorkspaceFileTextForPreview: vi.fn().mockResolvedValue(''),
|
||||
}
|
||||
})
|
||||
|
||||
import {
|
||||
buildPreviewContentUpdate,
|
||||
decodeJsonStringPrefix,
|
||||
extractEditContent,
|
||||
runStreamLoop,
|
||||
} from '@/lib/copilot/request/go/stream'
|
||||
import { AbortReason, createEvent, hasAbortMarker } from '@/lib/copilot/request/session'
|
||||
import { RequestTraceV1Outcome, TraceCollector } from '@/lib/copilot/request/trace'
|
||||
import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types'
|
||||
|
||||
function createSseResponse(events: unknown[]): Response {
|
||||
const payload = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join('')
|
||||
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(payload))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
},
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function createRawSseResponse(payload: string): Response {
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(payload))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
},
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function createStreamingContext(): StreamingContext {
|
||||
return {
|
||||
messageId: 'msg-1',
|
||||
accumulatedContent: '',
|
||||
finalAssistantContent: '',
|
||||
sawMainToolCall: false,
|
||||
contentBlocks: [],
|
||||
toolCalls: new Map(),
|
||||
pendingToolPromises: new Map(),
|
||||
currentThinkingBlock: null,
|
||||
subagentThinkingBlocks: new Map(),
|
||||
isInThinkingBlock: false,
|
||||
subAgentContent: {},
|
||||
subAgentToolCalls: {},
|
||||
pendingContent: '',
|
||||
streamComplete: false,
|
||||
wasAborted: false,
|
||||
errors: [],
|
||||
activeFileIntents: new Map(),
|
||||
trace: new TraceCollector(),
|
||||
}
|
||||
}
|
||||
|
||||
describe('copilot go stream helpers', () => {
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal('fetch', vi.fn())
|
||||
resolveWorkspaceFileReferenceMock.mockReset()
|
||||
resolveWorkspaceFileReferenceMock.mockResolvedValue(null)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('decodes complete escapes and stops at incomplete unicode escapes', () => {
|
||||
expect(decodeJsonStringPrefix('hello\\nworld')).toBe('hello\nworld')
|
||||
expect(decodeJsonStringPrefix('emoji \\u263A')).toBe('emoji ☺')
|
||||
expect(decodeJsonStringPrefix('partial \\u26')).toBe('partial ')
|
||||
})
|
||||
|
||||
it('extracts the streamed edit_content prefix from partial JSON', () => {
|
||||
expect(extractEditContent('{"content":"hello\\nwor')).toBe('hello\nwor')
|
||||
expect(extractEditContent('{"content":"tab\\tvalue"}')).toBe('tab\tvalue')
|
||||
})
|
||||
|
||||
it('emits full snapshots for append (sidebar viewer uses replace mode; no delta merge)', () => {
|
||||
expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'append')).toEqual({
|
||||
content: 'hello world',
|
||||
contentMode: 'snapshot',
|
||||
lastSnapshotAt: 200,
|
||||
})
|
||||
})
|
||||
|
||||
it('emits deltas for update when the preview extends the previous text', () => {
|
||||
expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'update')).toEqual({
|
||||
content: ' world',
|
||||
contentMode: 'delta',
|
||||
lastSnapshotAt: 100,
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to snapshots for patches and divergent content', () => {
|
||||
expect(buildPreviewContentUpdate('hello', 'goodbye', 100, 200, 'update')).toEqual({
|
||||
content: 'goodbye',
|
||||
contentMode: 'snapshot',
|
||||
lastSnapshotAt: 200,
|
||||
})
|
||||
|
||||
expect(buildPreviewContentUpdate('hello', 'hello world', 100, 200, 'patch')).toEqual({
|
||||
content: 'hello world',
|
||||
contentMode: 'snapshot',
|
||||
lastSnapshotAt: 200,
|
||||
})
|
||||
})
|
||||
|
||||
it('hydrates path-based workspace_file edits into file preview events before edit_content streams', async () => {
|
||||
resolveWorkspaceFileReferenceMock.mockResolvedValue({
|
||||
id: 'file-1',
|
||||
name: 'notes.md',
|
||||
})
|
||||
|
||||
const workspaceFileCall = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'workspace-file-path-1',
|
||||
toolName: 'workspace_file',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.call,
|
||||
arguments: {
|
||||
operation: 'update',
|
||||
target: { kind: 'path', path: 'files/notes.md' },
|
||||
title: 'Update notes',
|
||||
},
|
||||
},
|
||||
})
|
||||
const workspaceFileResult = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '2',
|
||||
seq: 2,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'workspace-file-path-1',
|
||||
toolName: 'workspace_file',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.result,
|
||||
success: true,
|
||||
output: {
|
||||
success: true,
|
||||
data: { id: 'file-1', name: 'notes.md', operation: 'update' },
|
||||
},
|
||||
},
|
||||
})
|
||||
const editContentDelta = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '3',
|
||||
seq: 3,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'edit-content-path-1',
|
||||
toolName: 'edit_content',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.args_delta,
|
||||
argumentsDelta: '{"content":"hello world',
|
||||
},
|
||||
})
|
||||
const editContentResult = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '4',
|
||||
seq: 4,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'edit-content-path-1',
|
||||
toolName: 'edit_content',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.result,
|
||||
success: true,
|
||||
output: {
|
||||
success: true,
|
||||
data: { id: 'file-1', name: 'notes.md' },
|
||||
},
|
||||
},
|
||||
})
|
||||
const complete = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '5',
|
||||
seq: 5,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.complete,
|
||||
payload: {
|
||||
status: MothershipStreamV1CompletionStatus.complete,
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createSseResponse([
|
||||
workspaceFileCall,
|
||||
workspaceFileResult,
|
||||
editContentDelta,
|
||||
editContentResult,
|
||||
complete,
|
||||
])
|
||||
)
|
||||
|
||||
const onEvent = vi.fn()
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
workspaceId: 'workspace-1',
|
||||
messageId: 'msg-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
onEvent,
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
const previewEvents = onEvent.mock.calls
|
||||
.map(([event]) => event)
|
||||
.filter(
|
||||
(event) =>
|
||||
event.type === MothershipStreamV1EventType.tool && 'previewPhase' in event.payload
|
||||
)
|
||||
|
||||
expect(previewEvents.map((event) => event.payload.previewPhase)).toEqual([
|
||||
'file_preview_start',
|
||||
'file_preview_target',
|
||||
'file_preview_content',
|
||||
'file_preview_complete',
|
||||
])
|
||||
expect(previewEvents[1].payload).toMatchObject({
|
||||
previewPhase: 'file_preview_target',
|
||||
target: { kind: 'file_id', fileId: 'file-1', fileName: 'notes.md' },
|
||||
})
|
||||
expect(previewEvents[2].payload).toMatchObject({
|
||||
previewPhase: 'file_preview_content',
|
||||
fileId: 'file-1',
|
||||
targetKind: 'file_id',
|
||||
content: 'hello world',
|
||||
})
|
||||
expect(previewEvents[3].payload).toMatchObject({
|
||||
previewPhase: 'file_preview_complete',
|
||||
fileId: 'file-1',
|
||||
})
|
||||
expect(resolveWorkspaceFileReferenceMock).toHaveBeenCalledWith('workspace-1', 'files/notes.md')
|
||||
})
|
||||
|
||||
it('resolves workflow alias paths to the backing file before streaming previews', async () => {
|
||||
resolveWorkspaceFileReferenceMock.mockResolvedValue({
|
||||
id: 'changelog-file-1',
|
||||
name: 'workflow-1.md',
|
||||
})
|
||||
|
||||
const workspaceFileCall = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'workspace-file-alias-1',
|
||||
toolName: 'workspace_file',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.call,
|
||||
arguments: {
|
||||
operation: 'append',
|
||||
target: { kind: 'path', path: 'workflows/My%20Workflow/changelog.md' },
|
||||
title: 'Update changelog',
|
||||
},
|
||||
},
|
||||
})
|
||||
const editContentDelta = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '2',
|
||||
seq: 2,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'edit-content-alias-1',
|
||||
toolName: 'edit_content',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.args_delta,
|
||||
argumentsDelta: '{"content":"\\n- Added a workflow step',
|
||||
},
|
||||
})
|
||||
const editContentResult = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '3',
|
||||
seq: 3,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'edit-content-alias-1',
|
||||
toolName: 'edit_content',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.result,
|
||||
success: true,
|
||||
output: {
|
||||
success: true,
|
||||
data: { id: 'changelog-file-1', name: 'workflow-1.md' },
|
||||
},
|
||||
},
|
||||
})
|
||||
const complete = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '4',
|
||||
seq: 4,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.complete,
|
||||
payload: {
|
||||
status: MothershipStreamV1CompletionStatus.complete,
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createSseResponse([workspaceFileCall, editContentDelta, editContentResult, complete])
|
||||
)
|
||||
|
||||
const onEvent = vi.fn()
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
workspaceId: 'workspace-1',
|
||||
messageId: 'msg-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
onEvent,
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
const previewEvents = onEvent.mock.calls
|
||||
.map(([event]) => event)
|
||||
.filter(
|
||||
(event) =>
|
||||
event.type === MothershipStreamV1EventType.tool && 'previewPhase' in event.payload
|
||||
)
|
||||
|
||||
expect(previewEvents.map((event) => event.payload.previewPhase)).toEqual([
|
||||
'file_preview_start',
|
||||
'file_preview_target',
|
||||
'file_preview_content',
|
||||
'file_preview_complete',
|
||||
])
|
||||
expect(previewEvents[1].payload).toMatchObject({
|
||||
previewPhase: 'file_preview_target',
|
||||
target: { kind: 'file_id', fileId: 'changelog-file-1', fileName: 'workflow-1.md' },
|
||||
})
|
||||
expect(previewEvents[2].payload).toMatchObject({
|
||||
previewPhase: 'file_preview_content',
|
||||
fileId: 'changelog-file-1',
|
||||
targetKind: 'file_id',
|
||||
content: '\n- Added a workflow step',
|
||||
})
|
||||
expect(previewEvents[3].payload).toMatchObject({
|
||||
previewPhase: 'file_preview_complete',
|
||||
fileId: 'changelog-file-1',
|
||||
})
|
||||
expect(resolveWorkspaceFileReferenceMock).toHaveBeenCalledWith(
|
||||
'workspace-1',
|
||||
'workflows/My%20Workflow/changelog.md'
|
||||
)
|
||||
})
|
||||
|
||||
it('drops duplicate tool_result events before forwarding them', async () => {
|
||||
const toolResult = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: {
|
||||
toolCallId: 'tool-result-dedupe',
|
||||
toolName: 'search_online',
|
||||
executor: MothershipStreamV1ToolExecutor.sim,
|
||||
mode: MothershipStreamV1ToolMode.async,
|
||||
phase: MothershipStreamV1ToolPhase.result,
|
||||
success: true,
|
||||
output: { value: 'ok' },
|
||||
},
|
||||
})
|
||||
const complete = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '2',
|
||||
seq: 2,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.complete,
|
||||
payload: {
|
||||
status: MothershipStreamV1CompletionStatus.complete,
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([toolResult, toolResult, complete]))
|
||||
|
||||
const onEvent = vi.fn()
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
onEvent,
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
|
||||
MothershipStreamV1EventType.tool,
|
||||
MothershipStreamV1EventType.complete,
|
||||
])
|
||||
expect(onEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
payload: expect.objectContaining({
|
||||
toolCallId: 'tool-result-dedupe',
|
||||
phase: MothershipStreamV1ToolPhase.result,
|
||||
}),
|
||||
})
|
||||
)
|
||||
expect(context.toolCalls.get('tool-result-dedupe')).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 'tool-result-dedupe',
|
||||
name: 'search_online',
|
||||
status: MothershipStreamV1ToolOutcome.success,
|
||||
result: { success: true, output: { value: 'ok' } },
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('does not retry transient backend statuses because stream requests are not idempotent', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(new Response('bad gateway', { status: 502 }))
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
name: 'CopilotBackendError',
|
||||
status: 502,
|
||||
body: 'bad gateway',
|
||||
})
|
||||
|
||||
expect(fetch).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not retry non-transient backend statuses before the SSE stream opens', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(new Response('limit reached', { status: 402 }))
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('Usage limit reached')
|
||||
|
||||
expect(fetch).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not retry network errors because Go may already be executing the request', async () => {
|
||||
vi.mocked(fetch).mockRejectedValueOnce(new TypeError('fetch failed'))
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('fetch failed')
|
||||
|
||||
expect(fetch).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('fails closed when the shared stream ends before a terminal event', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('Copilot backend stream ended before a terminal event')
|
||||
expect(
|
||||
context.errors.some((message) =>
|
||||
message.includes('Copilot backend stream ended before a terminal event')
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('reclassifies as aborted when the body closes without terminal but the abort marker is set', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
expect(hasAbortMarker).toHaveBeenCalledWith(context.messageId)
|
||||
expect(context.wasAborted).toBe(true)
|
||||
expect(
|
||||
context.errors.some((message) =>
|
||||
message.includes('Copilot backend stream ended before a terminal event')
|
||||
)
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('invokes onAbortObserved with MarkerObservedAtBodyClose when reclassifying via the abort marker', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
const onAbortObserved = vi.fn()
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
onAbortObserved,
|
||||
})
|
||||
|
||||
expect(onAbortObserved).toHaveBeenCalledTimes(1)
|
||||
expect(onAbortObserved).toHaveBeenCalledWith(AbortReason.MarkerObservedAtBodyClose)
|
||||
expect(context.wasAborted).toBe(true)
|
||||
})
|
||||
|
||||
it('does not invoke onAbortObserved when no abort marker is present at body close', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockResolvedValueOnce(false)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
const onAbortObserved = vi.fn()
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
onAbortObserved,
|
||||
})
|
||||
).rejects.toThrow('Copilot backend stream ended before a terminal event')
|
||||
|
||||
expect(onAbortObserved).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('still fails closed when the body closes without terminal and the abort marker check throws', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockRejectedValueOnce(new Error('redis unavailable'))
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('Copilot backend stream ended before a terminal event')
|
||||
expect(context.wasAborted).toBe(false)
|
||||
})
|
||||
|
||||
it('fails closed when the shared stream receives an invalid event', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createSseResponse([
|
||||
{
|
||||
v: 1,
|
||||
type: MothershipStreamV1EventType.tool,
|
||||
seq: 1,
|
||||
ts: '2026-01-01T00:00:00.000Z',
|
||||
stream: { streamId: 'stream-1', cursor: '1' },
|
||||
payload: {
|
||||
phase: MothershipStreamV1ToolPhase.result,
|
||||
},
|
||||
},
|
||||
])
|
||||
)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('Received invalid stream event on shared path')
|
||||
expect(
|
||||
context.errors.some((message) =>
|
||||
message.includes('Received invalid stream event on shared path')
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('fails closed when the shared stream receives malformed JSON', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createRawSseResponse('data: {"v":1,"type":"text","payload":\n\n')
|
||||
)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('Failed to parse SSE event JSON')
|
||||
expect(
|
||||
context.errors.some((message) => message.includes('Failed to parse SSE event JSON'))
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('records a split canonical request id and go trace id from the stream envelope', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createSseResponse([
|
||||
{
|
||||
v: 1,
|
||||
type: MothershipStreamV1EventType.text,
|
||||
seq: 1,
|
||||
ts: '2026-01-01T00:00:00.000Z',
|
||||
stream: { streamId: 'stream-1', cursor: '1' },
|
||||
trace: {
|
||||
requestId: 'sim-request-1',
|
||||
goTraceId: 'go-trace-1',
|
||||
},
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'hello',
|
||||
},
|
||||
},
|
||||
createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '2',
|
||||
seq: 2,
|
||||
requestId: 'sim-request-1',
|
||||
type: MothershipStreamV1EventType.complete,
|
||||
payload: {
|
||||
status: MothershipStreamV1CompletionStatus.complete,
|
||||
},
|
||||
}),
|
||||
])
|
||||
)
|
||||
|
||||
const context = createStreamingContext()
|
||||
context.requestId = 'sim-request-1'
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
expect(context.requestId).toBe('sim-request-1')
|
||||
expect(
|
||||
context.trace.build({
|
||||
outcome: RequestTraceV1Outcome.success,
|
||||
simRequestId: 'sim-request-1',
|
||||
}).goTraceId
|
||||
).toBe('go-trace-1')
|
||||
})
|
||||
|
||||
it('records span identity on the subagent block from the scope', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createSseResponse([
|
||||
createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.span,
|
||||
scope: {
|
||||
lane: 'subagent',
|
||||
agentId: 'deploy',
|
||||
parentToolCallId: 'tc-deploy-inner',
|
||||
spanId: 'S2',
|
||||
parentSpanId: 'S1',
|
||||
},
|
||||
payload: {
|
||||
kind: 'subagent',
|
||||
event: 'start',
|
||||
agent: 'deploy',
|
||||
data: { tool_call_id: 'tc-deploy-inner', nested: true },
|
||||
},
|
||||
}),
|
||||
createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '2',
|
||||
seq: 2,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.complete,
|
||||
payload: { status: MothershipStreamV1CompletionStatus.complete },
|
||||
}),
|
||||
])
|
||||
)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
const subagentBlock = context.contentBlocks.find((block) => block.type === 'subagent')
|
||||
expect(subagentBlock).toBeDefined()
|
||||
expect(subagentBlock?.spanId).toBe('S2')
|
||||
expect(subagentBlock?.parentSpanId).toBe('S1')
|
||||
expect(subagentBlock?.parentToolCallId).toBe('tc-deploy-inner')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,708 @@
|
||||
import { type Context, SpanStatusCode } from '@opentelemetry/api'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { ORCHESTRATION_TIMEOUT_MS } from '@/lib/copilot/constants'
|
||||
import {
|
||||
MothershipStreamV1EventType,
|
||||
MothershipStreamV1SpanLifecycleEvent,
|
||||
} from '@/lib/copilot/generated/mothership-stream-v1'
|
||||
import { CopilotSseCloseReason } from '@/lib/copilot/generated/trace-attribute-values-v1'
|
||||
import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
|
||||
import { TraceEvent } from '@/lib/copilot/generated/trace-events-v1'
|
||||
import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
|
||||
import { fetchGo } from '@/lib/copilot/request/go/fetch'
|
||||
import {
|
||||
buildPreviewContentUpdate,
|
||||
createFilePreviewAdapterState,
|
||||
decodeJsonStringPrefix,
|
||||
extractEditContent,
|
||||
processFilePreviewStreamEvent,
|
||||
} from '@/lib/copilot/request/go/file-preview-adapter'
|
||||
import { FatalSseEventError, processSSEStream } from '@/lib/copilot/request/go/parser'
|
||||
import {
|
||||
handleSubagentRouting,
|
||||
prePersistClientExecutableToolCall,
|
||||
sseHandlers,
|
||||
subAgentHandlers,
|
||||
} from '@/lib/copilot/request/handlers'
|
||||
import {
|
||||
flushSubagentThinkingBlock,
|
||||
flushThinkingBlock,
|
||||
} from '@/lib/copilot/request/handlers/types'
|
||||
import { getCopilotTracer } from '@/lib/copilot/request/otel'
|
||||
import {
|
||||
AbortReason,
|
||||
eventToStreamEvent,
|
||||
hasAbortMarker,
|
||||
isSubagentSpanStreamEvent,
|
||||
parsePersistedStreamEventEnvelope,
|
||||
} from '@/lib/copilot/request/session'
|
||||
import { shouldSkipToolCallEvent, shouldSkipToolResultEvent } from '@/lib/copilot/request/sse-utils'
|
||||
import type {
|
||||
ExecutionContext,
|
||||
OrchestratorOptions,
|
||||
StreamEvent,
|
||||
StreamingContext,
|
||||
} from '@/lib/copilot/request/types'
|
||||
|
||||
const logger = createLogger('CopilotGoStream')
|
||||
|
||||
export { buildPreviewContentUpdate, decodeJsonStringPrefix, extractEditContent }
|
||||
|
||||
type JsonRecord = Record<string, unknown>
|
||||
|
||||
type SubagentSpanData = {
|
||||
pending?: boolean
|
||||
toolCallId?: string
|
||||
}
|
||||
|
||||
function asJsonRecord(value: unknown): JsonRecord | undefined {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as JsonRecord)
|
||||
: undefined
|
||||
}
|
||||
|
||||
function parseSubagentSpanData(value: unknown): SubagentSpanData | undefined {
|
||||
const data = asJsonRecord(value)
|
||||
if (!data) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const toolCallId = typeof data.tool_call_id === 'string' ? data.tool_call_id : undefined
|
||||
const pending = typeof data.pending === 'boolean' ? data.pending : undefined
|
||||
|
||||
return {
|
||||
...(toolCallId ? { toolCallId } : {}),
|
||||
...(pending !== undefined ? { pending } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotBackendError extends Error {
|
||||
status?: number
|
||||
body?: string
|
||||
|
||||
constructor(message: string, options?: { status?: number; body?: string }) {
|
||||
super(message)
|
||||
this.name = 'CopilotBackendError'
|
||||
this.status = options?.status
|
||||
this.body = options?.body
|
||||
}
|
||||
}
|
||||
|
||||
export class BillingLimitError extends Error {
|
||||
constructor(public readonly userId: string) {
|
||||
super('Usage limit reached')
|
||||
this.name = 'BillingLimitError'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for the shared stream processing loop.
|
||||
*/
|
||||
export interface StreamLoopOptions extends OrchestratorOptions {
|
||||
/**
|
||||
* Called for each normalized event BEFORE standard handler dispatch.
|
||||
* Return true to skip the default handler for this event.
|
||||
*/
|
||||
onBeforeDispatch?: (event: StreamEvent, context: StreamingContext) => boolean | undefined
|
||||
/**
|
||||
* Called when the Go backend's trace ID (go_trace_id) is first received via SSE.
|
||||
*/
|
||||
onGoTraceId?: (goTraceId: string) => void
|
||||
otelContext?: Context
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the SSE stream processing loop against the Go backend.
|
||||
*
|
||||
* Handles: fetch -> parse -> normalize -> dedupe -> subagent routing -> handler dispatch.
|
||||
* Callers provide the fetch URL/options and can intercept events via onBeforeDispatch.
|
||||
* Feature-specific normalization runs through dedicated adapters before the raw event is forwarded.
|
||||
*/
|
||||
export async function runStreamLoop(
|
||||
fetchUrl: string,
|
||||
fetchOptions: RequestInit,
|
||||
context: StreamingContext,
|
||||
execContext: ExecutionContext,
|
||||
options: StreamLoopOptions
|
||||
): Promise<void> {
|
||||
const { timeout = ORCHESTRATION_TIMEOUT_MS, abortSignal } = options
|
||||
const filePreviewAdapterState = createFilePreviewAdapterState()
|
||||
|
||||
const pathname = new URL(fetchUrl).pathname
|
||||
const requestBodyBytes = estimateBodyBytes(fetchOptions.body)
|
||||
const fetchSpan = context.trace.startSpan(`HTTP Request → ${pathname}`, 'sim.http.fetch', {
|
||||
url: fetchUrl,
|
||||
method: fetchOptions.method ?? 'GET',
|
||||
requestBodyBytes,
|
||||
})
|
||||
const fetchStart = performance.now()
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetchGo(fetchUrl, {
|
||||
...fetchOptions,
|
||||
signal: abortSignal,
|
||||
otelContext: options.otelContext,
|
||||
spanName: `sim → go ${pathname}`,
|
||||
operation: 'stream',
|
||||
attributes: {
|
||||
[TraceAttr.CopilotStream]: true,
|
||||
...(requestBodyBytes ? { [TraceAttr.HttpRequestContentLength]: requestBodyBytes } : {}),
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
fetchSpan.attributes = {
|
||||
...(fetchSpan.attributes ?? {}),
|
||||
headersMs: Math.round(performance.now() - fetchStart),
|
||||
}
|
||||
context.trace.endSpan(fetchSpan, abortSignal?.aborted ? 'cancelled' : 'error')
|
||||
throw error
|
||||
}
|
||||
const headersElapsedMs = Math.round(performance.now() - fetchStart)
|
||||
fetchSpan.attributes = {
|
||||
...(fetchSpan.attributes ?? {}),
|
||||
status: response.status,
|
||||
headersMs: headersElapsedMs,
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
context.trace.endSpan(fetchSpan, 'error')
|
||||
const errorText = await response.text().catch(() => '')
|
||||
|
||||
if (response.status === 402) {
|
||||
throw new BillingLimitError(execContext.userId)
|
||||
}
|
||||
|
||||
throw new CopilotBackendError(
|
||||
`Copilot backend error (${response.status}): ${errorText || response.statusText}`,
|
||||
{ status: response.status, body: errorText || response.statusText }
|
||||
)
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
context.trace.endSpan(fetchSpan, 'error')
|
||||
throw new CopilotBackendError('Copilot backend response missing body')
|
||||
}
|
||||
|
||||
context.trace.endSpan(fetchSpan)
|
||||
|
||||
const bodySpan = context.trace.startSpan(`SSE Body → ${pathname}`, 'sim.http.stream_body', {
|
||||
url: fetchUrl,
|
||||
method: fetchOptions.method ?? 'GET',
|
||||
})
|
||||
|
||||
// Aggregate counters populated inline by the reader wrapper + onEvent
|
||||
// dispatcher below and flushed to both the legacy TraceCollector span
|
||||
// and the OTel read-loop span when the loop terminates. Kept as plain
|
||||
// JS variables (not span attrs) so incrementing them is free — we
|
||||
// only pay OTel cost once at span End().
|
||||
//
|
||||
// Idle-gap tracking is split two ways so we can tell apart
|
||||
// upstream-silent from we-were-busy:
|
||||
//
|
||||
// - `longestInboundGapMs`: biggest time between consecutive
|
||||
// `reader.read()` calls returning bytes. Upper bound on
|
||||
// "Go silent". Actually also includes Node waiting for main
|
||||
// thread free, so see dispatchMs below.
|
||||
// - `longestDispatchMs`: biggest time any single event handler
|
||||
// took between "event received" and "returned control". Upper
|
||||
// bound on "Sim was CPU-bound on a handler". If this is high
|
||||
// AND inbound gap is high at the same time, it's Sim. If only
|
||||
// inbound gap is high, it's upstream.
|
||||
// - `totalDispatchMs`: sum of all handler times. Helps gauge
|
||||
// whether handlers in aggregate ate a meaningful fraction of
|
||||
// the read loop.
|
||||
const counters = {
|
||||
bytes: 0,
|
||||
chunks: 0,
|
||||
events: 0,
|
||||
eventsByType: {
|
||||
session: 0,
|
||||
text: 0,
|
||||
tool: 0,
|
||||
span: 0,
|
||||
resource: 0,
|
||||
run: 0,
|
||||
error: 0,
|
||||
complete: 0,
|
||||
} as Record<MothershipStreamV1EventType, number>,
|
||||
firstEventMs: undefined as number | undefined,
|
||||
lastChunkMs: performance.now(),
|
||||
longestInboundGapMs: 0,
|
||||
longestDispatchMs: 0,
|
||||
totalDispatchMs: 0,
|
||||
}
|
||||
const bodyStart = performance.now()
|
||||
let endedOn: string = CopilotSseCloseReason.Terminal
|
||||
|
||||
// Wrap the body's reader so we can track per-chunk bytes and the gap
|
||||
// between chunks. `processSSEStream` consumes this reader exactly as
|
||||
// it would the raw one — no API changes there.
|
||||
const IDLE_GAP_EVENT_THRESHOLD_MS = 10000
|
||||
const rawReader = response.body.getReader()
|
||||
const reader: ReadableStreamDefaultReader<Uint8Array> = {
|
||||
async read() {
|
||||
const result = await rawReader.read()
|
||||
if (!result.done && result.value) {
|
||||
const now = performance.now()
|
||||
const gap = now - counters.lastChunkMs
|
||||
if (gap > counters.longestInboundGapMs) counters.longestInboundGapMs = gap
|
||||
counters.lastChunkMs = now
|
||||
counters.chunks += 1
|
||||
counters.bytes += result.value.byteLength
|
||||
}
|
||||
return result
|
||||
},
|
||||
cancel: (reason) => rawReader.cancel(reason),
|
||||
releaseLock: () => rawReader.releaseLock(),
|
||||
get closed() {
|
||||
return rawReader.closed
|
||||
},
|
||||
}
|
||||
const decoder = new TextDecoder()
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
context.errors.push('Request timed out')
|
||||
context.streamComplete = true
|
||||
endedOn = CopilotSseCloseReason.Timeout
|
||||
reader.cancel().catch(() => {})
|
||||
}, timeout)
|
||||
|
||||
try {
|
||||
await processSSEStream(reader, decoder, abortSignal, async (raw) => {
|
||||
// Track how long THIS handler invocation takes so we can tell
|
||||
// apart "Go was silent" from "we were CPU-bound on a handler".
|
||||
// `longestInboundGapMs` includes handler time (the next reader.read
|
||||
// doesn't run until the previous handler returns), so dispatch
|
||||
// time is the correction needed to isolate upstream silence.
|
||||
const dispatchStart = performance.now()
|
||||
try {
|
||||
if (counters.events === 0) {
|
||||
counters.firstEventMs = Math.round(performance.now() - bodyStart)
|
||||
}
|
||||
counters.events += 1
|
||||
if (abortSignal?.aborted) {
|
||||
context.wasAborted = true
|
||||
return true
|
||||
}
|
||||
|
||||
const parsedEvent = parsePersistedStreamEventEnvelope(raw)
|
||||
if (!parsedEvent.ok) {
|
||||
const detail = [parsedEvent.message, ...(parsedEvent.errors ?? [])]
|
||||
.filter(Boolean)
|
||||
.join('; ')
|
||||
const failureMessage = `Received invalid stream event on shared path: ${detail}`
|
||||
context.errors.push(failureMessage)
|
||||
logger.error('Received invalid stream event on shared path', {
|
||||
reason: parsedEvent.reason,
|
||||
message: parsedEvent.message,
|
||||
errors: parsedEvent.errors,
|
||||
})
|
||||
throw new FatalSseEventError(failureMessage)
|
||||
}
|
||||
|
||||
const envelope = parsedEvent.event
|
||||
const streamEvent = eventToStreamEvent(envelope)
|
||||
if (envelope.trace?.requestId) {
|
||||
const goTraceId = envelope.trace.goTraceId || envelope.trace.requestId
|
||||
context.trace.setGoTraceId(goTraceId)
|
||||
options.onGoTraceId?.(goTraceId)
|
||||
}
|
||||
|
||||
// Per-type counters for the copilot.sse.read_loop span. Bound set
|
||||
// (8 types) so this can never blow up into high cardinality.
|
||||
if (streamEvent.type in counters.eventsByType) {
|
||||
counters.eventsByType[streamEvent.type as MothershipStreamV1EventType] += 1
|
||||
}
|
||||
|
||||
// Surface the full error payload the moment it arrives on the wire. This
|
||||
// is the single chokepoint every error event passes through (main AND
|
||||
// subagent lanes), before subagent routing — which has no `error`
|
||||
// handler — would otherwise swallow it. The client only renders
|
||||
// `message`/`displayMessage`, so log `code`/`provider`/`data` (the raw
|
||||
// upstream provider error) here to explain a client-side "Stream error".
|
||||
if (streamEvent.type === MothershipStreamV1EventType.error) {
|
||||
const errorPayload = streamEvent.payload
|
||||
logger.error('Received error event from Go copilot stream', {
|
||||
path: pathname,
|
||||
lane: streamEvent.scope?.lane ?? 'main',
|
||||
parentToolCallId: streamEvent.scope?.parentToolCallId,
|
||||
agentId: streamEvent.scope?.agentId,
|
||||
code: errorPayload.code,
|
||||
provider: errorPayload.provider,
|
||||
message: errorPayload.message,
|
||||
error: errorPayload.error,
|
||||
displayMessage: errorPayload.displayMessage,
|
||||
data: errorPayload.data,
|
||||
requestId: context.requestId,
|
||||
messageId: context.messageId,
|
||||
})
|
||||
}
|
||||
|
||||
if (shouldSkipToolCallEvent(streamEvent) || shouldSkipToolResultEvent(streamEvent)) {
|
||||
return
|
||||
}
|
||||
|
||||
await processFilePreviewStreamEvent({
|
||||
streamId: envelope.stream.streamId,
|
||||
streamEvent,
|
||||
context,
|
||||
execContext,
|
||||
options,
|
||||
state: filePreviewAdapterState,
|
||||
})
|
||||
|
||||
await prePersistClientExecutableToolCall(streamEvent, context)
|
||||
|
||||
try {
|
||||
await options.onEvent?.(streamEvent)
|
||||
} catch (error) {
|
||||
logger.warn('Failed to forward stream event', {
|
||||
type: streamEvent.type,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
|
||||
// Yield a macrotask so Node.js flushes the HTTP response buffer to
|
||||
// the browser. Microtask yields (await Promise.resolve()) are not
|
||||
// enough — the I/O layer needs a full event loop tick to write.
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
|
||||
if (options.onBeforeDispatch?.(streamEvent, context)) {
|
||||
return context.streamComplete || undefined
|
||||
}
|
||||
|
||||
if (isSubagentSpanStreamEvent(streamEvent)) {
|
||||
const spanData = parseSubagentSpanData(streamEvent.payload.data)
|
||||
const toolCallId = streamEvent.scope?.parentToolCallId || spanData?.toolCallId
|
||||
// Deterministic nesting identity. spanId / parentSpanId are the
|
||||
// primary keys; the toolCallId-keyed stack below is the legacy
|
||||
// fallback for streams that predate span identity.
|
||||
const spanId = streamEvent.scope?.spanId
|
||||
const parentSpanId = streamEvent.scope?.parentSpanId
|
||||
const subagentName = streamEvent.payload.agent
|
||||
const spanEvt = streamEvent.payload.event
|
||||
const isPendingPause = spanData?.pending === true
|
||||
// A subagent lifecycle boundary breaks the main thinking stream.
|
||||
// Flush any open thinking block into contentBlocks BEFORE we push
|
||||
// the `subagent` marker, or the persisted order ends up
|
||||
// [subagent, thinking] and the UI renders the subagent group
|
||||
// above a thinking block that actually happened first.
|
||||
flushSubagentThinkingBlock(context)
|
||||
flushThinkingBlock(context)
|
||||
if (spanEvt === MothershipStreamV1SpanLifecycleEvent.start) {
|
||||
if (toolCallId) {
|
||||
context.subAgentContent[toolCallId] ??= ''
|
||||
context.subAgentToolCalls[toolCallId] ??= []
|
||||
}
|
||||
if (toolCallId && subagentName) {
|
||||
const openParents = (context.openSubagentParents ??= new Set<string>())
|
||||
if (!openParents.has(toolCallId)) {
|
||||
openParents.add(toolCallId)
|
||||
context.contentBlocks.push({
|
||||
type: 'subagent',
|
||||
content: subagentName,
|
||||
parentToolCallId: toolCallId,
|
||||
...(spanId ? { spanId } : {}),
|
||||
...(parentSpanId ? { parentSpanId } : {}),
|
||||
timestamp: Date.now(),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
logger.warn('subagent start missing toolCallId or agent name', {
|
||||
hasToolCallId: Boolean(toolCallId),
|
||||
hasSubagentName: Boolean(subagentName),
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
if (spanEvt === MothershipStreamV1SpanLifecycleEvent.end) {
|
||||
if (isPendingPause) {
|
||||
return
|
||||
}
|
||||
if (!toolCallId) {
|
||||
logger.warn('subagent end missing toolCallId')
|
||||
}
|
||||
if (toolCallId) {
|
||||
for (let i = context.contentBlocks.length - 1; i >= 0; i--) {
|
||||
const b = context.contentBlocks[i]
|
||||
if (
|
||||
b.type === 'subagent' &&
|
||||
b.endedAt === undefined &&
|
||||
b.parentToolCallId === toolCallId
|
||||
) {
|
||||
b.endedAt = Date.now()
|
||||
break
|
||||
}
|
||||
}
|
||||
context.openSubagentParents?.delete(toolCallId)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Subagent-lane events are routed ONLY by their own scope. A valid one
|
||||
// (has parentToolCallId) goes to the subagent handler; a malformed one
|
||||
// (missing parentToolCallId — Go always stamps it, so this is defensive)
|
||||
// is DROPPED rather than falling through to the main handler, which would
|
||||
// merge foreign subagent text/tools into the durable main assistant
|
||||
// message and mis-attribute it.
|
||||
if (streamEvent.scope?.lane === 'subagent') {
|
||||
if (handleSubagentRouting(streamEvent, context)) {
|
||||
const handler = subAgentHandlers[streamEvent.type]
|
||||
if (handler) {
|
||||
await handler(streamEvent, context, execContext, options)
|
||||
}
|
||||
}
|
||||
return context.streamComplete || undefined
|
||||
}
|
||||
|
||||
const handler = sseHandlers[streamEvent.type]
|
||||
if (handler) {
|
||||
await handler(streamEvent, context, execContext, options)
|
||||
}
|
||||
return context.streamComplete || undefined
|
||||
} finally {
|
||||
const dispatchMs = performance.now() - dispatchStart
|
||||
counters.totalDispatchMs += dispatchMs
|
||||
if (dispatchMs > counters.longestDispatchMs) counters.longestDispatchMs = dispatchMs
|
||||
}
|
||||
})
|
||||
|
||||
if (!context.streamComplete && !abortSignal?.aborted && !context.wasAborted) {
|
||||
let abortRequested = false
|
||||
try {
|
||||
abortRequested = await hasAbortMarker(context.messageId)
|
||||
} catch (error) {
|
||||
logger.warn('Failed to read abort marker at body close', {
|
||||
streamId: context.messageId,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
|
||||
if (abortRequested) {
|
||||
options.onAbortObserved?.(AbortReason.MarkerObservedAtBodyClose)
|
||||
context.wasAborted = true
|
||||
endedOn = CopilotSseCloseReason.Aborted
|
||||
} else {
|
||||
const streamPath = new URL(fetchUrl).pathname
|
||||
const message = `Copilot backend stream ended before a terminal event on ${streamPath}`
|
||||
context.errors.push(message)
|
||||
logger.error('Copilot backend stream ended before a terminal event', {
|
||||
path: streamPath,
|
||||
requestId: context.requestId,
|
||||
messageId: context.messageId,
|
||||
})
|
||||
endedOn = CopilotSseCloseReason.ClosedNoTerminal
|
||||
throw new CopilotBackendError(message, { status: 503 })
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof FatalSseEventError && !context.errors.includes(error.message)) {
|
||||
context.errors.push(error.message)
|
||||
}
|
||||
if (endedOn === CopilotSseCloseReason.Terminal) {
|
||||
endedOn =
|
||||
error instanceof CopilotBackendError
|
||||
? CopilotSseCloseReason.BackendError
|
||||
: error instanceof BillingLimitError
|
||||
? CopilotSseCloseReason.BillingLimit
|
||||
: CopilotSseCloseReason.Error
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
if (abortSignal?.aborted) {
|
||||
context.wasAborted = true
|
||||
await reader.cancel().catch(() => {})
|
||||
if (endedOn === CopilotSseCloseReason.Terminal) {
|
||||
endedOn = CopilotSseCloseReason.Aborted
|
||||
}
|
||||
}
|
||||
// An abort or error can tear down the loop mid-thinking. Flush any
|
||||
// open thinking blocks so partial-persistence on /chat/stop sees
|
||||
// them in contentBlocks with endedAt stamped, instead of silently
|
||||
// dropping the in-flight reasoning.
|
||||
flushSubagentThinkingBlock(context)
|
||||
flushThinkingBlock(context)
|
||||
clearTimeout(timeoutId)
|
||||
|
||||
// Legacy TraceCollector span (consumed by the in-memory trace
|
||||
// collector, kept for backwards compatibility with existing
|
||||
// tooling). The real OTel span is stamped below.
|
||||
const bodyDurationMs = Math.round(performance.now() - bodyStart)
|
||||
bodySpan.attributes = {
|
||||
...(bodySpan.attributes ?? {}),
|
||||
eventsReceived: counters.events,
|
||||
firstEventMs: counters.firstEventMs,
|
||||
endedOn,
|
||||
durationMs: bodyDurationMs,
|
||||
}
|
||||
context.trace.endSpan(
|
||||
bodySpan,
|
||||
endedOn === CopilotSseCloseReason.Terminal
|
||||
? 'ok'
|
||||
: endedOn === CopilotSseCloseReason.Aborted
|
||||
? 'cancelled'
|
||||
: 'error'
|
||||
)
|
||||
|
||||
// Real OTel span for Tempo/Grafana. Stamped aggregate-only so
|
||||
// there is no per-chunk OTel cost — one span per read loop with
|
||||
// integer counters, plus a bounded set of events.
|
||||
//
|
||||
// `expectedTerminal` = "the caller considered this leg the FINAL
|
||||
// leg and genuinely expected a terminal event on the wire." We
|
||||
// derive it from `context.streamComplete` MINUS the tool-pause
|
||||
// case: when the server emits a `run.checkpoint_pause`, its
|
||||
// handler also sets `streamComplete=true` to stop the read loop
|
||||
// cleanly, but no `complete` SSE event is ever sent in that
|
||||
// case — that's the tool-pause protocol, not a missing terminal.
|
||||
// `awaitingAsyncContinuation` is set by the same handler, so
|
||||
// its presence distinguishes "tool pause, no terminal expected"
|
||||
// from "caller thought stream was done but server never said so"
|
||||
// (= the real disappeared-response bug class).
|
||||
const expectedTerminal = context.streamComplete && !context.awaitingAsyncContinuation
|
||||
stampSseReadLoopSpan(bodyStart, counters, endedOn, fetchUrl, pathname, {
|
||||
idleGapEventThresholdMs: IDLE_GAP_EVENT_THRESHOLD_MS,
|
||||
expectedTerminal,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function estimateBodyBytes(body: BodyInit | null | undefined): number {
|
||||
if (!body) {
|
||||
return 0
|
||||
}
|
||||
if (typeof body === 'string') {
|
||||
return body.length
|
||||
}
|
||||
if (body instanceof ArrayBuffer) {
|
||||
return body.byteLength
|
||||
}
|
||||
if (ArrayBuffer.isView(body)) {
|
||||
return body.byteLength
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type SseReadLoopCounters = {
|
||||
bytes: number
|
||||
chunks: number
|
||||
events: number
|
||||
eventsByType: Record<MothershipStreamV1EventType, number>
|
||||
firstEventMs: number | undefined
|
||||
longestInboundGapMs: number
|
||||
longestDispatchMs: number
|
||||
totalDispatchMs: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Ship a one-shot `copilot.sse.read_loop` OTel span with the aggregate
|
||||
* counters collected during the read loop. Uses `startTime` so the
|
||||
* span's duration reflects the actual loop wall clock even though we
|
||||
* only talk to OTel once at the end.
|
||||
*
|
||||
* Deliberately synchronous, no per-chunk span calls: total OTel cost
|
||||
* per read loop is fixed (~10 attrs + up to 3 events), independent of
|
||||
* chunk count.
|
||||
*/
|
||||
function stampSseReadLoopSpan(
|
||||
startPerfMs: number,
|
||||
counters: SseReadLoopCounters,
|
||||
closeReason: string,
|
||||
fetchUrl: string,
|
||||
pathname: string,
|
||||
opts: { idleGapEventThresholdMs: number; expectedTerminal: boolean }
|
||||
): void {
|
||||
// Translate performance.now() values into wall-clock Date values so
|
||||
// the span's timestamps land in real time (OTel accepts both, but we
|
||||
// need to pair startTime with a matching "now" for .end()).
|
||||
const nowPerf = performance.now()
|
||||
const nowWall = Date.now()
|
||||
const startWall = nowWall - (nowPerf - startPerfMs)
|
||||
|
||||
const terminalEventSeen = counters.eventsByType.complete > 0 || counters.eventsByType.error > 0
|
||||
// `terminal_event_missing` is the single-attribute dashboard signal
|
||||
// for the "disappeared response" bug class: the caller considered
|
||||
// this leg to be the final one (`context.streamComplete === true`)
|
||||
// but no terminal `complete` or `error` event arrived on the wire.
|
||||
// Tool-pause legs have expectedTerminal=false and never trip this, so
|
||||
// dashboards can filter on `{ .copilot.sse.terminal_event_missing = true }`
|
||||
// without false positives.
|
||||
const terminalEventMissing = opts.expectedTerminal && !terminalEventSeen
|
||||
|
||||
const tracer = getCopilotTracer()
|
||||
const span = tracer.startSpan(TraceSpan.CopilotSseReadLoop, {
|
||||
startTime: startWall,
|
||||
attributes: {
|
||||
[TraceAttr.HttpUrl]: fetchUrl,
|
||||
[TraceAttr.HttpPath]: pathname,
|
||||
[TraceAttr.CopilotSseBytesReceived]: counters.bytes,
|
||||
[TraceAttr.CopilotSseChunksReceived]: counters.chunks,
|
||||
[TraceAttr.CopilotSseEventsReceived]: counters.events,
|
||||
[TraceAttr.CopilotSseEventsSession]: counters.eventsByType.session,
|
||||
[TraceAttr.CopilotSseEventsText]: counters.eventsByType.text,
|
||||
[TraceAttr.CopilotSseEventsTool]: counters.eventsByType.tool,
|
||||
[TraceAttr.CopilotSseEventsSpan]: counters.eventsByType.span,
|
||||
[TraceAttr.CopilotSseEventsResource]: counters.eventsByType.resource,
|
||||
[TraceAttr.CopilotSseEventsRun]: counters.eventsByType.run,
|
||||
[TraceAttr.CopilotSseEventsError]: counters.eventsByType.error,
|
||||
[TraceAttr.CopilotSseEventsComplete]: counters.eventsByType.complete,
|
||||
[TraceAttr.CopilotSseLongestInboundGapMs]: Math.round(counters.longestInboundGapMs),
|
||||
[TraceAttr.CopilotSseLongestDispatchMs]: Math.round(counters.longestDispatchMs),
|
||||
[TraceAttr.CopilotSseTotalDispatchMs]: Math.round(counters.totalDispatchMs),
|
||||
[TraceAttr.CopilotSseCloseReason]: closeReason,
|
||||
[TraceAttr.CopilotSseExpectedTerminal]: opts.expectedTerminal,
|
||||
[TraceAttr.CopilotSseTerminalEventSeen]: terminalEventSeen,
|
||||
[TraceAttr.CopilotSseTerminalEventMissing]: terminalEventMissing,
|
||||
},
|
||||
})
|
||||
|
||||
if (counters.firstEventMs !== undefined) {
|
||||
span.setAttribute(TraceAttr.CopilotSseFirstEventMs, counters.firstEventMs)
|
||||
// Anchor the event to the moment the first SSE event was actually
|
||||
// received (startWall + firstEventMs), not `now`, so a trace
|
||||
// waterfall shows the diamond at the TTFT point — not at span end.
|
||||
span.addEvent(
|
||||
TraceEvent.CopilotSseFirstEvent,
|
||||
{ [TraceAttr.CopilotSseFirstEventMs]: counters.firstEventMs },
|
||||
startWall + counters.firstEventMs
|
||||
)
|
||||
}
|
||||
// Fire the idle-gap event when the INBOUND gap (time between TCP
|
||||
// reads returning bytes) exceeds the threshold. This is the
|
||||
// "upstream was silent or Sim was CPU-bound" signal; dispatch time
|
||||
// on its own doesn't warrant an event because it's within our
|
||||
// control and visible on a dedicated attribute.
|
||||
if (counters.longestInboundGapMs >= opts.idleGapEventThresholdMs) {
|
||||
span.addEvent(TraceEvent.CopilotSseIdleGapExceeded, {
|
||||
[TraceAttr.CopilotSseLongestInboundGapMs]: Math.round(counters.longestInboundGapMs),
|
||||
[TraceAttr.CopilotSseLongestDispatchMs]: Math.round(counters.longestDispatchMs),
|
||||
})
|
||||
}
|
||||
if (terminalEventSeen) {
|
||||
span.addEvent(TraceEvent.CopilotSseTerminalEventReceived)
|
||||
}
|
||||
|
||||
// Span status: only mark ERROR for real failures. User aborts and
|
||||
// clean terminals stay UNSET so dashboards filtering `status=error`
|
||||
// don't light up for normal cancellations. Tool-pause legs (caller
|
||||
// didn't set streamComplete) are NOT errors even though they have
|
||||
// no complete event.
|
||||
if (terminalEventMissing) {
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: 'SSE read loop finished without terminal event (caller expected one)',
|
||||
})
|
||||
} else if (
|
||||
closeReason !== CopilotSseCloseReason.Terminal &&
|
||||
closeReason !== CopilotSseCloseReason.Aborted
|
||||
) {
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: `SSE read loop ended with reason: ${closeReason}`,
|
||||
})
|
||||
}
|
||||
|
||||
span.end(nowWall)
|
||||
}
|
||||
Reference in New Issue
Block a user