chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,46 @@
import type { z } from 'zod'
export interface ServerToolContext {
userId: string
workspaceId?: string
userPermission?: string
chatId?: string
messageId?: string
/**
* The invoking subagent's channel id (its outer tool_use id). Used to scope
* the workspace_file -> edit_content intent handoff to a single file subagent
* so two file agents writing concurrently never consume each other's pending
* intent. Undefined for main-agent tool calls (which never overlap).
*/
parentToolCallId?: string
abortSignal?: AbortSignal
/** Fires only on explicit user stop, never on passive transport disconnect. */
userStopSignal?: AbortSignal
}
export function assertServerToolNotAborted(
context?: ServerToolContext,
message = 'Request aborted before tool mutation could be applied.'
): void {
if (context?.userStopSignal?.aborted) {
const reason = context.userStopSignal.reason
? ` (reason: ${String(context.userStopSignal.reason)})`
: ''
throw new Error(`${message}${reason}`)
}
}
/**
* Base interface for server-side copilot tools.
*
* Tools can optionally declare Zod schemas for input/output validation.
* If provided, the router validates automatically.
*/
export interface BaseServerTool<TArgs = unknown, TResult = unknown> {
name: string
execute(args: TArgs, context?: ServerToolContext): Promise<TResult>
/** Optional Zod schema for input validation */
inputSchema?: z.ZodType<TArgs>
/** Optional Zod schema for output validation */
outputSchema?: z.ZodType<TResult>
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,53 @@
import { createLogger } from '@sim/logger'
import { z } from 'zod'
import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags'
import { getAllBlocks } from '@/blocks/registry'
import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check'
export const GetTriggerBlocksInput = z.object({})
export const GetTriggerBlocksResult = z.object({
triggerBlockIds: z.array(z.string()),
})
export const getTriggerBlocksServerTool: BaseServerTool<
ReturnType<typeof GetTriggerBlocksInput.parse>,
ReturnType<typeof GetTriggerBlocksResult.parse>
> = {
name: 'get_trigger_blocks',
inputSchema: GetTriggerBlocksInput,
outputSchema: GetTriggerBlocksResult,
async execute(_args: unknown, context?: { userId: string; workspaceId?: string }) {
const logger = createLogger('GetTriggerBlocksServerTool')
logger.debug('Executing get_trigger_blocks')
const permissionConfig =
context?.userId && context?.workspaceId
? await getUserPermissionConfig(context.userId, context.workspaceId)
: null
const allowedIntegrations =
permissionConfig?.allowedIntegrations ?? getAllowedIntegrationsFromEnv()
const triggerBlockIds: string[] = []
for (const blockConfig of getAllBlocks()) {
const blockType = blockConfig.type
if (blockConfig.hideFromToolbar) continue
if (allowedIntegrations != null && !allowedIntegrations.includes(blockType.toLowerCase()))
continue
if (blockConfig.category === 'triggers') {
triggerBlockIds.push(blockType)
} else if ('triggerAllowed' in blockConfig && blockConfig.triggerAllowed === true) {
triggerBlockIds.push(blockType)
} else if (blockConfig.subBlocks?.some((subBlock) => subBlock.mode === 'trigger')) {
triggerBlockIds.push(blockType)
}
}
triggerBlockIds.sort()
logger.debug(`Found ${triggerBlockIds.length} trigger blocks`)
return GetTriggerBlocksResult.parse({ triggerBlockIds })
},
}
@@ -0,0 +1,59 @@
import { db } from '@sim/db'
import { docsEmbeddings } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { sql } from 'drizzle-orm'
import { SearchDocumentation } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
import { generateSearchEmbedding } from '@/lib/knowledge/embeddings'
interface DocsSearchParams {
query: string
topK?: number
threshold?: number
}
const DEFAULT_DOCS_SIMILARITY_THRESHOLD = 0.3
export const searchDocumentationServerTool: BaseServerTool<DocsSearchParams, any> = {
name: SearchDocumentation.id,
async execute(params: DocsSearchParams): Promise<any> {
const logger = createLogger('SearchDocumentationServerTool')
const { query, topK = 10, threshold } = params
if (!query || typeof query !== 'string') throw new Error('query is required')
logger.info('Executing docs search', { query, topK })
const similarityThreshold = threshold ?? DEFAULT_DOCS_SIMILARITY_THRESHOLD
const { embedding: queryEmbedding } = await generateSearchEmbedding(query)
if (!queryEmbedding || queryEmbedding.length === 0) {
return { results: [], query, totalResults: 0 }
}
const results = await db
.select({
chunkId: docsEmbeddings.chunkId,
chunkText: docsEmbeddings.chunkText,
sourceDocument: docsEmbeddings.sourceDocument,
sourceLink: docsEmbeddings.sourceLink,
headerText: docsEmbeddings.headerText,
headerLevel: docsEmbeddings.headerLevel,
similarity: sql<number>`1 - (${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector)`,
})
.from(docsEmbeddings)
.orderBy(sql`${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector`)
.limit(topK)
const filteredResults = results.filter((r) => r.similarity >= similarityThreshold)
const documentationResults = filteredResults.map((r, idx) => ({
id: idx + 1,
title: String(r.headerText || 'Untitled Section'),
url: String(r.sourceLink || '#'),
content: String(r.chunkText || ''),
similarity: r.similarity,
}))
logger.info('Docs search complete', { count: documentationResults.length })
return { results: documentationResults, query, totalResults: documentationResults.length }
},
}
@@ -0,0 +1,66 @@
import { createLogger } from '@sim/logger'
import { EnrichmentRun } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
import { getEnrichment } from '@/enrichments/registry'
import { runEnrichment } from '@/enrichments/run'
interface EnrichmentRunParams {
enrichmentId: string
inputs: Record<string, unknown>
}
interface EnrichmentRunResult {
matched: boolean
result: Record<string, unknown>
provider: string | null
/** Hosted-key cost surfaced for per-round billing (omitted for BYOK / free). */
_serviceCost?: { service: string; cost: number }
}
/**
* Direct one-off enrichment lookup. Runs the same provider cascade as table
* enrichments (`runEnrichment`) for a single entity and returns the result
* inline — no table required. The hosted-key cost is surfaced as `_serviceCost`
* so copilot's per-round billing charges for it, matching how the media tools
* bill (see image/generate-image.ts).
*/
export const enrichmentRunServerTool: BaseServerTool<EnrichmentRunParams, EnrichmentRunResult> = {
name: EnrichmentRun.id,
async execute(params: EnrichmentRunParams, context): Promise<EnrichmentRunResult> {
const logger = createLogger('EnrichmentRunServerTool')
const { enrichmentId, inputs } = params
if (!enrichmentId || typeof enrichmentId !== 'string') {
throw new Error('enrichmentId is required')
}
const workspaceId = context?.workspaceId
if (!workspaceId) {
throw new Error('workspaceId is required to run an enrichment')
}
const enrichment = getEnrichment(enrichmentId)
if (!enrichment) {
throw new Error(`Unknown enrichment "${enrichmentId}"`)
}
const { result, cost, error, provider } = await runEnrichment(enrichment, inputs ?? {}, {
workspaceId,
signal: context?.abortSignal,
})
const matched = Object.keys(result).length > 0
logger.info('Enrichment run', { enrichmentId, matched, provider, cost })
// A genuine "no match" returns normally (matched: false). Only surface an
// error when every provider that ran failed (infra/auth/rate-limit).
if (error && !matched) {
throw new Error(error)
}
return {
matched,
result,
provider,
...(cost > 0 ? { _serviceCost: { service: provider ?? enrichmentId, cost } } : {}),
}
},
}
@@ -0,0 +1,97 @@
import { createLogger } from '@sim/logger'
import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { isPlanAliasPath } from '@/lib/copilot/vfs/workflow-aliases'
import { inferContentType } from './workspace-file'
const logger = createLogger('CreateFileServerTool')
const CREATE_FILE_TOOL_ID = 'create_file'
interface CreateFileArgs {
fileName: string
contentType?: string
outputs?: { files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }> }
args?: Record<string, unknown>
}
interface CreateFileResult {
success: boolean
message: string
data?: {
id: string
name: string
contentType: string
vfsPath: string
backingVfsPath?: string
}
}
export const createFileServerTool: BaseServerTool<CreateFileArgs, CreateFileResult> = {
name: CREATE_FILE_TOOL_ID,
async execute(params: CreateFileArgs, context?: ServerToolContext): Promise<CreateFileResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
await ensureWorkspaceAccess(workspaceId, context.userId, 'write')
const nested = params.args
const fileName = params.fileName || (nested?.fileName as string) || ''
const explicitType = params.contentType || (nested?.contentType as string) || undefined
const outputFile = params.outputs?.files?.[0]
if (!outputFile?.path && !fileName) {
return { success: false, message: 'create_file requires outputs.files[0].path or fileName' }
}
const outputPath =
outputFile?.path ?? (fileName.startsWith('files/') ? fileName : `files/${fileName}`)
if (isPlanAliasPath(outputPath)) {
return {
success: false,
message:
'create_file does not initialize plan aliases; changelog.md is created automatically per workflow.',
}
}
const contentType = outputFile?.mimeType ?? inferContentType(outputPath, explicitType)
const emptyBuffer = Buffer.from('', 'utf-8')
assertServerToolNotAborted(context)
const result = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: {
path: outputPath,
mode: outputFile?.mode ?? 'create',
mimeType: outputFile?.mimeType,
},
buffer: emptyBuffer,
inferredMimeType: contentType,
})
logger.info('File created via create_file', {
fileId: result.id,
name: result.vfsPath,
contentType,
userId: context.userId,
})
return {
success: true,
message: `File "${result.vfsPath}" created successfully`,
data: {
id: result.id,
name: result.name,
contentType,
vfsPath: result.vfsPath,
backingVfsPath: result.backingVfsPath,
},
}
},
}
@@ -0,0 +1,107 @@
import { createLogger } from '@sim/logger'
import { DeleteFile } from '@/lib/copilot/generated/tool-catalog-v1'
import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import {
getWorkspaceFile,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { performDeleteWorkspaceFileItems } from '@/lib/workspace-files/orchestration'
const logger = createLogger('DeleteFileServerTool')
interface DeleteFileArgs {
paths?: string[]
path?: string
fileIds?: string[]
fileId?: string
args?: Record<string, unknown>
}
interface DeleteFileResult {
success: boolean
message: string
}
export const deleteFileServerTool: BaseServerTool<DeleteFileArgs, DeleteFileResult> = {
name: DeleteFile.id,
async execute(params: DeleteFileArgs, context?: ServerToolContext): Promise<DeleteFileResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
await ensureWorkspaceAccess(workspaceId, context.userId, 'write')
const nested = params.args
const paths: string[] =
params.paths ??
(nested?.paths as string[] | undefined) ??
[params.path || (nested?.path as string) || ''].filter(Boolean)
const legacyFileIds: string[] =
params.fileIds ??
(nested?.fileIds as string[] | undefined) ??
[params.fileId || (nested?.fileId as string) || ''].filter(Boolean)
if (paths.length === 0 && legacyFileIds.length === 0) {
return { success: false, message: 'paths is required' }
}
const deletable: { id: string; name: string }[] = []
const failed: string[] = []
for (const path of paths) {
const existingFile = await resolveWorkspaceFileReference(workspaceId, path)
if (!existingFile) {
failed.push(path)
continue
}
deletable.push({ id: existingFile.id, name: existingFile.name })
}
for (const fileId of legacyFileIds) {
const existingFile = await getWorkspaceFile(workspaceId, fileId)
if (!existingFile) {
failed.push(fileId)
continue
}
deletable.push({ id: fileId, name: existingFile.name })
}
if (deletable.length > 0) {
assertServerToolNotAborted(context)
const result = await performDeleteWorkspaceFileItems({
workspaceId,
userId: context.userId,
fileIds: deletable.map((file) => file.id),
})
if (!result.success) {
return { success: false, message: result.error || 'Failed to delete files' }
}
}
for (const file of deletable) {
logger.info('File deleted via delete_file', {
fileId: file.id,
name: file.name,
userId: context.userId,
})
}
const parts: string[] = []
if (deletable.length > 0)
parts.push(`Deleted: ${deletable.map((file) => file.name).join(', ')}`)
if (failed.length > 0) parts.push(`Not found: ${failed.join(', ')}`)
return {
success: deletable.length > 0,
message: parts.join('. '),
}
},
}
@@ -0,0 +1,75 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
vi.mock('@/lib/execution/e2b', () => ({
executeInE2B: vi.fn(),
executeShellInE2B: vi.fn(),
}))
vi.mock('@/lib/execution/languages', () => ({
CodeLanguage: { javascript: 'javascript', python: 'python' },
}))
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
getWorkspaceFile: vi.fn(),
fetchWorkspaceFileBuffer: vi.fn(),
}))
vi.mock('./doc-compiled-store', () => ({
loadCompiledDoc: vi.fn(),
storeCompiledDoc: vi.fn(),
}))
import { collectReferencedFileIds } from './doc-compile'
const ID = '550e8400-e29b-41d4-a716-446655440000'
describe('collectReferencedFileIds', () => {
it('captures the id from getFileBase64(...) with single or double quotes', () => {
expect(collectReferencedFileIds(`await getFileBase64('${ID}')`)).toEqual(new Set([ID]))
expect(collectReferencedFileIds(`getFileBase64("abc_def-1")`)).toEqual(new Set(['abc_def-1']))
})
it('captures the id from pptx addImage(slide, id, opts) (second arg)', () => {
const src = `await addImage(slide, '${ID}', { x: 1, y: 1, w: 2, h: 2 })`
expect(collectReferencedFileIds(src)).toEqual(new Set([ID]))
})
it('captures the id from docx addImage(id, opts) (first arg)', () => {
const src = `const img = await addImage('docx-img-1', { width: 200, height: 100 })`
expect(collectReferencedFileIds(src)).toEqual(new Set(['docx-img-1']))
})
it('captures the id from pdf drawImage(page, id, opts) (second arg)', () => {
const src = `await drawImage(page, 'pdf-img-2', { x: 0, y: 0, width: 100, height: 100 })`
expect(collectReferencedFileIds(src)).toEqual(new Set(['pdf-img-2']))
})
it('still supports the legacy /home/user/inputs/<id> path form', () => {
expect(collectReferencedFileIds(`fs.readFileSync('/home/user/inputs/legacy-1')`)).toEqual(
new Set(['legacy-1'])
)
})
it('collects and dedupes ids across multiple call sites', () => {
const src = `
await addImage(slide, 'logo-1', { x: 0, y: 0, w: 1, h: 1 });
const uri = await getFileBase64('logo-1');
await addImage(slide, 'crest-2', { x: 2, y: 0, w: 1, h: 1 });
`
expect(collectReferencedFileIds(src)).toEqual(new Set(['logo-1', 'crest-2']))
})
it('does not match id-like strings outside the image helpers', () => {
const src = `slide.addText('order ${ID} shipped', { x: 1, y: 1, w: 8, h: 1 })`
expect(collectReferencedFileIds(src)).toEqual(new Set())
})
it('does not match slide.addImage({ data }) — no fileId is present there', () => {
const src = `slide.addImage({ data: base64Data, x: 1, y: 1, w: 2, h: 2 })`
expect(collectReferencedFileIds(src)).toEqual(new Set())
})
it('returns an empty set when there are no image references', () => {
expect(collectReferencedFileIds(`slide.addText('hello', { x: 1, y: 1 })`)).toEqual(new Set())
})
})
@@ -0,0 +1,556 @@
import { createLogger } from '@sim/logger'
import { sha256Hex } from '@sim/security/hash'
import { getErrorMessage } from '@sim/utils/errors'
import { isE2BDocEnabled } from '@/lib/core/config/env-flags'
import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
import { executeInE2B, executeShellInE2B, type SandboxFile } from '@/lib/execution/e2b'
import { CodeLanguage } from '@/lib/execution/languages'
import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
import {
fetchWorkspaceFileBuffer,
getWorkspaceFile,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { getContentType } from '@/app/api/files/utils'
import type { SandboxTaskId } from '@/sandbox-tasks/registry'
import { loadCompiledDoc, storeCompiledDoc } from './doc-compiled-store'
const logger = createLogger('CopilotDocCompile')
/**
* Thrown when the user-authored Python script itself fails (raised an exception
* or produced no output) — i.e. an error the agent should fix by editing the
* script. Infra failures (E2B sandbox create/timeout, S3) propagate as plain
* Errors so callers can return 5xx instead of telling the agent its script was
* wrong.
*/
export class DocCompileUserError extends Error {
constructor(message: string) {
super(message)
this.name = 'DocCompileUserError'
}
}
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
const XLSX_MIME = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
const PDF_MIME = 'application/pdf'
// When the E2B doc sandbox is enabled, ALL four formats compile there: pptx/docx
// via Node (pptxgenjs/docx + react-icons/sharp icons), pdf/xlsx via Python
// (reportlab/openpyxl). Source MIMEs for the node engines match the isolated-vm
// JS path; the python engines have distinct markers.
export const PPTXGENJS_SOURCE_MIME = 'text/x-pptxgenjs'
export const DOCXJS_SOURCE_MIME = 'text/x-docxjs'
export const PYTHON_PDF_SOURCE_MIME = 'text/x-python-pdf'
export const PYTHON_XLSX_SOURCE_MIME = 'text/x-python-xlsx'
export type DocEngine = 'node' | 'python'
export interface E2BDocFormat {
ext: 'pptx' | 'docx' | 'pdf' | 'xlsx'
engine: DocEngine
formatName: 'PPTX' | 'DOCX' | 'PDF' | 'XLSX'
contentType: string
sourceMime: string
}
/**
* Resolves the E2B doc format + engine for a filename, or null for non-docs.
* pptx/docx → node, pdf/xlsx → python. Only meaningful when the E2B doc sandbox
* is enabled; callers gate on isE2BDocEnabled before using this.
*/
export async function getE2BDocFormat(fileName: string): Promise<E2BDocFormat | null> {
const l = fileName.toLowerCase()
if (l.endsWith('.pptx'))
return {
ext: 'pptx',
engine: 'node',
formatName: 'PPTX',
contentType: PPTX_MIME,
sourceMime: PPTXGENJS_SOURCE_MIME,
}
if (l.endsWith('.docx'))
return {
ext: 'docx',
engine: 'node',
formatName: 'DOCX',
contentType: DOCX_MIME,
sourceMime: DOCXJS_SOURCE_MIME,
}
if (l.endsWith('.pdf'))
return {
ext: 'pdf',
engine: 'python',
formatName: 'PDF',
contentType: PDF_MIME,
sourceMime: PYTHON_PDF_SOURCE_MIME,
}
// xlsx is gated behind the mothership-beta feature flag (like plans/changelog): the
// skill + prompt are gated on the Go side, and this is the single Sim chokepoint
// that keeps the compile/serve/check/recalc paths off for xlsx when beta is off.
if (l.endsWith('.xlsx') && (await isFeatureEnabled('mothership-beta')))
return {
ext: 'xlsx',
engine: 'python',
formatName: 'XLSX',
contentType: XLSX_MIME,
sourceMime: PYTHON_XLSX_SOURCE_MIME,
}
return null
}
// The skills reference workspace images by BARE file id through the injected
// helpers — `getFileBase64(id)`, `addImage(slide, id, ...)` (pptx),
// `addImage(id, ...)` (docx), `drawImage(page, id, ...)` (pdf) — never as a path.
// Capture the id from those call sites (skipping a leading slide/page argument),
// plus the legacy `/home/user/inputs/<id>` path, so referenced files are staged
// before the script runs. Without this the sandbox `getFileBase64` throws
// "file not staged" and every workspace-image embed silently fails.
const INPUT_PATH_RE = /\/home\/user\/inputs\/([A-Za-z0-9_-]+)/g
const FILE_HELPER_RE =
/\b(?:getFileBase64|addImage|drawImage)\(\s*(?:[A-Za-z_$][\w$]*\s*,\s*)?['"]([A-Za-z0-9_-]+)['"]/g
// The doc source is user/LLM-controlled, so bound how much it can pull into the
// sandbox: each `/home/user/inputs/<id>` reference is only ~35 bytes, so the
// source-size cap alone does not bound staging. These caps prevent an
// authenticated member from forcing thousands of (or very large) workspace files
// to be downloaded and base64-held in-process per compile request.
const MAX_STAGED_INPUTS = 20
const MAX_STAGED_FILE_BYTES = 25 * 1024 * 1024
const MAX_STAGED_TOTAL_BYTES = 50 * 1024 * 1024
/**
* Collects the workspace file ids a doc source references — from the injected
* image-helper call sites and the legacy `/home/user/inputs/<id>` path. Matching
* is scoped to the helper calls (not bare id-like strings in slide text), and the
* caller skips any id that does not resolve to a real file, so over-matching is
* harmless.
*/
export function collectReferencedFileIds(source: string): Set<string> {
const ids = new Set<string>()
for (const re of [INPUT_PATH_RE, FILE_HELPER_RE]) {
for (const match of source.matchAll(re)) {
if (match[1]) ids.add(match[1])
}
}
return ids
}
async function stageReferencedImages(source: string, workspaceId: string): Promise<SandboxFile[]> {
const ids = collectReferencedFileIds(source)
if (ids.size > MAX_STAGED_INPUTS) {
throw new Error(
`Too many referenced input files (${ids.size}); max ${MAX_STAGED_INPUTS}. Reference fewer files.`
)
}
const files: SandboxFile[] = []
let totalBytes = 0
for (const fileId of ids) {
let record: Awaited<ReturnType<typeof getWorkspaceFile>>
try {
record = await getWorkspaceFile(workspaceId, fileId)
} catch (err) {
logger.warn('Failed to resolve referenced image for doc compile', {
workspaceId,
fileId,
error: getErrorMessage(err),
})
continue
}
if (!record) continue
if (typeof record.size === 'number' && record.size > MAX_STAGED_FILE_BYTES) {
logger.warn('Skipping oversized referenced image for doc compile', {
workspaceId,
fileId,
size: record.size,
})
continue
}
if (totalBytes + (record.size ?? 0) > MAX_STAGED_TOTAL_BYTES) {
throw new Error(
`Referenced input files exceed the ${MAX_STAGED_TOTAL_BYTES} byte staging budget.`
)
}
let buffer: Buffer
try {
buffer = await fetchWorkspaceFileBuffer(record)
} catch (err) {
logger.warn('Failed to stage referenced image for doc compile', {
workspaceId,
fileId,
error: getErrorMessage(err),
})
continue
}
// Enforce the per-file cap on actual bytes too: record.size can be null/stale,
// in which case the pre-fetch check above is skipped and a single oversized
// file would otherwise be fully base64-held in memory.
if (buffer.length > MAX_STAGED_FILE_BYTES) {
logger.warn('Skipping oversized referenced image for doc compile (post-fetch)', {
workspaceId,
fileId,
size: buffer.length,
})
continue
}
// Budget check after the fetch (record.size may be unset/stale) — kept
// outside the catch above so it fails the compile rather than being skipped.
totalBytes += buffer.length
if (totalBytes > MAX_STAGED_TOTAL_BYTES) {
throw new Error(
`Referenced input files exceed the ${MAX_STAGED_TOTAL_BYTES} byte staging budget.`
)
}
files.push({
path: `/home/user/inputs/${fileId}`,
content: buffer.toString('base64'),
encoding: 'base64',
})
}
return files
}
const DOC_COMPILE_TIMEOUT_MS = 120_000
// Appended to xlsx compile scripts: LibreOffice recalculates formulas on
// load/convert and writes cached values, then we move the result back over
// output.xlsx so the binary read back has computed results (openpyxl alone omits
// them). Indented at column 0 so it concatenates cleanly after the user's script.
const XLSX_RECALC_SNIPPET = `
import subprocess as __sim_sp, shutil as __sim_sh, os as __sim_os
# Best-effort: bake cached formula values via LibreOffice. If recalc fails
# (soffice crash/timeout/unsupported), keep the openpyxl workbook as-is — it's
# still a valid file (formulas just lack cached values). Never fail the user's
# compile over an infra recalc failure.
try:
__sim_os.makedirs("/home/user/__recalc", exist_ok=True)
__sim_sp.run(
["soffice", "--headless", "--convert-to", "xlsx", "--outdir", "/home/user/__recalc", "/home/user/output.xlsx"],
check=True, timeout=120, capture_output=True,
)
__sim_sh.move("/home/user/__recalc/output.xlsx", "/home/user/output.xlsx")
except Exception as __sim_recalc_err:
print("xlsx recalc skipped:", __sim_recalc_err)
`.trim()
interface CompileArgs {
source: string
fileName: string
workspaceId: string
}
/**
* Compiles a Python document script to its binary in the dedicated E2B doc
* sandbox. The script must save to /home/user/output.<ext>; we read that back.
* Throws with a human-readable message when the script errors or writes nothing.
* Internal — callers use compileDoc (load-or-build + store).
*/
async function compileDocViaE2BPython(
{ source, workspaceId }: CompileArgs,
fmt: E2BDocFormat
): Promise<Buffer> {
const sandboxFiles = await stageReferencedImages(source, workspaceId)
const outputSandboxPath = `/home/user/output.${fmt.ext}`
// openpyxl writes formula strings but no cached values, so a web viewer (SheetJS)
// renders formula cells blank. Recalculate in place with LibreOffice (it
// evaluates formulas on load and writes cached values on convert) so the stored
// artifact — and everything that serves it — shows computed results. pdf is
// unaffected. Runs only after the user's script succeeds.
const code = fmt.ext === 'xlsx' ? `${source}\n${XLSX_RECALC_SNIPPET}` : source
const result = await executeInE2B({
code,
language: CodeLanguage.Python,
timeoutMs: DOC_COMPILE_TIMEOUT_MS,
sandboxFiles,
outputSandboxPath,
sandboxKind: 'doc',
})
if (result.error) {
// The script raised — a user-code error the agent should fix.
throw new DocCompileUserError(result.error)
}
if (!result.exportedFileContent) {
throw new DocCompileUserError(
`${fmt.formatName} generation produced no output. The script must save to ${outputSandboxPath}.`
)
}
return Buffer.from(result.exportedFileContent, 'base64')
}
// ── Node engine (pptxgenjs / docx) ──────────────────────────────────────────
// Preambles replicate the isolated-vm bootstraps as node globals: the injected
// `pptx`/`docx` instances, geometry constants, and fileId-based image helpers
// (reading staged /home/user/inputs/<id> files). pptx also gets `iconImage`
// (react-icons → sharp → PNG), which only works here because the E2B sandbox is
// a full Linux VM. The agent's edit_content source runs inside an async IIFE so
// top-level await (addImage/iconImage) works; the finalizer writes the binary.
const PPTX_NODE_PREAMBLE = `
const PptxGenJS = require('pptxgenjs');
const fs = require('fs');
globalThis.pptx = new PptxGenJS();
globalThis.pptx.layout = 'LAYOUT_16x9';
globalThis.SLIDE_W = 10; globalThis.SLIDE_H = 5.625;
globalThis.MARGIN = 0.5; globalThis.CONTENT_W = 9; globalThis.CONTENT_H = 3.8;
function __mime(b){ if(b.length>=2&&b[0]===0x89&&b[1]===0x50)return 'image/png'; if(b.length>=2&&b[0]===0xff&&b[1]===0xd8)return 'image/jpeg'; if(b.length>=3&&b[0]===0x47&&b[1]===0x49&&b[2]===0x46)return 'image/gif'; if(b.length>=12&&b.slice(0,4).toString('latin1')==='RIFF'&&b.slice(8,12).toString('latin1')==='WEBP')return 'image/webp'; return 'image/png'; }
globalThis.getFileBase64 = async function(fileId){ const p='/home/user/inputs/'+fileId; if(!fs.existsSync(p)) throw new Error('getFileBase64: file not staged: '+fileId); const b=fs.readFileSync(p); return __mime(b)+';base64,'+b.toString('base64'); };
globalThis.addImage = async function(slide, fileId, opts){ if(!opts||opts.x==null||opts.y==null||opts.w==null||opts.h==null) throw new Error('addImage: opts must include x, y, w, h'); const data=await globalThis.getFileBase64(fileId); slide.addImage(Object.assign({}, opts, { data })); };
globalThis.iconImage = async function(IconComponent, color, size){ const React=require('react'); const RDS=require('react-dom/server'); const sharp=require('sharp'); const svg=RDS.renderToStaticMarkup(React.createElement(IconComponent,{color:color||'#000000',size:String(size||256)})); const png=await sharp(Buffer.from(svg)).png().toBuffer(); return 'image/png;base64,'+png.toString('base64'); };
`.trim()
const DOCX_NODE_PREAMBLE = `
const docx = require('docx');
const fs = require('fs');
globalThis.docx = docx;
globalThis.__docxSections = [];
globalThis.__docxDocOptions = null;
globalThis.addSection = function(s){ globalThis.__docxSections.push(s); };
globalThis.PAGE_W = 12240; globalThis.PAGE_H = 15840; globalThis.MARGIN = 1440; globalThis.CONTENT_W = 9360;
globalThis.getFileBase64 = async function(fileId){ const p='/home/user/inputs/'+fileId; if(!fs.existsSync(p)) throw new Error('getFileBase64: file not staged: '+fileId); const b=fs.readFileSync(p); const m=(b[0]===0x89?'image/png':b[0]===0xff?'image/jpeg':b[0]===0x47?'image/gif':'image/png'); return 'data:'+m+';base64,'+b.toString('base64'); };
globalThis.addImage = async function(fileId, opts){ if(!opts||opts.width==null||opts.height==null) throw new Error('addImage: opts must include width and height'); const p='/home/user/inputs/'+fileId; if(!fs.existsSync(p)) throw new Error('addImage: file not staged: '+fileId); const b=fs.readFileSync(p); const ext=(b[0]===0x89?'png':b[0]===0xff?'jpg':b[0]===0x47?'gif':'png'); const { width, height, type:_t, data:_d, transformation:ut, ...rest } = opts; return new docx.ImageRun(Object.assign(rest, { data: b, type: ext, transformation: Object.assign({ width, height }, ut||{}) })); };
`.trim()
const PPTX_NODE_FINALIZE = `await globalThis.pptx.writeFile({ fileName: '/home/user/output.pptx' });`
const DOCX_NODE_FINALIZE = `
let doc = globalThis.doc;
if (!doc && globalThis.__docxSections.length > 0) doc = new docx.Document(Object.assign({}, globalThis.__docxDocOptions || {}, { sections: globalThis.__docxSections }));
if (!doc) throw new Error('No document created. Use addSection({ children: [...] }) for chunked writes, or set globalThis.doc.');
const __buf = await docx.Packer.toBuffer(doc);
fs.writeFileSync('/home/user/output.docx', __buf);
`.trim()
/**
* Compiles a pptx/docx document by running the agent's pptxgenjs/docx source in
* the E2B doc sandbox via Node. Mirrors compileDocViaE2BPython for the JS
* engines. Throws DocCompileUserError on a script error.
*/
async function compileDocViaE2BNode(
{ source, fileName, workspaceId }: CompileArgs,
ext: 'pptx' | 'docx'
): Promise<Buffer> {
const sandboxFiles = await stageReferencedImages(source, workspaceId)
const outputSandboxPath = `/home/user/output.${ext}`
const preamble = ext === 'pptx' ? PPTX_NODE_PREAMBLE : DOCX_NODE_PREAMBLE
const finalize = ext === 'pptx' ? PPTX_NODE_FINALIZE : DOCX_NODE_FINALIZE
const script = `${preamble}
;(async () => {
${source}
${finalize}
})().then(() => console.log('__DOC_OK__')).catch((e) => { console.error('__DOC_ERR__' + (e && e.message ? e.message : String(e))); process.exit(1); });
`
const result = await executeShellInE2B({
code: 'NODE_PATH=$(npm root -g) node /home/user/script.js',
envs: {},
timeoutMs: DOC_COMPILE_TIMEOUT_MS,
sandboxKind: 'doc',
sandboxFiles: [
...sandboxFiles,
{
path: '/home/user/script.js',
content: Buffer.from(script, 'utf-8').toString('base64'),
encoding: 'base64',
},
],
outputSandboxPath,
})
// Success requires the script to reach the finalizer (__DOC_OK__) AND produce
// the output file — a script that writes then throws must not persist a
// partial/corrupt artifact (mirrors the Python path).
const out = `${result.stdout || ''}\n${result.error || ''}`
const errMatch = out.match(/__DOC_ERR__([\s\S]*)/)
if (out.includes('__DOC_OK__') && result.exportedFileContent) {
return Buffer.from(result.exportedFileContent, 'base64')
}
if (errMatch) {
// The script ran and threw — a user-code error the agent should fix.
throw new DocCompileUserError(
`${ext.toUpperCase()} generation failed: ${errMatch[1]?.trim() || 'unknown error'}`
)
}
// No __DOC_OK__ and no __DOC_ERR__ → node never completed (sandbox died, command
// failure, or the output couldn't be read). That's a retriable system error, not
// the agent's code — surface it as a plain Error so callers don't tell the agent
// to "fix its code".
throw new Error(
`${ext.toUpperCase()} compile did not complete in the sandbox: ${result.error || 'no output produced'}`
)
}
/**
* Returns the compiled binary for a doc, building it once (via the right engine —
* Node for pptx/docx, Python for pdf/xlsx) if the source-hash artifact is not
* already in S3. Used by read paths (serve, render, compiled-check) so E2B runs
* at most once per distinct source.
*/
export async function compileDoc(
args: CompileArgs
): Promise<{ buffer: Buffer; contentType: string }> {
const { source, fileName, workspaceId } = args
const fmt = await getE2BDocFormat(fileName)
if (!fmt) throw new Error(`Unsupported document format: ${fileName}`)
const existing = await loadCompiledDoc(workspaceId, source, fmt.ext)
if (existing) return { buffer: existing, contentType: fmt.contentType }
const buffer =
fmt.engine === 'node'
? await compileDocViaE2BNode({ source, fileName, workspaceId }, fmt.ext as 'pptx' | 'docx')
: await compileDocViaE2BPython({ source, fileName, workspaceId }, fmt)
await storeCompiledDoc(workspaceId, source, fmt.ext, fmt.contentType, buffer)
return { buffer, contentType: fmt.contentType }
}
/**
* Loads a compiled doc artifact by extension when present, without compiling.
* Used by the serve route, which has the source + ext but no file record — a hit
* means the file is a generated doc whose binary is already built.
*/
export async function loadCompiledDocByExt(
workspaceId: string,
source: string,
ext: string
): Promise<{ buffer: Buffer; contentType: string } | null> {
const fmt = await getE2BDocFormat(`x.${ext}`)
if (!fmt) return null
const buffer = await loadCompiledDoc(workspaceId, source, fmt.ext)
return buffer ? { buffer, contentType: fmt.contentType } : null
}
const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04])
const PDF_MAGIC = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d]) // %PDF-
function bufferStartsWith(buffer: Buffer, magic: Buffer): boolean {
return buffer.length >= magic.length && buffer.subarray(0, magic.length).equals(magic)
}
/**
* How a read-only consumer (e.g. the public share route) should serve a stored doc
* WITHOUT compiling:
* - `passthrough` — serve the raw stored bytes as-is (a non-doc file, or an uploaded
* binary that already carries its format magic).
* - `artifact` — serve this prebuilt content-addressed compiled binary.
* - `unavailable` — a generated doc stored as source whose compiled artifact does
* not exist yet; the raw bytes are source, so serving them under the file's binary
* content type would be corrupt. The caller should signal "not ready" instead.
*/
export type ServableDoc =
| { kind: 'passthrough' }
| { kind: 'artifact'; buffer: Buffer; contentType: string }
| { kind: 'unavailable' }
export async function resolveServableDoc(
workspaceId: string,
storedBytes: Buffer,
fileName: string
): Promise<ServableDoc> {
const fmt = await getE2BDocFormat(fileName)
if (!fmt) return { kind: 'passthrough' }
const magic = fmt.ext === 'pdf' ? PDF_MAGIC : ZIP_MAGIC
if (bufferStartsWith(storedBytes, magic)) return { kind: 'passthrough' }
const artifact = await loadCompiledDocByExt(workspaceId, storedBytes.toString('utf-8'), fmt.ext)
return artifact ? { kind: 'artifact', ...artifact } : { kind: 'unavailable' }
}
interface CompilableFormat {
magic: Buffer
taskId: SandboxTaskId
contentType: string
}
const COMPILABLE_FORMATS: Record<string, CompilableFormat> = {
'.pptx': { magic: ZIP_MAGIC, taskId: 'pptx-generate', contentType: PPTX_MIME },
'.docx': { magic: ZIP_MAGIC, taskId: 'docx-generate', contentType: DOCX_MIME },
'.pdf': { magic: PDF_MAGIC, taskId: 'pdf-generate', contentType: PDF_MIME },
}
const MAX_COMPILED_DOC_CACHE = 10
const compiledDocCache = new Map<string, Buffer>()
function compiledCacheSet(key: string, buffer: Buffer): void {
if (compiledDocCache.size >= MAX_COMPILED_DOC_CACHE) {
compiledDocCache.delete(compiledDocCache.keys().next().value as string)
}
compiledDocCache.set(key, buffer)
}
/**
* Resolves the bytes a consumer should actually serve/attach for a stored file —
* the single source of truth shared by the file-serve route and every tool that
* downloads a workspace file (email attachments, uploads, provider file inputs).
*
* Generated docs (pdf/docx/pptx/xlsx) store their GENERATION SOURCE as the primary
* file; the rendered binary lives in a separate content-addressed artifact store.
* A naive raw-byte read therefore hands out source text under a `.pdf` name — the
* corruption every non-serve consumer used to ship. The file-serve route and the
* attachment download helper share this one function so they resolve identically.
* (The public read-only share route uses the non-compiling {@link resolveServableDoc}
* variant, which returns `unavailable` instead of throwing.) The swap:
*
* - Bytes already carry the format magic (`%PDF`/ZIP) → real uploaded/binary file,
* serve as-is.
* - Generated-doc source → load the content-addressed compiled artifact.
* - Artifact missing in the E2B regime → the doc is still being generated; throw
* {@link DocCompileUserError} so callers signal "not ready / retry" instead of
* shipping source.
* - E2B disabled → compile the committed JS source via isolated-vm (cached).
* - Non-doc files → pass through with the extension-derived content type.
*
* It never falls back to attaching the raw source bytes for a generated doc.
*/
export async function resolveServableDocBytes(args: {
rawBuffer: Buffer
fileName: string
workspaceId: string | undefined
ownerKey?: string
signal?: AbortSignal
}): Promise<{ buffer: Buffer; contentType: string }> {
const { rawBuffer, fileName, workspaceId, ownerKey, signal } = args
const ext = fileName.slice(fileName.lastIndexOf('.')).toLowerCase()
const extNoDot = ext.replace(/^\./, '')
const format = COMPILABLE_FORMATS[ext]
// xlsx isn't in COMPILABLE_FORMATS (no isolated-vm path), so match its ZIP magic
// explicitly alongside the table-driven formats.
const magic = format?.magic ?? (extNoDot === 'xlsx' ? ZIP_MAGIC : undefined)
if (magic && bufferStartsWith(rawBuffer, magic)) {
return { buffer: rawBuffer, contentType: getContentType(fileName) }
}
if (!format && extNoDot !== 'xlsx') {
return { buffer: rawBuffer, contentType: getContentType(fileName) }
}
const source = rawBuffer.toString('utf-8')
if (workspaceId) {
const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot)
if (stored) {
return { buffer: stored.buffer, contentType: stored.contentType }
}
if (isE2BDocEnabled && (await getE2BDocFormat(fileName))) {
throw new DocCompileUserError('Document is still being generated')
}
}
// Reaches here only for xlsx, which has no isolated-vm fallback.
if (!format) return { buffer: rawBuffer, contentType: getContentType(fileName) }
const cacheKey = sha256Hex(`${ext}${source}${workspaceId ?? ''}`)
const cached = compiledDocCache.get(cacheKey)
if (cached) {
return { buffer: cached, contentType: format.contentType }
}
const compiled = await runSandboxTask(
format.taskId,
{ code: source, workspaceId: workspaceId || '' },
{ ownerKey, signal }
)
compiledCacheSet(cacheKey, compiled)
return { buffer: compiled, contentType: format.contentType }
}
@@ -0,0 +1,71 @@
import { createHash } from 'node:crypto'
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { downloadFile, uploadFile } from '@/lib/uploads/core/storage-service'
const logger = createLogger('CopilotDocCompiledStore')
/**
* Compiled-artifact store for Python-generated documents.
*
* The Python doc path keeps the SOURCE as the primary file (the agent reads and
* edits it exactly like the JS path). The compiled binary is stored as its own
* S3 object, content-addressed by (workspaceId, sha256(source), ext) — the hash
* is in the key, so when the source changes the key changes. Every read path
* (serve, preview, /compiled) loads the artifact for the current source hash and
* recompiles only when it is absent. No fileId in the key means any site with
* the source (e.g. the serve route) can find it. S3 is cheap; stale artifacts
* are inert.
*/
function compiledArtifactKey(workspaceId: string, source: string, ext: string): string {
const hash = createHash('sha256').update(source, 'utf-8').digest('hex')
return `copilot-doc-compiled/${workspaceId}/${hash}.${ext}`
}
/** Loads the compiled binary for the current source, or null if not yet built. */
export async function loadCompiledDoc(
workspaceId: string,
source: string,
ext: string
): Promise<Buffer | null> {
const key = compiledArtifactKey(workspaceId, source, ext)
try {
return await downloadFile({ key, context: 'copilot' })
} catch {
return null
}
}
/**
* Stores the compiled binary as the source's associated S3 artifact.
*
* Throws on failure (does not swallow): the serve route is load-only and cannot
* self-heal a missing artifact, so a silent store failure would make a write
* report success while leaving the document unrenderable. Propagating lets the
* write fail honestly so the caller (and the agent) can retry.
*/
export async function storeCompiledDoc(
workspaceId: string,
source: string,
ext: string,
contentType: string,
binary: Buffer
): Promise<void> {
const key = compiledArtifactKey(workspaceId, source, ext)
try {
await uploadFile({
file: binary,
fileName: `doc.${ext}`,
contentType,
context: 'copilot',
customKey: key,
preserveKey: true,
})
} catch (err) {
logger.error('Failed to store compiled doc artifact', {
key,
error: getErrorMessage(err),
})
throw toError(err)
}
}
@@ -0,0 +1,113 @@
import { executeInE2B } from '@/lib/execution/e2b'
import { CodeLanguage } from '@/lib/execution/languages'
const EXTRACT_TIMEOUT_MS = 120_000
// Bound the text handed back to the agent so a huge document can't blow the
// context window; the agent gets a clear truncation marker if it hits the cap.
const MAX_EXTRACT_CHARS = 200_000
/** Binary document formats whose text/tables we can extract in the doc sandbox. */
const EXTRACTABLE_EXTS = new Set(['pdf', 'pptx', 'docx', 'xlsx'])
export function isExtractableDocExt(ext: string): boolean {
return EXTRACTABLE_EXTS.has(ext.toLowerCase())
}
export interface DocExtract {
text: string
truncated: boolean
}
/**
* Extracts readable text (and tables) from an uploaded binary document inside the
* E2B doc sandbox so the agent can read/reason over files it cannot otherwise see
* as source: pdf via pdfplumber, pptx via python-pptx, docx via python-docx, xlsx
* via openpyxl. Read-only — never mutates the file. Throws on sandbox/infra
* failure or an unparseable document.
*/
export async function extractDocText(args: { binary: Buffer; ext: string }): Promise<DocExtract> {
const ext = args.ext.toLowerCase()
if (!isExtractableDocExt(ext)) {
throw new Error(`Cannot extract text from .${ext} (supported: pdf, pptx, docx, xlsx)`)
}
const script = `
import json
ext = ${JSON.stringify(ext)}
inp = f"/home/user/input.{ext}"
out = []
if ext == "pdf":
import pdfplumber
with pdfplumber.open(inp) as pdf:
for i, page in enumerate(pdf.pages, 1):
out.append(f"--- Page {i} ---")
out.append(page.extract_text() or "")
for t in (page.extract_tables() or []):
out.append("[table] " + json.dumps(t, ensure_ascii=False))
elif ext == "pptx":
from pptx import Presentation
prs = Presentation(inp)
for i, slide in enumerate(prs.slides, 1):
out.append(f"--- Slide {i} ---")
for shape in slide.shapes:
if shape.has_text_frame and shape.text_frame.text.strip():
out.append(shape.text_frame.text)
if shape.has_table:
for row in shape.table.rows:
out.append(" | ".join(c.text for c in row.cells))
nf = slide.notes_slide.notes_text_frame if slide.has_notes_slide else None
notes = nf.text if nf is not None else ""
if notes.strip():
out.append("[notes] " + notes)
elif ext == "docx":
import docx
d = docx.Document(inp)
for p in d.paragraphs:
if p.text.strip():
out.append(p.text)
for tbl in d.tables:
for row in tbl.rows:
out.append(" | ".join(c.text for c in row.cells))
elif ext == "xlsx":
import openpyxl
wb = openpyxl.load_workbook(inp, data_only=True)
for ws in wb.worksheets:
out.append(f"--- Sheet {ws.title} ---")
# Cap rows so an inflated used-range can't blow up memory/output.
for ri, row in enumerate(ws.iter_rows(values_only=True)):
if ri >= 5000:
out.append("[... more rows truncated]")
break
out.append(",".join("" if v is None else str(v) for v in row))
# Bound the transferred text so a decompression bomb can't return gigabytes.
# Headroom over MAX_EXTRACT_CHARS so the TS-side truncation flag can still fire.
text = "\\n".join(out)[:${MAX_EXTRACT_CHARS + 20000}]
print("__SIM_RESULT__=" + json.dumps({"text": text}))
`.trim()
const result = await executeInE2B({
code: script,
language: CodeLanguage.Python,
timeoutMs: EXTRACT_TIMEOUT_MS,
sandboxKind: 'doc',
sandboxFiles: [
{
path: `/home/user/input.${ext}`,
content: args.binary.toString('base64'),
encoding: 'base64',
},
],
})
if (result.error) {
throw new Error(`Document extraction failed: ${result.error}`)
}
const payload = result.result as { text?: string } | null
const full = payload?.text ?? ''
const truncated = full.length > MAX_EXTRACT_CHARS
// The caller (VFS read) owns the user-facing truncation note; just return the
// bounded text + the flag here.
return { text: full.slice(0, MAX_EXTRACT_CHARS), truncated }
}
@@ -0,0 +1,122 @@
import { createLogger } from '@sim/logger'
import { executeInE2B } from '@/lib/execution/e2b'
import { CodeLanguage } from '@/lib/execution/languages'
import { compileDoc, DocCompileUserError } from './doc-compile'
const logger = createLogger('CopilotDocRecalc')
const RECALC_TIMEOUT_MS = 150_000
const MAX_REPORTED_ERRORS = 50
export interface XlsxCellError {
sheet: string
cell: string
error: string
}
export interface XlsxRecalcResult {
ok: boolean
errors: XlsxCellError[]
}
/**
* Scans an .xlsx workbook for spilled error values (#REF!/#DIV/0!/etc.) — the
* spreadsheet equivalent of the visual QA loop.
*
* Precondition: the binary must already be recalculated (cached values present).
* compileDoc bakes a LibreOffice recalc into every xlsx artifact, so the input
* here always has cached values — meaning we can read them with openpyxl
* (data_only) directly and skip a second LibreOffice cold-start. Throws on
* sandbox/infra failure.
*/
export async function recalcXlsx(args: {
binary: Buffer
workspaceId: string
}): Promise<XlsxRecalcResult> {
const script = `
import json, openpyxl
ERR = {"#REF!", "#DIV/0!", "#VALUE!", "#NAME?", "#N/A", "#NULL!", "#NUM!"}
# Input is already recalculated by compileDoc, so cached values are present.
wb = openpyxl.load_workbook("/home/user/input.xlsx", data_only=True)
errors = []
for ws in wb.worksheets:
for row in ws.iter_rows():
for c in row:
if isinstance(c.value, str) and c.value.strip() in ERR:
errors.append({"sheet": ws.title, "cell": c.coordinate, "error": c.value.strip()})
print("__SIM_RESULT__=" + json.dumps({"ok": len(errors) == 0, "errors": errors[:${MAX_REPORTED_ERRORS}]}))
`.trim()
const result = await executeInE2B({
code: script,
language: CodeLanguage.Python,
timeoutMs: RECALC_TIMEOUT_MS,
sandboxKind: 'doc',
sandboxFiles: [
{
path: '/home/user/input.xlsx',
content: args.binary.toString('base64'),
encoding: 'base64',
},
],
})
if (result.error) {
throw new Error(`Spreadsheet recalc failed: ${result.error}`)
}
const payload = result.result as XlsxRecalcResult | null
if (!payload || typeof payload.ok !== 'boolean') {
logger.warn('Recalc returned no structured result', { workspaceId: args.workspaceId })
return { ok: true, errors: [] }
}
return { ok: payload.ok, errors: Array.isArray(payload.errors) ? payload.errors : [] }
}
/** Single-line summary of the first few formula errors, for the compiled-check result. */
export function formatXlsxErrors(errors: XlsxCellError[]): string {
return `${errors.length} formula error(s): ${errors
.slice(0, 5)
.map((e) => `${e.sheet}!${e.cell}=${e.error}`)
.join(', ')}`
}
export interface CompiledCheckResult {
ok: boolean
error?: string
errors?: XlsxCellError[]
}
/**
* Compiles a generated doc (and, for xlsx, recalc-scans its formulas) to verify
* it builds — the shared body behind the /compiled-check route and the VFS
* compiled-check read. Returns { ok: false } only for a DocCompileUserError (the
* agent's script is wrong); infra failures (E2B/S3) rethrow so callers surface a
* 5xx instead of telling the agent to fix its script.
*/
export async function runE2BCompiledCheck(args: {
source: string
fileName: string
workspaceId: string
ext: string
}): Promise<CompiledCheckResult> {
try {
const compiled = await compileDoc({
source: args.source,
fileName: args.fileName,
workspaceId: args.workspaceId,
})
if (args.ext === 'xlsx') {
const recalc = await recalcXlsx({ binary: compiled.buffer, workspaceId: args.workspaceId })
return recalc.ok
? { ok: true }
: { ok: false, error: formatXlsxErrors(recalc.errors), errors: recalc.errors }
}
return { ok: true }
} catch (err) {
if (err instanceof DocCompileUserError) return { ok: false, error: err.message }
throw err
}
}
@@ -0,0 +1,109 @@
import { executeInE2B } from '@/lib/execution/e2b'
import { CodeLanguage } from '@/lib/execution/languages'
const RENDER_TIMEOUT_MS = 150_000
// Bound the visual-QA cost: cap pages and rasterization DPI so the JPEGs the
// file agent inspects stay small enough for vision input.
const MAX_RENDER_PAGES = 20
const RENDER_DPI = 110
/** Extensions LibreOffice can render to page images for the visual QA loop. */
const RENDERABLE_EXTS = new Set(['pptx', 'docx', 'pdf'])
export function isRenderableDocExt(ext: string): boolean {
return RENDERABLE_EXTS.has(ext.toLowerCase())
}
export interface DocRender {
/** A single contact-sheet grid JPEG of all pages, for the agent's visual QA. */
grid: Buffer
pageCount: number
}
/**
* Renders a compiled document binary to a single contact-sheet grid image inside
* the E2B doc sandbox (LibreOffice → PDF → poppler `pdftoppm` → Pillow tile).
* Works for any compiled binary regardless of which engine produced it
* (isolated-vm JS or E2B Python), so the visual QA loop covers pptx/docx/pdf
* uniformly. One grid image fits the VFS single-attachment read and gives the
* agent the whole deck/doc at a glance (mirrors Anthropic's thumbnail grid).
*
* Throws on a sandbox/infra failure or when the doc renders to zero pages.
*/
export async function renderDocToGrid(args: {
binary: Buffer
ext: string
workspaceId: string
}): Promise<DocRender> {
const ext = args.ext.toLowerCase()
if (!isRenderableDocExt(ext)) {
throw new Error(`Cannot render .${ext} to images (supported: pptx, docx, pdf)`)
}
const script = `
import subprocess, glob, base64, json
from PIL import Image
ext = ${JSON.stringify(ext)}
inp = f"/home/user/input.{ext}"
pdf = inp if ext == "pdf" else "/home/user/input.pdf"
if ext != "pdf":
subprocess.run(
["soffice", "--headless", "--convert-to", "pdf", "--outdir", "/home/user", inp],
check=True, timeout=120, capture_output=True,
)
subprocess.run(
["pdftoppm", "-jpeg", "-r", "${RENDER_DPI}", "-l", "${MAX_RENDER_PAGES}", pdf, "/home/user/page"],
check=True, timeout=120, capture_output=True,
)
paths = sorted(glob.glob("/home/user/page*.jpg"))[:${MAX_RENDER_PAGES}]
imgs = [Image.open(p).convert("RGB") for p in paths]
n = len(imgs)
if n == 0:
print("__SIM_RESULT__=" + json.dumps({"grid": None, "pageCount": 0}))
else:
cols = 1 if n == 1 else (2 if n <= 6 else 3)
rows = (n + cols - 1) // cols
cell_w = max(i.width for i in imgs)
cell_h = max(i.height for i in imgs)
pad = 12
grid = Image.new("RGB", (cols * cell_w + (cols + 1) * pad, rows * cell_h + (rows + 1) * pad), (240, 240, 240))
for idx, im in enumerate(imgs):
r, c = divmod(idx, cols)
grid.paste(im, (pad + c * (cell_w + pad), pad + r * (cell_h + pad)))
# Cap the grid's longest edge so the JPEG stays a reasonable vision input.
max_edge = 2200
if max(grid.size) > max_edge:
scale = max_edge / max(grid.size)
grid = grid.resize((int(grid.width * scale), int(grid.height * scale)))
grid.save("/home/user/grid.jpg", "JPEG", quality=80)
with open("/home/user/grid.jpg", "rb") as f:
print("__SIM_RESULT__=" + json.dumps({"grid": base64.b64encode(f.read()).decode(), "pageCount": n}))
`.trim()
const result = await executeInE2B({
code: script,
language: CodeLanguage.Python,
timeoutMs: RENDER_TIMEOUT_MS,
sandboxKind: 'doc',
sandboxFiles: [
{
path: `/home/user/input.${ext}`,
content: args.binary.toString('base64'),
encoding: 'base64',
},
],
})
if (result.error) {
throw new Error(`Document render failed: ${result.error}`)
}
const payload = result.result as { grid?: string | null; pageCount?: number } | null
if (!payload?.grid) {
throw new Error('Document render produced no pages')
}
return { grid: Buffer.from(payload.grid, 'base64'), pageCount: payload.pageCount ?? 0 }
}
@@ -0,0 +1,183 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { e2bFlag, betaFlag, mockLoadCompiledDoc, mockRunSandboxTask } = vi.hoisted(() => ({
e2bFlag: { value: true },
betaFlag: { value: false },
mockLoadCompiledDoc: vi.fn(),
mockRunSandboxTask: vi.fn(),
}))
vi.mock('@/lib/execution/e2b', () => ({
executeInE2B: vi.fn(),
executeShellInE2B: vi.fn(),
}))
vi.mock('@/lib/execution/languages', () => ({
CodeLanguage: { javascript: 'javascript', python: 'python' },
}))
vi.mock('@/lib/execution/sandbox/run-task', () => ({
runSandboxTask: mockRunSandboxTask,
}))
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
getWorkspaceFile: vi.fn(),
fetchWorkspaceFileBuffer: vi.fn(),
}))
vi.mock('./doc-compiled-store', () => ({
loadCompiledDoc: mockLoadCompiledDoc,
storeCompiledDoc: vi.fn(),
}))
vi.mock('@/lib/core/config/feature-flags', () => ({
isFeatureEnabled: vi.fn(async () => betaFlag.value),
}))
vi.mock('@/lib/core/config/env-flags', () => ({
get isE2BDocEnabled() {
return e2bFlag.value
},
}))
vi.mock('@/app/api/files/utils', () => ({
getContentType: (name: string) =>
name.endsWith('.pdf')
? 'application/pdf'
: name.endsWith('.txt')
? 'text/plain'
: 'application/octet-stream',
}))
import { DocCompileUserError, resolveServableDocBytes } from './doc-compile'
const WORKSPACE_ID = '550e8400-e29b-41d4-a716-446655440000'
const PDF_MAGIC = Buffer.from('%PDF-1.7\n...binary...')
const PDF_SOURCE = Buffer.from('from reportlab.pdfgen import canvas\n# generates a PDF', 'utf-8')
const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x00, 0x01])
const XLSX_SOURCE = Buffer.from('from openpyxl import Workbook\n# generates an xlsx', 'utf-8')
describe('resolveServableDocBytes', () => {
beforeEach(() => {
vi.clearAllMocks()
e2bFlag.value = true
betaFlag.value = false
})
it('swaps generated-doc source for the compiled artifact + binary content type', async () => {
const artifact = Buffer.from('%PDF-compiled-binary')
mockLoadCompiledDoc.mockResolvedValue(artifact)
const result = await resolveServableDocBytes({
rawBuffer: PDF_SOURCE,
fileName: 'report.pdf',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(artifact)
expect(result.contentType).toBe('application/pdf')
expect(mockLoadCompiledDoc).toHaveBeenCalledWith(
WORKSPACE_ID,
PDF_SOURCE.toString('utf-8'),
'pdf'
)
})
it('passes through a real binary PDF (carries the %PDF magic) without an artifact lookup', async () => {
const result = await resolveServableDocBytes({
rawBuffer: PDF_MAGIC,
fileName: 'uploaded.pdf',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(PDF_MAGIC)
expect(result.contentType).toBe('application/pdf')
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
})
it('throws DocCompileUserError when a generated doc artifact is not ready (E2B regime)', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
e2bFlag.value = true
await expect(
resolveServableDocBytes({
rawBuffer: PDF_SOURCE,
fileName: 'report.pdf',
workspaceId: WORKSPACE_ID,
})
).rejects.toBeInstanceOf(DocCompileUserError)
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
it('compiles via the sandbox when E2B is disabled and no artifact is stored', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
e2bFlag.value = false
const compiled = Buffer.from('%PDF-isolated-vm-binary')
mockRunSandboxTask.mockResolvedValue(compiled)
const result = await resolveServableDocBytes({
rawBuffer: PDF_SOURCE,
fileName: 'report.pdf',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(compiled)
expect(result.contentType).toBe('application/pdf')
expect(mockRunSandboxTask).toHaveBeenCalledWith(
'pdf-generate',
{ code: PDF_SOURCE.toString('utf-8'), workspaceId: WORKSPACE_ID },
expect.objectContaining({})
)
})
it('passes non-doc files through untouched with their extension content type', async () => {
const text = Buffer.from('hello world', 'utf-8')
const result = await resolveServableDocBytes({
rawBuffer: text,
fileName: 'notes.txt',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(text)
expect(result.contentType).toBe('text/plain')
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
})
it('passes through a real binary XLSX (ZIP magic) without an artifact lookup', async () => {
const result = await resolveServableDocBytes({
rawBuffer: ZIP_MAGIC,
fileName: 'sheet.xlsx',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(ZIP_MAGIC)
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
})
it('throws when a generated XLSX artifact is not ready (E2B + mothership-beta enabled)', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
e2bFlag.value = true
betaFlag.value = true
await expect(
resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: WORKSPACE_ID,
})
).rejects.toBeInstanceOf(DocCompileUserError)
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
it('returns raw XLSX source when there is no workspaceId (xlsx has no isolated-vm path)', async () => {
betaFlag.value = true
const result = await resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: undefined,
})
expect(result.buffer).toBe(XLSX_SOURCE)
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,230 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { z } from 'zod'
import { DownloadToWorkspaceFile } from '@/lib/copilot/generated/tool-catalog-v1'
import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import {
getExtensionFromMimeType,
getFileExtension,
getMimeTypeFromExtension,
} from '@/lib/uploads/utils/file-utils'
const logger = createLogger('DownloadToWorkspaceFileTool')
const MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024 // 50 MB
const DownloadToWorkspaceFileArgsSchema = z.object({
url: z.string().url(),
fileName: z.string().min(1).optional(),
outputs: z
.object({
files: z
.array(
z.object({
path: z.string().min(1),
mode: z.enum(['create', 'overwrite']).optional(),
mimeType: z.string().optional(),
})
)
.optional(),
})
.optional(),
})
const DownloadToWorkspaceFileResultSchema = z.object({
success: z.boolean(),
message: z.string(),
fileId: z.string().optional(),
fileName: z.string().optional(),
vfsPath: z.string().optional(),
downloadUrl: z.string().optional(),
})
type DownloadToWorkspaceFileArgs = z.infer<typeof DownloadToWorkspaceFileArgsSchema>
type DownloadToWorkspaceFileResult = z.infer<typeof DownloadToWorkspaceFileResultSchema>
function sanitizeFileName(fileName: string): string {
return fileName.replace(/[\\/:*?"<>|\u0000-\u001f]+/g, '_').trim()
}
function stripQueryAndHash(input: string): string {
return input.split('#')[0]?.split('?')[0] ?? input
}
function extractFileNameFromUrl(url: string): string | undefined {
try {
const pathname = new URL(url).pathname
const lastSegment = pathname.split('/').pop()
if (!lastSegment) return undefined
const decoded = decodeURIComponent(lastSegment)
return decoded && decoded !== '/' ? decoded : undefined
} catch {
return undefined
}
}
function extractFileNameFromContentDisposition(header: string | null): string | undefined {
if (!header) return undefined
const utf8Match = header.match(/filename\*\s*=\s*UTF-8''([^;]+)/i)
if (utf8Match?.[1]) {
try {
return decodeURIComponent(utf8Match[1].trim())
} catch {
return utf8Match[1].trim()
}
}
const quotedMatch = header.match(/filename\s*=\s*"([^"]+)"/i)
if (quotedMatch?.[1]) return quotedMatch[1].trim()
const bareMatch = header.match(/filename\s*=\s*([^;]+)/i)
if (bareMatch?.[1]) return bareMatch[1].trim()
return undefined
}
function resolveMimeType(
responseContentType: string | null,
candidateFileName?: string,
sourceUrl?: string
): string {
const headerMime = responseContentType?.split(';')[0]?.trim().toLowerCase()
if (headerMime && headerMime !== 'application/octet-stream') {
return headerMime
}
const fileName = candidateFileName || extractFileNameFromUrl(sourceUrl || '')
const ext = fileName ? getFileExtension(stripQueryAndHash(fileName)) : ''
return ext ? getMimeTypeFromExtension(ext) : 'application/octet-stream'
}
function ensureFileExtension(fileName: string, mimeType: string): string {
const ext = getFileExtension(stripQueryAndHash(fileName))
if (ext) return fileName
const inferredExt = getExtensionFromMimeType(mimeType)
return inferredExt ? `${fileName}.${inferredExt}` : fileName
}
function inferOutputFileName(
requestedFileName: string | undefined,
headers: { get(name: string): string | null },
url: string,
mimeType: string
): string {
const preferredName =
requestedFileName ||
extractFileNameFromContentDisposition(headers.get('content-disposition')) ||
extractFileNameFromUrl(url) ||
'downloaded-file'
const sanitized = sanitizeFileName(stripQueryAndHash(preferredName)) || 'downloaded-file'
return ensureFileExtension(sanitized, mimeType)
}
export const downloadToWorkspaceFileServerTool: BaseServerTool<
DownloadToWorkspaceFileArgs,
DownloadToWorkspaceFileResult
> = {
name: DownloadToWorkspaceFile.id,
inputSchema: DownloadToWorkspaceFileArgsSchema,
outputSchema: DownloadToWorkspaceFileResultSchema,
async execute(
params: DownloadToWorkspaceFileArgs,
context?: ServerToolContext
): Promise<DownloadToWorkspaceFileResult> {
const withMessageId = (message: string) =>
context?.messageId ? `${message} [messageId:${context.messageId}]` : message
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
await ensureWorkspaceAccess(workspaceId, context.userId, 'write')
try {
assertServerToolNotAborted(context)
// secureFetchWithValidation handles: DNS resolution, private IP blocking (via ipaddr.js),
// SSRF-safe redirect following, and streaming size enforcement
const response = await secureFetchWithValidation(params.url, {
maxResponseBytes: MAX_DOWNLOAD_BYTES,
})
if (!response.ok) {
return {
success: false,
message: `Download failed with status ${response.status} ${response.statusText}`,
}
}
const mimeType = resolveMimeType(
response.headers.get('content-type'),
params.fileName,
params.url
)
const outputFile = params.outputs?.files?.[0]
const fileName = inferOutputFileName(params.fileName, response.headers, params.url, mimeType)
const outputPath = outputFile?.path ?? `files/${fileName}`
assertServerToolNotAborted(context)
const arrayBuffer = await response.arrayBuffer()
const fileBuffer = Buffer.from(arrayBuffer)
if (fileBuffer.length === 0) {
return { success: false, message: 'Downloaded file is empty' }
}
assertServerToolNotAborted(context)
const written = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: {
path: outputPath,
mode: outputFile?.mode ?? 'create',
mimeType: outputFile?.mimeType,
},
buffer: fileBuffer,
inferredMimeType: outputFile?.mimeType ?? mimeType,
})
logger.info('Downloaded remote file to workspace', {
sourceUrl: params.url,
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
mimeType,
size: fileBuffer.length,
})
return {
success: true,
message: `Downloaded "${written.name}" to ${written.vfsPath} (${fileBuffer.length} bytes)`,
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
downloadUrl: written.downloadUrl,
}
} catch (error) {
const msg = getErrorMessage(error, 'Unknown error')
logger.error('Failed to download file to workspace', {
url: params.url,
error: msg,
})
return { success: false, message: `Failed to download file: ${msg}` }
}
},
}
@@ -0,0 +1,282 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { isE2BDocEnabled } from '@/lib/core/config/env-flags'
import { updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { getE2BDocFormat } from './doc-compile'
import { buildEmbeddedImageRefWarning } from './embedded-image-refs'
import { consumeLatestFileIntent } from './file-intent-store'
import { compileDocForWrite, getDocumentFormatInfo, inferContentType } from './workspace-file'
const logger = createLogger('EditContentServerTool')
type EditContentArgs = {
content: string
}
type EditContentResult = {
success: boolean
message: string
data?: Record<string, unknown>
}
export const editContentServerTool: BaseServerTool<EditContentArgs, EditContentResult> = {
name: 'edit_content',
async execute(params: EditContentArgs, context?: ServerToolContext): Promise<EditContentResult> {
if (!context?.userId) {
logger.error('Unauthorized attempt to use edit_content')
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
const raw = params as Record<string, unknown>
const nested = raw.args as Record<string, unknown> | undefined
const content =
typeof params.content === 'string'
? params.content
: typeof nested?.content === 'string'
? (nested.content as string)
: undefined
if (content === undefined) {
return { success: false, message: 'content is required for edit_content' }
}
// Consume the intent from THIS file subagent's channel (its outer tool_use
// id), not just the latest in the message — otherwise two file agents
// writing concurrently would each grab whichever workspace_file landed last
// and write their content into the wrong file. Falls back to latest-in-
// message when no channel id is present (main-agent / legacy calls).
const intent = await consumeLatestFileIntent(workspaceId, {
chatId: context.chatId,
messageId: context.messageId,
channelId: context.parentToolCallId,
})
if (!intent) {
return {
success: false,
message:
'No workspace_file context found. Call workspace_file first, wait for it to succeed, then call edit_content in the next step. Do not emit edit_content in parallel or in the same batch as workspace_file.',
}
}
try {
const { operation, fileRecord } = intent
const docInfo = getDocumentFormatInfo(fileRecord.name)
const e2bFmt = isE2BDocEnabled ? await getE2BDocFormat(fileRecord.name) : null
let finalContent: string
switch (operation) {
case 'append': {
const existing = intent.existingContent ?? ''
// The JS engines (isolated-vm and E2B-node pptx/docx) use the `{ ... }`
// block-append convention — block statements scope cleanly inside the
// compile wrapper. Python docs (pdf/xlsx) are a single cohesive script,
// so brace-wrapping would produce invalid Python; plain-concatenate.
// Brace-wrap appended content for the JS engines (isolated-vm and
// E2B-node pptx/docx); Python docs (pdf/xlsx) are one cohesive script.
const braceWrap = e2bFmt ? e2bFmt.engine === 'node' : docInfo.isDoc
finalContent = braceWrap
? existing
? `${existing}\n{\n${content}\n}`
: content
: existing
? `${existing}\n${content}`
: content
break
}
case 'update': {
finalContent = content
break
}
case 'patch': {
const existing = intent.existingContent ?? ''
if (!intent.edit) {
return { success: false, message: 'Patch intent missing edit metadata' }
}
if (intent.edit.strategy === 'search_replace') {
const search = intent.edit.search!
const firstIdx = existing.indexOf(search)
if (firstIdx === -1) {
return {
success: false,
message: `Patch failed: search string not found in file "${fileRecord.name}"`,
}
}
finalContent = intent.edit.replaceAll
? existing.split(search).join(content)
: existing.slice(0, firstIdx) + content + existing.slice(firstIdx + search.length)
} else if (intent.edit.strategy === 'anchored') {
const lines = existing.split('\n')
const defaultOccurrence = intent.edit.occurrence ?? 1
const findAnchorLine = (
anchor: string,
occurrence = defaultOccurrence,
afterIndex = -1
): { index: number; error?: string } => {
const trimmed = anchor.trim()
let count = 0
for (let i = afterIndex + 1; i < lines.length; i++) {
if (lines[i].trim() === trimmed) {
count++
if (count === occurrence) return { index: i }
}
}
if (count === 0) {
return {
index: -1,
error: `Anchor line not found in "${fileRecord.name}": "${anchor.slice(0, 100)}"`,
}
}
return {
index: -1,
error: `Anchor line occurrence ${occurrence} not found (only ${count} match${count > 1 ? 'es' : ''}) in "${fileRecord.name}": "${anchor.slice(0, 100)}"`,
}
}
if (intent.edit.mode === 'replace_between') {
if (!intent.edit.before_anchor || !intent.edit.after_anchor) {
return {
success: false,
message: 'replace_between requires before_anchor and after_anchor',
}
}
const before = findAnchorLine(intent.edit.before_anchor)
if (before.error) return { success: false, message: `Patch failed: ${before.error}` }
const after = findAnchorLine(
intent.edit.after_anchor,
defaultOccurrence,
before.index
)
if (after.error) return { success: false, message: `Patch failed: ${after.error}` }
if (after.index <= before.index) {
return {
success: false,
message: 'Patch failed: after_anchor must appear after before_anchor in the file',
}
}
const newLines = [
...lines.slice(0, before.index + 1),
...content.split('\n'),
...lines.slice(after.index),
]
finalContent = newLines.join('\n')
} else if (intent.edit.mode === 'insert_after') {
if (!intent.edit.anchor) {
return { success: false, message: 'insert_after requires anchor' }
}
const found = findAnchorLine(intent.edit.anchor)
if (found.error) return { success: false, message: `Patch failed: ${found.error}` }
const newLines = [
...lines.slice(0, found.index + 1),
...content.split('\n'),
...lines.slice(found.index + 1),
]
finalContent = newLines.join('\n')
} else if (intent.edit.mode === 'delete_between') {
if (!intent.edit.start_anchor || !intent.edit.end_anchor) {
return {
success: false,
message: 'delete_between requires start_anchor and end_anchor',
}
}
const start = findAnchorLine(intent.edit.start_anchor)
if (start.error) return { success: false, message: `Patch failed: ${start.error}` }
const end = findAnchorLine(intent.edit.end_anchor, defaultOccurrence, start.index)
if (end.error) return { success: false, message: `Patch failed: ${end.error}` }
if (end.index <= start.index) {
return {
success: false,
message: 'Patch failed: end_anchor must appear after start_anchor in the file',
}
}
const newLines = [...lines.slice(0, start.index), ...lines.slice(end.index)]
finalContent = newLines.join('\n')
} else {
return {
success: false,
message: `Unknown anchored patch mode: "${intent.edit.mode}"`,
}
}
} else {
return { success: false, message: `Unknown patch strategy: "${intent.edit.strategy}"` }
}
break
}
default:
return { success: false, message: `Unsupported operation in intent: ${operation}` }
}
// Compile once via the right engine (or isolated-vm fallback) and resolve
// the source MIME to store. Shared with the create path.
const compiled = await compileDocForWrite({
source: finalContent,
fileName: fileRecord.name,
workspaceId,
ownerKey: `user:${context.userId}`,
signal: context.abortSignal,
fallbackMime: inferContentType(fileRecord.name, intent.contentType),
})
if (!compiled.ok) {
return { success: false, message: compiled.message }
}
const fileBuffer = Buffer.from(finalContent, 'utf-8')
assertServerToolNotAborted(context)
await updateWorkspaceFileContent(
workspaceId,
intent.fileId,
context.userId,
fileBuffer,
compiled.sourceMime
)
const verb =
operation === 'append' ? 'appended to' : operation === 'update' ? 'updated' : 'patched'
logger.info(`Workspace file ${verb} via copilot (edit_content)`, {
fileId: intent.fileId,
name: fileRecord.name,
operation,
size: fileBuffer.length,
userId: context.userId,
})
// Flag any `/api/files/view/<id>` embeds the model just authored that won't render/export
// (non-workspace or missing), so it can self-correct on the next step.
const embedWarning = await buildEmbeddedImageRefWarning(content, workspaceId)
return {
success: true,
message: `File "${fileRecord.name}" ${verb} successfully (${fileBuffer.length} bytes)${embedWarning}`,
data: {
id: intent.fileId,
name: fileRecord.name,
size: fileBuffer.length,
contentType: compiled.sourceMime,
},
}
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error occurred')
logger.error('Error in edit_content tool', {
operation: intent.operation,
fileId: intent.fileId,
error: errorMessage,
userId: context.userId,
})
return {
success: false,
message: `Failed to apply content: ${errorMessage}`,
}
}
},
}
@@ -0,0 +1,43 @@
import { describe, expect, it } from 'vitest'
import {
extractEmbeddedImageIds,
extractEmbeddedImageKeys,
} from '@/lib/copilot/tools/server/files/embedded-image-refs'
const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png'
describe('extractEmbeddedImageIds', () => {
it('extracts unique ids from view-url and in-app-path embeds (wf_ and uuid)', () => {
const a = 'wf_YwDXi8eWOkTxn0sbgChlB'
const b = '4bdaf6c4-072e-464e-891d-b6af3b5fe2cc'
const content = `![x](/api/files/view/${a}) ![y](/workspace/W1/files/${b}) ![dup](/api/files/view/${a})`
expect(extractEmbeddedImageIds(content).sort()).toEqual([b, a].sort())
})
it('ignores serve-url, external, and plain content', () => {
expect(
extractEmbeddedImageIds(`![a](/api/files/serve/${encodeURIComponent(KEY)}) plain`)
).toEqual([])
})
it('caps the result at 50 ids', () => {
const content = Array.from(
{ length: 60 },
(_, i) => `/api/files/view/wf_${String(i).padStart(6, '0')}`
).join(' ')
expect(extractEmbeddedImageIds(content)).toHaveLength(50)
})
})
describe('extractEmbeddedImageKeys', () => {
it('extracts decoded workspace keys from serve-url embeds (encoded + s3/blob prefixed)', () => {
const content = `![a](/api/files/serve/${encodeURIComponent(KEY)}?context=workspace) ![b](/api/files/serve/s3/${encodeURIComponent(KEY)})`
expect(extractEmbeddedImageKeys(content)).toEqual([KEY])
})
it('drops non-workspace keys (e.g. public profile pictures) and view-url embeds', () => {
const content =
'![a](/api/files/serve/profile-pictures%2Fu1%2Favatar.png) ![b](/api/files/view/wf_abc)'
expect(extractEmbeddedImageKeys(content)).toEqual([])
})
})
@@ -0,0 +1,69 @@
import { getFileMetadataById } from '@/lib/uploads/server/metadata'
import { extractEmbeddedFileRefs } from '@/lib/uploads/utils/embedded-image-ref'
/** View-URL embed (`/api/files/view/<id>`) — the only form the file agent writes; see {@link findUnembeddableImageRefs}. */
const VIEW_EMBED_RE = /\/api\/files\/view\/([A-Za-z0-9_-]+)/g
/**
* De-duplicated workspace file **ids** embedded in `content` (view URL or in-app workspace path).
* Shares the {@link extractEmbeddedFileRefs} grammar with the frontend renderer so the referenced-by-doc
* gate authorizes exactly what the client links. Resolution and access are checked by the caller.
*/
export function extractEmbeddedImageIds(content: string): string[] {
return extractEmbeddedFileRefs(content).ids
}
/**
* De-duplicated workspace storage **keys** (`workspace/<wsId>/…`) embedded in `content` via the serve URL.
* Same shared grammar as {@link extractEmbeddedImageIds}.
*/
export function extractEmbeddedImageKeys(content: string): string[] {
return extractEmbeddedFileRefs(content).keys
}
/**
* Returns the ids of `/api/files/view/<id>` image embeds in `content` that will not render or survive a
* workspace export. An embed is valid only when its id resolves to a workspace file in this same
* workspace — the only thing the view route serves and an export can bundle. Every other case (missing,
* archived, a different workspace, or a non-`workspace` upload such as a chat-scoped `mothership` file)
* is flagged by id alone, without disclosing the referenced file's real context or owning workspace, so
* the result can't be used to probe files outside this workspace. Best-effort and never throws, so a
* content write is never blocked by this validation.
*/
export async function findUnembeddableImageRefs(
content: string,
workspaceId: string
): Promise<string[]> {
const ids = new Set<string>()
for (const match of content.matchAll(VIEW_EMBED_RE)) ids.add(match[1])
if (ids.size === 0) return []
const checked = await Promise.all(
[...ids].map(async (id): Promise<string | null> => {
try {
const record = await getFileMetadataById(id)
const embeddable = record?.context === 'workspace' && record.workspaceId === workspaceId
return embeddable ? null : id
} catch {
return null
}
})
)
return checked.filter((id): id is string => id !== null)
}
/**
* Builds an actionable suffix appended to a successful file-write tool result so the model can
* self-correct: only workspace files in this workspace embed, so any other reference must be re-saved
* into the workspace and re-referenced by the workspace file's id. Empty when there is nothing to flag.
*/
export async function buildEmbeddedImageRefWarning(
content: string,
workspaceId: string
): Promise<string> {
const ids = await findUnembeddableImageRefs(content, workspaceId)
if (ids.length === 0) return ''
const list = ids.map((id) => `/api/files/view/${id}`).join('; ')
return ` Warning: embedded image(s) will not render or export because they are not workspace files in this workspace — ${list}. Save each image as a workspace file (under files/) and reference it via /api/files/view/<workspace-file-id>.`
}
@@ -0,0 +1,515 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import {
CreateFileFolder,
DeleteFileFolder,
ListFileFolders,
MoveFile,
MoveFileFolder,
RenameFileFolder,
} from '@/lib/copilot/generated/tool-catalog-v1'
import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { decodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils'
import {
findWorkspaceFileFolderIdByPath,
getWorkspaceFileFolder,
listWorkspaceFileFolders,
type WorkspaceFileFolderRecord,
} from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import {
performCreateWorkspaceFileFolder,
performDeleteWorkspaceFileItems,
performMoveWorkspaceFileItems,
performUpdateWorkspaceFileFolder,
} from '@/lib/workspace-files/orchestration'
const logger = createLogger('FileFolderServerTools')
interface WorkspaceScopedArgs {
workspaceId?: string
args?: Record<string, unknown>
}
type ListFileFoldersArgs = WorkspaceScopedArgs
interface CreateFileFolderArgs extends WorkspaceScopedArgs {
path?: string
name?: string
parentId?: string | null
parentPath?: string | null
}
interface RenameFileFolderArgs extends WorkspaceScopedArgs {
path?: string
folderId?: string
name?: string
}
interface MoveFileFolderArgs extends WorkspaceScopedArgs {
path?: string
folderId?: string
destinationPath?: string | null
parentId?: string | null
}
interface DeleteFileFolderArgs extends WorkspaceScopedArgs {
paths?: string[]
path?: string
folderIds?: string[]
folderId?: string
}
interface MoveFileArgs extends WorkspaceScopedArgs {
paths?: string[]
path?: string
destinationPath?: string | null
fileIds?: string[]
fileId?: string
folderId?: string | null
}
interface FileFolderResult {
success: boolean
message: string
data?: unknown
}
function nested(params: WorkspaceScopedArgs): Record<string, unknown> | undefined {
return params.args && typeof params.args === 'object' ? params.args : undefined
}
function stringValue(value: unknown): string | undefined {
return typeof value === 'string' ? value : undefined
}
function stringArrayValue(value: unknown): string[] | undefined {
return Array.isArray(value)
? value.filter((item): item is string => typeof item === 'string')
: undefined
}
function nullableStringValue(value: unknown): string | null | undefined {
if (value === null) return null
if (typeof value !== 'string') return undefined
return value.trim() ? value : null
}
function stringListFromValues(...values: unknown[]): string[] {
for (const value of values) {
const arr = stringArrayValue(value)
if (arr && arr.length > 0) return arr
}
return values
.map((value) => stringValue(value))
.filter((value): value is string => Boolean(value))
}
function decodeFileFolderPath(path: string): string[] | null {
const trimmed = path.trim().replace(/\/+$/, '')
if (!trimmed || trimmed === 'files') return null
const withoutPrefix = trimmed.startsWith('files/') ? trimmed.slice('files/'.length) : trimmed
const withoutMarker = withoutPrefix.endsWith('/.folder')
? withoutPrefix.slice(0, -'/.folder'.length)
: withoutPrefix
const segments = decodeVfsPathSegments(withoutMarker).filter(Boolean)
return segments.length > 0 ? segments : null
}
async function resolveFolderIdFromPath(
workspaceId: string,
path: string,
label = 'Folder'
): Promise<string> {
const segments = decodeFileFolderPath(path)
if (!segments) throw new Error(`${label} path must identify a folder under files/`)
const folderId = await findWorkspaceFileFolderIdByPath(workspaceId, segments)
if (!folderId) throw new Error(`${label} not found at files/${segments.join('/')}`)
return folderId
}
async function resolveOptionalFolderId(
workspaceId: string,
value: unknown
): Promise<string | null | undefined> {
const raw = nullableStringValue(value)
if (raw === undefined) return undefined
if (raw === null) return null
const segments = decodeFileFolderPath(raw)
if (!segments) return null
const folderId = await findWorkspaceFileFolderIdByPath(workspaceId, segments)
if (!folderId) throw new Error(`Target folder not found at files/${segments.join('/')}`)
return folderId
}
async function resolveFileIdsFromPaths(
workspaceId: string,
paths: string[]
): Promise<{
fileIds: string[]
failed: string[]
}> {
const fileIds: string[] = []
const failed: string[] = []
for (const path of paths) {
const file = await resolveWorkspaceFileReference(workspaceId, path)
if (!file) {
failed.push(path)
continue
}
fileIds.push(file.id)
}
return { fileIds, failed }
}
async function resolveWorkspaceId(
params: WorkspaceScopedArgs,
context: ServerToolContext | undefined,
permission: 'read' | 'write'
): Promise<string | FileFolderResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const payload = nested(params)
const workspaceId =
stringValue(params.workspaceId) || stringValue(payload?.workspaceId) || context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
await ensureWorkspaceAccess(workspaceId, context.userId, permission)
return workspaceId
}
function folderLabel(folder: WorkspaceFileFolderRecord): string {
return folder.path || folder.name
}
export const listFileFoldersServerTool: BaseServerTool<ListFileFoldersArgs, FileFolderResult> = {
name: ListFileFolders.id,
async execute(
params: ListFileFoldersArgs,
context?: ServerToolContext
): Promise<FileFolderResult> {
try {
const workspaceId = await resolveWorkspaceId(params, context, 'read')
if (typeof workspaceId !== 'string') return workspaceId
const folders = await listWorkspaceFileFolders(workspaceId)
return {
success: true,
message:
folders.length === 1 ? 'Found 1 file folder' : `Found ${folders.length} file folders`,
data: { workspaceId, folders },
}
} catch (error) {
return { success: false, message: toError(error).message }
}
},
}
export const createFileFolderServerTool: BaseServerTool<CreateFileFolderArgs, FileFolderResult> = {
name: CreateFileFolder.id,
async execute(
params: CreateFileFolderArgs,
context?: ServerToolContext
): Promise<FileFolderResult> {
try {
const workspaceId = await resolveWorkspaceId(params, context, 'write')
if (typeof workspaceId !== 'string') return workspaceId
if (!context?.userId) throw new Error('Authentication required')
const payload = nested(params)
const rawPath = stringValue(params.path) || stringValue(payload?.path)
const pathSegments = rawPath ? decodeFileFolderPath(rawPath) : undefined
const name = (
pathSegments?.at(-1) ||
stringValue(params.name) ||
stringValue(payload?.name) ||
''
).trim()
if (!name) return { success: false, message: 'name is required' }
let parentId =
(await resolveOptionalFolderId(workspaceId, params.parentPath ?? payload?.parentPath)) ??
nullableStringValue(params.parentId ?? payload?.parentId) ??
null
if (pathSegments && pathSegments.length > 1) {
const resolvedParentId = await findWorkspaceFileFolderIdByPath(
workspaceId,
pathSegments.slice(0, -1)
)
if (!resolvedParentId) {
return {
success: false,
message: `Parent folder not found at files/${pathSegments.slice(0, -1).join('/')}`,
}
}
parentId = resolvedParentId
}
assertServerToolNotAborted(context)
const result = await performCreateWorkspaceFileFolder({
workspaceId,
userId: context.userId,
name,
parentId,
})
if (!result.success || !result.folder) {
return { success: false, message: result.error || 'Failed to create file folder' }
}
const { folder } = result
logger.info('File folder created via create_file_folder', {
workspaceId,
folderId: folder.id,
parentId,
userId: context.userId,
})
return {
success: true,
message: `Created file folder "${folderLabel(folder)}"`,
data: { folder },
}
} catch (error) {
return { success: false, message: toError(error).message }
}
},
}
export const renameFileFolderServerTool: BaseServerTool<RenameFileFolderArgs, FileFolderResult> = {
name: RenameFileFolder.id,
async execute(
params: RenameFileFolderArgs,
context?: ServerToolContext
): Promise<FileFolderResult> {
try {
const workspaceId = await resolveWorkspaceId(params, context, 'write')
if (typeof workspaceId !== 'string') return workspaceId
if (!context?.userId) throw new Error('Authentication required')
const payload = nested(params)
const folderPath = stringValue(params.path) || stringValue(payload?.path)
const folderId =
(folderPath ? await resolveFolderIdFromPath(workspaceId, folderPath) : undefined) ||
stringValue(params.folderId) ||
stringValue(payload?.folderId) ||
''
const name = (stringValue(params.name) || stringValue(payload?.name) || '').trim()
if (!folderId) return { success: false, message: 'path is required' }
if (!name) return { success: false, message: 'name is required' }
const existing = await getWorkspaceFileFolder(workspaceId, folderId)
if (!existing) return { success: false, message: 'Folder not found' }
assertServerToolNotAborted(context)
const result = await performUpdateWorkspaceFileFolder({
workspaceId,
folderId,
userId: context.userId,
name,
})
if (!result.success || !result.folder) {
return { success: false, message: result.error || 'Failed to rename file folder' }
}
const { folder } = result
logger.info('File folder renamed via rename_file_folder', {
workspaceId,
folderId,
oldName: existing.name,
name,
userId: context.userId,
})
return {
success: true,
message: `Renamed file folder "${folderLabel(existing)}" to "${folderLabel(folder)}"`,
data: { folder },
}
} catch (error) {
return { success: false, message: toError(error).message }
}
},
}
export const moveFileFolderServerTool: BaseServerTool<MoveFileFolderArgs, FileFolderResult> = {
name: MoveFileFolder.id,
async execute(
params: MoveFileFolderArgs,
context?: ServerToolContext
): Promise<FileFolderResult> {
try {
const workspaceId = await resolveWorkspaceId(params, context, 'write')
if (typeof workspaceId !== 'string') return workspaceId
if (!context?.userId) throw new Error('Authentication required')
const payload = nested(params)
const folderPath = stringValue(params.path) || stringValue(payload?.path)
const folderId =
(folderPath ? await resolveFolderIdFromPath(workspaceId, folderPath) : undefined) ||
stringValue(params.folderId) ||
stringValue(payload?.folderId) ||
''
if (!folderId) return { success: false, message: 'path is required' }
const parentId =
(await resolveOptionalFolderId(
workspaceId,
params.destinationPath ?? payload?.destinationPath
)) ??
nullableStringValue(params.parentId ?? payload?.parentId) ??
null
assertServerToolNotAborted(context)
const result = await performUpdateWorkspaceFileFolder({
workspaceId,
folderId,
userId: context.userId,
parentId,
})
if (!result.success || !result.folder) {
return { success: false, message: result.error || 'Failed to move file folder' }
}
const { folder } = result
logger.info('File folder moved via move_file_folder', {
workspaceId,
folderId,
parentId,
userId: context.userId,
})
return {
success: true,
message: parentId
? `Moved file folder "${folderLabel(folder)}"`
: `Moved file folder "${folderLabel(folder)}" to root`,
data: { folder },
}
} catch (error) {
return { success: false, message: toError(error).message }
}
},
}
export const deleteFileFolderServerTool: BaseServerTool<DeleteFileFolderArgs, FileFolderResult> = {
name: DeleteFileFolder.id,
async execute(
params: DeleteFileFolderArgs,
context?: ServerToolContext
): Promise<FileFolderResult> {
try {
const workspaceId = await resolveWorkspaceId(params, context, 'write')
if (typeof workspaceId !== 'string') return workspaceId
if (!context?.userId) throw new Error('Authentication required')
const payload = nested(params)
const paths = stringListFromValues(params.paths, payload?.paths, params.path, payload?.path)
const folderIds =
paths.length > 0
? await Promise.all(paths.map((path) => resolveFolderIdFromPath(workspaceId, path)))
: (params.folderIds ??
stringArrayValue(payload?.folderIds) ??
[stringValue(params.folderId) || stringValue(payload?.folderId) || ''].filter(Boolean))
if (folderIds.length === 0) return { success: false, message: 'paths is required' }
assertServerToolNotAborted(context)
const result = await performDeleteWorkspaceFileItems({
workspaceId,
userId: context.userId,
folderIds,
})
if (!result.success || !result.deletedItems) {
return { success: false, message: result.error || 'Failed to delete file folders' }
}
logger.info('File folders deleted via delete_file_folder', {
workspaceId,
folderIds,
folders: result.deletedItems.folders,
files: result.deletedItems.files,
userId: context.userId,
})
return {
success: result.deletedItems.folders > 0 || result.deletedItems.files > 0,
message: `Deleted ${result.deletedItems.folders} file folder${result.deletedItems.folders === 1 ? '' : 's'} and ${result.deletedItems.files} file${result.deletedItems.files === 1 ? '' : 's'}`,
data: result.deletedItems,
}
} catch (error) {
return { success: false, message: toError(error).message }
}
},
}
export const moveFileServerTool: BaseServerTool<MoveFileArgs, FileFolderResult> = {
name: MoveFile.id,
async execute(params: MoveFileArgs, context?: ServerToolContext): Promise<FileFolderResult> {
try {
const workspaceId = await resolveWorkspaceId(params, context, 'write')
if (typeof workspaceId !== 'string') return workspaceId
if (!context?.userId) throw new Error('Authentication required')
const payload = nested(params)
const paths = stringListFromValues(params.paths, payload?.paths, params.path, payload?.path)
const resolvedByPath =
paths.length > 0 ? await resolveFileIdsFromPaths(workspaceId, paths) : undefined
if (resolvedByPath?.failed.length) {
return {
success: false,
message: `Files not found: ${resolvedByPath.failed.join(', ')}`,
}
}
const fileIds =
resolvedByPath?.fileIds ??
params.fileIds ??
stringArrayValue(payload?.fileIds) ??
[stringValue(params.fileId) || stringValue(payload?.fileId) || ''].filter(Boolean)
if (fileIds.length === 0) return { success: false, message: 'paths is required' }
const folderId =
(await resolveOptionalFolderId(
workspaceId,
params.destinationPath ?? payload?.destinationPath
)) ??
nullableStringValue(params.folderId ?? payload?.folderId) ??
null
assertServerToolNotAborted(context)
const result = await performMoveWorkspaceFileItems({
workspaceId,
userId: context.userId,
fileIds,
targetFolderId: folderId,
})
if (!result.success || !result.movedItems) {
return { success: false, message: result.error || 'Failed to move files' }
}
logger.info('Files moved via move_file', {
workspaceId,
fileIds,
folderId,
movedFiles: result.movedItems.files,
userId: context.userId,
})
return {
success: result.movedItems.files > 0,
message: folderId
? `Moved ${result.movedItems.files} file${result.movedItems.files === 1 ? '' : 's'}`
: `Moved ${result.movedItems.files} file${result.movedItems.files === 1 ? '' : 's'} to root`,
data: result.movedItems,
}
} catch (error) {
return { success: false, message: toError(error).message }
}
},
}
@@ -0,0 +1,123 @@
import { generateShortId } from '@sim/utils/id'
import { describe, expect, it, vi } from 'vitest'
import {
consumeLatestFileIntent,
type PendingFileIntent,
storeFileIntent,
} from './file-intent-store'
// Force the in-memory store path so the test is deterministic and Redis-free.
vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: () => null }))
function makeIntent(overrides: Partial<PendingFileIntent>): PendingFileIntent {
return {
operation: 'update',
fileId: 'file-x',
workspaceId: 'ws-1',
userId: 'user-1',
chatId: 'chat-1',
messageId: 'msg-1',
fileRecord: { id: overrides.fileId ?? 'file-x' } as unknown as PendingFileIntent['fileRecord'],
createdAt: Date.now(),
...overrides,
}
}
function uniqueWorkspace(): string {
return `ws-${generateShortId()}`
}
describe('file-intent-store channel scoping', () => {
it('consumes the intent for the requesting channel, not the latest in the message', async () => {
const ws = uniqueWorkspace()
const scope = { chatId: 'chat-1', messageId: 'msg-1' }
// Two concurrent file subagents: A declares fileA on channel F1 first, then
// B declares fileB on channel F2 (later createdAt = the "latest" in message).
await storeFileIntent(
ws,
'fileA',
makeIntent({ workspaceId: ws, fileId: 'fileA', channelId: 'F1', createdAt: Date.now() })
)
await storeFileIntent(
ws,
'fileB',
makeIntent({
workspaceId: ws,
fileId: 'fileB',
channelId: 'F2',
createdAt: Date.now() + 1000,
})
)
// edit_content from channel F1 must get fileA — NOT the latest (fileB).
const a = await consumeLatestFileIntent(ws, { ...scope, channelId: 'F1' })
expect(a?.fileId).toBe('fileA')
// edit_content from channel F2 gets fileB.
const b = await consumeLatestFileIntent(ws, { ...scope, channelId: 'F2' })
expect(b?.fileId).toBe('fileB')
})
it('only consumes its own channel, leaving the sibling intent intact', async () => {
const ws = uniqueWorkspace()
const scope = { chatId: 'chat-1', messageId: 'msg-1' }
await storeFileIntent(
ws,
'fileA',
makeIntent({ workspaceId: ws, fileId: 'fileA', channelId: 'F1', createdAt: Date.now() })
)
await storeFileIntent(
ws,
'fileB',
makeIntent({
workspaceId: ws,
fileId: 'fileB',
channelId: 'F2',
createdAt: Date.now() + 1000,
})
)
await consumeLatestFileIntent(ws, { ...scope, channelId: 'F1' })
// The sibling (F2) is untouched and still consumable afterward.
const b = await consumeLatestFileIntent(ws, { ...scope, channelId: 'F2' })
expect(b?.fileId).toBe('fileB')
})
it('falls back to latest-in-message when no channelId (legacy / main-agent)', async () => {
const ws = uniqueWorkspace()
const scope = { chatId: 'chat-1', messageId: 'msg-1' }
await storeFileIntent(
ws,
'fileA',
makeIntent({ workspaceId: ws, fileId: 'fileA', channelId: 'F1', createdAt: Date.now() })
)
await storeFileIntent(
ws,
'fileB',
makeIntent({
workspaceId: ws,
fileId: 'fileB',
channelId: 'F2',
createdAt: Date.now() + 1000,
})
)
const latest = await consumeLatestFileIntent(ws, scope)
expect(latest?.fileId).toBe('fileB')
})
it('returns undefined when the requesting channel has no pending intent', async () => {
const ws = uniqueWorkspace()
await storeFileIntent(
ws,
'fileA',
makeIntent({ workspaceId: ws, fileId: 'fileA', channelId: 'F1', createdAt: Date.now() })
)
const none = await consumeLatestFileIntent(ws, {
chatId: 'chat-1',
messageId: 'msg-1',
channelId: 'F-absent',
})
expect(none).toBeUndefined()
})
})
@@ -0,0 +1,312 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
import { getRedisClient } from '@/lib/core/config/redis'
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
export type PendingFileIntent = {
operation: 'append' | 'update' | 'patch'
fileId: string
workspaceId: string
userId: string
chatId?: string
messageId?: string
// The invoking file subagent's channel id (its outer tool_use id). Lets
// edit_content consume the intent for ITS OWN file subagent instead of the
// latest in the message, so two file agents writing concurrently never cross
// their content into each other's file.
channelId?: string
fileRecord: WorkspaceFileRecord
existingContent?: string
edit?: {
strategy: string
search?: string
replaceAll?: boolean
mode?: string
occurrence?: number
before_anchor?: string
after_anchor?: string
anchor?: string
start_anchor?: string
end_anchor?: string
}
contentType?: string
title?: string
createdAt: number
}
export type FileIntentScope = {
chatId?: string
messageId?: string
// When set, consumeLatestFileIntent only considers intents from this subagent
// channel — the key to isolating concurrent file subagents. Omitted by callers
// that intentionally span the whole message (e.g. clearIntentsForWorkspace).
channelId?: string
}
const logger = createLogger('FileIntentStore')
const INTENT_TTL_MS = 60 * 60 * 1000
const INTENT_TTL_SECONDS = INTENT_TTL_MS / 1000
const REDIS_KEY_PREFIX = 'mothership_file_intent:'
const RETRY_DELAYS_MS = [0, 50, 150] as const
const memoryStore = new Map<string, PendingFileIntent>()
function buildKey(workspaceId: string, fileId: string): string {
return `${workspaceId}:${fileId}`
}
function getWorkspaceRedisKey(workspaceId: string): string {
return `${REDIS_KEY_PREFIX}${workspaceId}`
}
function scopeMatches(intent: PendingFileIntent, scope?: FileIntentScope): boolean {
return intent.chatId === scope?.chatId && intent.messageId === scope?.messageId
}
// Channel filter for consume: when a scope carries a channelId, only the
// matching file subagent's intent qualifies. No channelId => message-wide
// (legacy / main-agent) behavior. Deliberately separate from scopeMatches so
// clearIntentsForWorkspace keeps clearing every channel in a message.
function channelMatches(intent: PendingFileIntent, scope?: FileIntentScope): boolean {
return !scope?.channelId || intent.channelId === scope.channelId
}
function buildScopedField(fileId: string, scope?: FileIntentScope): string {
return `${scope?.chatId ?? ''}:${scope?.messageId ?? ''}:${fileId}`
}
function cleanupStale(): void {
const now = Date.now()
for (const [key, intent] of memoryStore) {
if (now - intent.createdAt > INTENT_TTL_MS) {
memoryStore.delete(key)
}
}
}
async function withRedisRetry<T>(
operation: string,
workspaceId: string,
work: (redis: NonNullable<ReturnType<typeof getRedisClient>>) => Promise<T>
): Promise<T> {
const redis = getRedisClient()
if (!redis) {
throw new Error('Redis client unavailable')
}
let lastError: unknown
for (let attempt = 0; attempt < RETRY_DELAYS_MS.length; attempt++) {
const delay = RETRY_DELAYS_MS[attempt]
if (delay > 0) {
await sleep(delay)
}
try {
return await work(redis)
} catch (error) {
lastError = error
logger.warn('Redis file intent operation failed', {
operation,
workspaceId,
attempt: attempt + 1,
error: toError(error).message,
})
}
}
throw lastError instanceof Error ? lastError : new Error(`${operation} failed`)
}
function isStale(intent: PendingFileIntent): boolean {
return Date.now() - intent.createdAt > INTENT_TTL_MS
}
function parseIntent(raw: string | null | undefined): PendingFileIntent | undefined {
if (!raw) return undefined
try {
const parsed = JSON.parse(raw) as PendingFileIntent
return isStale(parsed) ? undefined : parsed
} catch (error) {
logger.warn('Failed to parse file intent', {
error: toError(error).message,
})
return undefined
}
}
export async function storeFileIntent(
workspaceId: string,
fileId: string,
intent: PendingFileIntent
): Promise<void> {
const redis = getRedisClient()
if (!redis) {
cleanupStale()
memoryStore.set(buildKey(workspaceId, buildScopedField(fileId, intent)), intent)
return
}
await withRedisRetry('store_file_intent', workspaceId, async (client) => {
const key = getWorkspaceRedisKey(workspaceId)
const pipeline = client.pipeline()
pipeline.hset(key, buildScopedField(fileId, intent), JSON.stringify(intent))
pipeline.expire(key, INTENT_TTL_SECONDS)
await pipeline.exec()
})
}
async function consumeFileIntent(
workspaceId: string,
fileId: string,
scope?: FileIntentScope
): Promise<PendingFileIntent | undefined> {
const redis = getRedisClient()
if (!redis) {
const key = buildKey(workspaceId, buildScopedField(fileId, scope))
const intent = memoryStore.get(key)
if (intent) {
memoryStore.delete(key)
}
return intent
}
const raw = await withRedisRetry('consume_file_intent', workspaceId, async (client) => {
const key = getWorkspaceRedisKey(workspaceId)
const field = buildScopedField(fileId, scope)
const value = await client.hget(key, field)
if (value !== null) {
await client.hdel(key, field)
}
return value
})
return parseIntent(raw)
}
export async function peekFileIntent(
workspaceId: string,
fileId: string,
scope?: FileIntentScope
): Promise<PendingFileIntent | undefined> {
const redis = getRedisClient()
if (!redis) {
cleanupStale()
return memoryStore.get(buildKey(workspaceId, buildScopedField(fileId, scope)))
}
const raw = await withRedisRetry('peek_file_intent', workspaceId, async (client) => {
const key = getWorkspaceRedisKey(workspaceId)
return client.hget(key, buildScopedField(fileId, scope))
})
const intent = parseIntent(raw)
if (!intent && raw !== null) {
await withRedisRetry('clear_stale_file_intent', workspaceId, async (client) => {
await client.hdel(getWorkspaceRedisKey(workspaceId), buildScopedField(fileId, scope))
})
}
return intent
}
export async function consumeLatestFileIntent(
workspaceId: string,
scope?: FileIntentScope
): Promise<PendingFileIntent | undefined> {
const redis = getRedisClient()
if (!redis) {
cleanupStale()
let latest: PendingFileIntent | undefined
let latestKey: string | undefined
for (const [key, intent] of memoryStore) {
if (
intent.workspaceId === workspaceId &&
scopeMatches(intent, scope) &&
channelMatches(intent, scope)
) {
if (!latest || intent.createdAt > latest.createdAt) {
latest = intent
latestKey = key
}
}
}
if (latestKey) {
memoryStore.delete(latestKey)
}
return latest
}
const entries = await withRedisRetry('read_workspace_file_intents', workspaceId, async (client) =>
client.hgetall(getWorkspaceRedisKey(workspaceId))
)
let latest: PendingFileIntent | undefined
let latestField: string | undefined
const staleFields: string[] = []
for (const [field, raw] of Object.entries(entries)) {
const parsed = parseIntent(raw)
if (!parsed) {
staleFields.push(field)
continue
}
if (!scopeMatches(parsed, scope) || !channelMatches(parsed, scope)) {
continue
}
if (!latest || parsed.createdAt > latest.createdAt) {
latest = parsed
latestField = field
}
}
const fieldsToDelete = latestField ? [...staleFields, latestField] : staleFields
if (fieldsToDelete.length > 0) {
await withRedisRetry('delete_workspace_file_intents', workspaceId, async (client) => {
await client.hdel(getWorkspaceRedisKey(workspaceId), ...fieldsToDelete)
})
}
return latest
}
export async function clearIntentsForWorkspace(
workspaceId: string,
scope?: FileIntentScope
): Promise<number> {
const redis = getRedisClient()
if (!redis) {
let cleared = 0
for (const [key, intent] of memoryStore) {
if (intent.workspaceId === workspaceId && (!scope || scopeMatches(intent, scope))) {
memoryStore.delete(key)
cleared++
}
}
return cleared
}
const key = getWorkspaceRedisKey(workspaceId)
if (!scope) {
const count = await withRedisRetry(
'count_workspace_file_intents',
workspaceId,
async (client) => client.hlen(key)
)
await withRedisRetry('clear_workspace_file_intents', workspaceId, async (client) => {
await client.del(key)
})
return count
}
const entries = await withRedisRetry('read_workspace_file_intents', workspaceId, async (client) =>
client.hgetall(key)
)
const fieldsToDelete: string[] = []
for (const [field, raw] of Object.entries(entries)) {
const parsed = parseIntent(raw)
if (parsed && scopeMatches(parsed, scope)) {
fieldsToDelete.push(field)
}
}
if (fieldsToDelete.length > 0) {
await withRedisRetry('clear_scoped_file_intents', workspaceId, async (client) => {
await client.hdel(key, ...fieldsToDelete)
})
}
return fieldsToDelete.length
}
@@ -0,0 +1,79 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { buildFilePreviewText } from '@/lib/copilot/tools/server/files/file-preview'
describe('buildFilePreviewText', () => {
it('returns the full streamed content for update previews', () => {
expect(
buildFilePreviewText({
operation: 'update',
streamedContent: '',
})
).toBe('')
expect(
buildFilePreviewText({
operation: 'update',
streamedContent: 'updated body',
})
).toBe('updated body')
})
it('builds append previews from the existing file content', () => {
expect(
buildFilePreviewText({
operation: 'append',
existingContent: 'line one',
streamedContent: 'line two',
})
).toBe('line one\nline two')
})
it('applies anchored replace_between previews', () => {
expect(
buildFilePreviewText({
operation: 'patch',
existingContent: ['# Title', 'before', 'after', 'footer'].join('\n'),
streamedContent: 'replacement',
edit: {
strategy: 'anchored',
mode: 'replace_between',
before_anchor: '# Title',
after_anchor: 'after',
},
})
).toBe(['# Title', 'replacement', 'after', 'footer'].join('\n'))
})
it('applies delete_between previews without streamed replacement text', () => {
expect(
buildFilePreviewText({
operation: 'patch',
existingContent: ['keep', 'start', 'remove me', 'end', 'keep too'].join('\n'),
streamedContent: '',
edit: {
strategy: 'anchored',
mode: 'delete_between',
start_anchor: 'start',
end_anchor: 'end',
},
})
).toBe(['keep', 'end', 'keep too'].join('\n'))
})
it('applies search_replace previews', () => {
expect(
buildFilePreviewText({
operation: 'patch',
existingContent: 'hello world',
streamedContent: 'sim',
edit: {
strategy: 'search_replace',
search: 'world',
},
})
).toBe('hello sim')
})
})
@@ -0,0 +1,193 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import {
fetchWorkspaceFileBuffer,
getWorkspaceFile,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
const logger = createLogger('CopilotFilePreview')
type FilePreviewEdit = {
strategy?: string
search?: string
replaceAll?: boolean
mode?: string
occurrence?: number
before_anchor?: string
after_anchor?: string
anchor?: string
start_anchor?: string
end_anchor?: string
}
interface BuildFilePreviewTextOptions {
operation: 'create' | 'append' | 'update' | 'patch'
streamedContent: string
existingContent?: string
edit?: Record<string, unknown>
}
function findAnchorIndex(lines: string[], anchor: string, occurrence = 1, afterIndex = -1): number {
const trimmed = anchor.trim()
let count = 0
for (let i = afterIndex + 1; i < lines.length; i++) {
if (lines[i].trim() === trimmed) {
count++
if (count === occurrence) return i
}
}
return -1
}
function extractPatchPreview(
streamedContent: string,
existingContent: string,
edit?: Record<string, unknown>
): string | undefined {
const strategy = typeof edit?.strategy === 'string' ? edit.strategy : undefined
const lines = existingContent.split('\n')
const occurrence =
typeof edit?.occurrence === 'number' && Number.isFinite(edit.occurrence) ? edit.occurrence : 1
if (strategy === 'search_replace') {
const search = typeof edit?.search === 'string' ? edit.search : ''
if (!search) return undefined
if (edit?.replaceAll === true) {
return existingContent.split(search).join(streamedContent)
}
const firstIdx = existingContent.indexOf(search)
if (firstIdx === -1) return undefined
return (
existingContent.slice(0, firstIdx) +
streamedContent +
existingContent.slice(firstIdx + search.length)
)
}
const mode = typeof edit?.mode === 'string' ? edit.mode : undefined
if (!mode) return undefined
if (mode === 'replace_between') {
const beforeAnchor = typeof edit?.before_anchor === 'string' ? edit.before_anchor : undefined
const afterAnchor = typeof edit?.after_anchor === 'string' ? edit.after_anchor : undefined
if (!beforeAnchor || !afterAnchor) return undefined
const beforeIdx = findAnchorIndex(lines, beforeAnchor, occurrence)
const afterIdx = findAnchorIndex(lines, afterAnchor, occurrence, beforeIdx)
if (beforeIdx === -1 || afterIdx === -1 || afterIdx <= beforeIdx) return undefined
const spliced = [
...lines.slice(0, beforeIdx + 1),
...(streamedContent.length > 0 ? streamedContent.split('\n') : []),
...lines.slice(afterIdx),
]
return spliced.join('\n')
}
if (mode === 'insert_after') {
const anchor = typeof edit?.anchor === 'string' ? edit.anchor : undefined
if (!anchor) return undefined
const anchorIdx = findAnchorIndex(lines, anchor, occurrence)
if (anchorIdx === -1) return undefined
const spliced = [
...lines.slice(0, anchorIdx + 1),
...(streamedContent.length > 0 ? streamedContent.split('\n') : []),
...lines.slice(anchorIdx + 1),
]
return spliced.join('\n')
}
if (mode === 'delete_between') {
const startAnchor = typeof edit?.start_anchor === 'string' ? edit.start_anchor : undefined
const endAnchor = typeof edit?.end_anchor === 'string' ? edit.end_anchor : undefined
if (!startAnchor || !endAnchor) return undefined
const startIdx = findAnchorIndex(lines, startAnchor, occurrence)
const endIdx = findAnchorIndex(lines, endAnchor, occurrence, startIdx)
if (startIdx === -1 || endIdx === -1 || endIdx <= startIdx) return undefined
const spliced = [...lines.slice(0, startIdx), ...lines.slice(endIdx)]
return spliced.join('\n')
}
return undefined
}
function shouldApplyPatchPreview(streamedContent: string, edit?: Record<string, unknown>): boolean {
const strategy = typeof edit?.strategy === 'string' ? edit.strategy : undefined
const mode = typeof edit?.mode === 'string' ? edit.mode : undefined
if (strategy === 'anchored' && mode === 'delete_between') {
return true
}
return streamedContent.length > 0
}
function buildAppendPreview(existingContent: string, incomingContent: string): string {
if (incomingContent.length === 0) return existingContent
if (existingContent.length === 0) return incomingContent
return `${existingContent}\n${incomingContent}`
}
/**
* Reads the current UTF-8 text of a workspace file for streaming previews.
*
* Preview runs in the SSE loop on `workspace_file` **call** events, which are
* processed **before** the async tool executor persists {@link storeFileIntent}.
* Loading the base here avoids a race where `edit_content` `args_delta` arrives
* before Redis holds `existingContent`, which would make append previews look like
* full-file replacement until the intent landed.
*/
export async function loadWorkspaceFileTextForPreview(
workspaceId: string,
fileId: string
): Promise<string | undefined> {
try {
const record = await getWorkspaceFile(workspaceId, fileId)
if (!record) return undefined
const buffer = await fetchWorkspaceFileBuffer(record)
return buffer.toString('utf-8')
} catch (error) {
logger.warn('Failed to load workspace file text for preview', {
workspaceId,
fileId,
error: toError(error).message,
})
return undefined
}
}
export function buildFilePreviewText({
operation,
streamedContent,
existingContent,
edit,
}: BuildFilePreviewTextOptions): string | undefined {
if (operation === 'update') {
return streamedContent
}
if (operation === 'create') {
return streamedContent
}
if (operation === 'append') {
if (existingContent !== undefined) {
return buildAppendPreview(existingContent, streamedContent)
}
return streamedContent
}
if (existingContent === undefined) {
return undefined
}
if (!shouldApplyPatchPreview(streamedContent, edit)) {
return undefined
}
return extractPatchPreview(streamedContent, existingContent, edit)
}
@@ -0,0 +1,92 @@
import { createLogger } from '@sim/logger'
import { RenameFile } from '@/lib/copilot/generated/tool-catalog-v1'
import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import {
getWorkspaceFile,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { performRenameWorkspaceFile } from '@/lib/workspace-files/orchestration'
import { validateFlatWorkspaceFileName } from './workspace-file'
const logger = createLogger('RenameFileServerTool')
interface RenameFileArgs {
path?: string
fileId?: string
newName: string
args?: Record<string, unknown>
}
interface RenameFileResult {
success: boolean
message: string
data?: {
id: string
name: string
}
}
export const renameFileServerTool: BaseServerTool<RenameFileArgs, RenameFileResult> = {
name: RenameFile.id,
async execute(params: RenameFileArgs, context?: ServerToolContext): Promise<RenameFileResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
await ensureWorkspaceAccess(workspaceId, context.userId, 'write')
const nested = params.args
const path = params.path || (nested?.path as string) || ''
const legacyFileId = params.fileId || (nested?.fileId as string) || ''
const newName = params.newName || (nested?.newName as string) || ''
const targetRef = path || legacyFileId
if (!targetRef) return { success: false, message: 'path is required' }
const nameError = validateFlatWorkspaceFileName(newName)
if (nameError) return { success: false, message: nameError }
const existingFile = path
? await resolveWorkspaceFileReference(workspaceId, path)
: await getWorkspaceFile(workspaceId, legacyFileId)
if (!existingFile) {
return { success: false, message: `File not found: ${targetRef}` }
}
const fileId = existingFile.id
assertServerToolNotAborted(context)
const result = await performRenameWorkspaceFile({
workspaceId,
fileId,
name: newName,
userId: context.userId,
})
if (!result.success) {
return { success: false, message: result.error || 'Failed to rename file' }
}
logger.info('File renamed via rename_file', {
fileId,
oldName: existingFile.name,
newName,
userId: context.userId,
})
return {
success: true,
message: `File renamed from "${existingFile.name}" to "${newName}"`,
data: {
id: fileId,
name: newName,
},
}
},
}
@@ -0,0 +1,663 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { truncate } from '@sim/utils/string'
import { WorkspaceFile } from '@/lib/copilot/generated/tool-catalog-v1'
import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { ensureWorkflowAliasBacking } from '@/lib/copilot/vfs/workflow-alias-backing'
import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-alias-resolver'
import { isPlanAliasPath } from '@/lib/copilot/vfs/workflow-aliases'
import { isE2BDocEnabled } from '@/lib/core/config/env-flags'
import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
import { ensureWorkspaceFileFolderPath } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
import {
fetchWorkspaceFileBuffer as downloadWsFile,
getWorkspaceFile,
getWorkspaceFileByName,
resolveWorkspaceFileReference,
uploadWorkspaceFile,
type WorkspaceFileRecord,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import {
performDeleteWorkspaceFileItems,
performRenameWorkspaceFile,
} from '@/lib/workspace-files/orchestration'
import type { SandboxTaskId } from '@/sandbox-tasks/registry'
import {
compileDoc,
DOCXJS_SOURCE_MIME,
DocCompileUserError,
getE2BDocFormat,
PPTXGENJS_SOURCE_MIME,
} from './doc-compile'
import { buildEmbeddedImageRefWarning } from './embedded-image-refs'
import { storeFileIntent } from './file-intent-store'
const logger = createLogger('WorkspaceFileServerTool')
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
const PDF_MIME = 'application/pdf'
// Single source of the JS source MIMEs is doc-compile.ts; reuse to avoid drift.
const PPTX_SOURCE_MIME = PPTXGENJS_SOURCE_MIME
const DOCX_SOURCE_MIME = DOCXJS_SOURCE_MIME
const PDF_SOURCE_MIME = 'text/x-pdflibjs'
type WorkspaceFileOperation = 'create' | 'append' | 'update' | 'delete' | 'rename' | 'patch'
type WorkspaceFileTarget =
| {
kind: 'new_file'
fileName: string
fileId?: string
}
| {
kind: 'file_id'
fileId: string
fileName?: string
}
| {
kind: 'path'
path: string
fileName?: string
}
type WorkspaceFileEdit =
| {
strategy: 'search_replace'
search: string
replace: string
replaceAll?: boolean
}
| {
strategy: 'anchored'
mode: 'replace_between' | 'insert_after' | 'delete_between'
occurrence?: number
before_anchor?: string
after_anchor?: string
start_anchor?: string
end_anchor?: string
anchor?: string
content?: string
}
type WorkspaceFileArgs = {
operation: WorkspaceFileOperation
target?: WorkspaceFileTarget
title?: string
content?: string
contentType?: string
newName?: string
edit?: WorkspaceFileEdit
}
type WorkspaceFileResult = {
success: boolean
message: string
data?: Record<string, unknown>
}
const EXT_TO_MIME: Record<string, string> = {
'.txt': 'text/plain',
'.md': 'text/markdown',
'.html': 'text/html',
'.json': 'application/json',
'.csv': 'text/csv',
'.pptx': PPTX_MIME,
'.docx': DOCX_MIME,
'.pdf': PDF_MIME,
}
export function inferContentType(fileName: string, explicitType?: string): string {
if (explicitType) return explicitType
const ext = fileName.slice(fileName.lastIndexOf('.')).toLowerCase()
return EXT_TO_MIME[ext] || 'text/plain'
}
export function validateFlatWorkspaceFileName(fileName: string): string | null {
const trimmed = fileName.trim()
if (!trimmed) return 'File name cannot be empty'
const segments = trimmed.split('/').map((segment) => segment.trim())
if (segments.some((segment) => !segment)) {
return 'File path cannot contain empty segments'
}
if (segments.some((segment) => segment === '.' || segment === '..' || segment.includes('\\'))) {
return 'File path cannot contain dot segments or backslashes'
}
return null
}
export function splitWorkspaceFilePath(fileName: string): {
folderSegments: string[]
leafName: string
} {
const segments = fileName
.trim()
.split('/')
.map((segment) => segment.trim())
.filter(Boolean)
return {
folderSegments: segments.slice(0, -1),
leafName: segments[segments.length - 1] ?? '',
}
}
export interface DocumentFormatInfo {
isDoc: boolean
formatName?: 'PPTX' | 'DOCX' | 'PDF'
sourceMime?: string
taskId?: SandboxTaskId
}
export function getDocumentFormatInfo(fileName: string): DocumentFormatInfo {
const lowerName = fileName.toLowerCase()
if (lowerName.endsWith('.pptx')) {
return {
isDoc: true,
formatName: 'PPTX',
sourceMime: PPTX_SOURCE_MIME,
taskId: 'pptx-generate',
}
}
if (lowerName.endsWith('.docx')) {
return {
isDoc: true,
formatName: 'DOCX',
sourceMime: DOCX_SOURCE_MIME,
taskId: 'docx-generate',
}
}
if (lowerName.endsWith('.pdf')) {
return {
isDoc: true,
formatName: 'PDF',
sourceMime: PDF_SOURCE_MIME,
taskId: 'pdf-generate',
}
}
return { isDoc: false }
}
export type CompileForWriteResult =
| { ok: true; sourceMime: string }
| { ok: false; message: string }
/**
* Shared write-time doc handling for create + edit_content: validates and builds
* the document (E2B doc sandbox when enabled — Node pptx/docx, Python pdf/xlsx —
* else isolated-vm JS) and returns the source MIME to store, or a user-facing
* failure message. Non-doc files resolve to `fallbackMime`. Compilation happens
* here exactly once per write; the artifact is content-addressed so a read can
* later just load it.
*/
export async function compileDocForWrite(args: {
source: string
fileName: string
workspaceId: string
ownerKey: string
signal?: AbortSignal
fallbackMime: string
}): Promise<CompileForWriteResult> {
const { source, fileName, workspaceId, ownerKey, signal, fallbackMime } = args
const docInfo = getDocumentFormatInfo(fileName)
const e2bFmt = isE2BDocEnabled ? await getE2BDocFormat(fileName) : null
if (!e2bFmt && fileName.toLowerCase().endsWith('.xlsx')) {
return {
ok: false,
message: isE2BDocEnabled
? 'Excel (.xlsx) generation is currently behind the mothership-beta feature flag and is not available.'
: 'Excel (.xlsx) generation requires the E2B document sandbox, which is not enabled in this environment.',
}
}
if (e2bFmt) {
// compileDoc is load-or-build, so an identical re-write reuses the cached
// binary instead of re-running E2B.
try {
await compileDoc({ source, fileName, workspaceId })
} catch (err) {
if (err instanceof DocCompileUserError) {
return {
ok: false,
message: `${e2bFmt.formatName} generation failed: ${err.message}. Fix the code and retry.`,
}
}
return {
ok: false,
message: `${e2bFmt.formatName} generation failed due to a system error: ${toError(err).message}. Retry shortly.`,
}
}
return { ok: true, sourceMime: e2bFmt.sourceMime }
}
if (docInfo.isDoc) {
try {
await runSandboxTask(docInfo.taskId!, { code: source, workspaceId }, { ownerKey, signal })
} catch (err) {
return {
ok: false,
message: `${docInfo.formatName} generation failed: ${toError(err).message}. Fix the code and retry.`,
}
}
return { ok: true, sourceMime: docInfo.sourceMime! }
}
return { ok: true, sourceMime: fallbackMime }
}
export const workspaceFileServerTool: BaseServerTool<WorkspaceFileArgs, WorkspaceFileResult> = {
name: WorkspaceFile.id,
async execute(
params: WorkspaceFileArgs,
context?: ServerToolContext
): Promise<WorkspaceFileResult> {
const withMessageId = (message: string) =>
context?.messageId ? `${message} [messageId:${context.messageId}]` : message
if (!context?.userId) {
logger.error('Unauthorized attempt to access workspace files')
throw new Error('Authentication required')
}
const raw = params as Record<string, unknown>
const nested = raw.args as Record<string, unknown> | undefined
const normalized: WorkspaceFileArgs =
params.operation && params.target
? params
: nested && typeof nested === 'object'
? {
operation: (nested.operation ?? raw.operation) as WorkspaceFileOperation,
target: (nested.target ?? raw.target) as WorkspaceFileTarget | undefined,
title: (nested.title ?? raw.title) as string | undefined,
content: (nested.content ?? raw.content) as string | undefined,
contentType: (nested.contentType ?? raw.contentType) as string | undefined,
newName: (nested.newName ?? raw.newName) as string | undefined,
edit: (nested.edit ?? raw.edit) as WorkspaceFileEdit | undefined,
}
: params
const { operation } = normalized
const workspaceId = context.workspaceId
const resolveExistingTarget = async (
target: WorkspaceFileTarget | undefined,
operationName: string
): Promise<{ fileRecord?: WorkspaceFileRecord; vfsPath?: string; error?: string }> => {
if (!target || (target.kind !== 'path' && target.kind !== 'file_id')) {
return { error: `${operationName} requires target.kind=path with target.path` }
}
let fileRecord: WorkspaceFileRecord | null = null
let vfsPath: string | undefined
if (target.kind === 'path') {
const alias = await resolveWorkflowAliasForWorkspace({
workspaceId: workspaceId!,
path: target.path,
})
if (!alias && isPlanAliasPath(target.path)) {
return { error: `Unsupported plan alias path or missing workflow: ${target.path}` }
}
if (alias) {
if (alias.kind === 'plans_dir') {
return { error: `Plan alias directory is not a file: ${target.path}` }
}
fileRecord = await resolveWorkspaceFileReference(workspaceId!, alias.backingPath)
if (!fileRecord && alias.kind === 'changelog') {
await ensureWorkflowAliasBacking({
workspaceId: workspaceId!,
userId: context.userId,
workflowId: alias.workflowId,
workflowName: alias.workflowName,
})
fileRecord = await resolveWorkspaceFileReference(workspaceId!, alias.backingPath)
}
vfsPath = alias.aliasPath
} else {
fileRecord = await resolveWorkspaceFileReference(workspaceId!, target.path)
vfsPath = target.path
}
} else {
fileRecord = await getWorkspaceFile(workspaceId!, target.fileId)
}
if (!fileRecord) {
const ref = target.kind === 'path' ? target.path : target.fileId
return { error: `File not found: ${ref}` }
}
if (target.fileName && target.fileName !== fileRecord.name) {
return {
error: `Target mismatch: "${target.fileName}" does not match resolved file "${fileRecord.name}"`,
}
}
return { fileRecord, vfsPath }
}
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
try {
await ensureWorkspaceAccess(workspaceId, context.userId, 'write')
switch (operation) {
case 'create': {
const target = normalized.target
if (!target || target.kind !== 'new_file') {
return {
success: false,
message: 'create requires target.kind=new_file with target.fileName',
}
}
const { folderSegments, leafName } = splitWorkspaceFilePath(target.fileName)
const fileName = leafName
const content = normalized.content ?? ''
const explicitType = normalized.contentType
const fileNameValidationError = validateFlatWorkspaceFileName(target.fileName)
if (fileNameValidationError) return { success: false, message: fileNameValidationError }
const folderId = await ensureWorkspaceFileFolderPath({
workspaceId,
userId: context.userId,
pathSegments: folderSegments,
})
const existingFile = await getWorkspaceFileByName(workspaceId, fileName, { folderId })
if (existingFile) {
return { success: false, message: `File "${target.fileName}" already exists` }
}
const compiled = await compileDocForWrite({
source: content,
fileName,
workspaceId,
ownerKey: `user:${context.userId}`,
signal: context.abortSignal,
fallbackMime: inferContentType(fileName, explicitType),
})
if (!compiled.ok) {
return { success: false, message: compiled.message }
}
const contentType = compiled.sourceMime
const fileBuffer = Buffer.from(content, 'utf-8')
assertServerToolNotAborted(context)
const result = await uploadWorkspaceFile(
workspaceId,
context.userId,
fileBuffer,
fileName,
contentType,
{ folderId }
)
logger.info('Workspace file created via copilot', {
fileId: result.id,
name: fileName,
size: fileBuffer.length,
contentType,
userId: context.userId,
})
const embedWarning = await buildEmbeddedImageRefWarning(content, workspaceId)
return {
success: true,
message: `File "${fileName}" created successfully (${fileBuffer.length} bytes)${embedWarning}`,
data: {
id: result.id,
name: result.name,
contentType,
size: fileBuffer.length,
downloadUrl: result.url,
},
}
}
case 'append': {
const target = normalized.target
const {
fileRecord: existingFile,
vfsPath,
error,
} = await resolveExistingTarget(target, 'append')
if (error || !existingFile) return { success: false, message: error || 'File not found' }
const currentBuffer = await downloadWsFile(existingFile)
await storeFileIntent(workspaceId, existingFile.id, {
operation: 'append',
fileId: existingFile.id,
workspaceId,
userId: context.userId,
chatId: context.chatId,
messageId: context.messageId,
channelId: context.parentToolCallId,
fileRecord: existingFile,
existingContent: currentBuffer.toString('utf-8'),
contentType: normalized.contentType,
title: normalized.title,
createdAt: Date.now(),
})
return {
success: true,
message: withMessageId(
`Intent set: append to "${existingFile.name}". Wait for this success result, then call edit_content in the next step with the content to write. Do not call edit_content in parallel.`
),
data: { id: existingFile.id, name: existingFile.name, vfsPath, operation: 'append' },
}
}
case 'update': {
const target = normalized.target
const { fileRecord, vfsPath, error } = await resolveExistingTarget(target, 'update')
if (error || !fileRecord) return { success: false, message: error || 'File not found' }
await storeFileIntent(workspaceId, fileRecord.id, {
operation: 'update',
fileId: fileRecord.id,
workspaceId,
userId: context.userId,
chatId: context.chatId,
messageId: context.messageId,
channelId: context.parentToolCallId,
fileRecord,
contentType: normalized.contentType,
title: normalized.title,
createdAt: Date.now(),
})
return {
success: true,
message: withMessageId(
`Intent set: update "${fileRecord.name}". Wait for this success result, then call edit_content in the next step with the replacement content. Do not call edit_content in parallel.`
),
data: { id: fileRecord.id, name: fileRecord.name, vfsPath, operation: 'update' },
}
}
case 'rename': {
const target = normalized.target
if (!target || target.kind !== 'file_id') {
return {
success: false,
message: 'rename requires target.kind=file_id with target.fileId',
}
}
if (!normalized.newName) {
return { success: false, message: 'newName is required for rename operation' }
}
const fileNameValidationError = validateFlatWorkspaceFileName(normalized.newName)
if (fileNameValidationError) return { success: false, message: fileNameValidationError }
const fileRecord = await getWorkspaceFile(workspaceId, target.fileId)
if (!fileRecord) {
return { success: false, message: `File with ID "${target.fileId}" not found` }
}
const oldName = fileRecord.name
assertServerToolNotAborted(context)
const result = await performRenameWorkspaceFile({
workspaceId,
fileId: target.fileId,
name: normalized.newName,
userId: context.userId,
})
if (!result.success) {
return { success: false, message: result.error || 'Failed to rename file' }
}
logger.info('Workspace file renamed via copilot', {
fileId: target.fileId,
oldName,
newName: normalized.newName,
userId: context.userId,
})
return {
success: true,
message: `File renamed from "${oldName}" to "${normalized.newName}"`,
data: { id: target.fileId, name: normalized.newName },
}
}
case 'delete': {
const target = normalized.target
if (!target || target.kind !== 'file_id') {
return {
success: false,
message: 'delete requires target.kind=file_id with target.fileId',
}
}
const fileRecord = await getWorkspaceFile(workspaceId, target.fileId)
if (!fileRecord) {
return { success: false, message: `File with ID "${target.fileId}" not found` }
}
assertServerToolNotAborted(context)
const result = await performDeleteWorkspaceFileItems({
workspaceId,
userId: context.userId,
fileIds: [target.fileId],
})
if (!result.success) {
return { success: false, message: result.error || 'Failed to delete file' }
}
logger.info('Workspace file deleted via copilot', {
fileId: target.fileId,
name: fileRecord.name,
userId: context.userId,
})
return {
success: true,
message: `File "${fileRecord.name}" deleted successfully`,
data: { id: target.fileId, name: fileRecord.name },
}
}
case 'patch': {
const target = normalized.target
if (!normalized.edit) {
return { success: false, message: 'edit is required for patch operation' }
}
const { fileRecord, vfsPath, error } = await resolveExistingTarget(target, 'patch')
if (error || !fileRecord) return { success: false, message: error || 'File not found' }
const currentBuffer = await downloadWsFile(fileRecord)
const existingContent = currentBuffer.toString('utf-8')
if (normalized.edit.strategy === 'search_replace') {
const search = normalized.edit.search
const firstIdx = existingContent.indexOf(search)
if (firstIdx === -1) {
return {
success: false,
message: `Patch failed: search string not found in file "${fileRecord.name}". Search: "${truncate(search, 100)}"`,
}
}
if (
!normalized.edit.replaceAll &&
existingContent.indexOf(search, firstIdx + 1) !== -1
) {
return {
success: false,
message: `Patch failed: search string is ambiguous — found at multiple locations in "${fileRecord.name}". Use a longer unique search string or replaceAll.`,
}
}
} else if (normalized.edit.strategy === 'anchored') {
if (!normalized.edit.mode) {
return { success: false, message: 'anchored strategy requires mode' }
}
} else {
return {
success: false,
message: `Unknown patch strategy: "${(normalized.edit as { strategy?: string }).strategy}"`,
}
}
await storeFileIntent(workspaceId, fileRecord.id, {
operation: 'patch',
fileId: fileRecord.id,
workspaceId,
userId: context.userId,
chatId: context.chatId,
messageId: context.messageId,
channelId: context.parentToolCallId,
fileRecord,
existingContent,
edit: {
strategy: normalized.edit.strategy,
...(normalized.edit.strategy === 'search_replace'
? {
search: normalized.edit.search,
replaceAll: normalized.edit.replaceAll,
}
: {
mode: normalized.edit.mode,
occurrence: normalized.edit.occurrence,
before_anchor: normalized.edit.before_anchor,
after_anchor: normalized.edit.after_anchor,
anchor: normalized.edit.anchor,
start_anchor: normalized.edit.start_anchor,
end_anchor: normalized.edit.end_anchor,
}),
},
contentType: normalized.contentType,
title: normalized.title,
createdAt: Date.now(),
})
return {
success: true,
message: withMessageId(
`Intent set: patch "${fileRecord.name}" (${normalized.edit.strategy}). Wait for this success result, then call edit_content in the next step with the replacement/insert content. Do not call edit_content in parallel.`
),
data: { id: fileRecord.id, name: fileRecord.name, vfsPath, operation: 'patch' },
}
}
default:
return {
success: false,
message: `Unknown operation: ${operation}. Supported: create, append, update, patch, rename, delete.`,
}
}
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error occurred')
logger.error('Error in workspace_file tool', {
operation,
error: errorMessage,
userId: context.userId,
})
return {
success: false,
message: `Failed to ${operation} file: ${errorMessage}`,
}
}
},
}
@@ -0,0 +1,49 @@
import Ajv, { type ErrorObject, type ValidateFunction } from 'ajv'
import { TOOL_RUNTIME_SCHEMAS } from '@/lib/copilot/generated/tool-schemas-v1'
const ajv = new Ajv({
allErrors: true,
strict: false,
})
const validatorCache = new Map<string, ValidateFunction>()
function formatErrors(errors: ErrorObject[] | null | undefined): string {
if (!errors || errors.length === 0) return 'unknown validation error'
return errors
.slice(0, 5)
.map((error) => `${error.instancePath || '/'} ${error.message || 'is invalid'}`.trim())
.join('; ')
}
function getValidator(
toolName: string,
schemaKind: 'parameters' | 'resultSchema'
): ValidateFunction | null {
const cacheKey = `${toolName}:${schemaKind}`
const cached = validatorCache.get(cacheKey)
if (cached) return cached
const schema = TOOL_RUNTIME_SCHEMAS[toolName]?.[schemaKind]
if (!schema) return null
const validator = ajv.compile(schema as object)
validatorCache.set(cacheKey, validator)
return validator
}
export function validateGeneratedToolPayload<T>(
toolName: string,
schemaKind: 'parameters' | 'resultSchema',
payload: T
): T {
const validator = getValidator(toolName, schemaKind)
if (!validator) return payload
if (!validator(payload)) {
const label = schemaKind === 'parameters' ? 'input' : 'output'
throw new Error(`${toolName} ${label} validation failed: ${formatErrors(validator.errors)}`)
}
return payload
}
@@ -0,0 +1,205 @@
import { GoogleGenAI, type Part } from '@google/genai'
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { GenerateImage } from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
import {
fetchWorkspaceFileBuffer,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
const logger = createLogger('GenerateImageTool')
const NANO_BANANA_MODEL = 'gemini-3.1-flash-image-preview'
const NANO_BANANA_IMAGE_COST_USD = 0.101
const ASPECT_RATIO_TO_SIZE: Record<string, string> = {
'1:1': '1024x1024',
'16:9': '1536x1024',
'9:16': '1024x1536',
'4:3': '1024x768',
'3:4': '768x1024',
}
interface GenerateImageArgs {
prompt: string
inputs?: { files?: Array<{ path: string }> }
aspectRatio?: string
outputs?: {
files?: Array<{
path: string
mode?: 'create' | 'overwrite'
mimeType?: string
}>
}
}
interface GenerateImageResult {
success: boolean
message: string
fileId?: string
fileName?: string
vfsPath?: string
downloadUrl?: string
_serviceCost?: { service: string; cost: number }
}
export const generateImageServerTool: BaseServerTool<GenerateImageArgs, GenerateImageResult> = {
name: GenerateImage.id,
async execute(
params: GenerateImageArgs,
context?: ServerToolContext
): Promise<GenerateImageResult> {
const withMessageId = (message: string) =>
context?.messageId ? `${message} [messageId:${context.messageId}]` : message
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
const { prompt } = params
if (!prompt) {
return { success: false, message: 'prompt is required' }
}
try {
const apiKey = getRotatingApiKey('gemini')
const ai = new GoogleGenAI({ apiKey })
const aspectRatio = params.aspectRatio || '1:1'
const sizeHint = ASPECT_RATIO_TO_SIZE[aspectRatio]
const parts: Part[] = []
const referencePaths = params.inputs?.files?.map((file) => file.path) ?? []
if (referencePaths.length) {
for (const filePath of referencePaths) {
try {
const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath)
if (fileRecord) {
const buffer = await fetchWorkspaceFileBuffer(fileRecord)
const base64 = buffer.toString('base64')
const mime = fileRecord.type || 'image/png'
parts.push({
inlineData: { mimeType: mime, data: base64 },
})
logger.info('Loaded reference image', {
filePath,
name: fileRecord.name,
size: buffer.length,
mimeType: mime,
})
} else {
logger.warn('Reference file not found, skipping', { filePath })
}
} catch (err) {
logger.warn('Failed to load reference image, skipping', {
filePath,
error: toError(err).message,
})
}
}
}
const sizeInstruction = sizeHint
? ` Generate the image at ${sizeHint} resolution with a ${aspectRatio} aspect ratio.`
: ''
parts.push({ text: prompt + sizeInstruction })
logger.info('Generating image with Nano Banana 2', {
model: NANO_BANANA_MODEL,
aspectRatio,
promptLength: prompt.length,
referenceImageCount: referencePaths.length,
})
const response = await ai.models.generateContent({
model: NANO_BANANA_MODEL,
contents: [{ role: 'user', parts }],
config: {
responseModalities: ['IMAGE', 'TEXT'],
},
})
let imageBase64: string | undefined
let mimeType = 'image/png'
if (response.candidates?.[0]?.content?.parts) {
for (const part of response.candidates[0].content.parts) {
if (part.inlineData?.data) {
imageBase64 = part.inlineData.data
if (part.inlineData.mimeType) {
mimeType = part.inlineData.mimeType
}
break
}
}
}
if (!imageBase64) {
const textParts = response.candidates?.[0]?.content?.parts
?.filter((p) => p.text)
.map((p) => p.text)
.join(' ')
return {
success: false,
message: `Image generation returned no image data. ${textParts ? `Model response: ${textParts.slice(0, 500)}` : 'No response from model.'}`,
}
}
const ext = mimeType.includes('jpeg') || mimeType.includes('jpg') ? '.jpg' : '.png'
const outputFile = params.outputs?.files?.[0]
const outputPath = outputFile?.path || `files/generated-image${ext}`
const imageBuffer = Buffer.from(imageBase64, 'base64')
const mode = outputFile?.mode ?? 'create'
assertServerToolNotAborted(context)
const written = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: {
path: outputPath,
mode,
mimeType: outputFile?.mimeType,
},
buffer: imageBuffer,
inferredMimeType: mimeType,
})
logger.info('Generated image saved', {
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
size: imageBuffer.length,
mimeType,
})
return {
success: true,
message: `Image ${referencePaths.length ? 'edited' : 'generated'} and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${imageBuffer.length} bytes)`,
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
downloadUrl: written.downloadUrl,
_serviceCost: { service: 'nano_banana_2', cost: NANO_BANANA_IMAGE_COST_USD },
}
} catch (error) {
const msg = getErrorMessage(error, 'Unknown error')
logger.error('Image generation failed', { error: msg })
return { success: false, message: `Failed to generate image: ${msg}` }
}
},
}
@@ -0,0 +1,228 @@
import { db } from '@sim/db'
import { jobExecutionLogs } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { and, desc, eq } from 'drizzle-orm'
import { GetScheduledTaskLogs } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool'
import type { TraceSpan } from '@/lib/logs/types'
import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils'
const logger = createLogger('GetJobLogsServerTool')
interface GetJobLogsArgs {
jobId: string
executionId?: string
limit?: number
includeDetails?: boolean
workspaceId?: string
}
interface ToolCallDetail {
name: string
input: unknown
output: unknown
error?: string
duration: number
}
interface JobLogEntry {
executionId: string
status: string
trigger: string
startedAt: string
endedAt: string | null
durationMs: number | null
error?: string
toolCalls?: ToolCallDetail[]
output?: unknown
cost?: unknown
tokens?: unknown
}
/**
* Walks the trace-span tree and collects tool invocations from both data shapes:
* - New: `type: 'tool'` spans nested under agent blocks in `children`.
* - Legacy: a `toolCalls` array hanging off the agent span directly (pre-unification).
*/
function collectToolCalls(spans: TraceSpan[] | undefined): ToolCallDetail[] {
if (!spans?.length) return []
const collected: ToolCallDetail[] = []
const visit = (span: TraceSpan) => {
if (span.type === 'tool') {
const output = span.output as { result?: unknown } | undefined
collected.push({
name: span.name || 'unknown',
input: span.input ?? {},
output: output?.result ?? span.output,
error: span.status === 'error' ? errorMessageFromSpan(span) : undefined,
duration: span.duration || 0,
})
return
}
if (span.toolCalls?.length) {
for (const tc of span.toolCalls) {
collected.push({
name: tc.name || 'unknown',
input: tc.input ?? {},
output: tc.output ?? undefined,
error: tc.error || undefined,
duration: tc.duration || 0,
})
}
}
if (span.children?.length) {
for (const child of span.children) visit(child)
}
}
for (const span of spans) visit(span)
return collected
}
function errorMessageFromSpan(span: TraceSpan): string | undefined {
const out = span.output as { error?: unknown } | undefined
if (typeof out?.error === 'string') return out.error
return undefined
}
function extractOutputAndError(
executionData: { traceSpans?: TraceSpan[] } & Record<string, unknown>
): {
output: unknown
error: string | undefined
toolCalls: ToolCallDetail[]
cost: unknown
tokens: unknown
} {
const traceSpans = executionData?.traceSpans ?? []
const mainSpan = traceSpans[0]
const toolCalls = collectToolCalls(traceSpans)
const output = mainSpan?.output || executionData?.finalOutput || undefined
const cost = mainSpan?.cost || executionData?.cost || undefined
const tokens = mainSpan?.tokens || undefined
const errorMsg =
mainSpan?.status === 'error'
? mainSpan?.output?.error || executionData?.error
: executionData?.error || undefined
return {
output,
error: errorMsg
? typeof errorMsg === 'string'
? errorMsg
: JSON.stringify(errorMsg)
: undefined,
toolCalls,
cost,
tokens,
}
}
export const getJobLogsServerTool: BaseServerTool<GetJobLogsArgs, JobLogEntry[]> = {
name: GetScheduledTaskLogs.id,
async execute(rawArgs: GetJobLogsArgs, context?: ServerToolContext): Promise<JobLogEntry[]> {
const withMessageId = (message: string) =>
context?.messageId ? `${message} [messageId:${context.messageId}]` : message
const {
jobId,
executionId,
limit = 3,
includeDetails = false,
workspaceId,
} = rawArgs || ({} as GetJobLogsArgs)
if (!jobId || typeof jobId !== 'string') {
throw new Error('jobId is required')
}
if (!context?.userId) {
throw new Error('Unauthorized access')
}
const wsId = workspaceId || context.workspaceId
if (!wsId) {
throw new Error('Workspace context required')
}
const access = await checkWorkspaceAccess(wsId, context.userId)
if (!access.hasAccess) {
throw new Error('Unauthorized workspace access')
}
const clampedLimit = Math.min(Math.max(1, limit), 5)
logger.info('Fetching job logs', {
jobId,
executionId,
limit: clampedLimit,
includeDetails,
})
const conditions = [
eq(jobExecutionLogs.scheduleId, jobId),
eq(jobExecutionLogs.workspaceId, wsId),
]
if (executionId) {
conditions.push(eq(jobExecutionLogs.executionId, executionId))
}
const rows = await db
.select({
id: jobExecutionLogs.id,
executionId: jobExecutionLogs.executionId,
status: jobExecutionLogs.status,
level: jobExecutionLogs.level,
trigger: jobExecutionLogs.trigger,
startedAt: jobExecutionLogs.startedAt,
endedAt: jobExecutionLogs.endedAt,
totalDurationMs: jobExecutionLogs.totalDurationMs,
executionData: jobExecutionLogs.executionData,
cost: jobExecutionLogs.cost,
})
.from(jobExecutionLogs)
.where(and(...conditions))
.orderBy(desc(jobExecutionLogs.startedAt))
.limit(executionId ? 1 : clampedLimit)
const entries: JobLogEntry[] = rows.map((row) => {
const executionData = row.executionData as any
const details = includeDetails ? extractOutputAndError(executionData) : null
const entry: JobLogEntry = {
executionId: row.executionId,
status: row.status,
trigger: row.trigger,
startedAt: row.startedAt.toISOString(),
endedAt: row.endedAt ? row.endedAt.toISOString() : null,
durationMs: row.totalDurationMs ?? null,
}
if (details) {
if (details.error) entry.error = details.error
if (details.toolCalls.length > 0) entry.toolCalls = details.toolCalls
if (details.output) entry.output = details.output
if (details.cost) entry.cost = details.cost
if (details.tokens) entry.tokens = details.tokens
} else {
const errorMsg = executionData?.error || executionData?.traceSpans?.[0]?.output?.error
if (row.status === 'error' && errorMsg) {
entry.error = typeof errorMsg === 'string' ? errorMsg : JSON.stringify(errorMsg)
}
}
return entry
})
logger.info('Job logs prepared', {
jobId,
count: entries.length,
resultSizeKB: Math.round(JSON.stringify(entries).length / 1024),
})
return entries
},
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,159 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { Ffmpeg } from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { type FfmpegOperation, type MediaFile, runFfmpegOperation } from '@/lib/media/ffmpeg'
import {
fetchWorkspaceFileBuffer,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
const logger = createLogger('FfmpegTool')
const VALID_OPERATIONS: FfmpegOperation[] = [
'overlay_audio',
'mux',
'mix_audio',
'concat',
'trim',
'scale_pad',
'overlay_image',
'add_text',
'fade',
'extract_audio',
'convert',
'thumbnail',
'probe',
]
interface FfmpegArgs {
operation: FfmpegOperation
inputs?: { files?: Array<{ path: string }> }
text?: string
position?: string
start?: number
end?: number
width?: number
height?: number
aspectRatio?: string
volume?: number
musicVolume?: number
loopToVideo?: boolean
format?: string
outputs?: {
files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }>
}
}
interface FfmpegResult {
success: boolean
message: string
fileId?: string
fileName?: string
vfsPath?: string
downloadUrl?: string
probe?: unknown
}
export const ffmpegServerTool: BaseServerTool<FfmpegArgs, FfmpegResult> = {
name: Ffmpeg.id,
async execute(params: FfmpegArgs, context?: ServerToolContext): Promise<FfmpegResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
if (!VALID_OPERATIONS.includes(params.operation)) {
return { success: false, message: `Invalid operation "${params.operation}".` }
}
const inputPaths = params.inputs?.files?.map((f) => f.path) ?? []
if (inputPaths.length === 0) {
return { success: false, message: 'At least one input file is required in inputs.files' }
}
try {
const mediaFiles: MediaFile[] = []
for (const filePath of inputPaths) {
const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath)
if (!fileRecord) {
return { success: false, message: `Input file not found: ${filePath}` }
}
const buffer = await fetchWorkspaceFileBuffer(fileRecord)
mediaFiles.push({
buffer,
mimeType: fileRecord.type || 'application/octet-stream',
name: fileRecord.name,
})
}
assertServerToolNotAborted(context)
const result = await runFfmpegOperation(params.operation, mediaFiles, {
text: params.text,
position: params.position,
start: params.start,
end: params.end,
width: params.width,
height: params.height,
aspectRatio: params.aspectRatio,
volume: params.volume,
musicVolume: params.musicVolume,
loopToVideo: params.loopToVideo,
format: params.format,
})
// probe reports metadata only — no file written.
if (params.operation === 'probe') {
return {
success: true,
message: `Probed ${mediaFiles[0]?.name ?? inputPaths[0]}: ${JSON.stringify(result.probe)}`,
probe: result.probe,
}
}
if (!result.buffer || !result.ext) {
return { success: false, message: `ffmpeg ${params.operation} produced no output` }
}
const outputFile = params.outputs?.files?.[0]
const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}`
const mode = outputFile?.mode ?? 'create'
assertServerToolNotAborted(context)
const written = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: { path: outputPath, mode, mimeType: outputFile?.mimeType },
buffer: result.buffer,
inferredMimeType: result.contentType || 'application/octet-stream',
})
logger.info('ffmpeg operation completed', {
operation: params.operation,
vfsPath: written.vfsPath,
size: result.buffer.length,
})
return {
success: true,
message: `${params.operation} completed and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes)`,
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
downloadUrl: written.downloadUrl,
}
} catch (error) {
const msg = getErrorMessage(error, 'Unknown error')
logger.error('ffmpeg operation failed', { operation: params.operation, error: msg })
return { success: false, message: `ffmpeg ${params.operation} failed: ${msg}` }
}
},
}
@@ -0,0 +1,152 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { GenerateAudio } from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { type AudioType, generateFalAudio } from '@/lib/media/falai-audio'
import {
fetchWorkspaceFileBuffer,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
const logger = createLogger('GenerateAudioTool')
const VALID_TYPES: AudioType[] = ['speech', 'music', 'sfx']
interface GenerateAudioArgs {
prompt: string
type?: string
model?: string
voice?: string
duration?: number
/** For music: explicit lyrics for a vocal track. */
lyrics?: string
/** For music: true = instrumental (default), false = vocal track. */
instrumental?: boolean
/** Optional reference voice sample (workspace audio file) for zero-shot voice cloning. */
inputs?: { files?: Array<{ path: string }> }
outputs?: {
files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }>
}
}
interface GenerateAudioResult {
success: boolean
message: string
fileId?: string
fileName?: string
vfsPath?: string
downloadUrl?: string
_serviceCost?: { service: string; cost: number }
}
function audioExtFromContentType(contentType: string): string {
if (contentType.includes('wav')) return 'wav'
if (contentType.includes('mp4') || contentType.includes('m4a')) return 'm4a'
if (contentType.includes('ogg')) return 'ogg'
if (contentType.includes('flac')) return 'flac'
if (contentType.includes('aac')) return 'aac'
if (contentType.includes('opus')) return 'opus'
return 'mp3'
}
export const generateAudioServerTool: BaseServerTool<GenerateAudioArgs, GenerateAudioResult> = {
name: GenerateAudio.id,
async execute(
params: GenerateAudioArgs,
context?: ServerToolContext
): Promise<GenerateAudioResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
if (!params.prompt) {
return { success: false, message: 'prompt is required' }
}
const type = (params.type || 'speech') as AudioType
if (!VALID_TYPES.includes(type)) {
return {
success: false,
message: `Invalid type "${params.type}". Must be one of: ${VALID_TYPES.join(', ')}`,
}
}
// Voice cloning: a reference sample clones that voice into the generated speech.
let voiceSampleDataUri: string | undefined
const samplePath = params.inputs?.files?.[0]?.path
if (samplePath) {
const sample = await resolveWorkspaceFileReference(workspaceId, samplePath)
if (!sample) {
return { success: false, message: `Voice sample not found: ${samplePath}` }
}
const sampleBuffer = await fetchWorkspaceFileBuffer(sample)
const sampleMime = sample.type || 'audio/mpeg'
voiceSampleDataUri = `data:${sampleMime};base64,${sampleBuffer.toString('base64')}`
}
try {
logger.info('Generating audio', {
type,
model: params.model,
promptLength: params.prompt.length,
voiceClone: Boolean(voiceSampleDataUri),
})
const result = await generateFalAudio({
prompt: params.prompt,
type,
model: params.model,
voice: params.voice,
duration: params.duration,
lyrics: params.lyrics,
instrumental: params.instrumental,
voiceSampleDataUri,
})
const outputFile = params.outputs?.files?.[0]
const ext = audioExtFromContentType(result.contentType)
const outputPath = outputFile?.path || `files/generated-audio.${ext}`
const mode = outputFile?.mode ?? 'create'
assertServerToolNotAborted(context)
const written = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: { path: outputPath, mode, mimeType: outputFile?.mimeType },
buffer: result.buffer,
inferredMimeType: result.contentType,
})
logger.info('Generated audio saved', {
fileId: written.id,
vfsPath: written.vfsPath,
size: result.buffer.length,
type,
model: result.model,
})
return {
success: true,
message: `${type === 'speech' ? 'Speech' : type === 'music' ? 'Music' : 'Sound effect'} generated and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes, model ${result.model})`,
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
downloadUrl: written.downloadUrl,
_serviceCost: { service: 'falai_audio', cost: result.cost.costDollars },
}
} catch (error) {
const msg = getErrorMessage(error, 'Unknown error')
logger.error('Audio generation failed', { error: msg })
return { success: false, message: `Failed to generate audio: ${msg}` }
}
},
}
@@ -0,0 +1,127 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { GenerateVideo } from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { generateFalVideo } from '@/lib/media/falai-video'
import {
fetchWorkspaceFileBuffer,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
const logger = createLogger('GenerateVideoTool')
interface GenerateVideoArgs {
prompt: string
model?: string
aspectRatio?: string
resolution?: string
duration?: number
generateAudio?: boolean
negativePrompt?: string
promptOptimizer?: boolean
inputs?: { files?: Array<{ path: string }> }
outputs?: {
files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }>
}
}
interface GenerateVideoResult {
success: boolean
message: string
fileId?: string
fileName?: string
vfsPath?: string
downloadUrl?: string
_serviceCost?: { service: string; cost: number }
}
export const generateVideoServerTool: BaseServerTool<GenerateVideoArgs, GenerateVideoResult> = {
name: GenerateVideo.id,
async execute(
params: GenerateVideoArgs,
context?: ServerToolContext
): Promise<GenerateVideoResult> {
if (!context?.userId) {
throw new Error('Authentication required')
}
const workspaceId = context.workspaceId
if (!workspaceId) {
return { success: false, message: 'Workspace ID is required' }
}
if (!params.prompt) {
return { success: false, message: 'prompt is required' }
}
try {
let imageDataUri: string | undefined
const refPath = params.inputs?.files?.[0]?.path
if (refPath) {
const fileRecord = await resolveWorkspaceFileReference(workspaceId, refPath)
if (!fileRecord) {
return { success: false, message: `Reference image not found: ${refPath}` }
}
const buffer = await fetchWorkspaceFileBuffer(fileRecord)
const mime = fileRecord.type || 'image/png'
imageDataUri = `data:${mime};base64,${buffer.toString('base64')}`
}
logger.info('Generating video', {
model: params.model || 'veo-3.1-fast',
promptLength: params.prompt.length,
imageToVideo: Boolean(imageDataUri),
})
const result = await generateFalVideo({
prompt: params.prompt,
model: params.model,
aspectRatio: params.aspectRatio,
resolution: params.resolution,
duration: params.duration,
generateAudio: params.generateAudio,
negativePrompt: params.negativePrompt,
promptOptimizer: params.promptOptimizer,
imageDataUri,
})
const outputFile = params.outputs?.files?.[0]
const outputPath = outputFile?.path || 'files/generated-video.mp4'
const mode = outputFile?.mode ?? 'create'
assertServerToolNotAborted(context)
const written = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: { path: outputPath, mode, mimeType: outputFile?.mimeType },
buffer: result.buffer,
inferredMimeType: result.contentType,
})
logger.info('Generated video saved', {
fileId: written.id,
vfsPath: written.vfsPath,
size: result.buffer.length,
model: result.model,
})
return {
success: true,
message: `Video generated and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes, model ${result.model})`,
fileId: written.id,
fileName: written.name,
vfsPath: written.vfsPath,
downloadUrl: written.downloadUrl,
_serviceCost: { service: 'falai_video', cost: result.cost.costDollars },
}
} catch (error) {
const msg = getErrorMessage(error, 'Unknown error')
logger.error('Video generation failed', { error: msg })
return { success: false, message: `Failed to generate video: ${msg}` }
}
},
}
@@ -0,0 +1,123 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { SearchOnline } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
import { env } from '@/lib/core/config/env'
import { executeTool } from '@/tools'
interface OnlineSearchParams {
query: string
num?: number
type?: string
gl?: string
hl?: string
}
interface SearchResult {
title: string
link: string
snippet: string
date?: string
position?: number
}
interface SearchResponse {
results: SearchResult[]
query: string
type: string
totalResults: number
source: 'exa' | 'serper'
}
export const searchOnlineServerTool: BaseServerTool<OnlineSearchParams, SearchResponse> = {
name: SearchOnline.id,
async execute(params: OnlineSearchParams): Promise<SearchResponse> {
const logger = createLogger('SearchOnlineServerTool')
const { query, num = 10, type = 'search', gl, hl } = params
if (!query || typeof query !== 'string') throw new Error('query is required')
const hasExaApiKey = Boolean(env.EXA_API_KEY && String(env.EXA_API_KEY).length > 0)
const hasSerperApiKey = Boolean(env.SERPER_API_KEY && String(env.SERPER_API_KEY).length > 0)
logger.debug('Performing online search', { queryLength: query.length, num, type })
// Try Exa first if available
if (hasExaApiKey) {
try {
const exaResult = await executeTool('exa_search', {
query,
numResults: num,
type: 'auto',
apiKey: env.EXA_API_KEY ?? '',
})
const output = exaResult.output as
| {
results?: Array<{
title?: string
url?: string
text?: string
summary?: string
publishedDate?: string
}>
}
| undefined
const exaResults = output?.results ?? []
if (exaResult.success && exaResults.length > 0) {
const transformedResults: SearchResult[] = exaResults.map((result, index) => ({
title: result.title ?? '',
link: result.url ?? '',
snippet: result.text ?? result.summary ?? '',
date: result.publishedDate,
position: index + 1,
}))
return {
results: transformedResults,
query,
type,
totalResults: transformedResults.length,
source: 'exa',
}
}
logger.debug('exa_search returned no results, falling back to Serper')
} catch (exaError) {
logger.warn('exa_search failed, falling back to Serper', {
error: toError(exaError).message,
})
}
}
if (!hasSerperApiKey) {
throw new Error('No search API keys available (EXA_API_KEY or SERPER_API_KEY required)')
}
const toolParams = {
query,
num,
type,
gl,
hl,
apiKey: env.SERPER_API_KEY ?? '',
}
const result = await executeTool('serper_search', toolParams)
const output = result.output as { searchResults?: SearchResult[] } | undefined
const results = output?.searchResults ?? []
if (!result.success) {
const errorMsg = (result as { error?: string }).error ?? 'Search failed'
throw new Error(errorMsg)
}
return {
results,
query,
type,
totalResults: results.length,
source: 'serper',
}
},
}
+280
View File
@@ -0,0 +1,280 @@
import { createLogger } from '@sim/logger'
import { z } from 'zod'
import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility'
import {
CreateFile,
CreateFileFolder,
DeleteFile,
DeleteFileFolder,
DownloadToWorkspaceFile,
Ffmpeg,
GenerateAudio,
GenerateImage,
GenerateVideo,
KnowledgeBase,
ManageCredential,
ManageCustomTool,
ManageMcpTool,
ManageSkill,
MoveFile,
MoveFileFolder,
RenameFile,
RenameFileFolder,
UserTable,
WorkspaceFile,
} from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { getBlocksMetadataServerTool } from '@/lib/copilot/tools/server/blocks/get-blocks-metadata-tool'
import { getTriggerBlocksServerTool } from '@/lib/copilot/tools/server/blocks/get-trigger-blocks'
import { searchDocumentationServerTool } from '@/lib/copilot/tools/server/docs/search-documentation'
import { enrichmentRunServerTool } from '@/lib/copilot/tools/server/enrichment/enrichment-run'
import { createFileServerTool } from '@/lib/copilot/tools/server/files/create-file'
import { deleteFileServerTool } from '@/lib/copilot/tools/server/files/delete-file'
import { downloadToWorkspaceFileServerTool } from '@/lib/copilot/tools/server/files/download-to-workspace-file'
import { editContentServerTool } from '@/lib/copilot/tools/server/files/edit-content'
import {
createFileFolderServerTool,
deleteFileFolderServerTool,
listFileFoldersServerTool,
moveFileFolderServerTool,
moveFileServerTool,
renameFileFolderServerTool,
} from '@/lib/copilot/tools/server/files/file-folders'
import { renameFileServerTool } from '@/lib/copilot/tools/server/files/rename-file'
import { workspaceFileServerTool } from '@/lib/copilot/tools/server/files/workspace-file'
import { validateGeneratedToolPayload } from '@/lib/copilot/tools/server/generated-schema'
import { generateImageServerTool } from '@/lib/copilot/tools/server/image/generate-image'
import { getJobLogsServerTool } from '@/lib/copilot/tools/server/jobs/get-job-logs'
import { knowledgeBaseServerTool } from '@/lib/copilot/tools/server/knowledge/knowledge-base'
import { ffmpegServerTool } from '@/lib/copilot/tools/server/media/ffmpeg'
import { generateAudioServerTool } from '@/lib/copilot/tools/server/media/generate-audio'
import { generateVideoServerTool } from '@/lib/copilot/tools/server/media/generate-video'
import { searchOnlineServerTool } from '@/lib/copilot/tools/server/other/search-online'
import { userTableServerTool } from '@/lib/copilot/tools/server/table/user-table'
import { getCredentialsServerTool } from '@/lib/copilot/tools/server/user/get-credentials'
import { setEnvironmentVariablesServerTool } from '@/lib/copilot/tools/server/user/set-environment-variables'
import { editWorkflowServerTool } from '@/lib/copilot/tools/server/workflow/edit-workflow'
import { queryLogsServerTool } from '@/lib/copilot/tools/server/workflow/query-logs'
import { listCustomBlocksWithInputsForWorkspace } from '@/lib/workflows/custom-blocks/operations'
import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay'
import { withBlockVisibility } from '@/blocks/visibility/server-context'
export type ExecuteResponseSuccess = z.output<typeof ExecuteResponseSuccessSchema>
const ExecuteResponseSuccessSchema = z.object({
success: z.literal(true),
result: z.unknown(),
})
const logger = createLogger('ServerToolRouter')
/**
* Tools that resolve blocks through the registry (`getBlock`/`getAllBlocks`) and
* must run inside the custom-block overlay so `custom_block_*` types resolve.
*/
const CUSTOM_BLOCK_OVERLAY_TOOLS = new Set(['edit_workflow', 'get_blocks_metadata'])
/**
* DISCOVERY tools that must run inside the viewer's block-visibility context so
* gated (preview / kill-switched) blocks disappear from what the agent can
* list. Deliberately a DIFFERENT set from {@link CUSTOM_BLOCK_OVERLAY_TOOLS}:
* `edit_workflow` is excluded because its registry use is functional
* (find-by-type over clones, never a discovery listing) and gating it would
* only risk leaking display projections into persisted state.
*/
const VISIBILITY_GATED_TOOLS = new Set(['get_blocks_metadata', 'get_trigger_blocks'])
const WRITE_ACTIONS: Record<string, string[]> = {
[KnowledgeBase.id]: [
'create',
'add_file',
'update',
'delete',
'delete_document',
'update_document',
'create_tag',
'update_tag',
'delete_tag',
'add_connector',
'update_connector',
'delete_connector',
'sync_connector',
],
[UserTable.id]: [
'create',
'create_from_file',
'import_file',
'delete',
'insert_row',
'batch_insert_rows',
'update_row',
'batch_update_rows',
'delete_row',
'batch_delete_rows',
'update_rows_by_filter',
'delete_rows_by_filter',
'add_column',
'rename_column',
'delete_column',
'update_column',
'add_enrichment',
],
[ManageCustomTool.id]: ['add', 'edit', 'delete'],
[ManageMcpTool.id]: ['add', 'edit', 'delete'],
[ManageSkill.id]: ['add', 'edit', 'delete'],
[ManageCredential.id]: ['rename', 'delete'],
[WorkspaceFile.id]: ['create', 'append', 'update', 'delete', 'rename', 'patch'],
[editContentServerTool.name]: ['*'],
[CreateFile.id]: ['*'],
[RenameFile.id]: ['*'],
[DeleteFile.id]: ['*'],
[MoveFile.id]: ['*'],
[CreateFileFolder.id]: ['*'],
[RenameFileFolder.id]: ['*'],
[MoveFileFolder.id]: ['*'],
[DeleteFileFolder.id]: ['*'],
[DownloadToWorkspaceFile.id]: ['*'],
[GenerateImage.id]: ['generate'],
[GenerateVideo.id]: ['generate'],
[GenerateAudio.id]: ['generate'],
[Ffmpeg.id]: ['*'],
// Paid external-provider lookups (hosted-key cost), like the media tools.
[enrichmentRunServerTool.name]: ['*'],
}
function isWritePermission(userPermission: string): boolean {
return userPermission === 'write' || userPermission === 'admin'
}
function isWriteAction(toolName: string, action: string | undefined): boolean {
const writeActions = WRITE_ACTIONS[toolName]
if (!writeActions) return false
// '*' means the tool is always a write operation regardless of action field
if (writeActions.includes('*')) return true
return Boolean(action && writeActions.includes(action))
}
/** Registry of all server tools. Tools self-declare their validation schemas. */
const baseServerToolRegistry: Record<string, BaseServerTool> = {
[getBlocksMetadataServerTool.name]: getBlocksMetadataServerTool,
[getTriggerBlocksServerTool.name]: getTriggerBlocksServerTool,
[editWorkflowServerTool.name]: editWorkflowServerTool,
[queryLogsServerTool.name]: queryLogsServerTool,
[getJobLogsServerTool.name]: getJobLogsServerTool,
[searchDocumentationServerTool.name]: searchDocumentationServerTool,
[searchOnlineServerTool.name]: searchOnlineServerTool,
[setEnvironmentVariablesServerTool.name]: setEnvironmentVariablesServerTool,
[getCredentialsServerTool.name]: getCredentialsServerTool,
[knowledgeBaseServerTool.name]: knowledgeBaseServerTool,
[enrichmentRunServerTool.name]: enrichmentRunServerTool,
[userTableServerTool.name]: userTableServerTool,
[workspaceFileServerTool.name]: workspaceFileServerTool,
[editContentServerTool.name]: editContentServerTool,
[createFileServerTool.name]: createFileServerTool,
[renameFileServerTool.name]: renameFileServerTool,
[deleteFileServerTool.name]: deleteFileServerTool,
[moveFileServerTool.name]: moveFileServerTool,
[listFileFoldersServerTool.name]: listFileFoldersServerTool,
[createFileFolderServerTool.name]: createFileFolderServerTool,
[renameFileFolderServerTool.name]: renameFileFolderServerTool,
[moveFileFolderServerTool.name]: moveFileFolderServerTool,
[deleteFileFolderServerTool.name]: deleteFileFolderServerTool,
[downloadToWorkspaceFileServerTool.name]: downloadToWorkspaceFileServerTool,
[generateImageServerTool.name]: generateImageServerTool,
[generateVideoServerTool.name]: generateVideoServerTool,
[generateAudioServerTool.name]: generateAudioServerTool,
[ffmpegServerTool.name]: ffmpegServerTool,
}
function getServerToolRegistry(): Record<string, BaseServerTool> {
return baseServerToolRegistry
}
export function getRegisteredServerToolNames(): string[] {
return Object.keys(getServerToolRegistry())
}
export async function routeExecution(
toolName: string,
payload: unknown,
context?: ServerToolContext
): Promise<unknown> {
const tool = getServerToolRegistry()[toolName]
if (!tool) {
throw new Error(`Unknown server tool: ${toolName}`)
}
logger.debug(
context?.messageId ? `Routing to tool [messageId:${context.messageId}]` : 'Routing to tool',
{ toolName }
)
// Action-level permission enforcement for mixed read/write tools
if (WRITE_ACTIONS[toolName]) {
const p = payload as Record<string, unknown>
const action = (p?.operation ?? p?.action) as string | undefined
if (isWriteAction(toolName, action) && !isWritePermission(context?.userPermission ?? '')) {
const actionLabel = action ? `'${action}' on ` : ''
throw new Error(
`Permission denied: ${actionLabel}${toolName} requires write access. You have '${context?.userPermission ?? 'none'}' permission.`
)
}
}
assertServerToolNotAborted(
context,
`User stop signal aborted ${toolName} before payload normalization`
)
// Go injects chatId/workspaceId and may wrap the model's args inside a
// nested "args" object. Unwrap that before validation so the generated
// JSON Schema sees the flat tool contract shape.
let normalizedPayload = payload ?? {}
if (
normalizedPayload &&
typeof normalizedPayload === 'object' &&
!Array.isArray(normalizedPayload)
) {
const raw = normalizedPayload as Record<string, unknown>
if (raw.args && typeof raw.args === 'object' && !raw.operation) {
const nested = raw.args as Record<string, unknown>
normalizedPayload = { ...nested, ...raw, args: undefined }
}
}
const args = tool.inputSchema
? tool.inputSchema.parse(normalizedPayload)
: validateGeneratedToolPayload(toolName, 'parameters', normalizedPayload)
assertServerToolNotAborted(context, `User stop signal aborted ${toolName} after validation`)
// Execute. The registry-dependent tools resolve blocks via getBlock/getAllBlocks;
// wrap them in the custom-block overlay for the workspace's org so `custom_block_*`
// types resolve (metadata lookup + edit-workflow validation) instead of being
// rejected as unknown, and wrap discovery tools in the viewer's block-visibility
// context so gated blocks stay hidden. The two ALS scopes are independent and
// nest in either order. Other tools skip the extra queries.
let run = () => tool.execute(args, context)
if (VISIBILITY_GATED_TOOLS.has(toolName) && context?.userId) {
// Memoized per (userId, workspaceId) ~30s — a multi-tool turn resolves once.
const vis = await getBlockVisibilityForCopilot(context.userId, context.workspaceId)
const inner = run
run = () => withBlockVisibility(vis, inner)
}
if (CUSTOM_BLOCK_OVERLAY_TOOLS.has(toolName) && context?.workspaceId) {
const rows = await listCustomBlocksWithInputsForWorkspace(context.workspaceId)
const inner = run
run = () => withCustomBlockOverlay(rows, inner)
}
const result = await run()
// Validate output if tool declares a schema; otherwise fall back to the
// generated JSON schema contract emitted from Go.
return tool.outputSchema
? tool.outputSchema.parse(result)
: validateGeneratedToolPayload(toolName, 'resultSchema', result)
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,134 @@
/**
* @vitest-environment node
*
* Regression test: the credentials response must expose only display metadata,
* never the connected account's OAuth access/refresh token.
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const SECRET_ACCESS_TOKEN = 'ya29.a0SECRET_GOOGLE_BEARER_TOKEN_DO_NOT_LEAK'
const { selectMock, getAllOAuthServicesMock, getPersonalAndWorkspaceEnvMock, decodeJwtMock } =
vi.hoisted(() => ({
selectMock: vi.fn(),
getAllOAuthServicesMock: vi.fn(),
getPersonalAndWorkspaceEnvMock: vi.fn(),
decodeJwtMock: vi.fn(),
}))
vi.mock('@sim/db', () => ({
db: { select: selectMock },
}))
vi.mock('@/lib/oauth', () => ({
getAllOAuthServices: getAllOAuthServicesMock,
}))
vi.mock('@/lib/environment/utils', () => ({
getPersonalAndWorkspaceEnv: getPersonalAndWorkspaceEnvMock,
}))
vi.mock('jose', () => ({
decodeJwt: decodeJwtMock,
}))
import { getCredentialsServerTool } from './get-credentials'
/**
* Wires the two sequential `db.select()` reads the tool performs:
* 1. `select().from(account).where()` → account rows (awaited directly)
* 2. `select({...}).from(user).where().limit(1)` → user row
*/
function wireDb(accountRows: unknown[], userRows: Array<{ email: string }>) {
const whereThenable = {
then: (resolve: (rows: unknown[]) => unknown) => resolve(accountRows),
limit: () => Promise.resolve(userRows),
}
const builder = { from: () => builder, where: () => whereThenable }
selectMock.mockReturnValue(builder)
}
describe('getCredentialsServerTool', () => {
beforeEach(() => {
vi.clearAllMocks()
wireDb(
[
{
id: 'acct-google-1',
providerId: 'google-default',
accountId: '1234567890',
idToken: 'jwt-token',
accessToken: SECRET_ACCESS_TOKEN,
refreshToken: 'refresh-secret',
updatedAt: new Date('2026-04-17T02:26:05.546Z'),
},
],
[{ email: 'brent@cellular.so' }]
)
getAllOAuthServicesMock.mockReturnValue([
{
providerId: 'google-default',
name: 'Google',
description: 'Google account',
baseProvider: 'google',
},
{
providerId: 'slack',
name: 'Slack',
description: 'Slack workspace',
baseProvider: 'slack',
},
])
getPersonalAndWorkspaceEnvMock.mockResolvedValue({
personalEncrypted: {},
workspaceEncrypted: {},
conflicts: [],
})
decodeJwtMock.mockReturnValue({ email: 'brent@cellular.so' })
})
it('never returns access tokens for connected OAuth credentials', async () => {
const result = await getCredentialsServerTool.execute({}, { userId: 'user-1' })
const credentials = result.oauth.connected.credentials
expect(credentials).toHaveLength(1)
for (const credential of credentials) {
expect(credential).not.toHaveProperty('accessToken')
expect(credential).not.toHaveProperty('refreshToken')
expect(credential).not.toHaveProperty('idToken')
}
})
it('returns only masked display metadata for each credential', async () => {
const result = await getCredentialsServerTool.execute({}, { userId: 'user-1' })
expect(result.oauth.connected.credentials[0]).toEqual({
id: 'acct-google-1',
name: 'brent@cellular.so',
provider: 'google-default',
serviceName: 'Google',
lastUsed: '2026-04-17T02:26:05.546Z',
isDefault: true,
})
})
it('does not leak the token value anywhere in the serialized response', async () => {
const result = await getCredentialsServerTool.execute({}, { userId: 'user-1' })
expect(JSON.stringify(result)).not.toContain(SECRET_ACCESS_TOKEN)
expect(JSON.stringify(result)).not.toContain('refresh-secret')
})
it('rejects unauthenticated callers without touching the database', async () => {
await expect(getCredentialsServerTool.execute({}, undefined)).rejects.toThrow(
'Authentication required'
)
expect(selectMock).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,199 @@
import { db } from '@sim/db'
import { account, user } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { eq } from 'drizzle-orm'
import { decodeJwt } from 'jose'
import { createPermissionError, verifyWorkflowAccess } from '@/lib/copilot/auth/permissions'
import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
import { getAccessibleOAuthCredentials } from '@/lib/credentials/environment'
import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils'
import { getAllOAuthServices } from '@/lib/oauth'
import { checkWorkspaceAccess, type WorkspaceAccess } from '@/lib/workspaces/permissions/utils'
interface GetCredentialsParams {
workflowId?: string
}
export const getCredentialsServerTool: BaseServerTool<GetCredentialsParams, any> = {
name: 'get_credentials',
async execute(params: GetCredentialsParams, context?: { userId: string }): Promise<any> {
const logger = createLogger('GetCredentialsServerTool')
if (!context?.userId) {
logger.error('Unauthorized attempt to access credentials - no authenticated user context')
throw new Error('Authentication required')
}
const authenticatedUserId = context.userId
let workspaceId: string | undefined
if (params?.workflowId) {
const { hasAccess, workspaceId: wId } = await verifyWorkflowAccess(
authenticatedUserId,
params.workflowId
)
if (!hasAccess) {
const errorMessage = createPermissionError('access credentials in')
logger.error('Unauthorized attempt to access credentials', {
workflowId: params.workflowId,
authenticatedUserId,
})
throw new Error(errorMessage)
}
workspaceId = wId
}
const userId = authenticatedUserId
// Resolve workspace access once and thread it into both credential lookups
// below; each would otherwise re-resolve the same workspace-admin status.
const workspaceAccess: WorkspaceAccess | undefined = workspaceId
? await checkWorkspaceAccess(workspaceId, userId)
: undefined
logger.info('Fetching credentials for authenticated user', {
userId,
hasWorkflowId: !!params?.workflowId,
})
// Fetch OAuth credentials
const accounts = await db.select().from(account).where(eq(account.userId, userId))
const userRecord = await db
.select({ email: user.email })
.from(user)
.where(eq(user.id, userId))
.limit(1)
const userEmail = userRecord.length > 0 ? userRecord[0]?.email : null
// Get all available OAuth services
const allOAuthServices = getAllOAuthServices()
// Track connected provider IDs
const connectedProviderIds = new Set<string>()
const connectedCredentials: Array<{
id: string
name: string
provider: string
serviceName: string
lastUsed: string
isDefault: boolean
}> = []
for (const acc of accounts) {
const providerId = acc.providerId
connectedProviderIds.add(providerId)
const [baseProvider, featureType = 'default'] = providerId.split('-')
let displayName = ''
if (acc.idToken) {
try {
const decoded = decodeJwt<{ email?: string; name?: string }>(acc.idToken)
displayName = decoded.email || decoded.name || ''
} catch (error) {
logger.warn('Failed to decode JWT id token', {
error: toError(error).message,
})
}
}
if (!displayName && baseProvider === 'github') displayName = `${acc.accountId} (GitHub)`
if (!displayName && userEmail) displayName = userEmail
if (!displayName) displayName = `${acc.accountId} (${baseProvider})`
// Find the service name for this provider ID
const service = allOAuthServices.find((s) => s.providerId === providerId)
const serviceName = service?.name ?? providerId
connectedCredentials.push({
id: acc.id,
name: displayName,
provider: providerId,
serviceName,
lastUsed: acc.updatedAt.toISOString(),
isDefault: featureType === 'default',
})
}
// Surface workspace-shared OAuth/service-account credentials the user can use,
// including those they reach as a derived workspace admin (not just their own
// personal account connections). Keyed by credential id so the agent references
// the workspace credential, not a legacy account id.
if (workspaceId) {
const sharedCredentials = await getAccessibleOAuthCredentials(workspaceId, userId, {
isWorkspaceAdmin: workspaceAccess?.canAdmin ?? false,
})
const seenCredentialIds = new Set(connectedCredentials.map((c) => c.id))
for (const cred of sharedCredentials) {
if (seenCredentialIds.has(cred.id)) continue
connectedProviderIds.add(cred.providerId)
const [, featureType = 'default'] = cred.providerId.split('-')
connectedCredentials.push({
id: cred.id,
name: cred.displayName,
provider: cred.providerId,
serviceName:
allOAuthServices.find((s) => s.providerId === cred.providerId)?.name ?? cred.providerId,
lastUsed: cred.updatedAt.toISOString(),
isDefault: featureType === 'default',
})
}
}
// Build list of not connected services
const notConnectedServices = allOAuthServices
.filter((service) => !connectedProviderIds.has(service.providerId))
.map((service) => ({
providerId: service.providerId,
name: service.name,
description: service.description,
baseProvider: service.baseProvider,
}))
// Fetch environment variables from both personal and workspace
const envResult = await getPersonalAndWorkspaceEnv(
userId,
workspaceId,
workspaceAccess ? { workspaceAccess } : undefined
)
// Get all unique variable names from both personal and workspace
const personalVarNames = Object.keys(envResult.personalEncrypted)
const workspaceVarNames = Object.keys(envResult.workspaceEncrypted)
const allVarNames = [...new Set([...personalVarNames, ...workspaceVarNames])]
logger.info('Fetched credentials', {
userId,
workspaceId,
connectedCount: connectedCredentials.length,
notConnectedCount: notConnectedServices.length,
personalEnvVarCount: personalVarNames.length,
workspaceEnvVarCount: workspaceVarNames.length,
totalEnvVarCount: allVarNames.length,
conflicts: envResult.conflicts,
})
return {
oauth: {
connected: {
credentials: connectedCredentials,
total: connectedCredentials.length,
},
notConnected: {
services: notConnectedServices,
total: notConnectedServices.length,
},
},
environment: {
variableNames: allVarNames,
count: allVarNames.length,
personalVariables: personalVarNames,
workspaceVariables: workspaceVarNames,
conflicts: envResult.conflicts,
},
}
},
}
@@ -0,0 +1,99 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
ensureWorkflowAccessMock,
ensureWorkspaceAccessMock,
getDefaultWorkspaceIdMock,
upsertPersonalEnvVarsMock,
upsertWorkspaceEnvVarsMock,
} = vi.hoisted(() => ({
ensureWorkflowAccessMock: vi.fn(),
ensureWorkspaceAccessMock: vi.fn(),
getDefaultWorkspaceIdMock: vi.fn(),
upsertPersonalEnvVarsMock: vi.fn(),
upsertWorkspaceEnvVarsMock: vi.fn(),
}))
vi.mock('@/lib/copilot/tools/handlers/access', () => ({
ensureWorkflowAccess: ensureWorkflowAccessMock,
ensureWorkspaceAccess: ensureWorkspaceAccessMock,
getDefaultWorkspaceId: getDefaultWorkspaceIdMock,
}))
vi.mock('@/lib/environment/utils', () => ({
upsertPersonalEnvVars: upsertPersonalEnvVarsMock,
upsertWorkspaceEnvVars: upsertWorkspaceEnvVarsMock,
}))
import { setEnvironmentVariablesServerTool } from './set-environment-variables'
describe('setEnvironmentVariablesServerTool', () => {
beforeEach(() => {
vi.clearAllMocks()
ensureWorkflowAccessMock.mockResolvedValue({
workflow: { id: 'wf-1', workspaceId: 'ws-from-workflow' },
})
ensureWorkspaceAccessMock.mockResolvedValue(undefined)
getDefaultWorkspaceIdMock.mockResolvedValue('ws-default')
upsertPersonalEnvVarsMock.mockResolvedValue({ added: ['API_KEY'], updated: [] })
upsertWorkspaceEnvVarsMock.mockResolvedValue(['API_KEY'])
})
it('defaults to workspace scope and uses the current workspace context', async () => {
const result = await setEnvironmentVariablesServerTool.execute(
{
variables: [{ name: 'API_KEY', value: 'secret' }],
},
{
userId: 'user-1',
workspaceId: 'ws-1',
}
)
expect(ensureWorkspaceAccessMock).toHaveBeenCalledWith('ws-1', 'user-1', 'write')
expect(upsertWorkspaceEnvVarsMock).toHaveBeenCalledWith('ws-1', { API_KEY: 'secret' }, 'user-1')
expect(upsertPersonalEnvVarsMock).not.toHaveBeenCalled()
expect(result.scope).toBe('workspace')
expect(result.workspaceId).toBe('ws-1')
})
it('supports explicit personal scope', async () => {
const result = await setEnvironmentVariablesServerTool.execute(
{
scope: 'personal',
variables: [{ name: 'API_KEY', value: 'secret' }],
},
{
userId: 'user-1',
workspaceId: 'ws-1',
}
)
expect(upsertPersonalEnvVarsMock).toHaveBeenCalledWith('user-1', { API_KEY: 'secret' })
expect(upsertWorkspaceEnvVarsMock).not.toHaveBeenCalled()
expect(ensureWorkspaceAccessMock).not.toHaveBeenCalled()
expect(result.scope).toBe('personal')
})
it('falls back to the default workspace when none is in context', async () => {
await setEnvironmentVariablesServerTool.execute(
{
variables: [{ name: 'API_KEY', value: 'secret' }],
},
{
userId: 'user-1',
}
)
expect(getDefaultWorkspaceIdMock).toHaveBeenCalledWith('user-1')
expect(upsertWorkspaceEnvVarsMock).toHaveBeenCalledWith(
'ws-default',
{ API_KEY: 'secret' },
'user-1'
)
})
})
@@ -0,0 +1,151 @@
import { createLogger } from '@sim/logger'
import { z } from 'zod'
import { SetEnvironmentVariables } from '@/lib/copilot/generated/tool-catalog-v1'
import {
ensureWorkflowAccess,
ensureWorkspaceAccess,
getDefaultWorkspaceId,
} from '@/lib/copilot/tools/handlers/access'
import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool'
import { upsertPersonalEnvVars, upsertWorkspaceEnvVars } from '@/lib/environment/utils'
type EnvironmentVariableInputValue = string | number | boolean | null | undefined
interface EnvironmentVariableInput {
name: string
value: EnvironmentVariableInputValue
}
interface SetEnvironmentVariablesParams {
variables: Record<string, EnvironmentVariableInputValue> | EnvironmentVariableInput[]
scope?: 'personal' | 'workspace'
workflowId?: string
workspaceId?: string
}
interface SetEnvironmentVariablesResult {
message: string
scope: 'personal' | 'workspace'
workspaceId?: string
variableCount: number
variableNames: string[]
addedVariables: string[]
updatedVariables: string[]
workspaceUpdatedVariables: string[]
}
const EnvVarSchema = z.object({ variables: z.record(z.string(), z.string()) })
function normalizeVariables(
input: Record<string, EnvironmentVariableInputValue> | EnvironmentVariableInput[]
): Record<string, string> {
if (Array.isArray(input)) {
return input.reduce(
(acc, item) => {
if (item && typeof item.name === 'string') {
acc[item.name] = String(item.value ?? '')
}
return acc
},
{} as Record<string, string>
)
}
return Object.fromEntries(
Object.entries(input || {}).map(([k, v]) => [k, String(v ?? '')])
) as Record<string, string>
}
async function resolveWorkspaceId(
params: SetEnvironmentVariablesParams,
context: ServerToolContext | undefined,
userId: string
): Promise<string> {
if (params.workflowId) {
const { workflow } = await ensureWorkflowAccess(params.workflowId, userId, 'write')
if (!workflow.workspaceId) {
throw new Error(`Workflow ${params.workflowId} is not associated with a workspace`)
}
return workflow.workspaceId
}
const workspaceId = params.workspaceId ?? context?.workspaceId
if (workspaceId) {
await ensureWorkspaceAccess(workspaceId, userId, 'write')
return workspaceId
}
return getDefaultWorkspaceId(userId)
}
export const setEnvironmentVariablesServerTool: BaseServerTool<
SetEnvironmentVariablesParams,
SetEnvironmentVariablesResult
> = {
name: SetEnvironmentVariables.id,
async execute(
params: SetEnvironmentVariablesParams,
context?: ServerToolContext
): Promise<SetEnvironmentVariablesResult> {
const logger = createLogger('SetEnvironmentVariablesServerTool')
if (!context?.userId) {
logger.error(
'Unauthorized attempt to set environment variables - no authenticated user context'
)
throw new Error('Authentication required')
}
const authenticatedUserId = context.userId
const { variables } = params || ({} as SetEnvironmentVariablesParams)
const scope = params.scope === 'personal' ? 'personal' : 'workspace'
const normalized = normalizeVariables(variables || {})
const { variables: validatedVariables } = EnvVarSchema.parse({ variables: normalized })
const variableNames = Object.keys(validatedVariables)
const added: string[] = []
const updated: string[] = []
let workspaceUpdated: string[] = []
let resolvedWorkspaceId: string | undefined
if (scope === 'workspace') {
resolvedWorkspaceId = await resolveWorkspaceId(params, context, authenticatedUserId)
workspaceUpdated = await upsertWorkspaceEnvVars(
resolvedWorkspaceId,
validatedVariables,
authenticatedUserId
)
} else {
const result = await upsertPersonalEnvVars(authenticatedUserId, validatedVariables)
added.push(...result.added)
updated.push(...result.updated)
}
const totalProcessed = added.length + updated.length + workspaceUpdated.length
logger.info('Saved environment variables', {
userId: authenticatedUserId,
scope,
addedCount: added.length,
updatedCount: updated.length,
workspaceUpdatedCount: workspaceUpdated.length,
workspaceId: resolvedWorkspaceId,
})
const parts: string[] = []
if (added.length > 0) parts.push(`${added.length} personal secret(s) added`)
if (updated.length > 0) parts.push(`${updated.length} personal secret(s) updated`)
if (workspaceUpdated.length > 0)
parts.push(`${workspaceUpdated.length} workspace secret(s) updated`)
return {
message: `Successfully processed ${totalProcessed} secret(s): ${parts.join(', ')}`,
scope,
workspaceId: resolvedWorkspaceId,
variableCount: variableNames.length,
variableNames,
addedVariables: added,
updatedVariables: updated,
workspaceUpdatedVariables: workspaceUpdated,
}
},
}
@@ -0,0 +1,72 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
import { createBlockFromParams } from './builders'
const agentBlockConfig = {
type: 'agent',
name: 'Agent',
outputs: {
content: { type: 'string', description: 'Default content output' },
},
subBlocks: [{ id: 'responseFormat', type: 'response-format' }],
}
const conditionBlockConfig = {
type: 'condition',
name: 'Condition',
outputs: {},
subBlocks: [{ id: 'conditions', type: 'condition-input' }],
}
vi.mock('@/blocks/registry', () => ({
getAllBlocks: () => [agentBlockConfig, conditionBlockConfig],
getBlock: (type: string) =>
type === 'agent' ? agentBlockConfig : type === 'condition' ? conditionBlockConfig : undefined,
}))
describe('createBlockFromParams', () => {
it('derives agent outputs from responseFormat when outputs are not provided', () => {
const block = createBlockFromParams('b-agent', {
type: 'agent',
name: 'Agent',
inputs: {
responseFormat: {
type: 'object',
properties: {
answer: {
type: 'string',
description: 'Structured answer text',
},
},
required: ['answer'],
},
},
triggerMode: false,
})
expect(block.outputs.answer).toBeDefined()
expect(block.outputs.answer.type).toBe('string')
})
it('preserves configured subblock types and normalizes condition branch ids', () => {
const block = createBlockFromParams('condition-1', {
type: 'condition',
name: 'Condition 1',
inputs: {
conditions: JSON.stringify([
{ id: 'arbitrary-if', title: 'if', value: 'true' },
{ id: 'arbitrary-else', title: 'else', value: '' },
]),
},
triggerMode: false,
})
expect(block.subBlocks.conditions.type).toBe('condition-input')
const parsed = JSON.parse(block.subBlocks.conditions.value)
expect(parsed[0].id).toBe('condition-1-if')
expect(parsed[1].id).toBe('condition-1-else')
})
})
@@ -0,0 +1,765 @@
import { createLogger } from '@sim/logger'
import { generateId, isValidUuid } from '@sim/utils/id'
import { sortObjectKeysDeep } from '@sim/utils/object'
import type { PermissionGroupConfig } from '@/lib/permission-groups/types'
import { getEffectiveBlockOutputs } from '@/lib/workflows/blocks/block-outputs'
import {
buildCanonicalIndex,
buildDefaultCanonicalModes,
isCanonicalPair,
} from '@/lib/workflows/subblocks/visibility'
import { hasTriggerCapability } from '@/lib/workflows/triggers/trigger-utils'
import { getAllBlocks } from '@/blocks/registry'
import type { BlockConfig } from '@/blocks/types'
import { TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants'
import type { EditWorkflowOperation, SkippedItem, ValidationError } from './types'
import { logSkippedItem } from './types'
import {
validateInputsForBlock,
validateSourceHandleForBlock,
validateTargetHandle,
} from './validation'
/**
* Helper to create a block state from operation params
*/
export function createBlockFromParams(
blockId: string,
params: any,
parentId?: string,
errorsCollector?: ValidationError[],
permissionConfig?: PermissionGroupConfig | null,
skippedItems?: SkippedItem[]
): any {
const blockConfig = getAllBlocks().find((b) => b.type === params.type)
// Validate inputs against block configuration
let validatedInputs: Record<string, any> | undefined
if (params.inputs) {
const result = validateInputsForBlock(params.type, params.inputs, blockId)
validatedInputs = result.validInputs
if (errorsCollector && result.errors.length > 0) {
errorsCollector.push(...result.errors)
}
}
// Determine outputs based on trigger mode
const triggerMode = params.triggerMode || false
const isTriggerCapable = blockConfig ? hasTriggerCapability(blockConfig) : false
const effectiveTriggerMode = Boolean(triggerMode && isTriggerCapable)
let outputs: Record<string, any>
if (params.outputs) {
outputs = params.outputs
} else if (blockConfig) {
const subBlocks: Record<string, any> = {}
if (validatedInputs) {
Object.entries(validatedInputs).forEach(([key, value]) => {
// Skip runtime subblock IDs when computing outputs
if (TRIGGER_RUNTIME_SUBBLOCK_IDS.includes(key)) {
return
}
subBlocks[key] = { id: key, type: 'short-input', value: value }
})
}
outputs = getEffectiveBlockOutputs(params.type, subBlocks, {
triggerMode: effectiveTriggerMode,
preferToolOutputs: !effectiveTriggerMode,
})
} else {
outputs = {}
}
const blockState: any = {
id: blockId,
type: params.type,
name: params.name,
position: { x: 0, y: 0 },
enabled: params.enabled !== undefined ? params.enabled : true,
horizontalHandles: true,
advancedMode: params.advancedMode || false,
height: 0,
triggerMode: triggerMode,
subBlocks: {},
outputs: outputs,
data: parentId ? { parentId, extent: 'parent' as const } : {},
locked: false,
}
// Add validated inputs as subBlocks
if (validatedInputs) {
Object.entries(validatedInputs).forEach(([key, value]) => {
if (TRIGGER_RUNTIME_SUBBLOCK_IDS.includes(key)) {
return
}
let sanitizedValue = value
// Normalize array subblocks with id fields (inputFormat, table rows, etc.)
if (shouldNormalizeArrayIds(key)) {
sanitizedValue = normalizeArrayWithIds(value)
if (JSON_STRING_SUBBLOCK_KEYS.has(key)) {
sanitizedValue = JSON.stringify(sanitizedValue)
}
}
sanitizedValue = normalizeConditionRouterIds(blockId, key, sanitizedValue)
// Special handling for tools - normalize and filter disallowed
if (key === 'tools' && Array.isArray(value)) {
sanitizedValue = filterDisallowedTools(
normalizeTools(value),
permissionConfig ?? null,
blockId,
skippedItems ?? []
)
}
// Special handling for responseFormat - normalize to ensure consistent format
if (key === 'responseFormat' && value) {
sanitizedValue = normalizeResponseFormat(value)
}
const subBlockDef = blockConfig?.subBlocks.find((subBlock) => subBlock.id === key)
blockState.subBlocks[key] = {
id: key,
type: subBlockDef?.type || 'short-input',
value: sanitizedValue,
}
})
}
// Set up subBlocks from block configuration
if (blockConfig) {
blockConfig.subBlocks.forEach((subBlock) => {
if (!blockState.subBlocks[subBlock.id]) {
blockState.subBlocks[subBlock.id] = {
id: subBlock.id,
type: subBlock.type,
value: null,
}
} else {
blockState.subBlocks[subBlock.id].type = subBlock.type
}
})
const defaultModes = buildDefaultCanonicalModes(blockConfig.subBlocks)
if (Object.keys(defaultModes).length > 0) {
if (!blockState.data) blockState.data = {}
blockState.data.canonicalModes = defaultModes
}
if (validatedInputs) {
updateCanonicalModesForInputs(blockState, Object.keys(validatedInputs), blockConfig)
}
}
// Initialize default conditions/routes so edge handle validation works.
// The UI does this in the React component; we need to mirror it here.
if (params.type === 'condition' && !blockState.subBlocks.conditions?.value) {
blockState.subBlocks.conditions = {
id: 'conditions',
type: 'condition-input',
value: JSON.stringify([
{ id: generateId(), title: 'If', value: '' },
{ id: generateId(), title: 'Else', value: '' },
]),
}
} else if (params.type === 'router_v2' && !blockState.subBlocks.routes?.value) {
blockState.subBlocks.routes = {
id: 'routes',
type: 'router-input',
value: JSON.stringify([{ id: generateId(), title: 'Route 1', value: '' }]),
}
}
return blockState
}
export function updateCanonicalModesForInputs(
block: { data?: { canonicalModes?: Record<string, 'basic' | 'advanced'> } },
inputKeys: string[],
blockConfig: BlockConfig
): void {
if (!blockConfig.subBlocks?.length) return
const canonicalIndex = buildCanonicalIndex(blockConfig.subBlocks)
const canonicalModeUpdates: Record<string, 'basic' | 'advanced'> = {}
for (const inputKey of inputKeys) {
const canonicalId = canonicalIndex.canonicalIdBySubBlockId[inputKey]
if (!canonicalId) continue
const group = canonicalIndex.groupsById[canonicalId]
if (!group || !isCanonicalPair(group)) continue
const isAdvanced = group.advancedIds.includes(inputKey)
const existingMode = canonicalModeUpdates[canonicalId]
if (!existingMode || isAdvanced) {
canonicalModeUpdates[canonicalId] = isAdvanced ? 'advanced' : 'basic'
}
}
if (Object.keys(canonicalModeUpdates).length > 0) {
if (!block.data) block.data = {}
if (!block.data.canonicalModes) block.data.canonicalModes = {}
Object.assign(block.data.canonicalModes, canonicalModeUpdates)
}
}
/**
* Normalize tools array by adding back fields that were sanitized for training
*/
export function normalizeTools(tools: any[]): any[] {
return tools.map((tool) => {
if (tool.type === 'custom-tool') {
// New reference format: minimal fields only
if (tool.customToolId && !tool.schema && !tool.code) {
return {
type: tool.type,
customToolId: tool.customToolId,
usageControl: tool.usageControl || 'auto',
isExpanded: tool.isExpanded ?? true,
}
}
// Legacy inline format: include all fields
const normalized: any = {
...tool,
params: tool.params || {},
isExpanded: tool.isExpanded ?? true,
}
// Ensure schema has proper structure (for inline format)
if (normalized.schema?.function) {
normalized.schema = {
type: 'function',
function: {
name: normalized.schema.function.name || tool.title, // Preserve name or derive from title
description: normalized.schema.function.description,
parameters: normalized.schema.function.parameters,
},
}
}
return normalized
}
// For other tool types, just ensure isExpanded exists
return {
...tool,
isExpanded: tool.isExpanded ?? true,
}
})
}
/**
* Subblock types that store arrays of objects with `id` fields.
* The LLM may generate arbitrary IDs which need to be converted to proper UUIDs.
*/
const ARRAY_WITH_ID_SUBBLOCK_TYPES = new Set([
'inputFormat', // input-format: Fields with id, name, type, value, collapsed
'headers', // table: Rows with id, cells (used for HTTP headers)
'params', // table: Rows with id, cells (used for query params)
'variables', // table or variables-input: Rows/assignments with id
'tagFilters', // knowledge-tag-filters: Filters with id, tagName, etc.
'documentTags', // document-tag-entry: Tags with id, tagName, etc.
'metrics', // eval-input: Metrics with id, name, description, range
'conditions', // condition-input: Condition branches with id, title, value
'routes', // router-input: Router routes with id, title, value
])
/**
* Subblock keys whose UI components expect a JSON string, not a raw array.
* After normalizeArrayWithIds returns an array, these must be re-stringified.
*/
export const JSON_STRING_SUBBLOCK_KEYS = new Set(['conditions', 'routes'])
/**
* Normalizes array subblock values by ensuring each item has a valid UUID.
* The LLM may generate arbitrary IDs like "input-desc-001" or "row-1" which need
* to be converted to proper UUIDs for consistency with UI-created items.
*/
export function normalizeArrayWithIds(value: unknown): any[] {
let arr: any[]
if (Array.isArray(value)) {
arr = value
} else if (typeof value === 'string') {
try {
const parsed = JSON.parse(value)
if (!Array.isArray(parsed)) return []
arr = parsed
} catch {
return []
}
} else {
return []
}
return arr.map((item: any) => {
if (!item || typeof item !== 'object') {
return item
}
const hasValidUUID = typeof item.id === 'string' && isValidUuid(item.id)
if (!hasValidUUID) {
return { ...item, id: generateId() }
}
return item
})
}
/**
* Checks if a subblock key should have its array items normalized with UUIDs.
*/
export function shouldNormalizeArrayIds(key: string): boolean {
return ARRAY_WITH_ID_SUBBLOCK_TYPES.has(key)
}
/**
* Normalizes condition/router branch IDs to use canonical block-scoped format.
* The LLM provides branch structure (if/else-if/else or routes) but should not
* have to generate the internal IDs -- we assign them based on the block ID.
*/
export function normalizeConditionRouterIds(blockId: string, key: string, value: unknown): unknown {
if (key !== 'conditions' && key !== 'routes') return value
let parsed: any[]
if (typeof value === 'string') {
try {
parsed = JSON.parse(value)
if (!Array.isArray(parsed)) return value
} catch {
return value
}
} else if (Array.isArray(value)) {
parsed = value
} else {
return value
}
let elseIfCounter = 0
const normalized = parsed.map((item, index) => {
if (!item || typeof item !== 'object') return item
let canonicalId: string
if (key === 'conditions') {
if (index === 0) {
canonicalId = `${blockId}-if`
} else if (index === parsed.length - 1) {
canonicalId = `${blockId}-else`
} else {
canonicalId = `${blockId}-else-if-${elseIfCounter}`
elseIfCounter++
}
} else {
canonicalId = `${blockId}-route${index + 1}`
}
return { ...item, id: canonicalId }
})
return typeof value === 'string' ? JSON.stringify(normalized) : normalized
}
/**
* Normalize responseFormat to ensure consistent storage
* Handles both string (JSON) and object formats
* Returns pretty-printed JSON for better UI readability
*/
export function normalizeResponseFormat(value: any): string {
try {
let obj = value
// If it's already a string, parse it first
if (typeof value === 'string') {
const trimmed = value.trim()
if (!trimmed) {
return ''
}
obj = JSON.parse(trimmed)
}
// If it's an object, stringify it with consistent formatting
if (obj && typeof obj === 'object') {
// Return pretty-printed with 2-space indentation for UI readability
// The sanitizer will normalize it to minified format for comparison
return JSON.stringify(sortObjectKeysDeep(obj), null, 2)
}
return String(value)
} catch {
// If parsing fails, return the original value as string
return String(value)
}
}
/**
* Creates a validated edge between two blocks.
* Returns true if edge was created, false if skipped due to validation errors.
*/
export function createValidatedEdge(
modifiedState: any,
sourceBlockId: string,
targetBlockId: string,
sourceHandle: string,
targetHandle: string,
operationType: string,
logger: ReturnType<typeof createLogger>,
skippedItems?: SkippedItem[]
): boolean {
if (!modifiedState.blocks[targetBlockId]) {
// The target doesn't exist yet. It may be created by a later operation in
// this batch or by a future edit_workflow call. Record the connection as
// pending on the source block (persisted in block.data) so it is resolved
// automatically once the target appears, instead of being silently dropped.
const pendingSource = modifiedState.blocks[sourceBlockId]
if (pendingSource) {
if (!pendingSource.data) pendingSource.data = {}
if (!pendingSource.data.pendingConnections) pendingSource.data.pendingConnections = {}
const pending = pendingSource.data.pendingConnections as Record<
string,
Array<{ target: string; targetHandle: string }>
>
if (!pending[sourceHandle]) pending[sourceHandle] = []
if (
!pending[sourceHandle].some(
(p) => p.target === targetBlockId && p.targetHandle === targetHandle
)
) {
pending[sourceHandle].push({ target: targetBlockId, targetHandle })
}
}
logger.warn(`Target block "${targetBlockId}" not found. Connection deferred until it exists.`, {
sourceBlockId,
targetBlockId,
sourceHandle,
})
skippedItems?.push({
type: 'invalid_edge_target',
operationType,
blockId: sourceBlockId,
reason: `Edge from "${sourceBlockId}" to "${targetBlockId}" deferred until the target block "${targetBlockId}" exists - if it is created later (in this or a following edit) the engine wires this edge automatically; if you did not intend to create "${targetBlockId}", fix the target id.`,
details: { sourceHandle, targetHandle, targetId: targetBlockId },
})
return false
}
const sourceBlock = modifiedState.blocks[sourceBlockId]
if (!sourceBlock) {
logger.warn(`Source block "${sourceBlockId}" not found. Edge skipped.`, {
sourceBlockId,
targetBlockId,
})
skippedItems?.push({
type: 'invalid_edge_source',
operationType,
blockId: sourceBlockId,
reason: `Edge from "${sourceBlockId}" to "${targetBlockId}" skipped - source block does not exist`,
details: { sourceHandle, targetHandle, targetId: targetBlockId },
})
return false
}
const sourceBlockType = sourceBlock.type
if (!sourceBlockType) {
logger.warn(`Source block "${sourceBlockId}" has no type. Edge skipped.`, {
sourceBlockId,
targetBlockId,
})
skippedItems?.push({
type: 'invalid_edge_source',
operationType,
blockId: sourceBlockId,
reason: `Edge from "${sourceBlockId}" to "${targetBlockId}" skipped - source block has no type`,
details: { sourceHandle, targetHandle, targetId: targetBlockId },
})
return false
}
const sourceValidation = validateSourceHandleForBlock(sourceHandle, sourceBlockType, sourceBlock)
if (!sourceValidation.valid) {
logger.warn(`Invalid source handle. Edge skipped.`, {
sourceBlockId,
targetBlockId,
sourceHandle,
error: sourceValidation.error,
})
skippedItems?.push({
type: 'invalid_source_handle',
operationType,
blockId: sourceBlockId,
reason: sourceValidation.error || `Invalid source handle "${sourceHandle}"`,
details: { sourceHandle, targetHandle, targetId: targetBlockId },
})
return false
}
const targetValidation = validateTargetHandle(targetHandle)
if (!targetValidation.valid) {
logger.warn(`Invalid target handle. Edge skipped.`, {
sourceBlockId,
targetBlockId,
targetHandle,
error: targetValidation.error,
})
skippedItems?.push({
type: 'invalid_target_handle',
operationType,
blockId: sourceBlockId,
reason: targetValidation.error || `Invalid target handle "${targetHandle}"`,
details: { sourceHandle, targetHandle, targetId: targetBlockId },
})
return false
}
// Use normalized handle if available (e.g., 'if' -> 'condition-{uuid}')
const finalSourceHandle = sourceValidation.normalizedHandle || sourceHandle
// Avoid creating duplicate edges (e.g., when a pending connection resolves to
// the same edge a later operation already created).
const edgeExists = (modifiedState.edges || []).some(
(e: any) =>
e.source === sourceBlockId &&
e.sourceHandle === finalSourceHandle &&
e.target === targetBlockId &&
e.targetHandle === targetHandle
)
if (edgeExists) return true
modifiedState.edges.push({
id: generateId(),
source: sourceBlockId,
sourceHandle: finalSourceHandle,
target: targetBlockId,
targetHandle,
type: 'default',
})
return true
}
/**
* Adds connections as edges for a block.
* Supports multiple target formats:
* - String: "target-block-id"
* - Object: { block: "target-block-id", handle?: "custom-target-handle" }
* - Array of strings or objects
*/
export function addConnectionsAsEdges(
modifiedState: any,
blockId: string,
connections: Record<string, any>,
logger: ReturnType<typeof createLogger>,
skippedItems?: SkippedItem[]
): void {
const normalizeHandle = (handle: string): string => {
if (handle === 'success') return 'source'
return handle
}
Object.entries(connections).forEach(([rawHandle, targets]) => {
if (targets === null) return
const sourceHandle = normalizeHandle(rawHandle)
const addEdgeForTarget = (targetBlock: string, targetHandle?: string) => {
createValidatedEdge(
modifiedState,
blockId,
targetBlock,
sourceHandle,
targetHandle || 'target',
'add_edge',
logger,
skippedItems
)
}
if (typeof targets === 'string') {
addEdgeForTarget(targets)
} else if (Array.isArray(targets)) {
targets.forEach((target: any) => {
if (typeof target === 'string') {
addEdgeForTarget(target)
} else if (target?.block) {
addEdgeForTarget(target.block, target.handle)
}
})
} else if (typeof targets === 'object' && targets?.block) {
addEdgeForTarget(targets.block, targets.handle)
}
})
}
export function applyTriggerConfigToBlockSubblocks(block: any, triggerConfig: Record<string, any>) {
if (!block?.subBlocks || !triggerConfig || typeof triggerConfig !== 'object') {
return
}
Object.entries(triggerConfig).forEach(([configKey, configValue]) => {
const existingSubblock = block.subBlocks[configKey]
if (existingSubblock) {
const existingValue = existingSubblock.value
const valuesEqual =
typeof existingValue === 'object' || typeof configValue === 'object'
? JSON.stringify(existingValue) === JSON.stringify(configValue)
: existingValue === configValue
if (valuesEqual) {
return
}
block.subBlocks[configKey] = {
...existingSubblock,
value: configValue,
}
} else {
block.subBlocks[configKey] = {
id: configKey,
type: 'short-input',
value: configValue,
}
}
})
}
/**
* Filters out tools that are not allowed by the permission group config
* Returns both the allowed tools and any skipped tool items for logging
*/
export function filterDisallowedTools(
tools: any[],
permissionConfig: PermissionGroupConfig | null,
blockId: string,
skippedItems: SkippedItem[]
): any[] {
if (!permissionConfig) {
return tools
}
const allowedTools: any[] = []
for (const tool of tools) {
if (tool.type === 'custom-tool' && permissionConfig.disableCustomTools) {
logSkippedItem(skippedItems, {
type: 'tool_not_allowed',
operationType: 'add',
blockId,
reason: `Custom tool "${tool.title || tool.customToolId || 'unknown'}" is not allowed by permission group - tool not added`,
details: { toolType: 'custom-tool', toolId: tool.customToolId },
})
continue
}
if (tool.type === 'mcp' && permissionConfig.disableMcpTools) {
logSkippedItem(skippedItems, {
type: 'tool_not_allowed',
operationType: 'add',
blockId,
reason: `MCP tool "${tool.title || 'unknown'}" is not allowed by permission group - tool not added`,
details: { toolType: 'mcp', serverId: tool.params?.serverId },
})
continue
}
allowedTools.push(tool)
}
return allowedTools
}
/**
* Normalizes block IDs in operations to ensure they are valid UUIDs.
* The LLM may generate human-readable IDs like "web_search" or "research_agent"
* which need to be converted to proper UUIDs for database compatibility.
*
* Returns the normalized operations and a mapping from old IDs to new UUIDs.
*/
export function normalizeBlockIdsInOperations(operations: EditWorkflowOperation[]): {
normalizedOperations: EditWorkflowOperation[]
idMapping: Map<string, string>
} {
const logger = createLogger('EditWorkflowServerTool')
const idMapping = new Map<string, string>()
// First pass: collect all non-UUID block_ids from add/insert operations
for (const op of operations) {
if (op.operation_type === 'add' || op.operation_type === 'insert_into_subflow') {
if (op.block_id && !isValidUuid(op.block_id)) {
const newId = generateId()
idMapping.set(op.block_id, newId)
logger.debug('Normalizing block ID', { oldId: op.block_id, newId })
}
}
}
if (idMapping.size === 0) {
return { normalizedOperations: operations, idMapping }
}
logger.info('Normalizing block IDs in operations', {
normalizedCount: idMapping.size,
mappings: Object.fromEntries(idMapping),
})
// Helper to replace an ID if it's in the mapping
const replaceId = (id: string | undefined): string | undefined => {
if (!id) return id
return idMapping.get(id) ?? id
}
// Second pass: update all references to use new UUIDs
const normalizedOperations = operations.map((op) => {
const normalized: EditWorkflowOperation = {
...op,
block_id: replaceId(op.block_id) ?? op.block_id,
}
if (op.params) {
normalized.params = { ...op.params }
// Update subflowId references (for insert_into_subflow)
if (normalized.params.subflowId) {
normalized.params.subflowId = replaceId(normalized.params.subflowId)
}
// Update connection references
if (normalized.params.connections) {
const normalizedConnections: Record<string, any> = {}
for (const [handle, targets] of Object.entries(normalized.params.connections)) {
if (typeof targets === 'string') {
normalizedConnections[handle] = replaceId(targets)
} else if (Array.isArray(targets)) {
normalizedConnections[handle] = targets.map((t) => {
if (typeof t === 'string') return replaceId(t)
if (t && typeof t === 'object' && t.block) {
return { ...t, block: replaceId(t.block) }
}
return t
})
} else if (targets && typeof targets === 'object' && (targets as any).block) {
normalizedConnections[handle] = { ...targets, block: replaceId((targets as any).block) }
} else {
normalizedConnections[handle] = targets
}
}
normalized.params.connections = normalizedConnections
}
// Update nestedNodes block IDs
if (normalized.params.nestedNodes) {
const normalizedNestedNodes: Record<string, any> = {}
for (const [childId, childBlock] of Object.entries(normalized.params.nestedNodes)) {
const newChildId = replaceId(childId) ?? childId
normalizedNestedNodes[newChildId] = childBlock
}
normalized.params.nestedNodes = normalizedNestedNodes
}
}
return normalized
})
return { normalizedOperations, idMapping }
}
@@ -0,0 +1,379 @@
import { createLogger } from '@sim/logger'
import type { PermissionGroupConfig } from '@/lib/permission-groups/types'
import { isValidKey } from '@/lib/workflows/sanitization/key-validation'
import { validateEdges } from '@/stores/workflows/workflow/edge-validation'
import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils'
import {
addConnectionsAsEdges,
createValidatedEdge,
normalizeBlockIdsInOperations,
} from './builders'
import {
handleAddOperation,
handleDeleteOperation,
handleEditOperation,
handleExtractFromSubflowOperation,
handleInsertIntoSubflowOperation,
} from './operations'
import type {
ApplyOperationsResult,
EditWorkflowOperation,
OperationContext,
ValidationError,
} from './types'
import { logSkippedItem, type SkippedItem } from './types'
const logger = createLogger('EditWorkflowServerTool')
type OperationHandler = (op: EditWorkflowOperation, ctx: OperationContext) => void
const OPERATION_HANDLERS: Record<EditWorkflowOperation['operation_type'], OperationHandler> = {
delete: handleDeleteOperation,
extract_from_subflow: handleExtractFromSubflowOperation,
add: handleAddOperation,
insert_into_subflow: handleInsertIntoSubflowOperation,
edit: handleEditOperation,
}
/**
* Topologically sort insert operations to ensure parents are created before children
* Returns sorted array where parent inserts always come before child inserts
*/
export function topologicalSortInserts(
inserts: EditWorkflowOperation[],
adds: EditWorkflowOperation[]
): EditWorkflowOperation[] {
if (inserts.length === 0) return []
// Build a map of blockId -> operation for quick lookup
const insertMap = new Map<string, EditWorkflowOperation>()
inserts.forEach((op) => insertMap.set(op.block_id, op))
// Build a set of blocks being added (potential parents)
const addedBlocks = new Set(adds.map((op) => op.block_id))
// Build dependency graph: block -> blocks that depend on it
const dependents = new Map<string, Set<string>>()
const dependencies = new Map<string, Set<string>>()
inserts.forEach((op) => {
const blockId = op.block_id
const parentId = op.params?.subflowId
dependencies.set(blockId, new Set())
if (parentId) {
// Track dependency if parent is being inserted OR being added
// This ensures children wait for parents regardless of operation type
const parentBeingCreated = insertMap.has(parentId) || addedBlocks.has(parentId)
if (parentBeingCreated) {
// Only add dependency if parent is also being inserted (not added)
// Because adds run before inserts, added parents are already created
if (insertMap.has(parentId)) {
dependencies.get(blockId)!.add(parentId)
if (!dependents.has(parentId)) {
dependents.set(parentId, new Set())
}
dependents.get(parentId)!.add(blockId)
}
}
}
})
// Topological sort using Kahn's algorithm
const sorted: EditWorkflowOperation[] = []
const queue: string[] = []
// Start with nodes that have no dependencies (or depend only on added blocks)
inserts.forEach((op) => {
const deps = dependencies.get(op.block_id)!
if (deps.size === 0) {
queue.push(op.block_id)
}
})
while (queue.length > 0) {
const blockId = queue.shift()!
const op = insertMap.get(blockId)
if (op) {
sorted.push(op)
}
// Remove this node from dependencies of others
const children = dependents.get(blockId)
if (children) {
children.forEach((childId) => {
const childDeps = dependencies.get(childId)!
childDeps.delete(blockId)
if (childDeps.size === 0) {
queue.push(childId)
}
})
}
}
// If sorted length doesn't match input, there's a cycle (shouldn't happen with valid operations)
// Just append remaining operations
if (sorted.length < inserts.length) {
inserts.forEach((op) => {
if (!sorted.includes(op)) {
sorted.push(op)
}
})
}
return sorted
}
function orderOperations(operations: EditWorkflowOperation[]): EditWorkflowOperation[] {
/**
* Reorder operations to ensure correct execution sequence:
* 1. delete - Remove blocks first to free up IDs and clean state
* 2. extract_from_subflow - Extract blocks from subflows before modifications
* 3. add - Create new blocks (sorted by connection dependencies)
* 4. insert_into_subflow - Insert blocks into subflows (sorted by parent dependency)
* 5. edit - Edit existing blocks last, so connections to newly added blocks work
*/
const deletes = operations.filter((op) => op.operation_type === 'delete')
const extracts = operations.filter((op) => op.operation_type === 'extract_from_subflow')
const adds = operations.filter((op) => op.operation_type === 'add')
const inserts = operations.filter((op) => op.operation_type === 'insert_into_subflow')
const edits = operations.filter((op) => op.operation_type === 'edit')
// Sort insert operations to ensure parents are inserted before children
const sortedInserts = topologicalSortInserts(inserts, adds)
return [...deletes, ...extracts, ...adds, ...sortedInserts, ...edits]
}
/**
* Apply operations directly to the workflow JSON state
*/
export function applyOperationsToWorkflowState(
workflowState: Record<string, unknown>,
operations: EditWorkflowOperation[],
permissionConfig: PermissionGroupConfig | null = null
): ApplyOperationsResult {
// Deep clone the workflow state to avoid mutations
const modifiedState = structuredClone(workflowState)
// Collect validation errors across all operations
const validationErrors: ValidationError[] = []
// Collect skipped items across all operations
const skippedItems: SkippedItem[] = []
// Normalize block IDs to UUIDs before processing
const { normalizedOperations } = normalizeBlockIdsInOperations(operations)
// Order operations for deterministic application
const orderedOperations = orderOperations(normalizedOperations)
logger.info('Applying operations to workflow:', {
totalOperations: orderedOperations.length,
operationTypes: orderedOperations.reduce((acc: Record<string, number>, op) => {
acc[op.operation_type] = (acc[op.operation_type] || 0) + 1
return acc
}, {}),
initialBlockCount: Object.keys((modifiedState as any).blocks || {}).length,
})
const ctx: OperationContext = {
modifiedState,
skippedItems,
validationErrors,
permissionConfig,
deferredConnections: [],
}
for (const operation of orderedOperations) {
const { operation_type, block_id } = operation
// CRITICAL: Validate block_id is a valid string and not "undefined"
// This prevents undefined keys from being set in the workflow state
if (!isValidKey(block_id)) {
logSkippedItem(skippedItems, {
type: 'missing_required_params',
operationType: operation_type,
blockId: String(block_id || 'invalid'),
reason: `Invalid block_id "${block_id}" (type: ${typeof block_id}) - operation skipped. Block IDs must be valid non-empty strings.`,
})
logger.error('Invalid block_id detected in operation', {
operation_type,
block_id,
block_id_type: typeof block_id,
})
continue
}
const handler = OPERATION_HANDLERS[operation_type]
if (!handler) continue
logger.debug(`Executing operation: ${operation_type} for block ${block_id}`, {
params: operation.params ? Object.keys(operation.params) : [],
currentBlockCount: Object.keys((modifiedState as any).blocks || {}).length,
})
handler(operation, ctx)
}
// Pass 2: Create all edges from deferred connections
// All blocks exist at this point, so forward references resolve correctly
if (ctx.deferredConnections.length > 0) {
logger.info('Processing deferred connections from add/insert operations', {
deferredConnectionCount: ctx.deferredConnections.length,
totalBlocks: Object.keys((modifiedState as any).blocks || {}).length,
})
for (const { blockId, connections } of ctx.deferredConnections) {
// Verify the source block still exists (it might have been deleted by a later operation)
if (!(modifiedState as any).blocks[blockId]) {
logger.warn('Source block no longer exists for deferred connection', {
blockId,
availableBlocks: Object.keys((modifiedState as any).blocks || {}),
})
continue
}
addConnectionsAsEdges(modifiedState, blockId, connections, logger, skippedItems)
}
logger.info('Finished processing deferred connections', {
totalEdges: (modifiedState as any).edges?.length,
})
}
// Pass 3: resolve pending connections whose target block now exists. These are
// forward-reference connections that were recorded (on block.data) when their
// target didn't exist yet — possibly in an earlier edit_workflow call. Now
// that this batch's blocks are all created, create the edges that resolve.
resolvePendingConnections(modifiedState, skippedItems)
// Remove edges that cross scope boundaries. This runs after all operations
// and deferred connections are applied so that every block has its final
// parentId. Running it per-operation would incorrectly drop edges between
// blocks that are both being moved into the same subflow in one batch.
removeInvalidScopeEdges(modifiedState, skippedItems)
// Regenerate loops and parallels after modifications
;(modifiedState as any).loops = generateLoopBlocks((modifiedState as any).blocks)
;(modifiedState as any).parallels = generateParallelBlocks((modifiedState as any).blocks)
// Validate all blocks have types before returning
const blocksWithoutType = Object.entries((modifiedState as any).blocks || {})
.filter(([_, block]: [string, any]) => !block.type || block.type === undefined)
.map(([id, block]: [string, any]) => ({ id, block }))
if (blocksWithoutType.length > 0) {
logger.error('Blocks without type after operations:', {
blocksWithoutType: blocksWithoutType.map(({ id, block }) => ({
id,
type: block.type,
name: block.name,
keys: Object.keys(block),
})),
})
// Attempt to fix by removing type-less blocks
blocksWithoutType.forEach(({ id }) => {
delete (modifiedState as any).blocks[id]
})
// Remove edges connected to removed blocks
const removedIds = new Set(blocksWithoutType.map(({ id }) => id))
;(modifiedState as any).edges = ((modifiedState as any).edges || []).filter(
(edge: any) => !removedIds.has(edge.source) && !removedIds.has(edge.target)
)
}
return { state: modifiedState, validationErrors, skippedItems }
}
/**
* Resolves pending forward-reference connections recorded on block.data.
*
* When a connection references a target block that does not exist yet, the edge
* is recorded as pending on the source block instead of being dropped. This runs
* after all blocks for the current batch exist (and picks up pending entries
* persisted from earlier edit_workflow calls), creating any edges whose target
* now exists and leaving still-unresolved entries pending.
*/
function resolvePendingConnections(modifiedState: any, skippedItems: SkippedItem[]): void {
const blocks = modifiedState.blocks || {}
for (const [sourceId, block] of Object.entries(blocks) as [string, any][]) {
const pending = block?.data?.pendingConnections as
| Record<string, Array<{ target: string; targetHandle: string }>>
| undefined
if (!pending) continue
for (const [handle, targets] of Object.entries(pending)) {
const stillPending: Array<{ target: string; targetHandle: string }> = []
for (const { target, targetHandle } of targets) {
if (blocks[target]) {
createValidatedEdge(
modifiedState,
sourceId,
target,
handle,
targetHandle || 'target',
'resolve_pending_connection',
logger,
skippedItems
)
} else {
stillPending.push({ target, targetHandle })
}
}
if (stillPending.length > 0) {
pending[handle] = stillPending
} else {
delete pending[handle]
}
}
if (Object.keys(pending).length === 0) {
block.data.pendingConnections = undefined
}
}
}
/**
* Removes edges that cross scope boundaries after all operations are applied.
* An edge is invalid if:
* - Either endpoint no longer exists (dangling reference)
* - The source and target are in incompatible scopes
* - A child block connects to its own parent container (non-handle edge)
*
* Valid scope relationships:
* - Same scope: both blocks share the same parentId
* - Container→child: source is the parent container of the target (start handles)
* - Child→container: target is the parent container of the source (end handles)
*/
function removeInvalidScopeEdges(modifiedState: any, skippedItems: SkippedItem[]): void {
const { valid, dropped } = validateEdges(modifiedState.edges || [], modifiedState.blocks || {})
modifiedState.edges = valid
if (dropped.length > 0) {
for (const { edge, reason } of dropped) {
logSkippedItem(skippedItems, {
type: 'invalid_edge_scope',
operationType: 'add_edge',
blockId: edge.source,
reason: `Edge from "${edge.source}" to "${edge.target}" skipped - ${reason}`,
details: {
edgeId: edge.id,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
targetId: edge.target,
},
})
}
logger.info('Removed invalid workflow edges', {
removed: dropped.length,
reasons: dropped.map(({ reason }) => reason),
})
}
}
@@ -0,0 +1,417 @@
import { db } from '@sim/db'
import { workflow as workflowTable } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import {
assertWorkflowMutable,
authorizeWorkflowByWorkspacePermission,
} from '@sim/platform-authz/workflow'
import { toError } from '@sim/utils/errors'
import { eq } from 'drizzle-orm'
import { EditWorkflow } from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import { env } from '@/lib/core/config/env'
import { getSocketServerUrl } from '@/lib/core/utils/urls'
import {
applyTargetedLayout,
getTargetedLayoutImpact,
transferBlockHeights,
} from '@/lib/workflows/autolayout'
import {
DEFAULT_HORIZONTAL_SPACING,
DEFAULT_VERTICAL_SPACING,
} from '@/lib/workflows/autolayout/constants'
import { extractAndPersistCustomTools } from '@/lib/workflows/persistence/custom-tools-persistence'
import {
loadWorkflowFromNormalizedTables,
saveWorkflowToNormalizedTables,
} from '@/lib/workflows/persistence/utils'
import { validateWorkflowState } from '@/lib/workflows/sanitization/validation'
import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check'
import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils'
import { normalizeWorkflowState } from '@/stores/workflows/workflow/validation'
import { applyOperationsToWorkflowState } from './engine'
import {
collectWorkflowFieldIssues,
formatWorkflowLintMessage,
hasWorkflowLintIssues,
lintEditedWorkflowState,
type WorkflowLintReport,
type WorkflowLintUnresolvedReference,
} from './lint'
import { type EditWorkflowParams, isDeferredSkippedItem, type ValidationError } from './types'
import {
collectUnresolvedAgentToolReferences,
collectUnresolvedReferences,
preValidateCredentialInputs,
UNRESOLVABLE_AT_LINT_NOTE,
} from './validation'
async function getCurrentWorkflowStateFromDb(
workflowId: string
): Promise<{ workflowState: any; subBlockValues: Record<string, Record<string, any>> }> {
const logger = createLogger('EditWorkflowServerTool')
const [workflowRecord] = await db
.select()
.from(workflowTable)
.where(eq(workflowTable.id, workflowId))
.limit(1)
if (!workflowRecord) throw new Error(`Workflow ${workflowId} not found in database`)
const normalized = await loadWorkflowFromNormalizedTables(workflowId)
if (!normalized) throw new Error('Workflow has no normalized data')
const { state: validatedState, warnings } = normalizeWorkflowState({
blocks: normalized.blocks,
edges: normalized.edges,
loops: normalized.loops || {},
parallels: normalized.parallels || {},
})
if (warnings.length > 0) {
logger.warn('Normalized workflow state loaded from DB for copilot', {
workflowId,
warningCount: warnings.length,
warnings,
})
}
const subBlockValues: Record<string, Record<string, any>> = {}
Object.entries(validatedState.blocks).forEach(([blockId, block]) => {
subBlockValues[blockId] = {}
Object.entries((block as any).subBlocks || {}).forEach(([subId, sub]) => {
if ((sub as any).value !== undefined) subBlockValues[blockId][subId] = (sub as any).value
})
})
return { workflowState: validatedState, subBlockValues }
}
export const editWorkflowServerTool: BaseServerTool<EditWorkflowParams, unknown> = {
name: EditWorkflow.id,
async execute(params: EditWorkflowParams, context?: ServerToolContext): Promise<unknown> {
const logger = createLogger('EditWorkflowServerTool')
const { operations, workflowId, currentUserWorkflow } = params
if (!Array.isArray(operations) || operations.length === 0) {
throw new Error('operations are required and must be an array')
}
if (!workflowId) throw new Error('workflowId is required')
if (!context?.userId) {
throw new Error('Unauthorized workflow access')
}
const authorization = await authorizeWorkflowByWorkspacePermission({
workflowId,
userId: context.userId,
action: 'write',
})
if (!authorization.allowed) {
throw new Error(authorization.message || 'Unauthorized workflow access')
}
await assertWorkflowMutable(workflowId)
const workspaceId = authorization.workflow?.workspaceId ?? undefined
const workflowName = authorization.workflow?.name ?? undefined
logger.info('Executing edit_workflow', {
operationCount: operations.length,
workflowId,
hasCurrentUserWorkflow: !!currentUserWorkflow,
chatId: context.chatId,
})
assertServerToolNotAborted(context)
let workflowState: any
if (currentUserWorkflow) {
try {
workflowState = JSON.parse(currentUserWorkflow)
} catch (error) {
logger.error('Failed to parse currentUserWorkflow', error)
throw new Error('Invalid currentUserWorkflow format')
}
} else {
const fromDb = await getCurrentWorkflowStateFromDb(workflowId)
workflowState = fromDb.workflowState
}
const permissionConfig =
context?.userId && workspaceId
? await getUserPermissionConfig(context.userId, workspaceId)
: null
// Pre-validate credential and apiKey inputs before applying operations
// This filters out invalid credentials and apiKeys for hosted models
let operationsToApply = operations
const credentialErrors: ValidationError[] = []
if (context?.userId) {
const { filteredOperations, errors: credErrors } = await preValidateCredentialInputs(
operations,
{ userId: context.userId, workspaceId },
workflowState
)
operationsToApply = filteredOperations
credentialErrors.push(...credErrors)
}
// Apply operations directly to the workflow state
const {
state: modifiedWorkflowState,
validationErrors,
skippedItems,
} = applyOperationsToWorkflowState(workflowState, operationsToApply, permissionConfig)
// Add credential validation errors
validationErrors.push(...credentialErrors)
// Resolve credential/resource references against the workspace (Tier 2).
// Includes oauth-input credentials and only the active canonical member, so a
// credential "set in basic mode but unresolved in the dropdown" is caught.
let unresolvedReferences: WorkflowLintUnresolvedReference[] = []
if (context?.userId) {
try {
unresolvedReferences = await collectUnresolvedReferences(modifiedWorkflowState, {
userId: context.userId,
workspaceId,
})
// Back-compat: also surface unresolved references through the input-validation channel.
validationErrors.push(
...unresolvedReferences.map((ref) => ({
blockId: ref.blockId,
blockType: ref.blockType ?? 'unknown',
field: ref.field,
value: ref.value,
error: ref.reason,
}))
)
} catch (error) {
logger.warn('Selector ID validation failed', {
error: toError(error).message,
})
}
// Resolve agent-block tool/skill references (custom tools, MCP servers,
// skills). A well-shaped entry whose id does not resolve is dropped at
// runtime, so the agent silently loses the tool/skill - surface it through
// the same lint + input-validation channels as credential/resource refs.
try {
const toolReferences = await collectUnresolvedAgentToolReferences(modifiedWorkflowState, {
userId: context.userId,
workspaceId,
})
unresolvedReferences.push(...toolReferences)
validationErrors.push(
...toolReferences.map((ref) => ({
blockId: ref.blockId,
blockType: ref.blockType ?? 'agent',
field: ref.field,
value: ref.value,
error: ref.reason,
}))
)
} catch (error) {
logger.warn('Agent tool/skill reference validation failed', {
error: toError(error).message,
})
}
}
// Validate the workflow state
const validation = validateWorkflowState(modifiedWorkflowState, { sanitize: true })
if (!validation.valid) {
logger.error('Edited workflow state is invalid', {
errors: validation.errors,
warnings: validation.warnings,
})
throw new Error(`Invalid edited workflow: ${validation.errors.join('; ')}`)
}
if (validation.warnings.length > 0) {
logger.warn('Edited workflow validation warnings', {
warnings: validation.warnings,
})
}
// Extract and persist custom tools to database (reuse workspaceId from selector validation)
if (context?.userId && workspaceId) {
try {
assertServerToolNotAborted(context)
const finalWorkflowState = validation.sanitizedState || modifiedWorkflowState
const { saved, errors } = await extractAndPersistCustomTools(
finalWorkflowState,
workspaceId,
context.userId
)
if (saved > 0) {
logger.info(`Persisted ${saved} custom tool(s) to database`, { workflowId })
}
if (errors.length > 0) {
logger.warn('Some custom tools failed to persist', { errors, workflowId })
}
} catch (error) {
logger.error('Failed to persist custom tools', { error, workflowId })
}
} else if (context?.userId && !workspaceId) {
logger.warn('Workflow has no workspaceId, skipping custom tools persistence', {
workflowId,
})
} else {
logger.warn('No userId in context - skipping custom tools persistence', { workflowId })
}
logger.info('edit_workflow successfully applied operations', {
operationCount: operations.length,
blocksCount: Object.keys(modifiedWorkflowState.blocks).length,
edgesCount: modifiedWorkflowState.edges.length,
inputValidationErrors: validationErrors.length,
skippedItemsCount: skippedItems.length,
schemaValidationErrors: validation.errors.length,
validationWarnings: validation.warnings.length,
})
// Format validation errors for LLM feedback
const inputErrors =
validationErrors.length > 0
? validationErrors.map((e) => `Block "${e.blockId}" (${e.blockType}): ${e.error}`)
: undefined
// Split engine skipped items into genuine failures vs benign, self-healing
// deferrals. A deferred forward-reference edge (invalid_edge_target) is NOT
// a failure: the engine wires it automatically once its target block exists
// (this call or a later one) via pendingConnections. Surfacing it through
// the same "skipped" failure channel as real skips makes a literal model
// thrash (re-issuing a self-healing op). Keep them in separate result fields
// and preserve item.type/details so the prompt can branch on a
// machine-readable category instead of pattern-matching prose.
const mapSkippedItem = (item: (typeof skippedItems)[number]) => ({
type: item.type,
operationType: item.operationType,
blockId: item.blockId,
reason: item.reason,
...(item.details && { details: item.details }),
})
const genuineSkippedItems = skippedItems.filter((item) => !isDeferredSkippedItem(item))
const deferredItems = skippedItems.filter((item) => isDeferredSkippedItem(item))
const skippedDetails =
genuineSkippedItems.length > 0 ? genuineSkippedItems.map(mapSkippedItem) : undefined
const deferredDetails = deferredItems.length > 0 ? deferredItems.map(mapSkippedItem) : undefined
// Persist the workflow state to the database
const finalWorkflowState = validation.sanitizedState || modifiedWorkflowState
const { layoutBlockIds, resizedBlockIds, shiftSourceBlockIds } = getTargetedLayoutImpact({
before: workflowState,
after: finalWorkflowState,
})
let layoutedBlocks = finalWorkflowState.blocks
if (layoutBlockIds.length > 0 || resizedBlockIds.length > 0 || shiftSourceBlockIds.length > 0) {
try {
transferBlockHeights(workflowState.blocks, finalWorkflowState.blocks)
layoutedBlocks = applyTargetedLayout(finalWorkflowState.blocks, finalWorkflowState.edges, {
changedBlockIds: layoutBlockIds,
resizedBlockIds,
shiftSourceBlockIds,
horizontalSpacing: DEFAULT_HORIZONTAL_SPACING,
verticalSpacing: DEFAULT_VERTICAL_SPACING,
})
} catch (error) {
logger.warn('Targeted autolayout failed, using default positions', {
workflowId,
error: toError(error).message,
})
}
}
const workflowStateForDb = {
blocks: layoutedBlocks,
edges: finalWorkflowState.edges,
loops: generateLoopBlocks(layoutedBlocks as any),
parallels: generateParallelBlocks(layoutedBlocks as any),
lastSaved: Date.now(),
isDeployed: false,
}
// Aggregate lint report: graph (sources/sinks/orphans/ports) + Tier-1 config
// (required + canonical-mode) + Tier-2 resolution (credential/resource IDs).
const graphLint = lintEditedWorkflowState(workflowStateForDb as any)
const fieldIssues = collectWorkflowFieldIssues(workflowStateForDb.blocks as any)
const workflowLint: WorkflowLintReport = {
...graphLint,
fieldIssues,
unresolvedReferences,
notes: unresolvedReferences.length > 0 ? [UNRESOLVABLE_AT_LINT_NOTE] : [],
}
const workflowLintMessage = hasWorkflowLintIssues(workflowLint)
? formatWorkflowLintMessage(workflowLint)
: undefined
assertServerToolNotAborted(context)
const saveResult = await saveWorkflowToNormalizedTables(workflowId, workflowStateForDb as any)
if (!saveResult.success) {
logger.error('Failed to persist workflow state to database', {
workflowId,
error: saveResult.error,
})
throw new Error(`Failed to save workflow: ${saveResult.error}`)
}
// Update workflow's lastSynced timestamp
assertServerToolNotAborted(context)
await db
.update(workflowTable)
.set({
lastSynced: new Date(),
updatedAt: new Date(),
})
.where(eq(workflowTable.id, workflowId))
logger.info('Workflow state persisted to database', { workflowId })
fetch(`${getSocketServerUrl()}/api/workflow-updated`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': env.INTERNAL_API_SECRET,
},
body: JSON.stringify({ workflowId }),
}).catch((error) => {
logger.warn('Failed to notify socket server of workflow update', { workflowId, error })
})
const sanitizationWarnings = validation.warnings.length > 0 ? validation.warnings : undefined
return {
success: true,
workflowId,
workflowName: workflowName ?? 'Workflow',
workflowState: { ...finalWorkflowState, blocks: layoutedBlocks },
workflowLint,
...(workflowLintMessage && { workflowLintMessage }),
...(inputErrors && {
inputValidationErrors: inputErrors,
inputValidationMessage: `${inputErrors.length} input(s) were rejected due to validation errors. The workflow was still updated with valid inputs only. Errors: ${inputErrors.join('; ')}`,
}),
...(skippedDetails && {
skippedItems: skippedDetails,
skippedItemsMessage: `${skippedDetails.length} operation(s) were skipped (not applied) and need attention. Each item includes a machine-readable "type" (e.g. block_not_found, block_locked, duplicate_block_name, invalid_block_type, invalid_source_handle, invalid_target_handle, invalid_edge_scope). Details: ${skippedDetails.map((item) => item.reason).join('; ')}`,
}),
...(deferredDetails && {
deferredConnections: deferredDetails,
deferredMessage: `${deferredDetails.length} edge(s) were deferred because their target block does not exist yet. This is NOT a failure and does NOT need fixing: the engine wires these edges automatically once the target block exists (in this edit or a later one). Do not re-issue them. Only act on a deferred edge if its target id was a typo or hallucination that you do not intend to create. Details: ${deferredDetails.map((item) => item.reason).join('; ')}`,
}),
...(sanitizationWarnings && {
sanitizationWarnings,
sanitizationMessage: `${sanitizationWarnings.length} field(s) were automatically sanitized: ${sanitizationWarnings.join('; ')}`,
}),
}
},
}
@@ -0,0 +1,320 @@
import { describe, expect, it } from 'vitest'
import { hasWorkflowLintIssues, lintEditedWorkflowState } from './lint'
function baseBlock(id: string, type: string, name: string, subBlocks: Record<string, any> = {}) {
return {
id,
type,
name,
enabled: true,
position: { x: 0, y: 0 },
subBlocks,
outputs: {},
}
}
describe('lintEditedWorkflowState', () => {
it('reports orphan blocks but allows unconnected condition/router branches', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
condition: baseBlock('condition', 'condition', 'Condition', {
conditions: {
value: JSON.stringify([
{ id: 'condition-if', title: 'if', value: 'true' },
{ id: 'condition-else', title: 'else', value: '' },
]),
},
}),
router: baseBlock('router', 'router_v2', 'Router', {
routes: {
value: [
{ id: 'route-1', title: 'Route 1', value: 'support' },
{ id: 'route-2', title: 'Route 2', value: 'sales' },
],
},
}),
agent: baseBlock('agent', 'agent', 'Agent'),
function: baseBlock('function', 'function', 'Orphan Function'),
note: baseBlock('note', 'note', 'Note'),
},
edges: [
{
id: 'edge-start-condition',
source: 'start',
sourceHandle: 'source',
target: 'condition',
targetHandle: 'target',
},
{
id: 'edge-start-router',
source: 'start',
sourceHandle: 'source',
target: 'router',
targetHandle: 'target',
},
{
id: 'edge-condition-agent',
source: 'condition',
sourceHandle: 'if',
target: 'agent',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
expect(lint.orphanBlocks).toEqual([
{ blockId: 'function', blockName: 'Orphan Function', blockType: 'function' },
])
expect(lint.emptyOutgoingPorts).toEqual([])
expect(lint.invalidBranchPorts).toEqual([])
expect(hasWorkflowLintIssues(lint)).toBe(true)
})
it('reports invalid branch handles and missing connection targets', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
condition: baseBlock('condition', 'condition', 'Condition', {
conditions: {
value: [{ id: 'condition-if', title: 'if', value: 'true' }],
},
}),
agent: baseBlock('agent', 'agent', 'Agent'),
},
edges: [
{
id: 'edge-start-condition',
source: 'start',
sourceHandle: 'source',
target: 'condition',
targetHandle: 'target',
},
{
id: 'edge-condition-agent',
source: 'condition',
sourceHandle: 'else',
target: 'agent',
targetHandle: 'target',
},
{
id: 'edge-agent-missing',
source: 'agent',
sourceHandle: 'source',
target: 'missing',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
expect(lint.invalidBranchPorts).toEqual([
expect.objectContaining({
blockId: 'condition',
sourceHandle: 'else',
}),
])
expect(lint.invalidConnectionTargets).toEqual([
expect.objectContaining({
sourceBlockId: 'agent',
targetBlockId: 'missing',
reason: 'Connection target block does not exist',
}),
])
expect(hasWorkflowLintIssues(lint)).toBe(true)
})
it('returns clean result when every active block and dynamic port is connected', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
router: baseBlock('router', 'router_v2', 'Router', {
routes: {
value: [{ id: 'route-1', title: 'Route 1', value: 'support' }],
},
}),
agent: baseBlock('agent', 'agent', 'Agent'),
},
edges: [
{
id: 'edge-start-router',
source: 'start',
sourceHandle: 'source',
target: 'router',
targetHandle: 'target',
},
{
id: 'edge-router-agent',
source: 'router',
sourceHandle: 'route-0',
target: 'agent',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
expect(lint).toEqual({
sources: [{ blockId: 'start', blockName: 'Start', blockType: 'starter' }],
sinks: [{ blockId: 'agent', blockName: 'Agent', blockType: 'agent' }],
orphanBlocks: [],
emptyOutgoingPorts: [],
invalidBranchPorts: [],
invalidConnectionTargets: [],
})
expect(hasWorkflowLintIssues(lint)).toBe(false)
})
it('objectively reports multiple sources without turning disconnected islands into an issue', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
fetch: baseBlock('fetch', 'function', 'FetchCurrentFiles'),
normalize: baseBlock('normalize', 'function', 'NormalizeFiles'),
slack: { ...baseBlock('slack', 'slack', 'SlackTrigger'), triggerMode: true },
filter: baseBlock('filter', 'condition', 'MessageFilter'),
},
edges: [
{
id: 'e1',
source: 'start',
sourceHandle: 'source',
target: 'fetch',
targetHandle: 'target',
},
{
id: 'e2',
source: 'fetch',
sourceHandle: 'source',
target: 'normalize',
targetHandle: 'target',
},
{
id: 'e3',
source: 'slack',
sourceHandle: 'source',
target: 'filter',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
expect(lint.sources.map((b) => b.blockId).sort()).toEqual(['slack', 'start'])
expect(lint.orphanBlocks).toEqual([])
expect(lint.emptyOutgoingPorts).toEqual([])
expect(hasWorkflowLintIssues(lint)).toBe(false)
})
it('reports sources and sinks (triggers are sources, terminals are sinks, notes excluded)', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
agent: baseBlock('agent', 'agent', 'Agent'),
end: baseBlock('end', 'function', 'End'),
note: baseBlock('note', 'note', 'Note'),
},
edges: [
{
id: 'e1',
source: 'start',
sourceHandle: 'source',
target: 'agent',
targetHandle: 'target',
},
{
id: 'e2',
source: 'agent',
sourceHandle: 'source',
target: 'end',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
// 'start' has no incoming edge -> a source, even though it is NOT an orphan (trigger).
expect(lint.sources).toEqual([{ blockId: 'start', blockName: 'Start', blockType: 'starter' }])
expect(lint.orphanBlocks).toEqual([])
// 'end' has no outgoing edge -> a sink.
expect(lint.sinks).toEqual([{ blockId: 'end', blockName: 'End', blockType: 'function' }])
// 'agent' has both in and out edges -> neither source nor sink.
expect(lint.sources.map((b) => b.blockId)).not.toContain('agent')
expect(lint.sinks.map((b) => b.blockId)).not.toContain('agent')
// 'note' is excluded from both even though it has no edges.
expect(lint.sources.map((b) => b.blockId)).not.toContain('note')
expect(lint.sinks.map((b) => b.blockId)).not.toContain('note')
})
it('warns when loop/parallel start ports are empty', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
loop: baseBlock('loop', 'loop', 'Loop'),
parallel: baseBlock('parallel', 'parallel', 'Parallel'),
},
edges: [
{
id: 'e1',
source: 'start',
sourceHandle: 'source',
target: 'loop',
targetHandle: 'target',
},
{
id: 'e2',
source: 'loop',
sourceHandle: 'loop-end-source',
target: 'parallel',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
expect(lint.emptyOutgoingPorts.map((port) => `${port.blockName}.${port.handle}`)).toEqual([
'Loop.loop-start-source',
'Parallel.parallel-start-source',
])
expect(hasWorkflowLintIssues(lint)).toBe(true)
})
it('treats loop/parallel start edges as internal so containers can still be sinks', () => {
const workflowState = {
blocks: {
start: baseBlock('start', 'starter', 'Start'),
loop: baseBlock('loop', 'loop', 'Loop'),
child: baseBlock('child', 'function', 'Loop Child'),
},
edges: [
{
id: 'e1',
source: 'start',
sourceHandle: 'source',
target: 'loop',
targetHandle: 'target',
},
{
id: 'e2',
source: 'loop',
sourceHandle: 'loop-start-source',
target: 'child',
targetHandle: 'target',
},
],
}
const lint = lintEditedWorkflowState(workflowState as any)
expect(lint.emptyOutgoingPorts).toEqual([])
expect(lint.sinks.map((block) => block.blockId).sort()).toEqual(['child', 'loop'])
expect(hasWorkflowLintIssues(lint)).toBe(false)
})
})
@@ -0,0 +1,377 @@
import { getBlock } from '@/blocks'
import { isTriggerBlockType } from '@/executor/constants'
import {
collectBlockFieldIssues,
extractBlockParams,
type InactiveModeValue,
} from '@/serializer/index'
import type { WorkflowState } from '@/stores/workflows/workflow/types'
import { validateConditionHandle, validateRouterHandle } from './validation'
type BlockState = {
id?: string
type?: string
name?: string
triggerMode?: boolean
subBlocks?: Record<string, { value?: unknown } | undefined>
}
type EdgeState = {
source?: string | null
sourceHandle?: string | null
target?: string | null
}
export interface WorkflowLintBlockRef {
blockId: string
blockName?: string
blockType?: string
}
export interface WorkflowLintEmptyOutgoingPort extends WorkflowLintBlockRef {
handle: string
label: string
}
export interface WorkflowLintInvalidBranchPort extends WorkflowLintBlockRef {
sourceHandle: string
reason: string
}
export interface WorkflowLintInvalidConnectionTarget {
sourceBlockId: string
sourceBlockName?: string
sourceHandle?: string
targetBlockId: string
reason: string
}
export interface WorkflowLintResult {
/** Every non-note block with no incoming edge (trigger blocks are naturally sources). */
sources: WorkflowLintBlockRef[]
/** Every non-note block with no outgoing edge. */
sinks: WorkflowLintBlockRef[]
orphanBlocks: WorkflowLintBlockRef[]
emptyOutgoingPorts: WorkflowLintEmptyOutgoingPort[]
invalidBranchPorts: WorkflowLintInvalidBranchPort[]
invalidConnectionTargets: WorkflowLintInvalidConnectionTarget[]
}
/** Tier-1 (sync, config) field issues for a single block. */
export interface WorkflowLintFieldIssue extends WorkflowLintBlockRef {
/** Required fields that resolve empty in the active mode. */
missingRequiredFields: string[]
/** Canonical pairs whose value is stranded on the inactive member (silently dropped). */
inactiveModeValues: InactiveModeValue[]
}
/** Tier-2 (async, DB) reference that does not resolve to an accessible entity. */
export interface WorkflowLintUnresolvedReference extends WorkflowLintBlockRef {
field: string
value: string | string[]
kind: 'credential' | 'resource' | 'custom-tool' | 'mcp-tool' | 'skill'
reason: string
}
/**
* Aggregate lint report: the graph lint plus the config (Tier 1) and resolution
* (Tier 2) checks. Returned in the edit_workflow result and written to lint.json.
*/
export interface WorkflowLintReport extends WorkflowLintResult {
fieldIssues: WorkflowLintFieldIssue[]
unresolvedReferences: WorkflowLintUnresolvedReference[]
notes: string[]
}
function blockRef(blockId: string, block: BlockState): WorkflowLintBlockRef {
return {
blockId,
blockName: block.name,
blockType: block.type,
}
}
function isWorkflowEntryBlock(block: BlockState) {
return Boolean(block.triggerMode) || isTriggerBlockType(block.type)
}
function requiredSubflowStartPort(block: BlockState) {
if (block.type === 'loop') {
return { handle: 'loop-start-source', label: 'loop-start-source' }
}
if (block.type === 'parallel') {
return { handle: 'parallel-start-source', label: 'parallel-start-source' }
}
return null
}
function countsAsExternalOutgoing(block: BlockState, sourceHandle?: string | null) {
if (block.type === 'loop') {
return sourceHandle !== 'loop-start-source'
}
if (block.type === 'parallel') {
return sourceHandle !== 'parallel-start-source'
}
return true
}
export function lintEditedWorkflowState(workflowState: Pick<WorkflowState, 'blocks' | 'edges'>) {
const blocks = (workflowState.blocks || {}) as Record<string, BlockState>
const edges = Array.isArray(workflowState.edges)
? (workflowState.edges as EdgeState[])
: ([] as EdgeState[])
const incomingEdgesByTarget = new Map<string, number>()
const outgoingEdgesBySource = new Set<string>()
const connectedDynamicHandles = new Map<string, Set<string>>()
const invalidBranchPorts: WorkflowLintInvalidBranchPort[] = []
const invalidConnectionTargets: WorkflowLintInvalidConnectionTarget[] = []
for (const edge of edges) {
const sourceBlockId = edge?.source || ''
const targetBlockId = edge?.target || ''
const sourceBlock = blocks[sourceBlockId]
const targetBlock = blocks[targetBlockId]
if (!sourceBlock || !targetBlock) {
invalidConnectionTargets.push({
sourceBlockId: sourceBlockId || 'unknown',
sourceBlockName: sourceBlock?.name,
sourceHandle: edge?.sourceHandle ?? undefined,
targetBlockId: targetBlockId || 'unknown',
reason: !sourceBlock
? 'Connection source block does not exist'
: 'Connection target block does not exist',
})
continue
}
incomingEdgesByTarget.set(targetBlockId, (incomingEdgesByTarget.get(targetBlockId) || 0) + 1)
if (countsAsExternalOutgoing(sourceBlock, edge?.sourceHandle)) {
outgoingEdgesBySource.add(sourceBlockId)
}
const sourceHandle = edge?.sourceHandle
if (!sourceHandle || sourceHandle === 'error') continue
if (sourceBlock.type === 'condition' || sourceBlock.type === 'router_v2') {
const validation =
sourceBlock.type === 'condition'
? validateConditionHandle(
sourceHandle,
sourceBlockId,
sourceBlock.subBlocks?.conditions?.value as string | any[]
)
: validateRouterHandle(
sourceHandle,
sourceBlockId,
sourceBlock.subBlocks?.routes?.value as string | any[]
)
if (!validation.valid) {
invalidBranchPorts.push({
...blockRef(sourceBlockId, sourceBlock),
sourceHandle,
reason: validation.error || `Invalid branch handle "${sourceHandle}"`,
})
continue
}
const normalizedHandle = validation.normalizedHandle || sourceHandle
const handles = connectedDynamicHandles.get(sourceBlockId) || new Set<string>()
handles.add(normalizedHandle)
connectedDynamicHandles.set(sourceBlockId, handles)
continue
}
const handles = connectedDynamicHandles.get(sourceBlockId) || new Set<string>()
handles.add(sourceHandle)
connectedDynamicHandles.set(sourceBlockId, handles)
}
const orphanBlocks = Object.entries(blocks)
.filter(([, block]) => block.type !== 'note' && !isWorkflowEntryBlock(block))
.filter(([blockId]) => !incomingEdgesByTarget.has(blockId))
.map(([blockId, block]) => blockRef(blockId, block))
// Structural descriptors (advisory, not "issues"): sources have no incoming
// edge (trigger blocks are naturally sources), sinks have no outgoing edge.
const sources = Object.entries(blocks)
.filter(([, block]) => block.type !== 'note')
.filter(([blockId]) => !incomingEdgesByTarget.has(blockId))
.map(([blockId, block]) => blockRef(blockId, block))
const sinks = Object.entries(blocks)
.filter(([, block]) => block.type !== 'note')
.filter(([blockId]) => !outgoingEdgesBySource.has(blockId))
.map(([blockId, block]) => blockRef(blockId, block))
const emptyOutgoingPorts = Object.entries(blocks).flatMap(([blockId, block]) => {
const handles = connectedDynamicHandles.get(blockId) || new Set<string>()
const requiredPort = requiredSubflowStartPort(block)
const ports = requiredPort ? [requiredPort] : []
return ports
.filter((port) => !handles.has(port.handle))
.map((port) => ({
...blockRef(blockId, block),
handle: port.handle,
label: port.label,
}))
})
return {
sources,
sinks,
orphanBlocks,
emptyOutgoingPorts,
invalidBranchPorts,
invalidConnectionTargets,
} satisfies WorkflowLintResult
}
/**
* Tier-1 config lint: per-block required-field and canonical-mode (inactive
* member) issues. Pure/sync. Uses the shared collector so results match the
* runtime serializer's required-field semantics. Skips notes and subflow
* containers, and blocks with no registry config.
*/
export function collectWorkflowFieldIssues(
blocks: WorkflowState['blocks'] | Record<string, unknown> | undefined
): WorkflowLintFieldIssue[] {
const results: WorkflowLintFieldIssue[] = []
for (const [blockId, block] of Object.entries(blocks || {})) {
const type = (block as { type?: string })?.type
if (!type || type === 'note' || type === 'loop' || type === 'parallel') continue
const blockConfig = getBlock(type)
if (!blockConfig) continue
let params: Record<string, any>
try {
params = extractBlockParams(block as any)
} catch {
continue
}
const { missingRequiredFields, inactiveModeValues } = collectBlockFieldIssues(
block as any,
blockConfig,
params
)
if (missingRequiredFields.length > 0 || inactiveModeValues.length > 0) {
results.push({
...blockRef(blockId, block as BlockState),
missingRequiredFields,
inactiveModeValues,
})
}
}
return results
}
type WorkflowLintIssueView = WorkflowLintResult & {
fieldIssues?: WorkflowLintFieldIssue[]
unresolvedReferences?: WorkflowLintUnresolvedReference[]
}
export function hasWorkflowLintIssues(lint: WorkflowLintIssueView) {
return (
lint.orphanBlocks.length > 0 ||
lint.emptyOutgoingPorts.length > 0 ||
lint.invalidBranchPorts.length > 0 ||
lint.invalidConnectionTargets.length > 0 ||
(lint.fieldIssues?.length ?? 0) > 0 ||
(lint.unresolvedReferences?.length ?? 0) > 0
)
}
export function formatWorkflowLintMessage(lint: WorkflowLintIssueView) {
const parts: string[] = []
if (lint.orphanBlocks.length > 0) {
parts.push(
`Blocks with no incoming edge: ${lint.orphanBlocks
.map((block) => `"${block.blockName || block.blockId}" (${block.blockType || 'unknown'})`)
.join(', ')}`
)
}
if (lint.emptyOutgoingPorts.length > 0) {
parts.push(
`Unconnected required subflow start ports: ${lint.emptyOutgoingPorts
.map((port) => `"${port.blockName || port.blockId}".${port.label}`)
.join(', ')}`
)
}
if (lint.invalidBranchPorts.length > 0) {
parts.push(
`Invalid condition/router branch handles: ${lint.invalidBranchPorts
.map((port) => `"${port.blockName || port.blockId}" uses "${port.sourceHandle}"`)
.join(', ')}`
)
}
if (lint.invalidConnectionTargets.length > 0) {
parts.push(
`Connections pointing at missing blocks: ${lint.invalidConnectionTargets
.map((edge) => `${edge.sourceBlockId} -> ${edge.targetBlockId}`)
.join(', ')}`
)
}
const fieldIssues = lint.fieldIssues ?? []
const missing = fieldIssues.filter((issue) => issue.missingRequiredFields.length > 0)
if (missing.length > 0) {
parts.push(
`Blocks missing required fields: ${missing
.map(
(issue) =>
`"${issue.blockName || issue.blockId}" (${issue.missingRequiredFields.join(', ')})`
)
.join(', ')}`
)
}
const inactive = fieldIssues.filter((issue) => issue.inactiveModeValues.length > 0)
if (inactive.length > 0) {
parts.push(
`Values set on the inactive field mode (they will not resolve): ${inactive
.map(
(issue) =>
`"${issue.blockName || issue.blockId}" (${issue.inactiveModeValues
.map(
(v) =>
`${v.inactiveMemberId}: move the value to "${v.activeMemberId ?? v.canonicalId}"`
)
.join('; ')})`
)
.join(', ')}`
)
}
const unresolved = lint.unresolvedReferences ?? []
const credResourceRefs = unresolved.filter(
(ref) => ref.kind === 'credential' || ref.kind === 'resource'
)
if (credResourceRefs.length > 0) {
parts.push(
`Credential/resource references that do not resolve: ${credResourceRefs
.map((ref) => `"${ref.blockName || ref.blockId}".${ref.field} (${ref.reason})`)
.join(', ')}`
)
}
const toolSkillRefs = unresolved.filter(
(ref) => ref.kind === 'custom-tool' || ref.kind === 'mcp-tool' || ref.kind === 'skill'
)
if (toolSkillRefs.length > 0) {
parts.push(
`Agent tool/skill references that do not resolve (they will not attach at runtime): ${toolSkillRefs
.map((ref) => `"${ref.blockName || ref.blockId}".${ref.field} (${ref.reason})`)
.join(', ')}`
)
}
return `Workflow lint found issues. Fix these before continuing: ${parts.join('; ')}`
}
@@ -0,0 +1,530 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
import { applyOperationsToWorkflowState } from './engine'
vi.mock('@/blocks/registry', () => ({
getAllBlocks: () => [
{
type: 'condition',
name: 'Condition',
subBlocks: [{ id: 'conditions', type: 'condition-input' }],
},
{
type: 'agent',
name: 'Agent',
subBlocks: [
{ id: 'systemPrompt', type: 'long-input' },
{ id: 'model', type: 'combobox' },
],
},
{
type: 'function',
name: 'Function',
subBlocks: [
{ id: 'code', type: 'code' },
{ id: 'language', type: 'dropdown' },
],
},
],
getBlock: (type: string) => {
const blocks: Record<string, any> = {
condition: {
type: 'condition',
name: 'Condition',
subBlocks: [{ id: 'conditions', type: 'condition-input' }],
},
agent: {
type: 'agent',
name: 'Agent',
subBlocks: [
{ id: 'systemPrompt', type: 'long-input' },
{ id: 'model', type: 'combobox' },
],
},
function: {
type: 'function',
name: 'Function',
subBlocks: [
{ id: 'code', type: 'code' },
{ id: 'language', type: 'dropdown' },
],
},
}
return blocks[type] || undefined
},
}))
function makeLoopWorkflow() {
return {
blocks: {
'loop-1': {
id: 'loop-1',
type: 'loop',
name: 'Loop 1',
position: { x: 0, y: 0 },
enabled: true,
subBlocks: {},
outputs: {},
data: { loopType: 'for', count: 5 },
},
'condition-1': {
id: 'condition-1',
type: 'condition',
name: 'Condition 1',
position: { x: 100, y: 100 },
enabled: true,
subBlocks: {
conditions: {
id: 'conditions',
type: 'condition-input',
value: JSON.stringify([
{ id: 'condition-1-if', title: 'if', value: 'true' },
{ id: 'condition-1-else', title: 'else', value: '' },
]),
},
},
outputs: {},
data: { parentId: 'loop-1', extent: 'parent' },
},
'agent-1': {
id: 'agent-1',
type: 'agent',
name: 'Agent 1',
position: { x: 300, y: 100 },
enabled: true,
subBlocks: {
systemPrompt: { id: 'systemPrompt', type: 'long-input', value: 'You are helpful' },
model: { id: 'model', type: 'combobox', value: 'gpt-4o' },
},
outputs: {},
data: { parentId: 'loop-1', extent: 'parent' },
},
},
edges: [
{
id: 'edge-1',
source: 'loop-1',
sourceHandle: 'loop-start-source',
target: 'condition-1',
targetHandle: 'target',
type: 'default',
},
{
id: 'edge-2',
source: 'condition-1',
sourceHandle: 'condition-condition-1-if',
target: 'agent-1',
targetHandle: 'target',
type: 'default',
},
],
loops: {},
parallels: {},
}
}
function makeNestedLoopWorkflow() {
return {
blocks: {
'outer-loop': {
id: 'outer-loop',
type: 'loop',
name: 'Outer Loop',
position: { x: 0, y: 0 },
enabled: true,
subBlocks: {},
outputs: {},
data: { loopType: 'for', count: 2 },
},
'inner-loop': {
id: 'inner-loop',
type: 'loop',
name: 'Inner Loop',
position: { x: 120, y: 80 },
enabled: true,
subBlocks: {},
outputs: {},
data: { parentId: 'outer-loop', extent: 'parent', loopType: 'for', count: 3 },
},
'inner-agent': {
id: 'inner-agent',
type: 'agent',
name: 'Inner Agent',
position: { x: 240, y: 120 },
enabled: true,
subBlocks: {
systemPrompt: { id: 'systemPrompt', type: 'long-input', value: 'Original prompt' },
model: { id: 'model', type: 'combobox', value: 'gpt-4o' },
},
outputs: {},
data: { parentId: 'inner-loop', extent: 'parent' },
},
},
edges: [
{
id: 'edge-outer-inner',
source: 'outer-loop',
sourceHandle: 'loop-start-source',
target: 'inner-loop',
targetHandle: 'target',
type: 'default',
},
{
id: 'edge-inner-agent',
source: 'inner-loop',
sourceHandle: 'loop-start-source',
target: 'inner-agent',
targetHandle: 'target',
type: 'default',
},
],
loops: {},
parallels: {},
}
}
describe('handleEditOperation nestedNodes merge', () => {
it('preserves existing child block IDs when editing a loop with nestedNodes', () => {
const workflow = makeLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'loop-1',
params: {
nestedNodes: {
'new-condition': {
type: 'condition',
name: 'Condition 1',
inputs: {
conditions: [
{ id: 'x', title: 'if', value: 'x > 1' },
{ id: 'y', title: 'else', value: '' },
],
},
},
'new-agent': {
type: 'agent',
name: 'Agent 1',
inputs: { systemPrompt: 'Updated prompt' },
},
},
},
},
])
expect(state.blocks['condition-1']).toBeDefined()
expect(state.blocks['agent-1']).toBeDefined()
expect(state.blocks['new-condition']).toBeUndefined()
expect(state.blocks['new-agent']).toBeUndefined()
})
it('preserves edges for matched children when connections are not provided', () => {
const workflow = makeLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'loop-1',
params: {
nestedNodes: {
x: { type: 'condition', name: 'Condition 1' },
y: { type: 'agent', name: 'Agent 1' },
},
},
},
])
const conditionEdge = state.edges.find((e: any) => e.source === 'condition-1')
expect(conditionEdge).toBeDefined()
})
it('removes children not present in incoming nestedNodes', () => {
const workflow = makeLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'loop-1',
params: {
nestedNodes: {
x: { type: 'condition', name: 'Condition 1' },
},
},
},
])
expect(state.blocks['condition-1']).toBeDefined()
expect(state.blocks['agent-1']).toBeUndefined()
const agentEdges = state.edges.filter(
(e: any) => e.source === 'agent-1' || e.target === 'agent-1'
)
expect(agentEdges).toHaveLength(0)
})
it('creates new children that do not match existing ones', () => {
const workflow = makeLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'loop-1',
params: {
nestedNodes: {
x: { type: 'condition', name: 'Condition 1' },
y: { type: 'agent', name: 'Agent 1' },
'new-func': { type: 'function', name: 'Function 1', inputs: { code: 'return 1' } },
},
},
},
])
expect(state.blocks['condition-1']).toBeDefined()
expect(state.blocks['agent-1']).toBeDefined()
const funcBlock = Object.values(state.blocks).find((b: any) => b.name === 'Function 1')
expect(funcBlock).toBeDefined()
expect((funcBlock as any).data?.parentId).toBe('loop-1')
})
it('updates inputs on matched children without changing their ID', () => {
const workflow = makeLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'loop-1',
params: {
nestedNodes: {
x: {
type: 'agent',
name: 'Agent 1',
inputs: { systemPrompt: 'New prompt' },
},
y: { type: 'condition', name: 'Condition 1' },
},
},
},
])
const agent = state.blocks['agent-1']
expect(agent).toBeDefined()
expect(agent.subBlocks.systemPrompt.value).toBe('New prompt')
})
it('recursively updates an existing nested loop and preserves grandchild IDs', () => {
const workflow = makeNestedLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'outer-loop',
params: {
nestedNodes: {
'new-inner-loop': {
type: 'loop',
name: 'Inner Loop',
inputs: {
loopType: 'forEach',
collection: '<start.input.items>',
},
nestedNodes: {
'new-inner-agent': {
type: 'agent',
name: 'Inner Agent',
inputs: { systemPrompt: 'Updated prompt' },
},
'new-helper': {
type: 'function',
name: 'Helper',
inputs: { code: 'return 1' },
},
},
},
},
},
},
])
expect(state.blocks['inner-loop']).toBeDefined()
expect(state.blocks['new-inner-loop']).toBeUndefined()
expect(state.blocks['inner-loop'].data.loopType).toBe('forEach')
expect(state.blocks['inner-loop'].data.collection).toBe('<start.input.items>')
expect(state.blocks['inner-agent']).toBeDefined()
expect(state.blocks['new-inner-agent']).toBeUndefined()
expect(state.blocks['inner-agent'].subBlocks.systemPrompt.value).toBe('Updated prompt')
const helperBlock = Object.values(state.blocks).find((block: any) => block.name === 'Helper') as
| any
| undefined
expect(helperBlock).toBeDefined()
expect(helperBlock?.data?.parentId).toBe('inner-loop')
})
it('removes grandchildren omitted from an existing nested loop update', () => {
const workflow = makeNestedLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'outer-loop',
params: {
nestedNodes: {
'new-inner-loop': {
type: 'loop',
name: 'Inner Loop',
nestedNodes: {
'new-helper': {
type: 'function',
name: 'Helper',
inputs: { code: 'return 1' },
},
},
},
},
},
},
])
expect(state.blocks['inner-loop']).toBeDefined()
expect(state.blocks['inner-agent']).toBeUndefined()
expect(
state.edges.some(
(edge: any) => edge.source === 'inner-agent' || edge.target === 'inner-agent'
)
).toBe(false)
const helperBlock = Object.values(state.blocks).find((block: any) => block.name === 'Helper')
expect(helperBlock).toBeDefined()
})
it('removes an unmatched nested container with all descendants and edges', () => {
const workflow = makeNestedLoopWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'edit',
block_id: 'outer-loop',
params: {
nestedNodes: {
replacement: {
type: 'function',
name: 'Replacement',
inputs: { code: 'return 2' },
},
},
},
},
])
expect(state.blocks['inner-loop']).toBeUndefined()
expect(state.blocks['inner-agent']).toBeUndefined()
expect(
state.edges.some(
(edge: any) =>
edge.source === 'inner-loop' ||
edge.target === 'inner-loop' ||
edge.source === 'inner-agent' ||
edge.target === 'inner-agent'
)
).toBe(false)
const replacementBlock = Object.values(state.blocks).find(
(block: any) => block.name === 'Replacement'
) as any
expect(replacementBlock).toBeDefined()
expect(replacementBlock.data?.parentId).toBe('outer-loop')
})
})
describe('forward-reference connections (pending resolution)', () => {
function makeMinimalWorkflow() {
return {
blocks: {
'start-1': {
id: 'start-1',
type: 'function',
name: 'Start',
position: { x: 0, y: 0 },
enabled: true,
subBlocks: {},
outputs: {},
data: {},
},
},
edges: [] as any[],
loops: {},
parallels: {},
}
}
// Valid UUIDs so block_ids are not normalized/remapped on add.
const BLOCK_A = '11111111-1111-4111-8111-111111111111'
const BLOCK_B = '22222222-2222-4222-8222-222222222222'
it('defers a connection to a not-yet-created block and resolves it on a later apply', () => {
const workflow = makeMinimalWorkflow()
// First apply: add block A connecting to block B, which does not exist yet.
const first = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'add',
block_id: BLOCK_A,
params: {
type: 'function',
name: 'Block A',
inputs: { code: 'return 1' },
connections: { source: BLOCK_B },
},
},
])
// No edge created yet; the connection is recorded as pending on block A.
expect(first.state.edges.some((e: any) => e.target === BLOCK_B)).toBe(false)
expect(first.state.blocks[BLOCK_A].data.pendingConnections.source).toEqual([
{ target: BLOCK_B, targetHandle: 'target' },
])
// Second apply (simulating a later edit_workflow call): add block B.
const second = applyOperationsToWorkflowState(first.state, [
{
operation_type: 'add',
block_id: BLOCK_B,
params: { type: 'function', name: 'Block B', inputs: { code: 'return 2' } },
},
])
// The pending edge is now created and the pending record cleared.
const edge = second.state.edges.find((e: any) => e.source === BLOCK_A && e.target === BLOCK_B)
expect(edge).toBeDefined()
expect(second.state.blocks[BLOCK_A].data?.pendingConnections).toBeUndefined()
})
it('resolves a forward-reference connection within a single apply regardless of operation order', () => {
const workflow = makeMinimalWorkflow()
const { state } = applyOperationsToWorkflowState(workflow, [
{
operation_type: 'add',
block_id: BLOCK_A,
params: {
type: 'function',
name: 'Block A',
inputs: { code: 'return 1' },
connections: { source: BLOCK_B },
},
},
{
operation_type: 'add',
block_id: BLOCK_B,
params: { type: 'function', name: 'Block B', inputs: { code: 'return 2' } },
},
])
const edge = state.edges.find((e: any) => e.source === BLOCK_A && e.target === BLOCK_B)
expect(edge).toBeDefined()
expect(state.blocks[BLOCK_A].data?.pendingConnections).toBeUndefined()
})
})
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,151 @@
import { createLogger } from '@sim/logger'
import type { PermissionGroupConfig } from '@/lib/permission-groups/types'
/** Selector subblock types that can be validated */
export const SELECTOR_TYPES = new Set([
'oauth-input',
'knowledge-base-selector',
'document-selector',
'file-selector',
'project-selector',
'channel-selector',
'folder-selector',
'mcp-server-selector',
'mcp-tool-selector',
'workflow-selector',
])
const validationLogger = createLogger('EditWorkflowValidation')
/**
* Validation error for a specific field
*/
export interface ValidationError {
blockId: string
blockType: string
field: string
value: any
error: string
}
/**
* Types of items that can be skipped during operation application
*/
export type SkippedItemType =
| 'block_not_found'
| 'invalid_block_type'
| 'block_not_allowed'
| 'block_locked'
| 'tool_not_allowed'
| 'invalid_edge_target'
| 'invalid_edge_source'
| 'invalid_edge_scope'
| 'invalid_source_handle'
| 'invalid_target_handle'
| 'invalid_subblock_field'
| 'missing_required_params'
| 'invalid_subflow_parent'
| 'nested_subflow_not_allowed'
| 'duplicate_block_name'
| 'reserved_block_name'
| 'duplicate_trigger'
| 'duplicate_single_instance_block'
/**
* Represents an item that was skipped during operation application
*/
export interface SkippedItem {
type: SkippedItemType
operationType: string
blockId: string
reason: string
details?: Record<string, any>
}
/**
* Skipped-item types that represent benign, SELF-HEALING deferrals rather than
* failures. A deferred forward-reference edge (`invalid_edge_target`) is
* recorded as a pending connection and wired automatically once its target
* block exists -- possibly on a later edit_workflow call. It must be surfaced to
* the model as informational, NOT through the "skipped/failed operation"
* channel; otherwise a literal model re-issues the self-healing operation in a
* loop. See `createValidatedEdge` (builders.ts) and `resolvePendingConnections`
* (engine.ts).
*/
export const DEFERRED_SKIPPED_ITEM_TYPES: ReadonlySet<SkippedItemType> = new Set([
'invalid_edge_target',
])
/** Whether a skipped item is a benign deferral (see DEFERRED_SKIPPED_ITEM_TYPES). */
export function isDeferredSkippedItem(item: SkippedItem): boolean {
return DEFERRED_SKIPPED_ITEM_TYPES.has(item.type)
}
/**
* Logs and records a skipped item
*/
export function logSkippedItem(skippedItems: SkippedItem[], item: SkippedItem): void {
validationLogger.warn(`Skipped ${item.operationType} operation: ${item.reason}`, {
type: item.type,
operationType: item.operationType,
blockId: item.blockId,
...(item.details && { details: item.details }),
})
skippedItems.push(item)
}
/**
* Result of input validation
*/
export interface ValidationResult {
validInputs: Record<string, any>
errors: ValidationError[]
}
/**
* Result of validating a single value
*/
export interface ValueValidationResult {
valid: boolean
value?: any
error?: ValidationError
}
export interface EditWorkflowOperation {
operation_type: 'add' | 'edit' | 'delete' | 'insert_into_subflow' | 'extract_from_subflow'
block_id: string
params?: Record<string, any>
}
export interface EditWorkflowParams {
operations: EditWorkflowOperation[]
workflowId: string
currentUserWorkflow?: string
}
export interface EdgeHandleValidationResult {
valid: boolean
error?: string
/** The normalized handle to use (e.g., simple 'if' normalized to 'condition-{uuid}') */
normalizedHandle?: string
}
/**
* Result of applying operations to workflow state
*/
export interface ApplyOperationsResult {
state: any
validationErrors: ValidationError[]
skippedItems: SkippedItem[]
}
export interface OperationContext {
modifiedState: any
skippedItems: SkippedItem[]
validationErrors: ValidationError[]
permissionConfig: PermissionGroupConfig | null
deferredConnections: Array<{
blockId: string
connections: Record<string, any>
}>
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,156 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { listLogsMock, fetchLogDetailMock, toOverviewMock, toFullMock, grepSpansMock } = vi.hoisted(
() => ({
listLogsMock: vi.fn(),
fetchLogDetailMock: vi.fn(),
toOverviewMock: vi.fn(),
toFullMock: vi.fn(),
grepSpansMock: vi.fn(),
})
)
vi.mock('@/lib/logs/list-logs', () => ({ listLogs: listLogsMock }))
vi.mock('@/lib/logs/fetch-log-detail', () => ({ fetchLogDetail: fetchLogDetailMock }))
vi.mock('@/lib/logs/log-views', () => ({
toOverview: toOverviewMock,
toFull: toFullMock,
grepSpans: grepSpansMock,
}))
vi.mock('@/lib/execution/payloads/large-execution-value', () => ({
collectLargeValueExecutionIds: vi.fn(() => []),
collectLargeValueKeys: vi.fn(() => []),
}))
import { queryLogsServerTool } from './query-logs'
const ctx = { userId: 'user-1', workspaceId: 'ws-1' }
function detail(overrides: Record<string, unknown> = {}) {
return {
executionId: 'exec-1',
workflowId: 'wf-1',
status: 'success',
trigger: 'manual',
cost: { total: 0.1 },
executionData: { totalDuration: 1234, traceSpans: [{ id: 's1' }] },
...overrides,
}
}
beforeEach(() => {
vi.clearAllMocks()
})
describe('queryLogsServerTool', () => {
it('list view delegates to listLogs with workspaceId and no view field', async () => {
listLogsMock.mockResolvedValue({ data: [{ id: 'log-1' }], nextCursor: null })
const result = await queryLogsServerTool.execute(
{ view: 'list', sortBy: 'date', sortOrder: 'desc', limit: 100 } as any,
ctx
)
expect(listLogsMock).toHaveBeenCalledTimes(1)
const [params, userId] = listLogsMock.mock.calls[0]
expect(userId).toBe('user-1')
expect(params.workspaceId).toBe('ws-1')
expect(params).not.toHaveProperty('view')
expect(result).toEqual({ data: [{ id: 'log-1' }], nextCursor: null })
})
it('overview view returns the projected span tree', async () => {
fetchLogDetailMock.mockResolvedValue(detail())
toOverviewMock.mockReturnValue([{ id: 's1', name: 'A' }])
const result: any = await queryLogsServerTool.execute(
{ view: 'overview', executionId: 'exec-1' } as any,
ctx
)
expect(result.executionId).toBe('exec-1')
expect(result.durationMs).toBe(1234)
expect(result.spans).toEqual([{ id: 's1', name: 'A' }])
expect(toFullMock).not.toHaveBeenCalled()
})
it('full view returns materialized spans', async () => {
fetchLogDetailMock.mockResolvedValue(detail())
toFullMock.mockResolvedValue([{ id: 's1', input: { a: 1 } }])
const result: any = await queryLogsServerTool.execute(
{ view: 'full', executionId: 'exec-1', blockId: 'blk-1' } as any,
ctx
)
expect(toFullMock).toHaveBeenCalledWith(expect.anything(), expect.anything(), {
blockId: 'blk-1',
blockName: undefined,
})
expect(result.spans).toEqual([{ id: 's1', input: { a: 1 } }])
expect(result.truncated).toBe(false)
})
it('full view falls back to overview when the result is too large', async () => {
fetchLogDetailMock.mockResolvedValue(detail())
const huge = 'x'.repeat(600 * 1024)
toFullMock.mockResolvedValue([{ id: 's1', output: huge }])
toOverviewMock.mockReturnValue([{ id: 's1', name: 'A' }])
const result: any = await queryLogsServerTool.execute(
{ view: 'full', executionId: 'exec-1' } as any,
ctx
)
expect(result.truncated).toBe(true)
expect(result.note).toContain('too large')
expect(result.spans).toEqual([{ id: 's1', name: 'A' }])
})
it('pattern runs grepSpans and returns matches', async () => {
fetchLogDetailMock.mockResolvedValue(detail())
grepSpansMock.mockResolvedValue({
matches: [{ spanId: 's1', name: 'A', field: 'output', snippet: '…timeout…' }],
truncated: false,
})
const result: any = await queryLogsServerTool.execute(
{ view: 'full', executionId: 'exec-1', pattern: 'timeout' } as any,
ctx
)
expect(grepSpansMock).toHaveBeenCalledTimes(1)
expect(result.pattern).toBe('timeout')
expect(result.matches).toHaveLength(1)
expect(toFullMock).not.toHaveBeenCalled()
})
it('returns not-found for an unknown executionId', async () => {
fetchLogDetailMock.mockResolvedValue(null)
const result: any = await queryLogsServerTool.execute(
{ view: 'overview', executionId: 'missing' } as any,
ctx
)
expect(result.ok).toBe(false)
expect(result.error).toContain('missing')
})
it('throws when unauthenticated', async () => {
await expect(
queryLogsServerTool.execute({ view: 'overview', executionId: 'exec-1' } as any, {} as any)
).rejects.toThrow('Unauthorized')
})
it('rejects overview/full without executionId via inputSchema', () => {
const schema = queryLogsServerTool.inputSchema!
expect(schema.safeParse({ view: 'overview', workspaceId: 'ws-1' }).success).toBe(false)
expect(schema.safeParse({ view: 'full', workspaceId: 'ws-1' }).success).toBe(false)
expect(
schema.safeParse({ view: 'overview', workspaceId: 'ws-1', executionId: 'e1' }).success
).toBe(true)
})
})
@@ -0,0 +1,199 @@
import { createLogger } from '@sim/logger'
import { z } from 'zod'
import { QueryLogs } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool'
import {
collectLargeValueExecutionIds,
collectLargeValueKeys,
} from '@/lib/execution/payloads/large-execution-value'
import { fetchLogDetail } from '@/lib/logs/fetch-log-detail'
import { type ListLogsParams, listLogs } from '@/lib/logs/list-logs'
import { grepSpans, type LogViewContext, toFull, toOverview } from '@/lib/logs/log-views'
import type { TraceSpan } from '@/lib/logs/types'
const logger = createLogger('QueryLogsServerTool')
/**
* Max serialized size for a `full` view result before falling back to the
* compact overview. Keeps a single tool result inline-able.
*/
const MAX_FULL_RESULT_BYTES = 512 * 1024
const comparisonOperator = z.enum(['=', '>', '<', '>=', '<=', '!='])
const listArgsSchema = z.object({
view: z.literal('list'),
workspaceId: z.string().optional(),
level: z.string().optional(),
workflowIds: z.string().optional(),
folderIds: z.string().optional(),
triggers: z.string().optional(),
startDate: z.string().optional(),
endDate: z.string().optional(),
search: z.string().optional(),
workflowName: z.string().optional(),
folderName: z.string().optional(),
executionId: z.string().optional(),
costOperator: comparisonOperator.optional(),
costValue: z.coerce.number().optional(),
durationOperator: comparisonOperator.optional(),
durationValue: z.coerce.number().optional(),
cursor: z.string().optional(),
limit: z.coerce.number().int().min(1).max(200).optional().default(100),
sortBy: z.enum(['date', 'duration', 'cost', 'status']).optional().default('date'),
sortOrder: z.enum(['asc', 'desc']).optional().default('desc'),
})
const overviewArgsSchema = z.object({
view: z.literal('overview'),
workspaceId: z.string().optional(),
executionId: z.string(),
pattern: z.string().optional(),
})
const fullArgsSchema = z.object({
view: z.literal('full'),
workspaceId: z.string().optional(),
executionId: z.string(),
blockId: z.string().optional(),
blockName: z.string().optional(),
pattern: z.string().optional(),
})
const queryLogsArgsSchema = z.discriminatedUnion('view', [
listArgsSchema,
overviewArgsSchema,
fullArgsSchema,
])
type QueryLogsArgs = z.infer<typeof queryLogsArgsSchema>
function resolveWorkspaceId(args: QueryLogsArgs, context?: ServerToolContext): string {
const workspaceId = args.workspaceId ?? context?.workspaceId
if (!workspaceId) {
throw new Error('workspaceId is required')
}
return workspaceId
}
function buildLogViewContext(
detail: {
workflowId: string | null
executionId: string
executionData?: unknown
},
workspaceId: string,
userId: string
): LogViewContext {
return {
workspaceId,
workflowId: detail.workflowId ?? undefined,
executionId: detail.executionId,
userId,
largeValueExecutionIds: collectLargeValueExecutionIds(detail.executionData),
largeValueKeys: collectLargeValueKeys(detail.executionData),
allowLargeValueWorkflowScope: true,
}
}
/**
* Consolidated execution/log read tool.
*
* - `view: "list"` — paginated execution summaries with the full Logs-UI filter
* set (reuses `listLogs`).
* - `view: "overview"` — a single execution's trace-span tree (timing + cost,
* no input/output).
* - `view: "full"` — a single execution's trace spans with materialized
* input/output, optionally scoped to one block via `blockId`/`blockName`.
* - `pattern` (with `overview`/`full`) — grep that execution's trace spans,
* streaming large values chunk-by-chunk.
*/
export const queryLogsServerTool: BaseServerTool<QueryLogsArgs, unknown> = {
name: QueryLogs.id,
inputSchema: queryLogsArgsSchema,
outputSchema: z.unknown(),
async execute(args: QueryLogsArgs, context?: ServerToolContext): Promise<unknown> {
if (!context?.userId) {
throw new Error('Unauthorized access')
}
const userId = context.userId
const workspaceId = resolveWorkspaceId(args, context)
if (args.view === 'list') {
const { view: _view, ...rest } = args
const params = { ...rest, workspaceId } as ListLogsParams
logger.info('query_logs list', { workspaceId, sortBy: params.sortBy })
return listLogs(params, userId)
}
// overview / full / grep — single execution by id
const detail = await fetchLogDetail({
userId,
workspaceId,
lookupColumn: 'executionId',
lookupValue: args.executionId,
})
if (!detail) {
return { ok: false, error: `Execution not found: ${args.executionId}` }
}
const execData = detail.executionData as
| { traceSpans?: TraceSpan[]; totalDuration?: number | null }
| undefined
const traceSpans = (execData?.traceSpans ?? []) as TraceSpan[]
const viewCtx = buildLogViewContext(detail, workspaceId, userId)
if (args.pattern) {
logger.info('query_logs grep', { workspaceId, executionId: args.executionId })
const { matches, truncated } = await grepSpans(traceSpans, args.pattern, viewCtx)
return {
executionId: detail.executionId,
workflowId: detail.workflowId,
status: detail.status,
pattern: args.pattern,
matches,
truncated,
}
}
if (args.view === 'overview') {
return {
executionId: detail.executionId,
workflowId: detail.workflowId,
status: detail.status,
trigger: detail.trigger,
durationMs: execData?.totalDuration ?? null,
cost: detail.cost ?? null,
spans: toOverview(traceSpans),
}
}
// full
const spans = await toFull(traceSpans, viewCtx, {
blockId: args.blockId,
blockName: args.blockName,
})
const result = {
executionId: detail.executionId,
workflowId: detail.workflowId,
status: detail.status,
trigger: detail.trigger,
cost: detail.cost ?? null,
spans,
truncated: false,
}
if (JSON.stringify(result).length > MAX_FULL_RESULT_BYTES) {
return {
executionId: detail.executionId,
workflowId: detail.workflowId,
status: detail.status,
truncated: true,
note: 'Full result too large; returning the compact overview. Scope with blockId/blockName, or use pattern to grep.',
spans: toOverview(traceSpans),
}
}
return result
},
}