chore: import upstream snapshot with attribution

apps/sim/lib/copilot/chat/payload.ts

@@ -0,0 +1,403 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { LRUCache } from 'lru-cache'
+import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
+import { isPaid } from '@/lib/billing/plan-helpers'
+import { getBlockVisibilityForCopilot, visibilitySignature } from '@/lib/copilot/block-visibility'
+import type { VfsSnapshotV1 } from '@/lib/copilot/generated/vfs-snapshot-v1'
+import {
+  filterExposedIntegrationTools,
+  getExposedIntegrationTools,
+} from '@/lib/copilot/integration-tools'
+import { getToolEntry } from '@/lib/copilot/tool-executor/router'
+import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions'
+import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils'
+import type { BlockVisibilityState } from '@/lib/core/config/block-visibility'
+import { isE2BDocEnabled, isHosted } from '@/lib/core/config/env-flags'
+import { buildUserSkillTool } from '@/lib/mothership/skills'
+import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+import { stripVersionSuffix } from '@/tools/utils'
+
+const logger = createLogger('CopilotChatPayload')
+const INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS = 5_000
+const INTEGRATION_TOOL_SCHEMA_CACHE_MAX_ENTRIES = 500
+
+interface BuildPayloadParams {
+  message: string
+  workflowId?: string
+  workflowName?: string
+  workspaceId?: string
+  userId: string
+  userMessageId: string
+  mode: string
+  model: string
+  provider?: string
+  contexts?: Array<{ type: string; content: string; tag?: string; path?: string }>
+  fileAttachments?: Array<{ id: string; key: string; size: number; [key: string]: unknown }>
+  commands?: string[]
+  chatId?: string
+  prefetch?: boolean
+  implicitFeedback?: string
+  workspaceContext?: string
+  vfs?: VfsSnapshotV1
+  userPermission?: string
+  /** Plan/flag-gated org capabilities (e.g. "custom-blocks") the mothership gates tools/prompts on. */
+  entitlements?: string[]
+  userTimezone?: string
+  userMetadata?: {
+    name?: string
+    email?: string
+    timezone?: string
+  }
+  includeMothershipTools?: boolean
+}
+
+export interface ToolSchema {
+  name: string
+  description: string
+  input_schema: Record<string, unknown>
+  defer_loading?: boolean
+  executeLocally?: boolean
+  params?: Record<string, unknown>
+  /** Canonical integration service/folder (e.g. "slack"), for server-side grouping. */
+  service?: string
+  oauth?: { required: boolean; provider: string }
+}
+
+interface BuildIntegrationToolSchemasOptions {
+  schemaSurface?: 'default' | 'copilot'
+}
+
+interface IntegrationToolSchemaCacheEntry {
+  promise: Promise<ToolSchema[]>
+}
+
+const integrationToolSchemaCache = new LRUCache<string, IntegrationToolSchemaCacheEntry>({
+  max: INTEGRATION_TOOL_SCHEMA_CACHE_MAX_ENTRIES,
+  ttl: INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS,
+})
+
+function getIntegrationToolSchemaCacheKey(
+  userId: string,
+  workspaceId: string | undefined,
+  schemaSurface: string,
+  visSignature: string
+): string {
+  // The visibility signature keys the entry to the viewer's gated projection —
+  // two users in one workspace with different preview reveals must not share.
+  return JSON.stringify([userId, workspaceId ?? null, schemaSurface, visSignature])
+}
+
+function cloneToolSchemas(toolSchemas: ToolSchema[]): ToolSchema[] {
+  return toolSchemas.map((tool) => {
+    const cloned: ToolSchema = {
+      ...tool,
+      input_schema: { ...tool.input_schema },
+    }
+    if (tool.params) cloned.params = { ...tool.params }
+    if (tool.oauth) cloned.oauth = { ...tool.oauth }
+    return cloned
+  })
+}
+
+export function clearIntegrationToolSchemaCacheForTests(): void {
+  integrationToolSchemaCache.clear()
+}
+
+/**
+ * Build deferred integration tool schemas from the Sim tool registry.
+ * Shared by the interactive chat payload builder and the non-interactive
+ * block execution route so both paths send the same tool definitions to Go.
+ *
+ * When `workspaceId` is provided the user's workspace permission config is
+ * loaded once and used to skip any tool whose owning block is not in the
+ * workspace's `allowedIntegrations` allowlist.
+ */
+export async function buildIntegrationToolSchemas(
+  userId: string,
+  messageId?: string,
+  options: BuildIntegrationToolSchemasOptions = { schemaSurface: 'copilot' },
+  workspaceId?: string
+): Promise<ToolSchema[]> {
+  const schemaSurface = options.schemaSurface ?? 'copilot'
+  const vis = await getBlockVisibilityForCopilot(userId, workspaceId)
+  const cacheKey = getIntegrationToolSchemaCacheKey(
+    userId,
+    workspaceId,
+    schemaSurface,
+    visibilitySignature(vis)
+  )
+  const cached = integrationToolSchemaCache.get(cacheKey)
+  if (cached) {
+    return cloneToolSchemas(await cached.promise)
+  }
+
+  const promise = buildIntegrationToolSchemasUncached(
+    userId,
+    messageId,
+    { schemaSurface },
+    workspaceId,
+    vis
+  ).catch((error) => {
+    integrationToolSchemaCache.delete(cacheKey)
+    throw error
+  })
+
+  integrationToolSchemaCache.set(cacheKey, {
+    promise,
+  })
+
+  return cloneToolSchemas(await promise)
+}
+
+async function buildIntegrationToolSchemasUncached(
+  userId: string,
+  messageId: string | undefined,
+  options: Required<BuildIntegrationToolSchemasOptions>,
+  workspaceId?: string,
+  vis: BlockVisibilityState | null = null
+): Promise<ToolSchema[]> {
+  const reqLogger = logger.withMetadata({ messageId })
+  const integrationTools: ToolSchema[] = []
+  try {
+    const { createUserToolSchema } = await import('@/tools/params')
+    let shouldAppendEmailTagline = false
+
+    try {
+      const subscription = await getHighestPrioritySubscription(userId)
+      shouldAppendEmailTagline = !subscription || !isPaid(subscription.plan)
+    } catch (error) {
+      reqLogger.warn('Failed to load subscription for copilot tool descriptions', {
+        userId,
+        error: toError(error).message,
+      })
+    }
+
+    let allowedIntegrations: Set<string> | null = null
+    let toolIdToBlockType: Map<string, string> | null = null
+    if (workspaceId) {
+      try {
+        const [{ getUserPermissionConfig }, { getAllBlocks }] = await Promise.all([
+          import('@/ee/access-control/utils/permission-check'),
+          import('@/blocks/registry'),
+        ])
+        const permissionConfig = await getUserPermissionConfig(userId, workspaceId)
+        if (permissionConfig?.allowedIntegrations) {
+          allowedIntegrations = new Set(
+            permissionConfig.allowedIntegrations.map((i) => i.toLowerCase())
+          )
+          toolIdToBlockType = new Map()
+          for (const blockConfig of getAllBlocks()) {
+            const access = blockConfig.tools?.access
+            if (!access) continue
+            for (const toolId of access) {
+              toolIdToBlockType.set(stripVersionSuffix(toolId), blockConfig.type.toLowerCase())
+            }
+          }
+        }
+      } catch (error) {
+        reqLogger.warn('Failed to load permission config for tool schema filter', {
+          userId,
+          workspaceId,
+          error: toError(error).message,
+        })
+      }
+    }
+
+    const exposedTools = filterExposedIntegrationTools(getExposedIntegrationTools(), vis)
+    for (const { toolId, config: toolConfig, service } of exposedTools) {
+      try {
+        if (allowedIntegrations && toolIdToBlockType) {
+          const owningBlock = toolIdToBlockType.get(stripVersionSuffix(toolId))
+          if (owningBlock && !allowedIntegrations.has(owningBlock)) {
+            continue
+          }
+        }
+        const userSchema = createUserToolSchema(toolConfig, {
+          surface: options.schemaSurface,
+        })
+        const catalogEntry = getToolEntry(toolId)
+        integrationTools.push({
+          name: toolId,
+          service,
+          description: getCopilotToolDescription(toolConfig, {
+            isHosted,
+            fallbackName: toolId,
+            appendEmailTagline: shouldAppendEmailTagline,
+          }),
+          input_schema: { ...userSchema },
+          defer_loading: true,
+          executeLocally:
+            catalogEntry?.clientExecutable === true || catalogEntry?.route === 'client',
+          ...(toolConfig.oauth?.required && {
+            oauth: {
+              required: true,
+              provider: toolConfig.oauth.provider,
+            },
+          }),
+        })
+      } catch (toolError) {
+        logger.warn(
+          messageId
+            ? `Failed to build schema for tool, skipping [messageId:${messageId}]`
+            : 'Failed to build schema for tool, skipping',
+          {
+            toolId,
+            error: toError(toolError).message,
+          }
+        )
+      }
+    }
+  } catch (error) {
+    logger.warn(
+      messageId
+        ? `Failed to build tool schemas [messageId:${messageId}]`
+        : 'Failed to build tool schemas',
+      {
+        error: toError(error).message,
+      }
+    )
+  }
+
+  return integrationTools
+}
+
+/**
+ * Build the request payload for the copilot backend.
+ */
+export async function buildCopilotRequestPayload(
+  params: BuildPayloadParams,
+  options: {
+    selectedModel: string
+  }
+): Promise<Record<string, unknown>> {
+  const {
+    message,
+    workflowId,
+    userId,
+    userMessageId,
+    mode,
+    provider,
+    contexts,
+    fileAttachments,
+    commands,
+    chatId,
+    prefetch,
+    implicitFeedback,
+  } = params
+
+  const selectedModel = options.selectedModel
+
+  const effectiveMode = mode === 'agent' ? 'build' : mode
+  const transportMode = effectiveMode === 'build' ? 'agent' : effectiveMode
+
+  // Track uploaded files in the DB and build context tags instead of base64 inlining
+  const uploadContexts: Array<{ type: string; content: string; tag?: string; path?: string }> = []
+  if (chatId && params.workspaceId && fileAttachments && fileAttachments.length > 0) {
+    for (const f of fileAttachments) {
+      const filename = (f.filename ?? f.name ?? 'file') as string
+      const mediaType = (f.media_type ?? f.mimeType ?? 'application/octet-stream') as string
+      try {
+        const { displayName } = await trackChatUpload(
+          params.workspaceId,
+          userId,
+          chatId,
+          f.key,
+          filename,
+          mediaType,
+          f.size
+        )
+        // Encode the read path per the percent-encoded VFS convention (matches
+        // files/ and the uploads glob output). The materialize_file `fileName`
+        // arg stays the raw display name — the upload resolver accepts both.
+        let encodedUploadName = displayName
+        try {
+          encodedUploadName = encodeVfsSegment(displayName)
+        } catch {
+          encodedUploadName = displayName
+        }
+        const lines = [
+          `File "${displayName}" (${mediaType}, ${f.size} bytes) uploaded.`,
+          `Read with: read("uploads/${encodedUploadName}")`,
+          `To save permanently: materialize_file(fileName: "${displayName}")`,
+        ]
+        if (displayName.endsWith('.json')) {
+          lines.push(
+            `To import as a workflow: materialize_file(fileName: "${displayName}", operation: "import")`
+          )
+        }
+        uploadContexts.push({
+          type: 'uploaded_file',
+          content: lines.join('\n'),
+        })
+      } catch (err) {
+        logger.warn('Failed to track chat upload', {
+          filename,
+          chatId,
+          error: toError(err).message,
+        })
+      }
+    }
+  }
+
+  const allContexts = [...(contexts ?? []), ...uploadContexts]
+
+  let integrationTools: ToolSchema[] = []
+  const mothershipTools: ToolSchema[] = []
+  const payloadLogger = logger.withMetadata({ messageId: userMessageId })
+
+  if (effectiveMode === 'build') {
+    integrationTools = await buildIntegrationToolSchemas(
+      userId,
+      userMessageId,
+      { schemaSurface: 'copilot' },
+      params.workspaceId
+    )
+
+    if (params.includeMothershipTools && params.workspaceId) {
+      // Expose all workspace user-created skills via the single load_user_skill
+      // tool. Available to every user; content is fetched sim-side when the
+      // model calls it.
+      try {
+        const userSkillTool = await buildUserSkillTool(params.workspaceId)
+        if (userSkillTool) mothershipTools.push(userSkillTool)
+      } catch (error) {
+        logger.warn('Failed to build load_user_skill tool', {
+          error: toError(error).message,
+        })
+      }
+    }
+  }
+
+  return {
+    message,
+    ...(workflowId ? { workflowId } : {}),
+    ...(params.workflowName ? { workflowName: params.workflowName } : {}),
+    ...(params.workspaceId ? { workspaceId: params.workspaceId } : {}),
+    userId,
+    ...(selectedModel ? { model: selectedModel } : {}),
+    ...(provider ? { provider } : {}),
+    mode: transportMode,
+    messageId: userMessageId,
+    ...(allContexts.length > 0 ? { context: allContexts } : {}),
+    ...(chatId ? { chatId } : {}),
+    ...(typeof prefetch === 'boolean' ? { prefetch } : {}),
+    ...(implicitFeedback ? { implicitFeedback } : {}),
+    ...(integrationTools.length > 0 ? { integrationTools } : {}),
+    ...(mothershipTools.length > 0 ? { mothershipTools } : {}),
+    ...(commands && commands.length > 0 ? { commands } : {}),
+    ...(params.workspaceContext ? { workspaceContext: params.workspaceContext } : {}),
+    ...(params.vfs ? { vfs: params.vfs } : {}),
+    ...(params.userPermission ? { userPermission: params.userPermission } : {}),
+    ...(params.entitlements?.length ? { entitlements: params.entitlements } : {}),
+    ...(params.userTimezone ? { userTimezone: params.userTimezone } : {}),
+    ...(params.userMetadata &&
+    (params.userMetadata.name || params.userMetadata.email || params.userMetadata.timezone)
+      ? { userMetadata: params.userMetadata }
+      : {}),
+    // Tell the copilot file subagent which document toolchain to write. Emitted
+    // only in Python mode so the JS path sends no new field (Go defaults to js).
+    ...(isE2BDocEnabled ? { docCompiler: 'python' } : {}),
+    isHosted,
+  }
+}