simstudioai--sim/apps/sim/lib/copilot/chat/payload.ts

import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { LRUCache } from 'lru-cache'
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
import { isPaid } from '@/lib/billing/plan-helpers'
import { getBlockVisibilityForCopilot, visibilitySignature } from '@/lib/copilot/block-visibility'
import type { VfsSnapshotV1 } from '@/lib/copilot/generated/vfs-snapshot-v1'
import {
  filterExposedIntegrationTools,
  getExposedIntegrationTools,
} from '@/lib/copilot/integration-tools'
import { getToolEntry } from '@/lib/copilot/tool-executor/router'
import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions'
import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils'
import type { BlockVisibilityState } from '@/lib/core/config/block-visibility'
import { isE2BDocEnabled, isHosted } from '@/lib/core/config/env-flags'
import { buildUserSkillTool } from '@/lib/mothership/skills'
import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { stripVersionSuffix } from '@/tools/utils'

const logger = createLogger('CopilotChatPayload')
const INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS = 5_000
const INTEGRATION_TOOL_SCHEMA_CACHE_MAX_ENTRIES = 500

interface BuildPayloadParams {
  message: string
  workflowId?: string
  workflowName?: string
  workspaceId?: string
  userId: string
  userMessageId: string
  mode: string
  model: string
  provider?: string
  contexts?: Array<{ type: string; content: string; tag?: string; path?: string }>
  fileAttachments?: Array<{ id: string; key: string; size: number; [key: string]: unknown }>
  commands?: string[]
  chatId?: string
  prefetch?: boolean
  implicitFeedback?: string
  workspaceContext?: string
  vfs?: VfsSnapshotV1
  userPermission?: string
  /** Plan/flag-gated org capabilities (e.g. "custom-blocks") the mothership gates tools/prompts on. */
  entitlements?: string[]
  userTimezone?: string
  userMetadata?: {
    name?: string
    email?: string
    timezone?: string
  }
  includeMothershipTools?: boolean
}

export interface ToolSchema {
  name: string
  description: string
  input_schema: Record<string, unknown>
  defer_loading?: boolean
  executeLocally?: boolean
  params?: Record<string, unknown>
  /** Canonical integration service/folder (e.g. "slack"), for server-side grouping. */
  service?: string
  oauth?: { required: boolean; provider: string }
}

interface BuildIntegrationToolSchemasOptions {
  schemaSurface?: 'default' | 'copilot'
}

interface IntegrationToolSchemaCacheEntry {
  promise: Promise<ToolSchema[]>
}

const integrationToolSchemaCache = new LRUCache<string, IntegrationToolSchemaCacheEntry>({
  max: INTEGRATION_TOOL_SCHEMA_CACHE_MAX_ENTRIES,
  ttl: INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS,
})

function getIntegrationToolSchemaCacheKey(
  userId: string,
  workspaceId: string | undefined,
  schemaSurface: string,
  visSignature: string
): string {
  // The visibility signature keys the entry to the viewer's gated projection —
  // two users in one workspace with different preview reveals must not share.
  return JSON.stringify([userId, workspaceId ?? null, schemaSurface, visSignature])
}

function cloneToolSchemas(toolSchemas: ToolSchema[]): ToolSchema[] {
  return toolSchemas.map((tool) => {
    const cloned: ToolSchema = {
      ...tool,
      input_schema: { ...tool.input_schema },
    }
    if (tool.params) cloned.params = { ...tool.params }
    if (tool.oauth) cloned.oauth = { ...tool.oauth }
    return cloned
  })
}

export function clearIntegrationToolSchemaCacheForTests(): void {
  integrationToolSchemaCache.clear()
}

/**
 * Build deferred integration tool schemas from the Sim tool registry.
 * Shared by the interactive chat payload builder and the non-interactive
 * block execution route so both paths send the same tool definitions to Go.
 *
 * When `workspaceId` is provided the user's workspace permission config is
 * loaded once and used to skip any tool whose owning block is not in the
 * workspace's `allowedIntegrations` allowlist.
 */
export async function buildIntegrationToolSchemas(
  userId: string,
  messageId?: string,
  options: BuildIntegrationToolSchemasOptions = { schemaSurface: 'copilot' },
  workspaceId?: string
): Promise<ToolSchema[]> {
  const schemaSurface = options.schemaSurface ?? 'copilot'
  const vis = await getBlockVisibilityForCopilot(userId, workspaceId)
  const cacheKey = getIntegrationToolSchemaCacheKey(
    userId,
    workspaceId,
    schemaSurface,
    visibilitySignature(vis)
  )
  const cached = integrationToolSchemaCache.get(cacheKey)
  if (cached) {
    return cloneToolSchemas(await cached.promise)
  }

  const promise = buildIntegrationToolSchemasUncached(
    userId,
    messageId,
    { schemaSurface },
    workspaceId,
    vis
  ).catch((error) => {
    integrationToolSchemaCache.delete(cacheKey)
    throw error
  })

  integrationToolSchemaCache.set(cacheKey, {
    promise,
  })

  return cloneToolSchemas(await promise)
}

async function buildIntegrationToolSchemasUncached(
  userId: string,
  messageId: string | undefined,
  options: Required<BuildIntegrationToolSchemasOptions>,
  workspaceId?: string,
  vis: BlockVisibilityState | null = null
): Promise<ToolSchema[]> {
  const reqLogger = logger.withMetadata({ messageId })
  const integrationTools: ToolSchema[] = []
  try {
    const { createUserToolSchema } = await import('@/tools/params')
    let shouldAppendEmailTagline = false

    try {
      const subscription = await getHighestPrioritySubscription(userId)
      shouldAppendEmailTagline = !subscription || !isPaid(subscription.plan)
    } catch (error) {
      reqLogger.warn('Failed to load subscription for copilot tool descriptions', {
        userId,
        error: toError(error).message,
      })
    }

    let allowedIntegrations: Set<string> | null = null
    let toolIdToBlockType: Map<string, string> | null = null
    if (workspaceId) {
      try {
        const [{ getUserPermissionConfig }, { getAllBlocks }] = await Promise.all([
          import('@/ee/access-control/utils/permission-check'),
          import('@/blocks/registry'),
        ])
        const permissionConfig = await getUserPermissionConfig(userId, workspaceId)
        if (permissionConfig?.allowedIntegrations) {
          allowedIntegrations = new Set(
            permissionConfig.allowedIntegrations.map((i) => i.toLowerCase())
          )
          toolIdToBlockType = new Map()
          for (const blockConfig of getAllBlocks()) {
            const access = blockConfig.tools?.access
            if (!access) continue
            for (const toolId of access) {
              toolIdToBlockType.set(stripVersionSuffix(toolId), blockConfig.type.toLowerCase())
            }
          }
        }
      } catch (error) {
        reqLogger.warn('Failed to load permission config for tool schema filter', {
          userId,
          workspaceId,
          error: toError(error).message,
        })
      }
    }

    const exposedTools = filterExposedIntegrationTools(getExposedIntegrationTools(), vis)
    for (const { toolId, config: toolConfig, service } of exposedTools) {
      try {
        if (allowedIntegrations && toolIdToBlockType) {
          const owningBlock = toolIdToBlockType.get(stripVersionSuffix(toolId))
          if (owningBlock && !allowedIntegrations.has(owningBlock)) {
            continue
          }
        }
        const userSchema = createUserToolSchema(toolConfig, {
          surface: options.schemaSurface,
        })
        const catalogEntry = getToolEntry(toolId)
        integrationTools.push({
          name: toolId,
          service,
          description: getCopilotToolDescription(toolConfig, {
            isHosted,
            fallbackName: toolId,
            appendEmailTagline: shouldAppendEmailTagline,
          }),
          input_schema: { ...userSchema },
          defer_loading: true,
          executeLocally:
            catalogEntry?.clientExecutable === true || catalogEntry?.route === 'client',
          ...(toolConfig.oauth?.required && {
            oauth: {
              required: true,
              provider: toolConfig.oauth.provider,
            },
          }),
        })
      } catch (toolError) {
        logger.warn(
          messageId
            ? `Failed to build schema for tool, skipping [messageId:${messageId}]`
            : 'Failed to build schema for tool, skipping',
          {
            toolId,
            error: toError(toolError).message,
          }
        )
      }
    }
  } catch (error) {
    logger.warn(
      messageId
        ? `Failed to build tool schemas [messageId:${messageId}]`
        : 'Failed to build tool schemas',
      {
        error: toError(error).message,
      }
    )
  }

  return integrationTools
}

/**
 * Build the request payload for the copilot backend.
 */
export async function buildCopilotRequestPayload(
  params: BuildPayloadParams,
  options: {
    selectedModel: string
  }
): Promise<Record<string, unknown>> {
  const {
    message,
    workflowId,
    userId,
    userMessageId,
    mode,
    provider,
    contexts,
    fileAttachments,
    commands,
    chatId,
    prefetch,
    implicitFeedback,
  } = params

  const selectedModel = options.selectedModel

  const effectiveMode = mode === 'agent' ? 'build' : mode
  const transportMode = effectiveMode === 'build' ? 'agent' : effectiveMode

  // Track uploaded files in the DB and build context tags instead of base64 inlining
  const uploadContexts: Array<{ type: string; content: string; tag?: string; path?: string }> = []
  if (chatId && params.workspaceId && fileAttachments && fileAttachments.length > 0) {
    for (const f of fileAttachments) {
      const filename = (f.filename ?? f.name ?? 'file') as string
      const mediaType = (f.media_type ?? f.mimeType ?? 'application/octet-stream') as string
      try {
        const { displayName } = await trackChatUpload(
          params.workspaceId,
          userId,
          chatId,
          f.key,
          filename,
          mediaType,
          f.size
        )
        // Encode the read path per the percent-encoded VFS convention (matches
        // files/ and the uploads glob output). The materialize_file `fileName`
        // arg stays the raw display name — the upload resolver accepts both.
        let encodedUploadName = displayName
        try {
          encodedUploadName = encodeVfsSegment(displayName)
        } catch {
          encodedUploadName = displayName
        }
        const lines = [
          `File "${displayName}" (${mediaType}, ${f.size} bytes) uploaded.`,
          `Read with: read("uploads/${encodedUploadName}")`,
          `To save permanently: materialize_file(fileName: "${displayName}")`,
        ]
        if (displayName.endsWith('.json')) {
          lines.push(
            `To import as a workflow: materialize_file(fileName: "${displayName}", operation: "import")`
          )
        }
        uploadContexts.push({
          type: 'uploaded_file',
          content: lines.join('\n'),
        })
      } catch (err) {
        logger.warn('Failed to track chat upload', {
          filename,
          chatId,
          error: toError(err).message,
        })
      }
    }
  }

  const allContexts = [...(contexts ?? []), ...uploadContexts]

  let integrationTools: ToolSchema[] = []
  const mothershipTools: ToolSchema[] = []
  const payloadLogger = logger.withMetadata({ messageId: userMessageId })

  if (effectiveMode === 'build') {
    integrationTools = await buildIntegrationToolSchemas(
      userId,
      userMessageId,
      { schemaSurface: 'copilot' },
      params.workspaceId
    )

    if (params.includeMothershipTools && params.workspaceId) {
      // Expose all workspace user-created skills via the single load_user_skill
      // tool. Available to every user; content is fetched sim-side when the
      // model calls it.
      try {
        const userSkillTool = await buildUserSkillTool(params.workspaceId)
        if (userSkillTool) mothershipTools.push(userSkillTool)
      } catch (error) {
        logger.warn('Failed to build load_user_skill tool', {
          error: toError(error).message,
        })
      }
    }
  }

  return {
    message,
    ...(workflowId ? { workflowId } : {}),
    ...(params.workflowName ? { workflowName: params.workflowName } : {}),
    ...(params.workspaceId ? { workspaceId: params.workspaceId } : {}),
    userId,
    ...(selectedModel ? { model: selectedModel } : {}),
    ...(provider ? { provider } : {}),
    mode: transportMode,
    messageId: userMessageId,
    ...(allContexts.length > 0 ? { context: allContexts } : {}),
    ...(chatId ? { chatId } : {}),
    ...(typeof prefetch === 'boolean' ? { prefetch } : {}),
    ...(implicitFeedback ? { implicitFeedback } : {}),
    ...(integrationTools.length > 0 ? { integrationTools } : {}),
    ...(mothershipTools.length > 0 ? { mothershipTools } : {}),
    ...(commands && commands.length > 0 ? { commands } : {}),
    ...(params.workspaceContext ? { workspaceContext: params.workspaceContext } : {}),
    ...(params.vfs ? { vfs: params.vfs } : {}),
    ...(params.userPermission ? { userPermission: params.userPermission } : {}),
    ...(params.entitlements?.length ? { entitlements: params.entitlements } : {}),
    ...(params.userTimezone ? { userTimezone: params.userTimezone } : {}),
    ...(params.userMetadata &&
    (params.userMetadata.name || params.userMetadata.email || params.userMetadata.timezone)
      ? { userMetadata: params.userMetadata }
      : {}),
    // Tell the copilot file subagent which document toolchain to write. Emitted
    // only in Python mode so the JS path sends no new field (Go defaults to js).
    ...(isE2BDocEnabled ? { docCompiler: 'python' } : {}),
    isHosted,
  }
}