chore: import upstream snapshot with attribution

apps/sim/lib/copilot/tools/server/media/ffmpeg.ts

@@ -0,0 +1,159 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { Ffmpeg } from '@/lib/copilot/generated/tool-catalog-v1'
+import {
+  assertServerToolNotAborted,
+  type BaseServerTool,
+  type ServerToolContext,
+} from '@/lib/copilot/tools/server/base-tool'
+import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
+import { type FfmpegOperation, type MediaFile, runFfmpegOperation } from '@/lib/media/ffmpeg'
+import {
+  fetchWorkspaceFileBuffer,
+  resolveWorkspaceFileReference,
+} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+
+const logger = createLogger('FfmpegTool')
+
+const VALID_OPERATIONS: FfmpegOperation[] = [
+  'overlay_audio',
+  'mux',
+  'mix_audio',
+  'concat',
+  'trim',
+  'scale_pad',
+  'overlay_image',
+  'add_text',
+  'fade',
+  'extract_audio',
+  'convert',
+  'thumbnail',
+  'probe',
+]
+
+interface FfmpegArgs {
+  operation: FfmpegOperation
+  inputs?: { files?: Array<{ path: string }> }
+  text?: string
+  position?: string
+  start?: number
+  end?: number
+  width?: number
+  height?: number
+  aspectRatio?: string
+  volume?: number
+  musicVolume?: number
+  loopToVideo?: boolean
+  format?: string
+  outputs?: {
+    files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }>
+  }
+}
+
+interface FfmpegResult {
+  success: boolean
+  message: string
+  fileId?: string
+  fileName?: string
+  vfsPath?: string
+  downloadUrl?: string
+  probe?: unknown
+}
+
+export const ffmpegServerTool: BaseServerTool<FfmpegArgs, FfmpegResult> = {
+  name: Ffmpeg.id,
+
+  async execute(params: FfmpegArgs, context?: ServerToolContext): Promise<FfmpegResult> {
+    if (!context?.userId) {
+      throw new Error('Authentication required')
+    }
+    const workspaceId = context.workspaceId
+    if (!workspaceId) {
+      return { success: false, message: 'Workspace ID is required' }
+    }
+    if (!VALID_OPERATIONS.includes(params.operation)) {
+      return { success: false, message: `Invalid operation "${params.operation}".` }
+    }
+
+    const inputPaths = params.inputs?.files?.map((f) => f.path) ?? []
+    if (inputPaths.length === 0) {
+      return { success: false, message: 'At least one input file is required in inputs.files' }
+    }
+
+    try {
+      const mediaFiles: MediaFile[] = []
+      for (const filePath of inputPaths) {
+        const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath)
+        if (!fileRecord) {
+          return { success: false, message: `Input file not found: ${filePath}` }
+        }
+        const buffer = await fetchWorkspaceFileBuffer(fileRecord)
+        mediaFiles.push({
+          buffer,
+          mimeType: fileRecord.type || 'application/octet-stream',
+          name: fileRecord.name,
+        })
+      }
+
+      assertServerToolNotAborted(context)
+      const result = await runFfmpegOperation(params.operation, mediaFiles, {
+        text: params.text,
+        position: params.position,
+        start: params.start,
+        end: params.end,
+        width: params.width,
+        height: params.height,
+        aspectRatio: params.aspectRatio,
+        volume: params.volume,
+        musicVolume: params.musicVolume,
+        loopToVideo: params.loopToVideo,
+        format: params.format,
+      })
+
+      // probe reports metadata only — no file written.
+      if (params.operation === 'probe') {
+        return {
+          success: true,
+          message: `Probed ${mediaFiles[0]?.name ?? inputPaths[0]}: ${JSON.stringify(result.probe)}`,
+          probe: result.probe,
+        }
+      }
+
+      if (!result.buffer || !result.ext) {
+        return { success: false, message: `ffmpeg ${params.operation} produced no output` }
+      }
+
+      const outputFile = params.outputs?.files?.[0]
+      const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}`
+      const mode = outputFile?.mode ?? 'create'
+
+      assertServerToolNotAborted(context)
+      const written = await writeWorkspaceFileByPath({
+        workspaceId,
+        userId: context.userId,
+        target: { path: outputPath, mode, mimeType: outputFile?.mimeType },
+        buffer: result.buffer,
+        inferredMimeType: result.contentType || 'application/octet-stream',
+      })
+
+      logger.info('ffmpeg operation completed', {
+        operation: params.operation,
+        vfsPath: written.vfsPath,
+        size: result.buffer.length,
+      })
+
+      return {
+        success: true,
+        message: `${params.operation} completed and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes)`,
+        fileId: written.id,
+        fileName: written.name,
+        vfsPath: written.vfsPath,
+        downloadUrl: written.downloadUrl,
+      }
+    } catch (error) {
+      const msg = getErrorMessage(error, 'Unknown error')
+      logger.error('ffmpeg operation failed', { operation: params.operation, error: msg })
+      return { success: false, message: `ffmpeg ${params.operation} failed: ${msg}` }
+    }
+  },
+}

apps/sim/lib/copilot/tools/server/media/generate-audio.ts

@@ -0,0 +1,152 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { GenerateAudio } from '@/lib/copilot/generated/tool-catalog-v1'
+import {
+  assertServerToolNotAborted,
+  type BaseServerTool,
+  type ServerToolContext,
+} from '@/lib/copilot/tools/server/base-tool'
+import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
+import { type AudioType, generateFalAudio } from '@/lib/media/falai-audio'
+import {
+  fetchWorkspaceFileBuffer,
+  resolveWorkspaceFileReference,
+} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+
+const logger = createLogger('GenerateAudioTool')
+
+const VALID_TYPES: AudioType[] = ['speech', 'music', 'sfx']
+
+interface GenerateAudioArgs {
+  prompt: string
+  type?: string
+  model?: string
+  voice?: string
+  duration?: number
+  /** For music: explicit lyrics for a vocal track. */
+  lyrics?: string
+  /** For music: true = instrumental (default), false = vocal track. */
+  instrumental?: boolean
+  /** Optional reference voice sample (workspace audio file) for zero-shot voice cloning. */
+  inputs?: { files?: Array<{ path: string }> }
+  outputs?: {
+    files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }>
+  }
+}
+
+interface GenerateAudioResult {
+  success: boolean
+  message: string
+  fileId?: string
+  fileName?: string
+  vfsPath?: string
+  downloadUrl?: string
+  _serviceCost?: { service: string; cost: number }
+}
+
+function audioExtFromContentType(contentType: string): string {
+  if (contentType.includes('wav')) return 'wav'
+  if (contentType.includes('mp4') || contentType.includes('m4a')) return 'm4a'
+  if (contentType.includes('ogg')) return 'ogg'
+  if (contentType.includes('flac')) return 'flac'
+  if (contentType.includes('aac')) return 'aac'
+  if (contentType.includes('opus')) return 'opus'
+  return 'mp3'
+}
+
+export const generateAudioServerTool: BaseServerTool<GenerateAudioArgs, GenerateAudioResult> = {
+  name: GenerateAudio.id,
+
+  async execute(
+    params: GenerateAudioArgs,
+    context?: ServerToolContext
+  ): Promise<GenerateAudioResult> {
+    if (!context?.userId) {
+      throw new Error('Authentication required')
+    }
+    const workspaceId = context.workspaceId
+    if (!workspaceId) {
+      return { success: false, message: 'Workspace ID is required' }
+    }
+    if (!params.prompt) {
+      return { success: false, message: 'prompt is required' }
+    }
+
+    const type = (params.type || 'speech') as AudioType
+    if (!VALID_TYPES.includes(type)) {
+      return {
+        success: false,
+        message: `Invalid type "${params.type}". Must be one of: ${VALID_TYPES.join(', ')}`,
+      }
+    }
+
+    // Voice cloning: a reference sample clones that voice into the generated speech.
+    let voiceSampleDataUri: string | undefined
+    const samplePath = params.inputs?.files?.[0]?.path
+    if (samplePath) {
+      const sample = await resolveWorkspaceFileReference(workspaceId, samplePath)
+      if (!sample) {
+        return { success: false, message: `Voice sample not found: ${samplePath}` }
+      }
+      const sampleBuffer = await fetchWorkspaceFileBuffer(sample)
+      const sampleMime = sample.type || 'audio/mpeg'
+      voiceSampleDataUri = `data:${sampleMime};base64,${sampleBuffer.toString('base64')}`
+    }
+
+    try {
+      logger.info('Generating audio', {
+        type,
+        model: params.model,
+        promptLength: params.prompt.length,
+        voiceClone: Boolean(voiceSampleDataUri),
+      })
+
+      const result = await generateFalAudio({
+        prompt: params.prompt,
+        type,
+        model: params.model,
+        voice: params.voice,
+        duration: params.duration,
+        lyrics: params.lyrics,
+        instrumental: params.instrumental,
+        voiceSampleDataUri,
+      })
+
+      const outputFile = params.outputs?.files?.[0]
+      const ext = audioExtFromContentType(result.contentType)
+      const outputPath = outputFile?.path || `files/generated-audio.${ext}`
+      const mode = outputFile?.mode ?? 'create'
+
+      assertServerToolNotAborted(context)
+      const written = await writeWorkspaceFileByPath({
+        workspaceId,
+        userId: context.userId,
+        target: { path: outputPath, mode, mimeType: outputFile?.mimeType },
+        buffer: result.buffer,
+        inferredMimeType: result.contentType,
+      })
+
+      logger.info('Generated audio saved', {
+        fileId: written.id,
+        vfsPath: written.vfsPath,
+        size: result.buffer.length,
+        type,
+        model: result.model,
+      })
+
+      return {
+        success: true,
+        message: `${type === 'speech' ? 'Speech' : type === 'music' ? 'Music' : 'Sound effect'} generated and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes, model ${result.model})`,
+        fileId: written.id,
+        fileName: written.name,
+        vfsPath: written.vfsPath,
+        downloadUrl: written.downloadUrl,
+        _serviceCost: { service: 'falai_audio', cost: result.cost.costDollars },
+      }
+    } catch (error) {
+      const msg = getErrorMessage(error, 'Unknown error')
+      logger.error('Audio generation failed', { error: msg })
+      return { success: false, message: `Failed to generate audio: ${msg}` }
+    }
+  },
+}

apps/sim/lib/copilot/tools/server/media/generate-video.ts

@@ -0,0 +1,127 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { GenerateVideo } from '@/lib/copilot/generated/tool-catalog-v1'
+import {
+  assertServerToolNotAborted,
+  type BaseServerTool,
+  type ServerToolContext,
+} from '@/lib/copilot/tools/server/base-tool'
+import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
+import { generateFalVideo } from '@/lib/media/falai-video'
+import {
+  fetchWorkspaceFileBuffer,
+  resolveWorkspaceFileReference,
+} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+
+const logger = createLogger('GenerateVideoTool')
+
+interface GenerateVideoArgs {
+  prompt: string
+  model?: string
+  aspectRatio?: string
+  resolution?: string
+  duration?: number
+  generateAudio?: boolean
+  negativePrompt?: string
+  promptOptimizer?: boolean
+  inputs?: { files?: Array<{ path: string }> }
+  outputs?: {
+    files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }>
+  }
+}
+
+interface GenerateVideoResult {
+  success: boolean
+  message: string
+  fileId?: string
+  fileName?: string
+  vfsPath?: string
+  downloadUrl?: string
+  _serviceCost?: { service: string; cost: number }
+}
+
+export const generateVideoServerTool: BaseServerTool<GenerateVideoArgs, GenerateVideoResult> = {
+  name: GenerateVideo.id,
+
+  async execute(
+    params: GenerateVideoArgs,
+    context?: ServerToolContext
+  ): Promise<GenerateVideoResult> {
+    if (!context?.userId) {
+      throw new Error('Authentication required')
+    }
+    const workspaceId = context.workspaceId
+    if (!workspaceId) {
+      return { success: false, message: 'Workspace ID is required' }
+    }
+    if (!params.prompt) {
+      return { success: false, message: 'prompt is required' }
+    }
+
+    try {
+      let imageDataUri: string | undefined
+      const refPath = params.inputs?.files?.[0]?.path
+      if (refPath) {
+        const fileRecord = await resolveWorkspaceFileReference(workspaceId, refPath)
+        if (!fileRecord) {
+          return { success: false, message: `Reference image not found: ${refPath}` }
+        }
+        const buffer = await fetchWorkspaceFileBuffer(fileRecord)
+        const mime = fileRecord.type || 'image/png'
+        imageDataUri = `data:${mime};base64,${buffer.toString('base64')}`
+      }
+
+      logger.info('Generating video', {
+        model: params.model || 'veo-3.1-fast',
+        promptLength: params.prompt.length,
+        imageToVideo: Boolean(imageDataUri),
+      })
+
+      const result = await generateFalVideo({
+        prompt: params.prompt,
+        model: params.model,
+        aspectRatio: params.aspectRatio,
+        resolution: params.resolution,
+        duration: params.duration,
+        generateAudio: params.generateAudio,
+        negativePrompt: params.negativePrompt,
+        promptOptimizer: params.promptOptimizer,
+        imageDataUri,
+      })
+
+      const outputFile = params.outputs?.files?.[0]
+      const outputPath = outputFile?.path || 'files/generated-video.mp4'
+      const mode = outputFile?.mode ?? 'create'
+
+      assertServerToolNotAborted(context)
+      const written = await writeWorkspaceFileByPath({
+        workspaceId,
+        userId: context.userId,
+        target: { path: outputPath, mode, mimeType: outputFile?.mimeType },
+        buffer: result.buffer,
+        inferredMimeType: result.contentType,
+      })
+
+      logger.info('Generated video saved', {
+        fileId: written.id,
+        vfsPath: written.vfsPath,
+        size: result.buffer.length,
+        model: result.model,
+      })
+
+      return {
+        success: true,
+        message: `Video generated and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes, model ${result.model})`,
+        fileId: written.id,
+        fileName: written.name,
+        vfsPath: written.vfsPath,
+        downloadUrl: written.downloadUrl,
+        _serviceCost: { service: 'falai_video', cost: result.cost.costDollars },
+      }
+    } catch (error) {
+      const msg = getErrorMessage(error, 'Unknown error')
+      logger.error('Video generation failed', { error: msg })
+      return { success: false, message: `Failed to generate video: ${msg}` }
+    }
+  },
+}