import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { GenerateAudio } from '@/lib/copilot/generated/tool-catalog-v1' import { assertServerToolNotAborted, type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { type AudioType, generateFalAudio } from '@/lib/media/falai-audio' import { fetchWorkspaceFileBuffer, resolveWorkspaceFileReference, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateAudioTool') const VALID_TYPES: AudioType[] = ['speech', 'music', 'sfx'] interface GenerateAudioArgs { prompt: string type?: string model?: string voice?: string duration?: number /** For music: explicit lyrics for a vocal track. */ lyrics?: string /** For music: true = instrumental (default), false = vocal track. */ instrumental?: boolean /** Optional reference voice sample (workspace audio file) for zero-shot voice cloning. */ inputs?: { files?: Array<{ path: string }> } outputs?: { files?: Array<{ path: string; mode?: 'create' | 'overwrite'; mimeType?: string }> } } interface GenerateAudioResult { success: boolean message: string fileId?: string fileName?: string vfsPath?: string downloadUrl?: string _serviceCost?: { service: string; cost: number } } function audioExtFromContentType(contentType: string): string { if (contentType.includes('wav')) return 'wav' if (contentType.includes('mp4') || contentType.includes('m4a')) return 'm4a' if (contentType.includes('ogg')) return 'ogg' if (contentType.includes('flac')) return 'flac' if (contentType.includes('aac')) return 'aac' if (contentType.includes('opus')) return 'opus' return 'mp3' } export const generateAudioServerTool: BaseServerTool = { name: GenerateAudio.id, async execute( params: GenerateAudioArgs, context?: ServerToolContext ): Promise { if (!context?.userId) { throw new Error('Authentication required') } const workspaceId = context.workspaceId if (!workspaceId) { return { success: false, message: 'Workspace ID is required' } } if (!params.prompt) { return { success: false, message: 'prompt is required' } } const type = (params.type || 'speech') as AudioType if (!VALID_TYPES.includes(type)) { return { success: false, message: `Invalid type "${params.type}". Must be one of: ${VALID_TYPES.join(', ')}`, } } // Voice cloning: a reference sample clones that voice into the generated speech. let voiceSampleDataUri: string | undefined const samplePath = params.inputs?.files?.[0]?.path if (samplePath) { const sample = await resolveWorkspaceFileReference(workspaceId, samplePath) if (!sample) { return { success: false, message: `Voice sample not found: ${samplePath}` } } const sampleBuffer = await fetchWorkspaceFileBuffer(sample) const sampleMime = sample.type || 'audio/mpeg' voiceSampleDataUri = `data:${sampleMime};base64,${sampleBuffer.toString('base64')}` } try { logger.info('Generating audio', { type, model: params.model, promptLength: params.prompt.length, voiceClone: Boolean(voiceSampleDataUri), }) const result = await generateFalAudio({ prompt: params.prompt, type, model: params.model, voice: params.voice, duration: params.duration, lyrics: params.lyrics, instrumental: params.instrumental, voiceSampleDataUri, }) const outputFile = params.outputs?.files?.[0] const ext = audioExtFromContentType(result.contentType) const outputPath = outputFile?.path || `files/generated-audio.${ext}` const mode = outputFile?.mode ?? 'create' assertServerToolNotAborted(context) const written = await writeWorkspaceFileByPath({ workspaceId, userId: context.userId, target: { path: outputPath, mode, mimeType: outputFile?.mimeType }, buffer: result.buffer, inferredMimeType: result.contentType, }) logger.info('Generated audio saved', { fileId: written.id, vfsPath: written.vfsPath, size: result.buffer.length, type, model: result.model, }) return { success: true, message: `${type === 'speech' ? 'Speech' : type === 'music' ? 'Music' : 'Sound effect'} generated and ${written.mode === 'overwrite' ? 'updated' : 'saved'} at "${written.vfsPath}" (${result.buffer.length} bytes, model ${result.model})`, fileId: written.id, fileName: written.name, vfsPath: written.vfsPath, downloadUrl: written.downloadUrl, _serviceCost: { service: 'falai_audio', cost: result.cost.costDollars }, } } catch (error) { const msg = getErrorMessage(error, 'Unknown error') logger.error('Audio generation failed', { error: msg }) return { success: false, message: `Failed to generate audio: ${msg}` } } }, }