import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { playHtOutputFormatSchema, ttsUnifiedToolContract, } from '@/lib/api/contracts/tools/media/tts' import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError, readResponseJsonWithLimit, readResponseTextWithLimit, readResponseToBufferWithLimit, } from '@/lib/core/utils/stream-limits' import { getBaseUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { StorageService } from '@/lib/uploads' import type { AzureTtsParams, CartesiaTtsParams, DeepgramTtsParams, ElevenLabsTtsUnifiedParams, GoogleTtsParams, OpenAiTtsParams, PlayHtTtsParams, TtsResponse, } from '@/tools/tts/types' import { getFileExtension, getMimeType } from '@/tools/tts/types' const logger = createLogger('TtsUnifiedProxyAPI') const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 const MAX_TTS_ERROR_BYTES = 64 * 1024 const MAX_TTS_JSON_BYTES = Math.ceil((MAX_TTS_AUDIO_BYTES * 4) / 3) + 256 * 1024 async function readTtsErrorJson( response: Response, label: string ): Promise> { return readResponseJsonWithLimit>(response, { maxBytes: MAX_TTS_ERROR_BYTES, label, }).catch(() => ({})) } function getTtsErrorMessage(error: Record, fallback: string): string { const nested = error.error if (typeof nested === 'object' && nested !== null && 'message' in nested) { const message = (nested as { message?: unknown }).message if (typeof message === 'string') return message } for (const key of ['message', 'err_msg', 'error_message', 'error', 'detail']) { const value = error[key] if (typeof value === 'string') return value if (typeof value === 'object' && value !== null && 'message' in value) { const message = (value as { message?: unknown }).message if (typeof message === 'string') return message } } return fallback } export const dynamic = 'force-dynamic' export const maxDuration = 60 // 1 minute export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateId() logger.info(`[${requestId}] TTS unified request started`) try { const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) if (!authResult.success) { logger.error('Authentication failed for TTS unified proxy:', authResult.error) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } const parsed = await parseRequest( ttsUnifiedToolContract, request, {}, { validationErrorResponse: (error) => { logger.warn(`[${requestId}] Invalid TTS unified request:`, error.issues) return validationErrorResponse( error, getValidationErrorMessage(error, 'Invalid request data') ) }, } ) if (!parsed.success) return parsed.response const body = parsed.data.body const { provider, text, apiKey, workspaceId, workflowId, executionId } = body const executionContext = workspaceId && workflowId && executionId ? { workspaceId, workflowId, executionId } : null logger.info(`[${requestId}] Processing TTS with ${provider}`, { hasExecutionContext: Boolean(executionContext), textLength: text.length, }) let audioBuffer: Buffer let format: string let mimeType: string let duration: number | undefined try { if (provider === 'openai') { const result = await synthesizeWithOpenAi({ text, apiKey, model: body.model, voice: body.voice as OpenAiTtsParams['voice'], responseFormat: body.responseFormat, speed: body.speed, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType } else if (provider === 'deepgram') { const result = await synthesizeWithDeepgram({ text, apiKey, model: body.voice, encoding: body.encoding, sampleRate: body.sampleRate, bitRate: body.bitRate, container: body.container, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType duration = result.duration } else if (provider === 'elevenlabs') { if (!body.voiceId) { return NextResponse.json( { error: 'voiceId is required for ElevenLabs provider' }, { status: 400 } ) } const voiceIdValidation = validateAlphanumericId(body.voiceId, 'voiceId') if (!voiceIdValidation.isValid) { return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 }) } const result = await synthesizeWithElevenLabs({ text, apiKey, voiceId: body.voiceId, modelId: body.modelId, stability: body.stability, similarityBoost: body.similarityBoost, style: body.style as number | undefined, useSpeakerBoost: body.useSpeakerBoost, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType } else if (provider === 'cartesia') { const result = await synthesizeWithCartesia({ text, apiKey, modelId: body.modelId, voice: body.voice, language: body.language, outputFormat: body.outputFormat && typeof body.outputFormat === 'object' && !Array.isArray(body.outputFormat) ? (body.outputFormat as CartesiaTtsParams['outputFormat']) : undefined, speed: body.speed, emotion: body.emotion, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType } else if (provider === 'google') { const result = await synthesizeWithGoogle({ text, apiKey, voiceId: body.voiceId, languageCode: body.languageCode, gender: body.gender, audioEncoding: body.audioEncoding, speakingRate: body.speakingRate, pitch: typeof body.pitch === 'number' ? body.pitch : undefined, volumeGainDb: body.volumeGainDb, sampleRateHertz: body.sampleRateHertz, effectsProfileId: body.effectsProfileId, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType } else if (provider === 'azure') { const result = await synthesizeWithAzure({ text, apiKey, voiceId: body.voiceId, region: body.region, outputFormat: typeof body.outputFormat === 'string' ? (body.outputFormat as AzureTtsParams['outputFormat']) : undefined, rate: body.rate, pitch: body.pitch as string | undefined, style: body.style as string | undefined, styleDegree: body.styleDegree, role: body.role, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType } else if (provider === 'playht') { if (!body.userId) { return NextResponse.json( { error: 'userId is required for PlayHT provider' }, { status: 400 } ) } const playHtOutputFormat = playHtOutputFormatSchema.safeParse(body.outputFormat) const result = await synthesizeWithPlayHT({ text, apiKey, userId: body.userId, voice: body.voice, quality: body.quality, outputFormat: playHtOutputFormat.success ? playHtOutputFormat.data : undefined, speed: body.speed, temperature: body.temperature, voiceGuidance: body.voiceGuidance, textGuidance: body.textGuidance, sampleRate: body.sampleRate, }) audioBuffer = result.audioBuffer format = result.format mimeType = result.mimeType } else { return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) } } catch (error) { logger.error(`[${requestId}] TTS synthesis failed:`, error) const errorMessage = getErrorMessage(error, 'TTS synthesis failed') return NextResponse.json( { error: errorMessage }, { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } const timestamp = Date.now() const fileExtension = getFileExtension(format) const fileName = `tts-${provider}-${timestamp}.${fileExtension}` if (executionContext) { const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') const userFile = await uploadExecutionFile( executionContext, audioBuffer, fileName, mimeType, authResult.userId ) logger.info(`[${requestId}] TTS audio stored in execution context:`, { executionId, fileName, size: userFile.size, }) const response: TtsResponse = { audioUrl: userFile.url, audioFile: userFile, characterCount: text.length, format, provider, } if (duration) { response.duration = duration } return NextResponse.json(response) } // Chat UI / copilot usage - no execution context const fileInfo = await StorageService.uploadFile({ file: audioBuffer, fileName, contentType: mimeType, context: 'copilot', }) const audioUrl = `${getBaseUrl()}${fileInfo.path}` logger.info(`[${requestId}] TTS audio stored in copilot context:`, { fileName, size: fileInfo.size, }) const response: TtsResponse = { audioUrl, characterCount: text.length, format, provider, } if (duration) { response.duration = duration } return NextResponse.json(response) } catch (error) { logger.error(`[${requestId}] TTS unified proxy error:`, error) const errorMessage = getErrorMessage(error, 'Unknown error') return NextResponse.json( { error: errorMessage }, { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } }) async function synthesizeWithOpenAi( params: OpenAiTtsParams ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { const { text, apiKey, model = 'tts-1', responseFormat = 'mp3', speed = 1.0 } = params const voice = (params.voice || 'alloy') as OpenAiTtsParams['voice'] const response = await fetch('https://api.openai.com/v1/audio/speech', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ model, voice, input: text, response_format: responseFormat, speed: Math.max(0.25, Math.min(4.0, speed)), }), }) if (!response.ok) { const error = await readTtsErrorJson(response, 'OpenAI TTS error response') const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`OpenAI TTS API error: ${errorMessage}`) } const audioBuffer = await readResponseToBufferWithLimit(response, { maxBytes: MAX_TTS_AUDIO_BYTES, label: 'OpenAI TTS audio response', }) const mimeType = getMimeType(responseFormat) return { audioBuffer, format: responseFormat, mimeType, } } async function synthesizeWithDeepgram( params: DeepgramTtsParams ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string; duration?: number }> { const { text, apiKey, model = 'aura-asteria-en', encoding = 'mp3', sampleRate, bitRate, container, } = params const queryParams = new URLSearchParams({ model: model, encoding: encoding, }) if (sampleRate && encoding === 'linear16') { queryParams.append('sample_rate', sampleRate.toString()) } if (bitRate) { queryParams.append('bit_rate', bitRate.toString()) } if (container && container !== 'none') { queryParams.append('container', container) } const response = await fetch(`https://api.deepgram.com/v1/speak?${queryParams.toString()}`, { method: 'POST', headers: { Authorization: `Token ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ text }), }) if (!response.ok) { const error = await readTtsErrorJson(response, 'Deepgram TTS error response') const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`Deepgram TTS API error: ${errorMessage}`) } const audioBuffer = await readResponseToBufferWithLimit(response, { maxBytes: MAX_TTS_AUDIO_BYTES, label: 'Deepgram TTS audio response', }) let finalFormat: string = encoding if (container === 'wav') { finalFormat = 'wav' } else if (container === 'ogg') { finalFormat = 'ogg' } const mimeType = getMimeType(finalFormat) return { audioBuffer, format: finalFormat, mimeType, } } async function synthesizeWithElevenLabs( params: ElevenLabsTtsUnifiedParams ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { const { text, apiKey, voiceId, modelId = 'eleven_turbo_v2_5', stability = 0.5, similarityBoost = 0.8, style, useSpeakerBoost = true, } = params const voiceSettings: any = { stability: Math.max(0, Math.min(1, stability)), similarity_boost: Math.max(0, Math.min(1, similarityBoost)), use_speaker_boost: useSpeakerBoost, } if (style !== undefined) { voiceSettings.style = Math.max(0, Math.min(1, style)) } const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, { method: 'POST', headers: { Accept: 'audio/mpeg', 'Content-Type': 'application/json', 'xi-api-key': apiKey, }, body: JSON.stringify({ text, model_id: modelId, voice_settings: voiceSettings, }), }) if (!response.ok) { const error = await readTtsErrorJson(response, 'ElevenLabs TTS error response') const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`ElevenLabs TTS API error: ${errorMessage}`) } const audioBuffer = await readResponseToBufferWithLimit(response, { maxBytes: MAX_TTS_AUDIO_BYTES, label: 'ElevenLabs TTS audio response', }) return { audioBuffer, format: 'mp3', mimeType: 'audio/mpeg', } } async function synthesizeWithCartesia( params: Partial ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { const { text, apiKey, modelId = 'sonic-3', voice, language = 'en', outputFormat, speed, emotion, } = params if (!text || !apiKey) { throw new Error('text and apiKey are required for Cartesia') } const requestBody: Record = { model_id: modelId, transcript: text, language, } if (voice) { requestBody.voice = { mode: 'id', id: voice, } } const generationConfig: Record = {} if (speed !== undefined) generationConfig.speed = speed if (emotion !== undefined) generationConfig.emotion = emotion if (Object.keys(generationConfig).length > 0) { requestBody.generation_config = generationConfig } if (outputFormat && typeof outputFormat === 'object') { requestBody.output_format = outputFormat } if (!requestBody.output_format) { requestBody.output_format = { container: 'wav', encoding: 'pcm_s16le', sample_rate: 24000, } } logger.info('Cartesia API request:', { model_id: requestBody.model_id, has_voice: !!requestBody.voice, language: requestBody.language, output_format: requestBody.output_format, has_generation_config: !!requestBody.generation_config, }) const response = await fetch('https://api.cartesia.ai/tts/bytes', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', 'Cartesia-Version': '2025-04-16', }, body: JSON.stringify(requestBody), }) if (!response.ok) { const error = await readTtsErrorJson(response, 'Cartesia TTS error response') const errorMessage = getTtsErrorMessage(error, response.statusText) const errorDetail = typeof error.detail === 'string' ? error.detail : '' logger.error('Cartesia API error details:', { status: response.status, error: errorMessage, detail: errorDetail, requestBody: JSON.stringify(requestBody), }) throw new Error( `Cartesia TTS API error: ${errorMessage}${errorDetail ? ` - ${errorDetail}` : ''}` ) } const audioBuffer = await readResponseToBufferWithLimit(response, { maxBytes: MAX_TTS_AUDIO_BYTES, label: 'Cartesia TTS audio response', }) const format = outputFormat && typeof outputFormat === 'object' && 'container' in outputFormat ? (outputFormat.container as string) : 'mp3' const mimeType = getMimeType(format) return { audioBuffer, format, mimeType, } } async function synthesizeWithGoogle( params: Partial ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { const { text, apiKey, voiceId, languageCode, gender, audioEncoding = 'MP3', speakingRate = 1.0, pitch = 0.0, volumeGainDb, sampleRateHertz, effectsProfileId, } = params if (!text || !apiKey || !languageCode) { throw new Error('text, apiKey, and languageCode are required for Google Cloud TTS') } const clampedSpeakingRate = Math.max(0.25, Math.min(2.0, speakingRate)) const audioConfig: Record = { audioEncoding, speakingRate: clampedSpeakingRate, pitch, } if (volumeGainDb !== undefined) { audioConfig.volumeGainDb = volumeGainDb } if (sampleRateHertz) { audioConfig.sampleRateHertz = sampleRateHertz } if (effectsProfileId && effectsProfileId.length > 0) { audioConfig.effectsProfileId = effectsProfileId } // Build voice config based on what's provided const voice: Record = { languageCode, } // If voiceId is provided, use it (it takes precedence over gender) if (voiceId) { voice.name = voiceId } // Only include gender if specified (don't default to NEUTRAL as it's not supported) if (gender) { voice.ssmlGender = gender } // If neither voiceId nor gender is provided, default to a specific voice if (!voiceId && !gender) { voice.name = 'en-US-Neural2-C' } const requestBody: Record = { input: { text }, voice, audioConfig, } const response = await fetch( `https://texttospeech.googleapis.com/v1/text:synthesize?key=${apiKey}`, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(requestBody), } ) if (!response.ok) { const error = await readTtsErrorJson(response, 'Google TTS error response') const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`Google Cloud TTS API error: ${errorMessage}`) } const data = await readResponseJsonWithLimit<{ audioContent?: string }>(response, { maxBytes: MAX_TTS_JSON_BYTES, label: 'Google TTS JSON response', }) const audioContent = data.audioContent if (!audioContent) { throw new Error('No audio content returned from Google Cloud TTS') } const audioBuffer = Buffer.from(audioContent, 'base64') assertKnownSizeWithinLimit(audioBuffer.length, MAX_TTS_AUDIO_BYTES, 'Google TTS audio response') const format = audioEncoding.toLowerCase().replace('_', '') const mimeType = getMimeType(format) return { audioBuffer, format, mimeType, } } async function synthesizeWithAzure( params: Partial ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { const { text, apiKey, voiceId = 'en-US-JennyNeural', region = 'eastus', outputFormat = 'audio-24khz-96kbitrate-mono-mp3', rate, pitch, style, styleDegree, role, } = params if (!text || !apiKey) { throw new Error('text and apiKey are required for Azure TTS') } const AZURE_REGION_RE = /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ if (!AZURE_REGION_RE.test(region)) { throw new Error( 'Invalid Azure region: must match /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ (e.g. eastus, westeurope)' ) } let ssml = `` if (style) { ssml += ` ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { const { text, apiKey, userId, voice, quality = 'standard', outputFormat = 'mp3', speed = 1.0, temperature, voiceGuidance, textGuidance, sampleRate, } = params if (!text || !apiKey || !userId) { throw new Error('text, apiKey, and userId are required for PlayHT') } const requestBody: Record = { text, quality, output_format: outputFormat, speed, } if (voice) requestBody.voice = voice if (temperature !== undefined) requestBody.temperature = temperature if (voiceGuidance !== undefined) requestBody.voice_guidance = voiceGuidance if (textGuidance !== undefined) requestBody.text_guidance = textGuidance if (sampleRate) requestBody.sample_rate = sampleRate const response = await fetch('https://api.play.ht/api/v2/tts/stream', { method: 'POST', headers: { AUTHORIZATION: apiKey, 'X-USER-ID': userId, 'Content-Type': 'application/json', }, body: JSON.stringify(requestBody), }) if (!response.ok) { const error = await readTtsErrorJson(response, 'PlayHT TTS error response') const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`PlayHT TTS API error: ${errorMessage}`) } const audioBuffer = await readResponseToBufferWithLimit(response, { maxBytes: MAX_TTS_AUDIO_BYTES, label: 'PlayHT TTS audio response', }) const format = outputFormat || 'mp3' const mimeType = getMimeType(format) return { audioBuffer, format, mimeType, } }