chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,174 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import type { NextRequest } from 'next/server'
|
||||
import { NextResponse } from 'next/server'
|
||||
import { ttsToolContract } from '@/lib/api/contracts/tools/media/tts'
|
||||
import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
|
||||
import { checkInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
|
||||
import { validateAlphanumericId } from '@/lib/core/security/input-validation'
|
||||
import {
|
||||
isPayloadSizeLimitError,
|
||||
readResponseToBufferWithLimit,
|
||||
} from '@/lib/core/utils/stream-limits'
|
||||
import { getBaseUrl } from '@/lib/core/utils/urls'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { StorageService } from '@/lib/uploads'
|
||||
|
||||
const logger = createLogger('ProxyTTSAPI')
|
||||
const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const authResult = await checkInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success) {
|
||||
logger.error('Authentication failed for TTS proxy:', authResult.error)
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(
|
||||
ttsToolContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) =>
|
||||
NextResponse.json(
|
||||
{ error: getValidationErrorMessage(error, 'Missing required parameters') },
|
||||
{ status: 400 }
|
||||
),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const {
|
||||
text,
|
||||
voiceId,
|
||||
apiKey,
|
||||
modelId,
|
||||
stability,
|
||||
similarityBoost,
|
||||
workspaceId,
|
||||
workflowId,
|
||||
executionId,
|
||||
} = parsed.data.body
|
||||
|
||||
const voiceIdValidation = validateAlphanumericId(voiceId, 'voiceId', 255)
|
||||
if (!voiceIdValidation.isValid) {
|
||||
logger.error(`Invalid voice ID: ${voiceIdValidation.error}`)
|
||||
return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 })
|
||||
}
|
||||
|
||||
// Check if this is an execution context (from workflow tool execution)
|
||||
const executionContext =
|
||||
workspaceId && workflowId && executionId ? { workspaceId, workflowId, executionId } : null
|
||||
logger.info('Proxying TTS request for voice:', {
|
||||
voiceId,
|
||||
hasExecutionContext: Boolean(executionContext),
|
||||
workspaceId,
|
||||
workflowId,
|
||||
executionId,
|
||||
})
|
||||
|
||||
const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`
|
||||
|
||||
const hasVoiceSetting = stability !== undefined || similarityBoost !== undefined
|
||||
const voiceSettings = hasVoiceSetting
|
||||
? {
|
||||
stability: stability ?? 0.5,
|
||||
similarity_boost: similarityBoost ?? 0.75,
|
||||
}
|
||||
: undefined
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'audio/mpeg',
|
||||
'Content-Type': 'application/json',
|
||||
'xi-api-key': apiKey,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
model_id: modelId,
|
||||
...(voiceSettings ? { voice_settings: voiceSettings } : {}),
|
||||
}),
|
||||
signal: AbortSignal.timeout(DEFAULT_EXECUTION_TIMEOUT_MS),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {})
|
||||
logger.error(`Failed to generate TTS: ${response.status} ${response.statusText}`)
|
||||
return NextResponse.json(
|
||||
{ error: `Failed to generate TTS: ${response.status} ${response.statusText}` },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'TTS audio response',
|
||||
signal: request.signal,
|
||||
})
|
||||
|
||||
if (audioBuffer.length === 0) {
|
||||
logger.error('Empty audio received from ElevenLabs')
|
||||
return NextResponse.json({ error: 'Empty audio received' }, { status: 422 })
|
||||
}
|
||||
|
||||
const timestamp = Date.now()
|
||||
|
||||
// Use execution storage for workflow tool calls, copilot for chat UI
|
||||
if (executionContext) {
|
||||
const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution')
|
||||
const fileName = `tts-${timestamp}.mp3`
|
||||
|
||||
const userFile = await uploadExecutionFile(
|
||||
executionContext,
|
||||
audioBuffer,
|
||||
fileName,
|
||||
'audio/mpeg',
|
||||
authResult.userId
|
||||
)
|
||||
|
||||
logger.info('TTS audio stored in execution context:', {
|
||||
executionId,
|
||||
fileName,
|
||||
size: userFile.size,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
audioFile: userFile,
|
||||
audioUrl: userFile.url,
|
||||
})
|
||||
}
|
||||
|
||||
// Chat UI usage - no execution context, use copilot context
|
||||
const fileName = `tts-${timestamp}.mp3`
|
||||
const fileInfo = await StorageService.uploadFile({
|
||||
file: audioBuffer,
|
||||
fileName,
|
||||
contentType: 'audio/mpeg',
|
||||
context: 'copilot',
|
||||
})
|
||||
|
||||
const audioUrl = `${getBaseUrl()}${fileInfo.path}`
|
||||
|
||||
logger.info('TTS audio stored in copilot context (chat UI):', {
|
||||
fileName,
|
||||
size: fileInfo.size,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
audioUrl,
|
||||
size: fileInfo.size,
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error('Error proxying TTS:', error)
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Internal Server Error: ${getErrorMessage(error, 'Unknown error')}`,
|
||||
},
|
||||
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,846 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import type { NextRequest } from 'next/server'
|
||||
import { NextResponse } from 'next/server'
|
||||
import {
|
||||
playHtOutputFormatSchema,
|
||||
ttsUnifiedToolContract,
|
||||
} from '@/lib/api/contracts/tools/media/tts'
|
||||
import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server'
|
||||
import { checkInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { validateAlphanumericId } from '@/lib/core/security/input-validation'
|
||||
import {
|
||||
assertKnownSizeWithinLimit,
|
||||
isPayloadSizeLimitError,
|
||||
readResponseJsonWithLimit,
|
||||
readResponseTextWithLimit,
|
||||
readResponseToBufferWithLimit,
|
||||
} from '@/lib/core/utils/stream-limits'
|
||||
import { getBaseUrl } from '@/lib/core/utils/urls'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { StorageService } from '@/lib/uploads'
|
||||
import type {
|
||||
AzureTtsParams,
|
||||
CartesiaTtsParams,
|
||||
DeepgramTtsParams,
|
||||
ElevenLabsTtsUnifiedParams,
|
||||
GoogleTtsParams,
|
||||
OpenAiTtsParams,
|
||||
PlayHtTtsParams,
|
||||
TtsResponse,
|
||||
} from '@/tools/tts/types'
|
||||
import { getFileExtension, getMimeType } from '@/tools/tts/types'
|
||||
|
||||
const logger = createLogger('TtsUnifiedProxyAPI')
|
||||
const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024
|
||||
const MAX_TTS_ERROR_BYTES = 64 * 1024
|
||||
const MAX_TTS_JSON_BYTES = Math.ceil((MAX_TTS_AUDIO_BYTES * 4) / 3) + 256 * 1024
|
||||
|
||||
async function readTtsErrorJson(
|
||||
response: Response,
|
||||
label: string
|
||||
): Promise<Record<string, unknown>> {
|
||||
return readResponseJsonWithLimit<Record<string, unknown>>(response, {
|
||||
maxBytes: MAX_TTS_ERROR_BYTES,
|
||||
label,
|
||||
}).catch(() => ({}))
|
||||
}
|
||||
|
||||
function getTtsErrorMessage(error: Record<string, unknown>, fallback: string): string {
|
||||
const nested = error.error
|
||||
if (typeof nested === 'object' && nested !== null && 'message' in nested) {
|
||||
const message = (nested as { message?: unknown }).message
|
||||
if (typeof message === 'string') return message
|
||||
}
|
||||
for (const key of ['message', 'err_msg', 'error_message', 'error', 'detail']) {
|
||||
const value = error[key]
|
||||
if (typeof value === 'string') return value
|
||||
if (typeof value === 'object' && value !== null && 'message' in value) {
|
||||
const message = (value as { message?: unknown }).message
|
||||
if (typeof message === 'string') return message
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const maxDuration = 60 // 1 minute
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateId()
|
||||
logger.info(`[${requestId}] TTS unified request started`)
|
||||
|
||||
try {
|
||||
const authResult = await checkInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success) {
|
||||
logger.error('Authentication failed for TTS unified proxy:', authResult.error)
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(
|
||||
ttsUnifiedToolContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) => {
|
||||
logger.warn(`[${requestId}] Invalid TTS unified request:`, error.issues)
|
||||
return validationErrorResponse(
|
||||
error,
|
||||
getValidationErrorMessage(error, 'Invalid request data')
|
||||
)
|
||||
},
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const body = parsed.data.body
|
||||
const { provider, text, apiKey, workspaceId, workflowId, executionId } = body
|
||||
|
||||
const executionContext =
|
||||
workspaceId && workflowId && executionId ? { workspaceId, workflowId, executionId } : null
|
||||
logger.info(`[${requestId}] Processing TTS with ${provider}`, {
|
||||
hasExecutionContext: Boolean(executionContext),
|
||||
textLength: text.length,
|
||||
})
|
||||
|
||||
let audioBuffer: Buffer
|
||||
let format: string
|
||||
let mimeType: string
|
||||
let duration: number | undefined
|
||||
|
||||
try {
|
||||
if (provider === 'openai') {
|
||||
const result = await synthesizeWithOpenAi({
|
||||
text,
|
||||
apiKey,
|
||||
model: body.model,
|
||||
voice: body.voice as OpenAiTtsParams['voice'],
|
||||
responseFormat: body.responseFormat,
|
||||
speed: body.speed,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
} else if (provider === 'deepgram') {
|
||||
const result = await synthesizeWithDeepgram({
|
||||
text,
|
||||
apiKey,
|
||||
model: body.voice,
|
||||
encoding: body.encoding,
|
||||
sampleRate: body.sampleRate,
|
||||
bitRate: body.bitRate,
|
||||
container: body.container,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
duration = result.duration
|
||||
} else if (provider === 'elevenlabs') {
|
||||
if (!body.voiceId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'voiceId is required for ElevenLabs provider' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const voiceIdValidation = validateAlphanumericId(body.voiceId, 'voiceId')
|
||||
if (!voiceIdValidation.isValid) {
|
||||
return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 })
|
||||
}
|
||||
const result = await synthesizeWithElevenLabs({
|
||||
text,
|
||||
apiKey,
|
||||
voiceId: body.voiceId,
|
||||
modelId: body.modelId,
|
||||
stability: body.stability,
|
||||
similarityBoost: body.similarityBoost,
|
||||
style: body.style as number | undefined,
|
||||
useSpeakerBoost: body.useSpeakerBoost,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
} else if (provider === 'cartesia') {
|
||||
const result = await synthesizeWithCartesia({
|
||||
text,
|
||||
apiKey,
|
||||
modelId: body.modelId,
|
||||
voice: body.voice,
|
||||
language: body.language,
|
||||
outputFormat:
|
||||
body.outputFormat &&
|
||||
typeof body.outputFormat === 'object' &&
|
||||
!Array.isArray(body.outputFormat)
|
||||
? (body.outputFormat as CartesiaTtsParams['outputFormat'])
|
||||
: undefined,
|
||||
speed: body.speed,
|
||||
emotion: body.emotion,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
} else if (provider === 'google') {
|
||||
const result = await synthesizeWithGoogle({
|
||||
text,
|
||||
apiKey,
|
||||
voiceId: body.voiceId,
|
||||
languageCode: body.languageCode,
|
||||
gender: body.gender,
|
||||
audioEncoding: body.audioEncoding,
|
||||
speakingRate: body.speakingRate,
|
||||
pitch: typeof body.pitch === 'number' ? body.pitch : undefined,
|
||||
volumeGainDb: body.volumeGainDb,
|
||||
sampleRateHertz: body.sampleRateHertz,
|
||||
effectsProfileId: body.effectsProfileId,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
} else if (provider === 'azure') {
|
||||
const result = await synthesizeWithAzure({
|
||||
text,
|
||||
apiKey,
|
||||
voiceId: body.voiceId,
|
||||
region: body.region,
|
||||
outputFormat:
|
||||
typeof body.outputFormat === 'string'
|
||||
? (body.outputFormat as AzureTtsParams['outputFormat'])
|
||||
: undefined,
|
||||
rate: body.rate,
|
||||
pitch: body.pitch as string | undefined,
|
||||
style: body.style as string | undefined,
|
||||
styleDegree: body.styleDegree,
|
||||
role: body.role,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
} else if (provider === 'playht') {
|
||||
if (!body.userId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'userId is required for PlayHT provider' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const playHtOutputFormat = playHtOutputFormatSchema.safeParse(body.outputFormat)
|
||||
const result = await synthesizeWithPlayHT({
|
||||
text,
|
||||
apiKey,
|
||||
userId: body.userId,
|
||||
voice: body.voice,
|
||||
quality: body.quality,
|
||||
outputFormat: playHtOutputFormat.success ? playHtOutputFormat.data : undefined,
|
||||
speed: body.speed,
|
||||
temperature: body.temperature,
|
||||
voiceGuidance: body.voiceGuidance,
|
||||
textGuidance: body.textGuidance,
|
||||
sampleRate: body.sampleRate,
|
||||
})
|
||||
audioBuffer = result.audioBuffer
|
||||
format = result.format
|
||||
mimeType = result.mimeType
|
||||
} else {
|
||||
return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] TTS synthesis failed:`, error)
|
||||
const errorMessage = getErrorMessage(error, 'TTS synthesis failed')
|
||||
return NextResponse.json(
|
||||
{ error: errorMessage },
|
||||
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
|
||||
)
|
||||
}
|
||||
|
||||
const timestamp = Date.now()
|
||||
const fileExtension = getFileExtension(format)
|
||||
const fileName = `tts-${provider}-${timestamp}.${fileExtension}`
|
||||
|
||||
if (executionContext) {
|
||||
const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution')
|
||||
|
||||
const userFile = await uploadExecutionFile(
|
||||
executionContext,
|
||||
audioBuffer,
|
||||
fileName,
|
||||
mimeType,
|
||||
authResult.userId
|
||||
)
|
||||
|
||||
logger.info(`[${requestId}] TTS audio stored in execution context:`, {
|
||||
executionId,
|
||||
fileName,
|
||||
size: userFile.size,
|
||||
})
|
||||
|
||||
const response: TtsResponse = {
|
||||
audioUrl: userFile.url,
|
||||
audioFile: userFile,
|
||||
characterCount: text.length,
|
||||
format,
|
||||
provider,
|
||||
}
|
||||
|
||||
if (duration) {
|
||||
response.duration = duration
|
||||
}
|
||||
|
||||
return NextResponse.json(response)
|
||||
}
|
||||
|
||||
// Chat UI / copilot usage - no execution context
|
||||
const fileInfo = await StorageService.uploadFile({
|
||||
file: audioBuffer,
|
||||
fileName,
|
||||
contentType: mimeType,
|
||||
context: 'copilot',
|
||||
})
|
||||
|
||||
const audioUrl = `${getBaseUrl()}${fileInfo.path}`
|
||||
|
||||
logger.info(`[${requestId}] TTS audio stored in copilot context:`, {
|
||||
fileName,
|
||||
size: fileInfo.size,
|
||||
})
|
||||
|
||||
const response: TtsResponse = {
|
||||
audioUrl,
|
||||
characterCount: text.length,
|
||||
format,
|
||||
provider,
|
||||
}
|
||||
|
||||
if (duration) {
|
||||
response.duration = duration
|
||||
}
|
||||
|
||||
return NextResponse.json(response)
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] TTS unified proxy error:`, error)
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
return NextResponse.json(
|
||||
{ error: errorMessage },
|
||||
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
async function synthesizeWithOpenAi(
|
||||
params: OpenAiTtsParams
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
||||
const { text, apiKey, model = 'tts-1', responseFormat = 'mp3', speed = 1.0 } = params
|
||||
const voice = (params.voice || 'alloy') as OpenAiTtsParams['voice']
|
||||
|
||||
const response = await fetch('https://api.openai.com/v1/audio/speech', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
voice,
|
||||
input: text,
|
||||
response_format: responseFormat,
|
||||
speed: Math.max(0.25, Math.min(4.0, speed)),
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readTtsErrorJson(response, 'OpenAI TTS error response')
|
||||
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
||||
throw new Error(`OpenAI TTS API error: ${errorMessage}`)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'OpenAI TTS audio response',
|
||||
})
|
||||
const mimeType = getMimeType(responseFormat)
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format: responseFormat,
|
||||
mimeType,
|
||||
}
|
||||
}
|
||||
|
||||
async function synthesizeWithDeepgram(
|
||||
params: DeepgramTtsParams
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string; duration?: number }> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
model = 'aura-asteria-en',
|
||||
encoding = 'mp3',
|
||||
sampleRate,
|
||||
bitRate,
|
||||
container,
|
||||
} = params
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
model: model,
|
||||
encoding: encoding,
|
||||
})
|
||||
|
||||
if (sampleRate && encoding === 'linear16') {
|
||||
queryParams.append('sample_rate', sampleRate.toString())
|
||||
}
|
||||
|
||||
if (bitRate) {
|
||||
queryParams.append('bit_rate', bitRate.toString())
|
||||
}
|
||||
|
||||
if (container && container !== 'none') {
|
||||
queryParams.append('container', container)
|
||||
}
|
||||
|
||||
const response = await fetch(`https://api.deepgram.com/v1/speak?${queryParams.toString()}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Token ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ text }),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readTtsErrorJson(response, 'Deepgram TTS error response')
|
||||
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
||||
throw new Error(`Deepgram TTS API error: ${errorMessage}`)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'Deepgram TTS audio response',
|
||||
})
|
||||
|
||||
let finalFormat: string = encoding
|
||||
if (container === 'wav') {
|
||||
finalFormat = 'wav'
|
||||
} else if (container === 'ogg') {
|
||||
finalFormat = 'ogg'
|
||||
}
|
||||
|
||||
const mimeType = getMimeType(finalFormat)
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format: finalFormat,
|
||||
mimeType,
|
||||
}
|
||||
}
|
||||
|
||||
async function synthesizeWithElevenLabs(
|
||||
params: ElevenLabsTtsUnifiedParams
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
voiceId,
|
||||
modelId = 'eleven_turbo_v2_5',
|
||||
stability = 0.5,
|
||||
similarityBoost = 0.8,
|
||||
style,
|
||||
useSpeakerBoost = true,
|
||||
} = params
|
||||
|
||||
const voiceSettings: any = {
|
||||
stability: Math.max(0, Math.min(1, stability)),
|
||||
similarity_boost: Math.max(0, Math.min(1, similarityBoost)),
|
||||
use_speaker_boost: useSpeakerBoost,
|
||||
}
|
||||
|
||||
if (style !== undefined) {
|
||||
voiceSettings.style = Math.max(0, Math.min(1, style))
|
||||
}
|
||||
|
||||
const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'audio/mpeg',
|
||||
'Content-Type': 'application/json',
|
||||
'xi-api-key': apiKey,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
model_id: modelId,
|
||||
voice_settings: voiceSettings,
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readTtsErrorJson(response, 'ElevenLabs TTS error response')
|
||||
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
||||
throw new Error(`ElevenLabs TTS API error: ${errorMessage}`)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'ElevenLabs TTS audio response',
|
||||
})
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format: 'mp3',
|
||||
mimeType: 'audio/mpeg',
|
||||
}
|
||||
}
|
||||
|
||||
async function synthesizeWithCartesia(
|
||||
params: Partial<CartesiaTtsParams>
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
modelId = 'sonic-3',
|
||||
voice,
|
||||
language = 'en',
|
||||
outputFormat,
|
||||
speed,
|
||||
emotion,
|
||||
} = params
|
||||
|
||||
if (!text || !apiKey) {
|
||||
throw new Error('text and apiKey are required for Cartesia')
|
||||
}
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
model_id: modelId,
|
||||
transcript: text,
|
||||
language,
|
||||
}
|
||||
|
||||
if (voice) {
|
||||
requestBody.voice = {
|
||||
mode: 'id',
|
||||
id: voice,
|
||||
}
|
||||
}
|
||||
|
||||
const generationConfig: Record<string, unknown> = {}
|
||||
if (speed !== undefined) generationConfig.speed = speed
|
||||
if (emotion !== undefined) generationConfig.emotion = emotion
|
||||
if (Object.keys(generationConfig).length > 0) {
|
||||
requestBody.generation_config = generationConfig
|
||||
}
|
||||
|
||||
if (outputFormat && typeof outputFormat === 'object') {
|
||||
requestBody.output_format = outputFormat
|
||||
}
|
||||
|
||||
if (!requestBody.output_format) {
|
||||
requestBody.output_format = {
|
||||
container: 'wav',
|
||||
encoding: 'pcm_s16le',
|
||||
sample_rate: 24000,
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Cartesia API request:', {
|
||||
model_id: requestBody.model_id,
|
||||
has_voice: !!requestBody.voice,
|
||||
language: requestBody.language,
|
||||
output_format: requestBody.output_format,
|
||||
has_generation_config: !!requestBody.generation_config,
|
||||
})
|
||||
|
||||
const response = await fetch('https://api.cartesia.ai/tts/bytes', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
'Cartesia-Version': '2025-04-16',
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readTtsErrorJson(response, 'Cartesia TTS error response')
|
||||
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
||||
const errorDetail = typeof error.detail === 'string' ? error.detail : ''
|
||||
logger.error('Cartesia API error details:', {
|
||||
status: response.status,
|
||||
error: errorMessage,
|
||||
detail: errorDetail,
|
||||
requestBody: JSON.stringify(requestBody),
|
||||
})
|
||||
throw new Error(
|
||||
`Cartesia TTS API error: ${errorMessage}${errorDetail ? ` - ${errorDetail}` : ''}`
|
||||
)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'Cartesia TTS audio response',
|
||||
})
|
||||
|
||||
const format =
|
||||
outputFormat && typeof outputFormat === 'object' && 'container' in outputFormat
|
||||
? (outputFormat.container as string)
|
||||
: 'mp3'
|
||||
const mimeType = getMimeType(format)
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format,
|
||||
mimeType,
|
||||
}
|
||||
}
|
||||
|
||||
async function synthesizeWithGoogle(
|
||||
params: Partial<GoogleTtsParams>
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
voiceId,
|
||||
languageCode,
|
||||
gender,
|
||||
audioEncoding = 'MP3',
|
||||
speakingRate = 1.0,
|
||||
pitch = 0.0,
|
||||
volumeGainDb,
|
||||
sampleRateHertz,
|
||||
effectsProfileId,
|
||||
} = params
|
||||
|
||||
if (!text || !apiKey || !languageCode) {
|
||||
throw new Error('text, apiKey, and languageCode are required for Google Cloud TTS')
|
||||
}
|
||||
|
||||
const clampedSpeakingRate = Math.max(0.25, Math.min(2.0, speakingRate))
|
||||
|
||||
const audioConfig: Record<string, unknown> = {
|
||||
audioEncoding,
|
||||
speakingRate: clampedSpeakingRate,
|
||||
pitch,
|
||||
}
|
||||
|
||||
if (volumeGainDb !== undefined) {
|
||||
audioConfig.volumeGainDb = volumeGainDb
|
||||
}
|
||||
if (sampleRateHertz) {
|
||||
audioConfig.sampleRateHertz = sampleRateHertz
|
||||
}
|
||||
if (effectsProfileId && effectsProfileId.length > 0) {
|
||||
audioConfig.effectsProfileId = effectsProfileId
|
||||
}
|
||||
|
||||
// Build voice config based on what's provided
|
||||
const voice: Record<string, unknown> = {
|
||||
languageCode,
|
||||
}
|
||||
|
||||
// If voiceId is provided, use it (it takes precedence over gender)
|
||||
if (voiceId) {
|
||||
voice.name = voiceId
|
||||
}
|
||||
|
||||
// Only include gender if specified (don't default to NEUTRAL as it's not supported)
|
||||
if (gender) {
|
||||
voice.ssmlGender = gender
|
||||
}
|
||||
|
||||
// If neither voiceId nor gender is provided, default to a specific voice
|
||||
if (!voiceId && !gender) {
|
||||
voice.name = 'en-US-Neural2-C'
|
||||
}
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
input: { text },
|
||||
voice,
|
||||
audioConfig,
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
`https://texttospeech.googleapis.com/v1/text:synthesize?key=${apiKey}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readTtsErrorJson(response, 'Google TTS error response')
|
||||
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
||||
throw new Error(`Google Cloud TTS API error: ${errorMessage}`)
|
||||
}
|
||||
|
||||
const data = await readResponseJsonWithLimit<{ audioContent?: string }>(response, {
|
||||
maxBytes: MAX_TTS_JSON_BYTES,
|
||||
label: 'Google TTS JSON response',
|
||||
})
|
||||
const audioContent = data.audioContent
|
||||
|
||||
if (!audioContent) {
|
||||
throw new Error('No audio content returned from Google Cloud TTS')
|
||||
}
|
||||
|
||||
const audioBuffer = Buffer.from(audioContent, 'base64')
|
||||
assertKnownSizeWithinLimit(audioBuffer.length, MAX_TTS_AUDIO_BYTES, 'Google TTS audio response')
|
||||
|
||||
const format = audioEncoding.toLowerCase().replace('_', '')
|
||||
const mimeType = getMimeType(format)
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format,
|
||||
mimeType,
|
||||
}
|
||||
}
|
||||
|
||||
async function synthesizeWithAzure(
|
||||
params: Partial<AzureTtsParams>
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
voiceId = 'en-US-JennyNeural',
|
||||
region = 'eastus',
|
||||
outputFormat = 'audio-24khz-96kbitrate-mono-mp3',
|
||||
rate,
|
||||
pitch,
|
||||
style,
|
||||
styleDegree,
|
||||
role,
|
||||
} = params
|
||||
|
||||
if (!text || !apiKey) {
|
||||
throw new Error('text and apiKey are required for Azure TTS')
|
||||
}
|
||||
|
||||
const AZURE_REGION_RE = /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/
|
||||
if (!AZURE_REGION_RE.test(region)) {
|
||||
throw new Error(
|
||||
'Invalid Azure region: must match /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ (e.g. eastus, westeurope)'
|
||||
)
|
||||
}
|
||||
|
||||
let ssml = `<speak version='1.0' xml:lang='en-US' xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts"><voice name='${voiceId}'>`
|
||||
|
||||
if (style) {
|
||||
ssml += `<mstts:express-as style='${style}'`
|
||||
if (styleDegree) ssml += ` styledegree='${styleDegree}'`
|
||||
if (role) ssml += ` role='${role}'`
|
||||
ssml += '>'
|
||||
}
|
||||
|
||||
if (rate || pitch) {
|
||||
ssml += '<prosody'
|
||||
if (rate) ssml += ` rate='${rate}'`
|
||||
if (pitch) ssml += ` pitch='${pitch}'`
|
||||
ssml += '>'
|
||||
}
|
||||
|
||||
ssml += text
|
||||
|
||||
if (rate || pitch) {
|
||||
ssml += '</prosody>'
|
||||
}
|
||||
|
||||
if (style) {
|
||||
ssml += '</mstts:express-as>'
|
||||
}
|
||||
|
||||
ssml += '</voice></speak>'
|
||||
|
||||
const response = await fetch(`https://${region}.tts.speech.microsoft.com/cognitiveservices/v1`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Ocp-Apim-Subscription-Key': apiKey,
|
||||
'Content-Type': 'application/ssml+xml',
|
||||
'X-Microsoft-OutputFormat': outputFormat,
|
||||
},
|
||||
body: ssml,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readResponseTextWithLimit(response, {
|
||||
maxBytes: MAX_TTS_ERROR_BYTES,
|
||||
label: 'Azure TTS error response',
|
||||
})
|
||||
throw new Error(`Azure TTS API error: ${error || response.statusText}`)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'Azure TTS audio response',
|
||||
})
|
||||
|
||||
const format = outputFormat.includes('mp3') ? 'mp3' : 'wav'
|
||||
const mimeType = getMimeType(format)
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format,
|
||||
mimeType,
|
||||
}
|
||||
}
|
||||
|
||||
async function synthesizeWithPlayHT(
|
||||
params: Partial<PlayHtTtsParams>
|
||||
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
userId,
|
||||
voice,
|
||||
quality = 'standard',
|
||||
outputFormat = 'mp3',
|
||||
speed = 1.0,
|
||||
temperature,
|
||||
voiceGuidance,
|
||||
textGuidance,
|
||||
sampleRate,
|
||||
} = params
|
||||
|
||||
if (!text || !apiKey || !userId) {
|
||||
throw new Error('text, apiKey, and userId are required for PlayHT')
|
||||
}
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
text,
|
||||
quality,
|
||||
output_format: outputFormat,
|
||||
speed,
|
||||
}
|
||||
|
||||
if (voice) requestBody.voice = voice
|
||||
if (temperature !== undefined) requestBody.temperature = temperature
|
||||
if (voiceGuidance !== undefined) requestBody.voice_guidance = voiceGuidance
|
||||
if (textGuidance !== undefined) requestBody.text_guidance = textGuidance
|
||||
if (sampleRate) requestBody.sample_rate = sampleRate
|
||||
|
||||
const response = await fetch('https://api.play.ht/api/v2/tts/stream', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
AUTHORIZATION: apiKey,
|
||||
'X-USER-ID': userId,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await readTtsErrorJson(response, 'PlayHT TTS error response')
|
||||
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
||||
throw new Error(`PlayHT TTS API error: ${errorMessage}`)
|
||||
}
|
||||
|
||||
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
||||
maxBytes: MAX_TTS_AUDIO_BYTES,
|
||||
label: 'PlayHT TTS audio response',
|
||||
})
|
||||
|
||||
const format = outputFormat || 'mp3'
|
||||
const mimeType = getMimeType(format)
|
||||
|
||||
return {
|
||||
audioBuffer,
|
||||
format,
|
||||
mimeType,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user