d25d482dc2
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
847 lines
24 KiB
TypeScript
847 lines
24 KiB
TypeScript
import { createLogger } from '@sim/logger'
|
|
import { getErrorMessage } from '@sim/utils/errors'
|
|
import { generateId } from '@sim/utils/id'
|
|
import type { NextRequest } from 'next/server'
|
|
import { NextResponse } from 'next/server'
|
|
import {
|
|
playHtOutputFormatSchema,
|
|
ttsUnifiedToolContract,
|
|
} from '@/lib/api/contracts/tools/media/tts'
|
|
import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server'
|
|
import { checkInternalAuth } from '@/lib/auth/hybrid'
|
|
import { validateAlphanumericId } from '@/lib/core/security/input-validation'
|
|
import {
|
|
assertKnownSizeWithinLimit,
|
|
isPayloadSizeLimitError,
|
|
readResponseJsonWithLimit,
|
|
readResponseTextWithLimit,
|
|
readResponseToBufferWithLimit,
|
|
} from '@/lib/core/utils/stream-limits'
|
|
import { getBaseUrl } from '@/lib/core/utils/urls'
|
|
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
|
import { StorageService } from '@/lib/uploads'
|
|
import type {
|
|
AzureTtsParams,
|
|
CartesiaTtsParams,
|
|
DeepgramTtsParams,
|
|
ElevenLabsTtsUnifiedParams,
|
|
GoogleTtsParams,
|
|
OpenAiTtsParams,
|
|
PlayHtTtsParams,
|
|
TtsResponse,
|
|
} from '@/tools/tts/types'
|
|
import { getFileExtension, getMimeType } from '@/tools/tts/types'
|
|
|
|
const logger = createLogger('TtsUnifiedProxyAPI')
|
|
const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024
|
|
const MAX_TTS_ERROR_BYTES = 64 * 1024
|
|
const MAX_TTS_JSON_BYTES = Math.ceil((MAX_TTS_AUDIO_BYTES * 4) / 3) + 256 * 1024
|
|
|
|
async function readTtsErrorJson(
|
|
response: Response,
|
|
label: string
|
|
): Promise<Record<string, unknown>> {
|
|
return readResponseJsonWithLimit<Record<string, unknown>>(response, {
|
|
maxBytes: MAX_TTS_ERROR_BYTES,
|
|
label,
|
|
}).catch(() => ({}))
|
|
}
|
|
|
|
function getTtsErrorMessage(error: Record<string, unknown>, fallback: string): string {
|
|
const nested = error.error
|
|
if (typeof nested === 'object' && nested !== null && 'message' in nested) {
|
|
const message = (nested as { message?: unknown }).message
|
|
if (typeof message === 'string') return message
|
|
}
|
|
for (const key of ['message', 'err_msg', 'error_message', 'error', 'detail']) {
|
|
const value = error[key]
|
|
if (typeof value === 'string') return value
|
|
if (typeof value === 'object' && value !== null && 'message' in value) {
|
|
const message = (value as { message?: unknown }).message
|
|
if (typeof message === 'string') return message
|
|
}
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
export const dynamic = 'force-dynamic'
|
|
export const maxDuration = 60 // 1 minute
|
|
|
|
export const POST = withRouteHandler(async (request: NextRequest) => {
|
|
const requestId = generateId()
|
|
logger.info(`[${requestId}] TTS unified request started`)
|
|
|
|
try {
|
|
const authResult = await checkInternalAuth(request, { requireWorkflowId: false })
|
|
if (!authResult.success) {
|
|
logger.error('Authentication failed for TTS unified proxy:', authResult.error)
|
|
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
|
}
|
|
|
|
const parsed = await parseRequest(
|
|
ttsUnifiedToolContract,
|
|
request,
|
|
{},
|
|
{
|
|
validationErrorResponse: (error) => {
|
|
logger.warn(`[${requestId}] Invalid TTS unified request:`, error.issues)
|
|
return validationErrorResponse(
|
|
error,
|
|
getValidationErrorMessage(error, 'Invalid request data')
|
|
)
|
|
},
|
|
}
|
|
)
|
|
if (!parsed.success) return parsed.response
|
|
|
|
const body = parsed.data.body
|
|
const { provider, text, apiKey, workspaceId, workflowId, executionId } = body
|
|
|
|
const executionContext =
|
|
workspaceId && workflowId && executionId ? { workspaceId, workflowId, executionId } : null
|
|
logger.info(`[${requestId}] Processing TTS with ${provider}`, {
|
|
hasExecutionContext: Boolean(executionContext),
|
|
textLength: text.length,
|
|
})
|
|
|
|
let audioBuffer: Buffer
|
|
let format: string
|
|
let mimeType: string
|
|
let duration: number | undefined
|
|
|
|
try {
|
|
if (provider === 'openai') {
|
|
const result = await synthesizeWithOpenAi({
|
|
text,
|
|
apiKey,
|
|
model: body.model,
|
|
voice: body.voice as OpenAiTtsParams['voice'],
|
|
responseFormat: body.responseFormat,
|
|
speed: body.speed,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
} else if (provider === 'deepgram') {
|
|
const result = await synthesizeWithDeepgram({
|
|
text,
|
|
apiKey,
|
|
model: body.voice,
|
|
encoding: body.encoding,
|
|
sampleRate: body.sampleRate,
|
|
bitRate: body.bitRate,
|
|
container: body.container,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
duration = result.duration
|
|
} else if (provider === 'elevenlabs') {
|
|
if (!body.voiceId) {
|
|
return NextResponse.json(
|
|
{ error: 'voiceId is required for ElevenLabs provider' },
|
|
{ status: 400 }
|
|
)
|
|
}
|
|
const voiceIdValidation = validateAlphanumericId(body.voiceId, 'voiceId')
|
|
if (!voiceIdValidation.isValid) {
|
|
return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 })
|
|
}
|
|
const result = await synthesizeWithElevenLabs({
|
|
text,
|
|
apiKey,
|
|
voiceId: body.voiceId,
|
|
modelId: body.modelId,
|
|
stability: body.stability,
|
|
similarityBoost: body.similarityBoost,
|
|
style: body.style as number | undefined,
|
|
useSpeakerBoost: body.useSpeakerBoost,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
} else if (provider === 'cartesia') {
|
|
const result = await synthesizeWithCartesia({
|
|
text,
|
|
apiKey,
|
|
modelId: body.modelId,
|
|
voice: body.voice,
|
|
language: body.language,
|
|
outputFormat:
|
|
body.outputFormat &&
|
|
typeof body.outputFormat === 'object' &&
|
|
!Array.isArray(body.outputFormat)
|
|
? (body.outputFormat as CartesiaTtsParams['outputFormat'])
|
|
: undefined,
|
|
speed: body.speed,
|
|
emotion: body.emotion,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
} else if (provider === 'google') {
|
|
const result = await synthesizeWithGoogle({
|
|
text,
|
|
apiKey,
|
|
voiceId: body.voiceId,
|
|
languageCode: body.languageCode,
|
|
gender: body.gender,
|
|
audioEncoding: body.audioEncoding,
|
|
speakingRate: body.speakingRate,
|
|
pitch: typeof body.pitch === 'number' ? body.pitch : undefined,
|
|
volumeGainDb: body.volumeGainDb,
|
|
sampleRateHertz: body.sampleRateHertz,
|
|
effectsProfileId: body.effectsProfileId,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
} else if (provider === 'azure') {
|
|
const result = await synthesizeWithAzure({
|
|
text,
|
|
apiKey,
|
|
voiceId: body.voiceId,
|
|
region: body.region,
|
|
outputFormat:
|
|
typeof body.outputFormat === 'string'
|
|
? (body.outputFormat as AzureTtsParams['outputFormat'])
|
|
: undefined,
|
|
rate: body.rate,
|
|
pitch: body.pitch as string | undefined,
|
|
style: body.style as string | undefined,
|
|
styleDegree: body.styleDegree,
|
|
role: body.role,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
} else if (provider === 'playht') {
|
|
if (!body.userId) {
|
|
return NextResponse.json(
|
|
{ error: 'userId is required for PlayHT provider' },
|
|
{ status: 400 }
|
|
)
|
|
}
|
|
const playHtOutputFormat = playHtOutputFormatSchema.safeParse(body.outputFormat)
|
|
const result = await synthesizeWithPlayHT({
|
|
text,
|
|
apiKey,
|
|
userId: body.userId,
|
|
voice: body.voice,
|
|
quality: body.quality,
|
|
outputFormat: playHtOutputFormat.success ? playHtOutputFormat.data : undefined,
|
|
speed: body.speed,
|
|
temperature: body.temperature,
|
|
voiceGuidance: body.voiceGuidance,
|
|
textGuidance: body.textGuidance,
|
|
sampleRate: body.sampleRate,
|
|
})
|
|
audioBuffer = result.audioBuffer
|
|
format = result.format
|
|
mimeType = result.mimeType
|
|
} else {
|
|
return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
|
|
}
|
|
} catch (error) {
|
|
logger.error(`[${requestId}] TTS synthesis failed:`, error)
|
|
const errorMessage = getErrorMessage(error, 'TTS synthesis failed')
|
|
return NextResponse.json(
|
|
{ error: errorMessage },
|
|
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
|
|
)
|
|
}
|
|
|
|
const timestamp = Date.now()
|
|
const fileExtension = getFileExtension(format)
|
|
const fileName = `tts-${provider}-${timestamp}.${fileExtension}`
|
|
|
|
if (executionContext) {
|
|
const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution')
|
|
|
|
const userFile = await uploadExecutionFile(
|
|
executionContext,
|
|
audioBuffer,
|
|
fileName,
|
|
mimeType,
|
|
authResult.userId
|
|
)
|
|
|
|
logger.info(`[${requestId}] TTS audio stored in execution context:`, {
|
|
executionId,
|
|
fileName,
|
|
size: userFile.size,
|
|
})
|
|
|
|
const response: TtsResponse = {
|
|
audioUrl: userFile.url,
|
|
audioFile: userFile,
|
|
characterCount: text.length,
|
|
format,
|
|
provider,
|
|
}
|
|
|
|
if (duration) {
|
|
response.duration = duration
|
|
}
|
|
|
|
return NextResponse.json(response)
|
|
}
|
|
|
|
// Chat UI / copilot usage - no execution context
|
|
const fileInfo = await StorageService.uploadFile({
|
|
file: audioBuffer,
|
|
fileName,
|
|
contentType: mimeType,
|
|
context: 'copilot',
|
|
})
|
|
|
|
const audioUrl = `${getBaseUrl()}${fileInfo.path}`
|
|
|
|
logger.info(`[${requestId}] TTS audio stored in copilot context:`, {
|
|
fileName,
|
|
size: fileInfo.size,
|
|
})
|
|
|
|
const response: TtsResponse = {
|
|
audioUrl,
|
|
characterCount: text.length,
|
|
format,
|
|
provider,
|
|
}
|
|
|
|
if (duration) {
|
|
response.duration = duration
|
|
}
|
|
|
|
return NextResponse.json(response)
|
|
} catch (error) {
|
|
logger.error(`[${requestId}] TTS unified proxy error:`, error)
|
|
const errorMessage = getErrorMessage(error, 'Unknown error')
|
|
return NextResponse.json(
|
|
{ error: errorMessage },
|
|
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
|
|
)
|
|
}
|
|
})
|
|
|
|
async function synthesizeWithOpenAi(
|
|
params: OpenAiTtsParams
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
|
const { text, apiKey, model = 'tts-1', responseFormat = 'mp3', speed = 1.0 } = params
|
|
const voice = (params.voice || 'alloy') as OpenAiTtsParams['voice']
|
|
|
|
const response = await fetch('https://api.openai.com/v1/audio/speech', {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${apiKey}`,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify({
|
|
model,
|
|
voice,
|
|
input: text,
|
|
response_format: responseFormat,
|
|
speed: Math.max(0.25, Math.min(4.0, speed)),
|
|
}),
|
|
})
|
|
|
|
if (!response.ok) {
|
|
const error = await readTtsErrorJson(response, 'OpenAI TTS error response')
|
|
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
|
throw new Error(`OpenAI TTS API error: ${errorMessage}`)
|
|
}
|
|
|
|
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
|
maxBytes: MAX_TTS_AUDIO_BYTES,
|
|
label: 'OpenAI TTS audio response',
|
|
})
|
|
const mimeType = getMimeType(responseFormat)
|
|
|
|
return {
|
|
audioBuffer,
|
|
format: responseFormat,
|
|
mimeType,
|
|
}
|
|
}
|
|
|
|
async function synthesizeWithDeepgram(
|
|
params: DeepgramTtsParams
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string; duration?: number }> {
|
|
const {
|
|
text,
|
|
apiKey,
|
|
model = 'aura-asteria-en',
|
|
encoding = 'mp3',
|
|
sampleRate,
|
|
bitRate,
|
|
container,
|
|
} = params
|
|
|
|
const queryParams = new URLSearchParams({
|
|
model: model,
|
|
encoding: encoding,
|
|
})
|
|
|
|
if (sampleRate && encoding === 'linear16') {
|
|
queryParams.append('sample_rate', sampleRate.toString())
|
|
}
|
|
|
|
if (bitRate) {
|
|
queryParams.append('bit_rate', bitRate.toString())
|
|
}
|
|
|
|
if (container && container !== 'none') {
|
|
queryParams.append('container', container)
|
|
}
|
|
|
|
const response = await fetch(`https://api.deepgram.com/v1/speak?${queryParams.toString()}`, {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Token ${apiKey}`,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify({ text }),
|
|
})
|
|
|
|
if (!response.ok) {
|
|
const error = await readTtsErrorJson(response, 'Deepgram TTS error response')
|
|
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
|
throw new Error(`Deepgram TTS API error: ${errorMessage}`)
|
|
}
|
|
|
|
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
|
maxBytes: MAX_TTS_AUDIO_BYTES,
|
|
label: 'Deepgram TTS audio response',
|
|
})
|
|
|
|
let finalFormat: string = encoding
|
|
if (container === 'wav') {
|
|
finalFormat = 'wav'
|
|
} else if (container === 'ogg') {
|
|
finalFormat = 'ogg'
|
|
}
|
|
|
|
const mimeType = getMimeType(finalFormat)
|
|
|
|
return {
|
|
audioBuffer,
|
|
format: finalFormat,
|
|
mimeType,
|
|
}
|
|
}
|
|
|
|
async function synthesizeWithElevenLabs(
|
|
params: ElevenLabsTtsUnifiedParams
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
|
const {
|
|
text,
|
|
apiKey,
|
|
voiceId,
|
|
modelId = 'eleven_turbo_v2_5',
|
|
stability = 0.5,
|
|
similarityBoost = 0.8,
|
|
style,
|
|
useSpeakerBoost = true,
|
|
} = params
|
|
|
|
const voiceSettings: any = {
|
|
stability: Math.max(0, Math.min(1, stability)),
|
|
similarity_boost: Math.max(0, Math.min(1, similarityBoost)),
|
|
use_speaker_boost: useSpeakerBoost,
|
|
}
|
|
|
|
if (style !== undefined) {
|
|
voiceSettings.style = Math.max(0, Math.min(1, style))
|
|
}
|
|
|
|
const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
|
|
method: 'POST',
|
|
headers: {
|
|
Accept: 'audio/mpeg',
|
|
'Content-Type': 'application/json',
|
|
'xi-api-key': apiKey,
|
|
},
|
|
body: JSON.stringify({
|
|
text,
|
|
model_id: modelId,
|
|
voice_settings: voiceSettings,
|
|
}),
|
|
})
|
|
|
|
if (!response.ok) {
|
|
const error = await readTtsErrorJson(response, 'ElevenLabs TTS error response')
|
|
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
|
throw new Error(`ElevenLabs TTS API error: ${errorMessage}`)
|
|
}
|
|
|
|
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
|
maxBytes: MAX_TTS_AUDIO_BYTES,
|
|
label: 'ElevenLabs TTS audio response',
|
|
})
|
|
|
|
return {
|
|
audioBuffer,
|
|
format: 'mp3',
|
|
mimeType: 'audio/mpeg',
|
|
}
|
|
}
|
|
|
|
async function synthesizeWithCartesia(
|
|
params: Partial<CartesiaTtsParams>
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
|
const {
|
|
text,
|
|
apiKey,
|
|
modelId = 'sonic-3',
|
|
voice,
|
|
language = 'en',
|
|
outputFormat,
|
|
speed,
|
|
emotion,
|
|
} = params
|
|
|
|
if (!text || !apiKey) {
|
|
throw new Error('text and apiKey are required for Cartesia')
|
|
}
|
|
|
|
const requestBody: Record<string, unknown> = {
|
|
model_id: modelId,
|
|
transcript: text,
|
|
language,
|
|
}
|
|
|
|
if (voice) {
|
|
requestBody.voice = {
|
|
mode: 'id',
|
|
id: voice,
|
|
}
|
|
}
|
|
|
|
const generationConfig: Record<string, unknown> = {}
|
|
if (speed !== undefined) generationConfig.speed = speed
|
|
if (emotion !== undefined) generationConfig.emotion = emotion
|
|
if (Object.keys(generationConfig).length > 0) {
|
|
requestBody.generation_config = generationConfig
|
|
}
|
|
|
|
if (outputFormat && typeof outputFormat === 'object') {
|
|
requestBody.output_format = outputFormat
|
|
}
|
|
|
|
if (!requestBody.output_format) {
|
|
requestBody.output_format = {
|
|
container: 'wav',
|
|
encoding: 'pcm_s16le',
|
|
sample_rate: 24000,
|
|
}
|
|
}
|
|
|
|
logger.info('Cartesia API request:', {
|
|
model_id: requestBody.model_id,
|
|
has_voice: !!requestBody.voice,
|
|
language: requestBody.language,
|
|
output_format: requestBody.output_format,
|
|
has_generation_config: !!requestBody.generation_config,
|
|
})
|
|
|
|
const response = await fetch('https://api.cartesia.ai/tts/bytes', {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${apiKey}`,
|
|
'Content-Type': 'application/json',
|
|
'Cartesia-Version': '2025-04-16',
|
|
},
|
|
body: JSON.stringify(requestBody),
|
|
})
|
|
|
|
if (!response.ok) {
|
|
const error = await readTtsErrorJson(response, 'Cartesia TTS error response')
|
|
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
|
const errorDetail = typeof error.detail === 'string' ? error.detail : ''
|
|
logger.error('Cartesia API error details:', {
|
|
status: response.status,
|
|
error: errorMessage,
|
|
detail: errorDetail,
|
|
requestBody: JSON.stringify(requestBody),
|
|
})
|
|
throw new Error(
|
|
`Cartesia TTS API error: ${errorMessage}${errorDetail ? ` - ${errorDetail}` : ''}`
|
|
)
|
|
}
|
|
|
|
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
|
maxBytes: MAX_TTS_AUDIO_BYTES,
|
|
label: 'Cartesia TTS audio response',
|
|
})
|
|
|
|
const format =
|
|
outputFormat && typeof outputFormat === 'object' && 'container' in outputFormat
|
|
? (outputFormat.container as string)
|
|
: 'mp3'
|
|
const mimeType = getMimeType(format)
|
|
|
|
return {
|
|
audioBuffer,
|
|
format,
|
|
mimeType,
|
|
}
|
|
}
|
|
|
|
async function synthesizeWithGoogle(
|
|
params: Partial<GoogleTtsParams>
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
|
const {
|
|
text,
|
|
apiKey,
|
|
voiceId,
|
|
languageCode,
|
|
gender,
|
|
audioEncoding = 'MP3',
|
|
speakingRate = 1.0,
|
|
pitch = 0.0,
|
|
volumeGainDb,
|
|
sampleRateHertz,
|
|
effectsProfileId,
|
|
} = params
|
|
|
|
if (!text || !apiKey || !languageCode) {
|
|
throw new Error('text, apiKey, and languageCode are required for Google Cloud TTS')
|
|
}
|
|
|
|
const clampedSpeakingRate = Math.max(0.25, Math.min(2.0, speakingRate))
|
|
|
|
const audioConfig: Record<string, unknown> = {
|
|
audioEncoding,
|
|
speakingRate: clampedSpeakingRate,
|
|
pitch,
|
|
}
|
|
|
|
if (volumeGainDb !== undefined) {
|
|
audioConfig.volumeGainDb = volumeGainDb
|
|
}
|
|
if (sampleRateHertz) {
|
|
audioConfig.sampleRateHertz = sampleRateHertz
|
|
}
|
|
if (effectsProfileId && effectsProfileId.length > 0) {
|
|
audioConfig.effectsProfileId = effectsProfileId
|
|
}
|
|
|
|
// Build voice config based on what's provided
|
|
const voice: Record<string, unknown> = {
|
|
languageCode,
|
|
}
|
|
|
|
// If voiceId is provided, use it (it takes precedence over gender)
|
|
if (voiceId) {
|
|
voice.name = voiceId
|
|
}
|
|
|
|
// Only include gender if specified (don't default to NEUTRAL as it's not supported)
|
|
if (gender) {
|
|
voice.ssmlGender = gender
|
|
}
|
|
|
|
// If neither voiceId nor gender is provided, default to a specific voice
|
|
if (!voiceId && !gender) {
|
|
voice.name = 'en-US-Neural2-C'
|
|
}
|
|
|
|
const requestBody: Record<string, unknown> = {
|
|
input: { text },
|
|
voice,
|
|
audioConfig,
|
|
}
|
|
|
|
const response = await fetch(
|
|
`https://texttospeech.googleapis.com/v1/text:synthesize?key=${apiKey}`,
|
|
{
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(requestBody),
|
|
}
|
|
)
|
|
|
|
if (!response.ok) {
|
|
const error = await readTtsErrorJson(response, 'Google TTS error response')
|
|
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
|
throw new Error(`Google Cloud TTS API error: ${errorMessage}`)
|
|
}
|
|
|
|
const data = await readResponseJsonWithLimit<{ audioContent?: string }>(response, {
|
|
maxBytes: MAX_TTS_JSON_BYTES,
|
|
label: 'Google TTS JSON response',
|
|
})
|
|
const audioContent = data.audioContent
|
|
|
|
if (!audioContent) {
|
|
throw new Error('No audio content returned from Google Cloud TTS')
|
|
}
|
|
|
|
const audioBuffer = Buffer.from(audioContent, 'base64')
|
|
assertKnownSizeWithinLimit(audioBuffer.length, MAX_TTS_AUDIO_BYTES, 'Google TTS audio response')
|
|
|
|
const format = audioEncoding.toLowerCase().replace('_', '')
|
|
const mimeType = getMimeType(format)
|
|
|
|
return {
|
|
audioBuffer,
|
|
format,
|
|
mimeType,
|
|
}
|
|
}
|
|
|
|
async function synthesizeWithAzure(
|
|
params: Partial<AzureTtsParams>
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
|
const {
|
|
text,
|
|
apiKey,
|
|
voiceId = 'en-US-JennyNeural',
|
|
region = 'eastus',
|
|
outputFormat = 'audio-24khz-96kbitrate-mono-mp3',
|
|
rate,
|
|
pitch,
|
|
style,
|
|
styleDegree,
|
|
role,
|
|
} = params
|
|
|
|
if (!text || !apiKey) {
|
|
throw new Error('text and apiKey are required for Azure TTS')
|
|
}
|
|
|
|
const AZURE_REGION_RE = /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/
|
|
if (!AZURE_REGION_RE.test(region)) {
|
|
throw new Error(
|
|
'Invalid Azure region: must match /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ (e.g. eastus, westeurope)'
|
|
)
|
|
}
|
|
|
|
let ssml = `<speak version='1.0' xml:lang='en-US' xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts"><voice name='${voiceId}'>`
|
|
|
|
if (style) {
|
|
ssml += `<mstts:express-as style='${style}'`
|
|
if (styleDegree) ssml += ` styledegree='${styleDegree}'`
|
|
if (role) ssml += ` role='${role}'`
|
|
ssml += '>'
|
|
}
|
|
|
|
if (rate || pitch) {
|
|
ssml += '<prosody'
|
|
if (rate) ssml += ` rate='${rate}'`
|
|
if (pitch) ssml += ` pitch='${pitch}'`
|
|
ssml += '>'
|
|
}
|
|
|
|
ssml += text
|
|
|
|
if (rate || pitch) {
|
|
ssml += '</prosody>'
|
|
}
|
|
|
|
if (style) {
|
|
ssml += '</mstts:express-as>'
|
|
}
|
|
|
|
ssml += '</voice></speak>'
|
|
|
|
const response = await fetch(`https://${region}.tts.speech.microsoft.com/cognitiveservices/v1`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Ocp-Apim-Subscription-Key': apiKey,
|
|
'Content-Type': 'application/ssml+xml',
|
|
'X-Microsoft-OutputFormat': outputFormat,
|
|
},
|
|
body: ssml,
|
|
})
|
|
|
|
if (!response.ok) {
|
|
const error = await readResponseTextWithLimit(response, {
|
|
maxBytes: MAX_TTS_ERROR_BYTES,
|
|
label: 'Azure TTS error response',
|
|
})
|
|
throw new Error(`Azure TTS API error: ${error || response.statusText}`)
|
|
}
|
|
|
|
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
|
maxBytes: MAX_TTS_AUDIO_BYTES,
|
|
label: 'Azure TTS audio response',
|
|
})
|
|
|
|
const format = outputFormat.includes('mp3') ? 'mp3' : 'wav'
|
|
const mimeType = getMimeType(format)
|
|
|
|
return {
|
|
audioBuffer,
|
|
format,
|
|
mimeType,
|
|
}
|
|
}
|
|
|
|
async function synthesizeWithPlayHT(
|
|
params: Partial<PlayHtTtsParams>
|
|
): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
|
|
const {
|
|
text,
|
|
apiKey,
|
|
userId,
|
|
voice,
|
|
quality = 'standard',
|
|
outputFormat = 'mp3',
|
|
speed = 1.0,
|
|
temperature,
|
|
voiceGuidance,
|
|
textGuidance,
|
|
sampleRate,
|
|
} = params
|
|
|
|
if (!text || !apiKey || !userId) {
|
|
throw new Error('text, apiKey, and userId are required for PlayHT')
|
|
}
|
|
|
|
const requestBody: Record<string, unknown> = {
|
|
text,
|
|
quality,
|
|
output_format: outputFormat,
|
|
speed,
|
|
}
|
|
|
|
if (voice) requestBody.voice = voice
|
|
if (temperature !== undefined) requestBody.temperature = temperature
|
|
if (voiceGuidance !== undefined) requestBody.voice_guidance = voiceGuidance
|
|
if (textGuidance !== undefined) requestBody.text_guidance = textGuidance
|
|
if (sampleRate) requestBody.sample_rate = sampleRate
|
|
|
|
const response = await fetch('https://api.play.ht/api/v2/tts/stream', {
|
|
method: 'POST',
|
|
headers: {
|
|
AUTHORIZATION: apiKey,
|
|
'X-USER-ID': userId,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(requestBody),
|
|
})
|
|
|
|
if (!response.ok) {
|
|
const error = await readTtsErrorJson(response, 'PlayHT TTS error response')
|
|
const errorMessage = getTtsErrorMessage(error, response.statusText)
|
|
throw new Error(`PlayHT TTS API error: ${errorMessage}`)
|
|
}
|
|
|
|
const audioBuffer = await readResponseToBufferWithLimit(response, {
|
|
maxBytes: MAX_TTS_AUDIO_BYTES,
|
|
label: 'PlayHT TTS audio response',
|
|
})
|
|
|
|
const format = outputFormat || 'mp3'
|
|
const mimeType = getMimeType(format)
|
|
|
|
return {
|
|
audioBuffer,
|
|
format,
|
|
mimeType,
|
|
}
|
|
}
|