wehub-resource-sync
548 lines
16 KiB
TypeScript
548 lines
16 KiB
TypeScript
/**
|
|
* Audio Provider Registry
|
|
*
|
|
* Defines providers that support audio endpoints:
|
|
* - /v1/audio/transcriptions (Whisper API)
|
|
* - /v1/audio/translations (Whisper translate-to-English API)
|
|
* - /v1/audio/speech (TTS API)
|
|
*/
|
|
|
|
interface AudioModel {
|
|
id: string;
|
|
name: string;
|
|
}
|
|
|
|
export interface AudioProvider {
|
|
id: string;
|
|
baseUrl: string;
|
|
authType: string;
|
|
authHeader: string;
|
|
format?: string;
|
|
supportedFormats?: string[];
|
|
async?: boolean;
|
|
models: AudioModel[];
|
|
}
|
|
|
|
export const AUDIO_TRANSCRIPTION_PROVIDERS: Record<string, AudioProvider> = {
|
|
vertex: {
|
|
id: "vertex",
|
|
baseUrl: "https://us-central1-aiplatform.googleapis.com/v1",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "vertex-gemini",
|
|
models: [
|
|
{ id: "gemini-2.5-flash", name: "Gemini 2.5 Flash (Vertex Transcribe)" },
|
|
{ id: "gemini-2.5-pro", name: "Gemini 2.5 Pro (Vertex Transcribe)" },
|
|
{ id: "gemini-2.0-flash", name: "Gemini 2.0 Flash (Vertex Transcribe)" },
|
|
],
|
|
},
|
|
|
|
openai: {
|
|
id: "openai",
|
|
baseUrl: "https://api.openai.com/v1/audio/transcriptions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
models: [
|
|
{ id: "whisper-1", name: "Whisper 1" },
|
|
{ id: "gpt-4o-transcription", name: "GPT-4o Transcription" },
|
|
],
|
|
},
|
|
|
|
cohere: {
|
|
id: "cohere",
|
|
baseUrl: "https://api.cohere.com/v2/audio/transcriptions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
models: [{ id: "cohere-transcribe-03-2026", name: "Cohere Transcribe 2026-03" }],
|
|
},
|
|
|
|
groq: {
|
|
id: "groq",
|
|
baseUrl: "https://api.groq.com/openai/v1/audio/transcriptions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
models: [
|
|
{ id: "whisper-large-v3", name: "Whisper Large v3" },
|
|
{ id: "whisper-large-v3-turbo", name: "Whisper Large v3 Turbo" },
|
|
{ id: "distil-whisper-large-v3-en", name: "Distil Whisper Large v3 EN" },
|
|
],
|
|
},
|
|
|
|
deepgram: {
|
|
id: "deepgram",
|
|
baseUrl: "https://api.deepgram.com/v1/listen",
|
|
authType: "apikey",
|
|
authHeader: "token",
|
|
format: "deepgram",
|
|
models: [
|
|
{ id: "nova-3", name: "Nova 3" },
|
|
{ id: "nova-2", name: "Nova 2" },
|
|
{ id: "whisper-large", name: "Whisper Large" },
|
|
],
|
|
},
|
|
|
|
pollinations: {
|
|
id: "pollinations",
|
|
baseUrl: "https://gen.pollinations.ai/v1/audio/transcriptions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "openai",
|
|
models: [{ id: "whisper", name: "Pollinations Whisper (Free)" }],
|
|
},
|
|
|
|
together: {
|
|
id: "together",
|
|
baseUrl: "https://api.together.xyz/v1/audio/transcriptions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "openai",
|
|
models: [
|
|
{ id: "openai/whisper-large-v3", name: "Whisper Large v3" },
|
|
{ id: "openai/whisper-large-v3-turbo", name: "Whisper Large v3 Turbo" },
|
|
],
|
|
},
|
|
|
|
assemblyai: {
|
|
id: "assemblyai",
|
|
baseUrl: "https://api.assemblyai.com/v2/transcript",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
async: true,
|
|
format: "assemblyai",
|
|
models: [
|
|
{ id: "universal-3-pro", name: "Universal 3 Pro" },
|
|
{ id: "universal-2", name: "Universal 2" },
|
|
],
|
|
},
|
|
|
|
nvidia: {
|
|
id: "nvidia",
|
|
baseUrl: "https://integrate.api.nvidia.com/v1/audio/transcriptions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "nvidia-asr",
|
|
models: [
|
|
{ id: "nvidia/parakeet-ctc-1.1b-asr", name: "Parakeet CTC 1.1B" },
|
|
{ id: "openai/whisper-large-v3", name: "Whisper Large v3 (NVIDIA)" },
|
|
],
|
|
},
|
|
|
|
huggingface: {
|
|
id: "huggingface",
|
|
baseUrl: "https://api-inference.huggingface.co/models",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "huggingface-asr",
|
|
models: [
|
|
{ id: "openai/whisper-large-v3-turbo", name: "Whisper Large v3 Turbo (HF)" },
|
|
{ id: "openai/whisper-large-v3", name: "Whisper Large v3 (HF)" },
|
|
],
|
|
},
|
|
|
|
qwen: {
|
|
id: "qwen",
|
|
baseUrl: "http://localhost:8000/v1/audio/transcriptions",
|
|
authType: "none",
|
|
authHeader: "none",
|
|
format: "openai",
|
|
models: [{ id: "qwen3-asr", name: "Qwen3 ASR" }],
|
|
},
|
|
|
|
kie: {
|
|
id: "kie",
|
|
baseUrl: "https://api.kie.ai",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "kie-audio",
|
|
models: [
|
|
{ id: "elevenlabs/speech-to-text", name: "ElevenLabs STT" },
|
|
{ id: "elevenlabs/audio-isolation", name: "ElevenLabs Audio Isolation" },
|
|
],
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Providers that expose an OpenAI-Whisper-compatible /audio/translations
|
|
* endpoint (translate-to-English). This is a narrower surface than
|
|
* transcription: only Whisper-family models support it, and there is no
|
|
* `language` input — output is always English regardless of source audio.
|
|
*/
|
|
export const AUDIO_TRANSLATION_PROVIDERS: Record<string, AudioProvider> = {
|
|
openai: {
|
|
id: "openai",
|
|
baseUrl: "https://api.openai.com/v1/audio/translations",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
models: [{ id: "whisper-1", name: "Whisper 1" }],
|
|
},
|
|
|
|
groq: {
|
|
id: "groq",
|
|
baseUrl: "https://api.groq.com/openai/v1/audio/translations",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
models: [{ id: "whisper-large-v3", name: "Whisper Large v3" }],
|
|
},
|
|
};
|
|
|
|
export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
|
|
vertex: {
|
|
id: "vertex",
|
|
baseUrl: "https://us-central1-aiplatform.googleapis.com/v1",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "vertex-gemini-tts",
|
|
models: [
|
|
{ id: "gemini-3.1-flash-tts-preview", name: "Gemini 3.1 Flash TTS (Vertex)" },
|
|
{ id: "gemini-2.5-flash-preview-tts", name: "Gemini 2.5 Flash TTS (Vertex)" },
|
|
{ id: "gemini-2.5-pro-preview-tts", name: "Gemini 2.5 Pro TTS (Vertex)" },
|
|
],
|
|
},
|
|
|
|
openai: {
|
|
id: "openai",
|
|
baseUrl: "https://api.openai.com/v1/audio/speech",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
models: [
|
|
{ id: "tts-1-hd", name: "TTS 1 HD" },
|
|
{ id: "tts-1", name: "TTS 1" },
|
|
{ id: "gpt-4o-mini-tts", name: "GPT-4o Mini TTS" },
|
|
],
|
|
},
|
|
|
|
hyperbolic: {
|
|
id: "hyperbolic",
|
|
baseUrl: "https://api.hyperbolic.xyz/v1/audio/generation",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "hyperbolic",
|
|
models: [{ id: "melo-tts", name: "Melo TTS" }],
|
|
},
|
|
|
|
deepgram: {
|
|
id: "deepgram",
|
|
baseUrl: "https://api.deepgram.com/v1/speak",
|
|
authType: "apikey",
|
|
authHeader: "token",
|
|
format: "deepgram",
|
|
models: [
|
|
{ id: "aura-asteria-en", name: "Aura Asteria (EN)" },
|
|
{ id: "aura-luna-en", name: "Aura Luna (EN)" },
|
|
{ id: "aura-stella-en", name: "Aura Stella (EN)" },
|
|
],
|
|
},
|
|
|
|
nvidia: {
|
|
id: "nvidia",
|
|
baseUrl: "https://integrate.api.nvidia.com/v1/audio/speech",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "nvidia-tts",
|
|
models: [
|
|
{ id: "nvidia/fastpitch", name: "FastPitch" },
|
|
{ id: "nvidia/tacotron2", name: "Tacotron2" },
|
|
],
|
|
},
|
|
|
|
elevenlabs: {
|
|
id: "elevenlabs",
|
|
baseUrl: "https://api.elevenlabs.io/v1/text-to-speech",
|
|
authType: "apikey",
|
|
authHeader: "xi-api-key",
|
|
format: "elevenlabs",
|
|
models: [
|
|
{ id: "eleven_multilingual_v2", name: "Eleven Multilingual v2" },
|
|
{ id: "eleven_turbo_v2_5", name: "Eleven Turbo v2.5" },
|
|
],
|
|
},
|
|
|
|
huggingface: {
|
|
id: "huggingface",
|
|
baseUrl: "https://api-inference.huggingface.co/models",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "huggingface-tts",
|
|
models: [
|
|
{ id: "canopylabs/orpheus-3b-0.1-ft", name: "Orpheus 3B" },
|
|
{ id: "ResembleAI/chatterbox", name: "Chatterbox" },
|
|
{ id: "hexgrad/Kokoro-82M", name: "Kokoro TTS" },
|
|
],
|
|
},
|
|
|
|
coqui: {
|
|
id: "coqui",
|
|
baseUrl: "http://localhost:5002/api/tts",
|
|
authType: "none",
|
|
authHeader: "none",
|
|
format: "coqui",
|
|
models: [{ id: "tts_models/en/ljspeech/tacotron2-DDC", name: "Tacotron2 DDC (LJSpeech)" }],
|
|
},
|
|
|
|
tortoise: {
|
|
id: "tortoise",
|
|
baseUrl: "http://localhost:5000/api/tts",
|
|
authType: "none",
|
|
authHeader: "none",
|
|
format: "tortoise",
|
|
models: [{ id: "tortoise-v2", name: "Tortoise v2" }],
|
|
},
|
|
|
|
qwen: {
|
|
id: "qwen",
|
|
baseUrl: "http://localhost:8000/v1/audio/speech",
|
|
authType: "none",
|
|
authHeader: "none",
|
|
format: "openai",
|
|
models: [{ id: "qwen3-tts", name: "Qwen3 TTS" }],
|
|
},
|
|
|
|
// ── Cloud TTS Providers (#248) ────────────────────────────────────────────
|
|
|
|
inworld: {
|
|
id: "inworld",
|
|
// POST https://api.inworld.ai/tts/v1/voice
|
|
// Auth: Authorization: Basic <api-key>
|
|
// Response: JSON { audioContent: "<base64>", contentType, sampleRateHertz }
|
|
baseUrl: "https://api.inworld.ai/tts/v1/voice",
|
|
authType: "apikey",
|
|
authHeader: "basic",
|
|
format: "inworld",
|
|
supportedFormats: ["mp3", "wav", "opus", "pcm"],
|
|
models: [
|
|
{ id: "inworld-tts-2", name: "Inworld TTS 2" },
|
|
{ id: "inworld-tts-1.5-mini", name: "Inworld TTS 1.5 Mini" },
|
|
],
|
|
},
|
|
|
|
cartesia: {
|
|
id: "cartesia",
|
|
// POST https://api.cartesia.ai/tts/bytes
|
|
// Auth: X-API-Key header, Cartesia-Version: 2024-06-10
|
|
// Response: binary audio bytes
|
|
baseUrl: "https://api.cartesia.ai/tts/bytes",
|
|
authType: "apikey",
|
|
authHeader: "x-api-key",
|
|
format: "cartesia",
|
|
models: [
|
|
{ id: "sonic-3", name: "Sonic 3" },
|
|
{ id: "sonic-2", name: "Sonic 2" },
|
|
],
|
|
},
|
|
|
|
playht: {
|
|
id: "playht",
|
|
// POST https://api.play.ht/api/v2/tts/stream
|
|
// Auth: X-USER-ID + Authorization: Bearer <api-key>
|
|
// Response: audio stream (mp3/wav)
|
|
baseUrl: "https://api.play.ht/api/v2/tts/stream",
|
|
authType: "apikey",
|
|
authHeader: "playht",
|
|
format: "playht",
|
|
models: [
|
|
{ id: "PlayDialog", name: "PlayDialog" },
|
|
{ id: "Play3.0-mini", name: "Play3.0 Mini" },
|
|
],
|
|
},
|
|
|
|
kie: {
|
|
id: "kie",
|
|
baseUrl: "https://api.kie.ai",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "kie-audio",
|
|
models: [
|
|
{ id: "elevenlabs/text-to-speech-multilingual-v2", name: "ElevenLabs TTS v2" },
|
|
{ id: "elevenlabs/text-to-speech-turbo-2-5", name: "ElevenLabs TTS Turbo 2.5" },
|
|
{ id: "elevenlabs/text-to-dialogue-v3", name: "ElevenLabs Text to Dialogue v3" },
|
|
{ id: "elevenlabs/sound-effect-v2", name: "ElevenLabs Sound Effect v2" },
|
|
],
|
|
},
|
|
|
|
"aws-polly": {
|
|
id: "aws-polly",
|
|
// POST https://polly.{region}.amazonaws.com/v1/speech
|
|
// Auth: AWS SigV4. The provider apiKey stores Secret Access Key; PSD stores accessKeyId/region.
|
|
baseUrl: "https://polly.us-east-1.amazonaws.com",
|
|
authType: "apikey",
|
|
authHeader: "aws-sigv4",
|
|
format: "aws-polly",
|
|
models: [
|
|
{ id: "standard", name: "Polly Standard" },
|
|
{ id: "neural", name: "Polly Neural" },
|
|
{ id: "long-form", name: "Polly Long-Form" },
|
|
{ id: "generative", name: "Polly Generative" },
|
|
],
|
|
},
|
|
pollinations: {
|
|
id: "pollinations",
|
|
baseUrl: "https://gen.pollinations.ai/v1/audio/speech",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "openai",
|
|
models: [{ id: "default", name: "Pollinations TTS (Free)" }],
|
|
},
|
|
|
|
minimax: {
|
|
id: "minimax",
|
|
baseUrl: "https://api.minimax.io/v1/t2a_v2",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "minimax-tts",
|
|
models: [{ id: "speech-2.8-hd", name: "Speech 2.8 HD" }],
|
|
},
|
|
|
|
together: {
|
|
id: "together",
|
|
baseUrl: "https://api.together.xyz/v1/audio/speech",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "openai",
|
|
models: [
|
|
{ id: "cartesia/sonic-2", name: "Cartesia Sonic 2" },
|
|
{ id: "hexgrad/Kokoro-82M", name: "Kokoro 82M" },
|
|
{ id: "canopylabs/orpheus-3b-0.1-ft", name: "Orpheus 3B" },
|
|
],
|
|
},
|
|
|
|
"xiaomi-mimo": {
|
|
id: "xiaomi-mimo",
|
|
baseUrl: "https://api.xiaomimimo.com/v1/chat/completions",
|
|
authType: "apikey",
|
|
authHeader: "bearer",
|
|
format: "xiaomi-mimo-tts",
|
|
supportedFormats: ["mp3", "wav"],
|
|
models: [
|
|
{ id: "mimo-v2.5-tts", name: "MiMo V2.5 TTS" },
|
|
{ id: "mimo-v2.5-tts-voicedesign", name: "MiMo V2.5 Voice Design" },
|
|
{ id: "mimo-v2.5-tts-voiceclone", name: "MiMo V2.5 Voice Clone" },
|
|
],
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Get transcription provider config by ID
|
|
*/
|
|
export function getTranscriptionProvider(providerId: string): AudioProvider | null {
|
|
return AUDIO_TRANSCRIPTION_PROVIDERS[providerId] || null;
|
|
}
|
|
|
|
/**
|
|
* Get translation provider config by ID
|
|
*/
|
|
export function getTranslationProvider(providerId: string): AudioProvider | null {
|
|
return AUDIO_TRANSLATION_PROVIDERS[providerId] || null;
|
|
}
|
|
|
|
/**
|
|
* Get speech provider config by ID
|
|
*/
|
|
export function getSpeechProvider(providerId: string): AudioProvider | null {
|
|
return AUDIO_SPEECH_PROVIDERS[providerId] || null;
|
|
}
|
|
|
|
export interface ProviderNodeRow {
|
|
prefix: string;
|
|
name: string;
|
|
baseUrl: string;
|
|
apiType?: string;
|
|
}
|
|
|
|
/**
|
|
* Build a dynamic AudioProvider from a provider_node DB entry.
|
|
* Only used for local providers (localhost/127.0.0.1) — remote nodes are
|
|
* excluded by the caller to prevent auth bypass and SSRF.
|
|
*/
|
|
export function buildDynamicAudioProvider(node: ProviderNodeRow, audioPath: string): AudioProvider {
|
|
if (!node.prefix || !node.baseUrl) {
|
|
throw new Error(`Invalid provider_node: missing prefix or baseUrl`);
|
|
}
|
|
const baseUrl = node.baseUrl.replace(/\/+$/, "");
|
|
return {
|
|
id: node.prefix,
|
|
baseUrl: `${baseUrl}${audioPath}`,
|
|
authType: "none",
|
|
authHeader: "none",
|
|
models: [],
|
|
};
|
|
}
|
|
|
|
function parseAudioModel(
|
|
modelStr: string | null,
|
|
registry: Record<string, AudioProvider>,
|
|
dynamicProviders?: AudioProvider[]
|
|
): { provider: string | null; model: string | null } {
|
|
if (!modelStr) return { provider: null, model: null };
|
|
|
|
// Phase 1: prefix match in hardcoded registry
|
|
for (const [providerId] of Object.entries(registry)) {
|
|
if (modelStr.startsWith(providerId + "/")) {
|
|
return { provider: providerId, model: modelStr.slice(providerId.length + 1) };
|
|
}
|
|
}
|
|
|
|
// Phase 2: bare model lookup in hardcoded registry
|
|
for (const [providerId, config] of Object.entries(registry)) {
|
|
if (config.models.some((m) => m.id === modelStr)) {
|
|
return { provider: providerId, model: modelStr };
|
|
}
|
|
}
|
|
|
|
// Phase 3: prefix match in dynamic providers (provider_nodes)
|
|
if (dynamicProviders) {
|
|
for (const dp of dynamicProviders) {
|
|
if (modelStr.startsWith(dp.id + "/")) {
|
|
return { provider: dp.id, model: modelStr.slice(dp.id.length + 1) };
|
|
}
|
|
}
|
|
}
|
|
|
|
return { provider: null, model: modelStr };
|
|
}
|
|
|
|
export function parseTranscriptionModel(
|
|
modelStr: string | null,
|
|
dynamicProviders?: AudioProvider[]
|
|
) {
|
|
return parseAudioModel(modelStr, AUDIO_TRANSCRIPTION_PROVIDERS, dynamicProviders);
|
|
}
|
|
|
|
export function parseSpeechModel(modelStr: string | null, dynamicProviders?: AudioProvider[]) {
|
|
return parseAudioModel(modelStr, AUDIO_SPEECH_PROVIDERS, dynamicProviders);
|
|
}
|
|
|
|
export function parseTranslationModel(modelStr: string | null, dynamicProviders?: AudioProvider[]) {
|
|
return parseAudioModel(modelStr, AUDIO_TRANSLATION_PROVIDERS, dynamicProviders);
|
|
}
|
|
|
|
/**
|
|
* Get all audio models as a flat list
|
|
*/
|
|
export function getAllAudioModels() {
|
|
const models = [];
|
|
|
|
for (const [providerId, config] of Object.entries(AUDIO_TRANSCRIPTION_PROVIDERS)) {
|
|
for (const model of config.models) {
|
|
models.push({
|
|
id: model.id.startsWith(`${providerId}/`) ? model.id : `${providerId}/${model.id}`,
|
|
name: model.name,
|
|
provider: providerId,
|
|
subtype: "transcription",
|
|
});
|
|
}
|
|
}
|
|
|
|
for (const [providerId, config] of Object.entries(AUDIO_SPEECH_PROVIDERS)) {
|
|
for (const model of config.models) {
|
|
models.push({
|
|
id: model.id.startsWith(`${providerId}/`) ? model.id : `${providerId}/${model.id}`,
|
|
name: model.name,
|
|
provider: providerId,
|
|
subtype: "speech",
|
|
});
|
|
}
|
|
}
|
|
|
|
return models;
|
|
}
|