wehub-resource-sync
324 lines
11 KiB
Python
324 lines
11 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
from dataclasses import dataclass, replace
|
|
from typing import Literal
|
|
|
|
import aiohttp
|
|
|
|
from livekit.agents import (
|
|
APIConnectionError,
|
|
APIStatusError,
|
|
APITimeoutError,
|
|
LanguageCode,
|
|
tts,
|
|
utils,
|
|
)
|
|
from livekit.agents.types import (
|
|
DEFAULT_API_CONNECT_OPTIONS,
|
|
NOT_GIVEN,
|
|
APIConnectOptions,
|
|
NotGivenOr,
|
|
)
|
|
from livekit.agents.utils import is_given
|
|
|
|
SUPPORTED_OUTPUT_FORMATS = {
|
|
8000: "raw-8khz-16bit-mono-pcm",
|
|
16000: "raw-16khz-16bit-mono-pcm",
|
|
22050: "raw-22050hz-16bit-mono-pcm",
|
|
24000: "raw-24khz-16bit-mono-pcm",
|
|
44100: "raw-44100hz-16bit-mono-pcm",
|
|
48000: "raw-48khz-16bit-mono-pcm",
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class ProsodyConfig:
|
|
rate: Literal["x-slow", "slow", "medium", "fast", "x-fast"] | float | None = None
|
|
volume: Literal["silent", "x-soft", "soft", "medium", "loud", "x-loud"] | float | None = None
|
|
pitch: Literal["x-low", "low", "medium", "high", "x-high"] | None = None
|
|
|
|
def validate(self) -> None:
|
|
if self.rate:
|
|
if isinstance(self.rate, float) and not 0.5 <= self.rate <= 2:
|
|
raise ValueError("Prosody rate must be between 0.5 and 2")
|
|
if isinstance(self.rate, str) and self.rate not in [
|
|
"x-slow",
|
|
"slow",
|
|
"medium",
|
|
"fast",
|
|
"x-fast",
|
|
]:
|
|
raise ValueError(
|
|
"Prosody rate must be one of 'x-slow', 'slow', 'medium', 'fast', 'x-fast'"
|
|
)
|
|
if self.volume:
|
|
if isinstance(self.volume, float) and not 0 <= self.volume <= 100:
|
|
raise ValueError("Prosody volume must be between 0 and 100")
|
|
if isinstance(self.volume, str) and self.volume not in [
|
|
"silent",
|
|
"x-soft",
|
|
"soft",
|
|
"medium",
|
|
"loud",
|
|
"x-loud",
|
|
]:
|
|
raise ValueError(
|
|
"Prosody volume must be one of 'silent', 'x-soft', 'soft', 'medium', 'loud', 'x-loud'" # noqa: E501
|
|
)
|
|
if self.pitch and self.pitch not in [
|
|
"x-low",
|
|
"low",
|
|
"medium",
|
|
"high",
|
|
"x-high",
|
|
]:
|
|
raise ValueError(
|
|
"Prosody pitch must be one of 'x-low', 'low', 'medium', 'high', 'x-high'"
|
|
)
|
|
|
|
def __post_init__(self) -> None:
|
|
self.validate()
|
|
|
|
|
|
@dataclass
|
|
class StyleConfig:
|
|
style: str
|
|
degree: float | None = None
|
|
|
|
def validate(self) -> None:
|
|
if self.degree is not None and not 0.1 <= self.degree <= 2.0:
|
|
raise ValueError("Style degree must be between 0.1 and 2.0")
|
|
|
|
def __post_init__(self) -> None:
|
|
self.validate()
|
|
|
|
|
|
@dataclass
|
|
class _TTSOptions:
|
|
sample_rate: int
|
|
subscription_key: str | None
|
|
region: str | None
|
|
voice: str
|
|
language: LanguageCode | None
|
|
speech_endpoint: str | None
|
|
deployment_id: str | None
|
|
prosody: NotGivenOr[ProsodyConfig]
|
|
style: NotGivenOr[StyleConfig]
|
|
lexicon_uri: NotGivenOr[str]
|
|
auth_token: str | None = None
|
|
|
|
def get_endpoint_url(self) -> str:
|
|
base = (
|
|
self.speech_endpoint
|
|
or f"https://{self.region}.tts.speech.microsoft.com/cognitiveservices/v1"
|
|
)
|
|
if self.deployment_id:
|
|
return f"{base}?deploymentId={self.deployment_id}"
|
|
return base
|
|
|
|
|
|
class TTS(tts.TTS):
|
|
def __init__(
|
|
self,
|
|
*,
|
|
voice: str = "en-US-JennyNeural",
|
|
language: str | None = None,
|
|
sample_rate: int = 24000,
|
|
prosody: NotGivenOr[ProsodyConfig] = NOT_GIVEN,
|
|
style: NotGivenOr[StyleConfig] = NOT_GIVEN,
|
|
lexicon_uri: NotGivenOr[str] = NOT_GIVEN,
|
|
speech_key: str | None = None,
|
|
speech_region: str | None = None,
|
|
speech_endpoint: str | None = None,
|
|
deployment_id: str | None = None,
|
|
speech_auth_token: str | None = None,
|
|
http_session: aiohttp.ClientSession | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
capabilities=tts.TTSCapabilities(streaming=False),
|
|
sample_rate=sample_rate,
|
|
num_channels=1,
|
|
)
|
|
if sample_rate not in SUPPORTED_OUTPUT_FORMATS:
|
|
raise ValueError(
|
|
f"Unsupported sample rate {sample_rate}. Supported: {list(SUPPORTED_OUTPUT_FORMATS)}" # noqa: E501
|
|
)
|
|
|
|
if not speech_key:
|
|
speech_key = os.environ.get("AZURE_SPEECH_KEY")
|
|
|
|
if not speech_region:
|
|
speech_region = os.environ.get("AZURE_SPEECH_REGION")
|
|
|
|
if not speech_endpoint:
|
|
speech_endpoint = os.environ.get("AZURE_SPEECH_ENDPOINT")
|
|
|
|
has_endpoint = bool(speech_endpoint)
|
|
has_key_and_region = bool(speech_key and speech_region)
|
|
has_token_and_region = bool(speech_auth_token and speech_region)
|
|
if not (has_endpoint or has_key_and_region or has_token_and_region):
|
|
raise ValueError(
|
|
"Authentication requires one of: speech_endpoint (AZURE_SPEECH_ENDPOINT), "
|
|
"speech_key & speech_region (AZURE_SPEECH_KEY & AZURE_SPEECH_REGION), "
|
|
"or speech_auth_token & speech_region."
|
|
)
|
|
|
|
if is_given(prosody):
|
|
prosody.validate()
|
|
if is_given(style):
|
|
style.validate()
|
|
|
|
self._session = http_session
|
|
self._opts = _TTSOptions(
|
|
sample_rate=sample_rate,
|
|
subscription_key=speech_key,
|
|
region=speech_region,
|
|
speech_endpoint=speech_endpoint,
|
|
voice=voice,
|
|
deployment_id=deployment_id,
|
|
language=LanguageCode(language) if language else None,
|
|
prosody=prosody,
|
|
style=style,
|
|
lexicon_uri=lexicon_uri,
|
|
auth_token=speech_auth_token,
|
|
)
|
|
|
|
@property
|
|
def model(self) -> str:
|
|
return "unknown"
|
|
|
|
@property
|
|
def provider(self) -> str:
|
|
return "Azure TTS"
|
|
|
|
def update_options(
|
|
self,
|
|
*,
|
|
voice: NotGivenOr[str] = NOT_GIVEN,
|
|
language: NotGivenOr[str] = NOT_GIVEN,
|
|
prosody: NotGivenOr[ProsodyConfig] = NOT_GIVEN,
|
|
style: NotGivenOr[StyleConfig] = NOT_GIVEN,
|
|
lexicon_uri: NotGivenOr[str] = NOT_GIVEN,
|
|
) -> None:
|
|
if is_given(voice):
|
|
self._opts.voice = voice
|
|
if is_given(language):
|
|
self._opts.language = LanguageCode(language)
|
|
if is_given(prosody):
|
|
prosody.validate()
|
|
self._opts.prosody = prosody
|
|
if is_given(style):
|
|
style.validate()
|
|
self._opts.style = style
|
|
if is_given(lexicon_uri):
|
|
self._opts.lexicon_uri = lexicon_uri
|
|
|
|
def _ensure_session(self) -> aiohttp.ClientSession:
|
|
if not self._session:
|
|
self._session = utils.http_context.http_session()
|
|
return self._session
|
|
|
|
def synthesize(
|
|
self,
|
|
text: str,
|
|
*,
|
|
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
|
|
) -> tts.ChunkedStream:
|
|
return ChunkedStream(tts=self, input_text=text, conn_options=conn_options)
|
|
|
|
|
|
class ChunkedStream(tts.ChunkedStream):
|
|
def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None:
|
|
super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
|
|
self._tts: TTS = tts
|
|
self._opts = replace(tts._opts)
|
|
|
|
def _build_ssml(self) -> str:
|
|
lang = self._opts.language or "en-US"
|
|
ssml = (
|
|
f'<speak version="1.0" '
|
|
f'xmlns="http://www.w3.org/2001/10/synthesis" '
|
|
f'xmlns:mstts="http://www.w3.org/2001/mstts" '
|
|
f'xml:lang="{lang}">'
|
|
)
|
|
ssml += f'<voice name="{self._opts.voice}">'
|
|
|
|
if is_given(self._opts.lexicon_uri):
|
|
ssml += f'<lexicon uri="{self._opts.lexicon_uri}"/>'
|
|
|
|
if is_given(self._opts.style):
|
|
degree = f' styledegree="{self._opts.style.degree}"' if self._opts.style.degree else ""
|
|
ssml += f'<mstts:express-as style="{self._opts.style.style}"{degree}>'
|
|
|
|
if is_given(self._opts.prosody):
|
|
p = self._opts.prosody
|
|
|
|
rate_attr = f' rate="{p.rate}"' if p.rate is not None else ""
|
|
vol_attr = f' volume="{p.volume}"' if p.volume is not None else ""
|
|
pitch_attr = f' pitch="{p.pitch}"' if p.pitch is not None else ""
|
|
ssml += f"<prosody{rate_attr}{vol_attr}{pitch_attr}>{self.input_text}</prosody>"
|
|
else:
|
|
ssml += self.input_text
|
|
|
|
if is_given(self._opts.style):
|
|
ssml += "</mstts:express-as>"
|
|
|
|
ssml += "</voice></speak>"
|
|
return ssml
|
|
|
|
async def _run(self, output_emitter: tts.AudioEmitter) -> None:
|
|
headers = {
|
|
"Content-Type": "application/ssml+xml",
|
|
"X-Microsoft-OutputFormat": SUPPORTED_OUTPUT_FORMATS[self._opts.sample_rate],
|
|
"User-Agent": "LiveKit Agents",
|
|
}
|
|
if self._opts.auth_token:
|
|
headers["Authorization"] = f"Bearer {self._opts.auth_token}"
|
|
|
|
elif self._opts.subscription_key:
|
|
headers["Ocp-Apim-Subscription-Key"] = self._opts.subscription_key
|
|
|
|
output_emitter.initialize(
|
|
request_id=utils.shortuuid(),
|
|
sample_rate=self._opts.sample_rate,
|
|
num_channels=1,
|
|
mime_type="audio/pcm",
|
|
)
|
|
|
|
try:
|
|
async with self._tts._ensure_session().post(
|
|
url=self._opts.get_endpoint_url(),
|
|
headers=headers,
|
|
data=self._build_ssml(),
|
|
timeout=aiohttp.ClientTimeout(total=30, sock_connect=self._conn_options.timeout),
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
async for data, _ in resp.content.iter_chunks():
|
|
output_emitter.push(data)
|
|
|
|
except asyncio.TimeoutError:
|
|
raise APITimeoutError() from None
|
|
except aiohttp.ClientResponseError as e:
|
|
raise APIStatusError(
|
|
message=e.message,
|
|
status_code=e.status,
|
|
request_id=None,
|
|
body=None,
|
|
) from None
|
|
except Exception as e:
|
|
raise APIConnectionError(str(e)) from e
|