livekit--agents/livekit-plugins/livekit-plugins-gradium/livekit/plugins/gradium/stt.py

# Copyright 2023 LiveKit, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


from __future__ import annotations

import asyncio
import base64
import dataclasses
import json
import os
import weakref
from dataclasses import dataclass
from typing import Any, Literal

import aiohttp

from livekit.agents import (
    DEFAULT_API_CONNECT_OPTIONS,
    APIConnectOptions,
    APIStatusError,
    LanguageCode,
    stt,
    utils,
)
from livekit.agents.types import NOT_GIVEN, NotGivenOr
from livekit.agents.utils import AudioBuffer, is_given
from livekit.rtc import AudioFrame

from .log import logger

STTEncoding = Literal["pcm_s16le"]

# Define bytes per frame for different encoding types
bytes_per_frame = {
    "pcm_s16le": 2,
}


SUPPORTED_SAMPLE_RATE = 24000


@dataclass
class STTOptions:
    sample_rate: int = SUPPORTED_SAMPLE_RATE
    buffer_size_seconds: float = 0.08
    encoding: str = "pcm_s16le"
    temperature: float | None = None
    # TODO(laurent): support language detection
    language: LanguageCode = LanguageCode("en")
    vad_threshold: float = 0.6
    vad_bucket: int | None = 2
    # When set, we flush the stt state on the first time the VAD triggers
    # in order to recover the text currently being processed as soon as possible.
    vad_flush: bool = True


class STT(stt.STT):
    def __init__(
        self,
        *,
        api_key: str | None = None,
        model_endpoint: str | None = "wss://api.gradium.ai/api/speech/asr",
        model_name: str = "default",
        sample_rate: int = SUPPORTED_SAMPLE_RATE,
        encoding: NotGivenOr[STTEncoding] = NOT_GIVEN,
        buffer_size_seconds: float = 0.08,
        http_session: aiohttp.ClientSession | None = None,
        vad_threshold: float = 0.9,
        vad_bucket: int | None = 2,
        vad_flush: bool = True,
        temperature: float | None = None,
        language: str = "en",
    ):
        super().__init__(
            capabilities=stt.STTCapabilities(
                streaming=True,
                interim_results=True,  # only final transcripts
                aligned_transcript=False,  # only chunk start times are available
                offline_recognize=False,
            ),
        )

        api_key = api_key or os.environ.get("GRADIUM_API_KEY")

        if sample_rate != SUPPORTED_SAMPLE_RATE:
            raise ValueError(f"Only {SUPPORTED_SAMPLE_RATE}Hz sample rate is supported")

        if not api_key:
            raise ValueError(
                "Gradium API key is required. "
                "Pass one in via the `api_key` parameter, "
                "or set it as the `GRADIUM_API_KEY` environment variable"
            )

        self._api_key = api_key

        model_endpoint = model_endpoint or os.environ.get("GRADIUM_MODEL_ENDPOINT")

        if not model_endpoint:
            raise ValueError(
                "The model endpoint is required, you can find it in the Gradium dashboard"
            )

        self._model_endpoint = model_endpoint
        self._model_name = model_name

        self._opts = STTOptions(
            sample_rate=sample_rate,
            buffer_size_seconds=buffer_size_seconds,
            vad_threshold=vad_threshold,
            vad_bucket=vad_bucket,
            vad_flush=vad_flush,
            temperature=temperature,
            language=LanguageCode(language),
        )

        if is_given(encoding):
            self._opts.encoding = encoding

        self._session = http_session
        self._streams = weakref.WeakSet[SpeechStream]()

    @property
    def model(self) -> str:
        return "unknown"

    @property
    def provider(self) -> str:
        return "Gradium"

    @property
    def session(self) -> aiohttp.ClientSession:
        if not self._session:
            self._session = utils.http_context.http_session()
        return self._session

    async def _recognize_impl(
        self,
        buffer: AudioBuffer,
        *,
        language: NotGivenOr[str] = NOT_GIVEN,
        conn_options: APIConnectOptions,
    ) -> stt.SpeechEvent:
        raise NotImplementedError("Not implemented")

    def stream(
        self,
        *,
        language: NotGivenOr[str] = NOT_GIVEN,
        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
    ) -> SpeechStream:
        config = dataclasses.replace(self._opts)
        if is_given(language):
            config.language = LanguageCode(language)
        stream = SpeechStream(
            stt=self,
            conn_options=conn_options,
            opts=config,
            api_key=self._api_key,
            model_endpoint=self._model_endpoint,
            model_name=self._model_name,
            http_session=self.session,
        )
        self._streams.add(stream)
        return stream

    def update_options(
        self,
        *,
        buffer_size_seconds: NotGivenOr[float] = NOT_GIVEN,
    ) -> None:
        if is_given(buffer_size_seconds):
            self._opts.buffer_size_seconds = buffer_size_seconds

        for stream in self._streams:
            stream.update_options(
                buffer_size_seconds=buffer_size_seconds,
            )


class SpeechStream(stt.SpeechStream):
    # Used to close websocket
    _CLOSE_MSG: str = json.dumps({"terminate_session": True})

    def __init__(
        self,
        *,
        stt: STT,
        opts: STTOptions,
        conn_options: APIConnectOptions,
        api_key: str,
        model_endpoint: str,
        model_name: str,
        http_session: aiohttp.ClientSession,
    ) -> None:
        super().__init__(stt=stt, conn_options=conn_options, sample_rate=opts.sample_rate)

        self._opts = opts
        self._api_key = api_key
        self._model_endpoint = model_endpoint
        self._model_name = model_name
        self._session = http_session
        self._speech_duration: float = 0

        self._reconnect_event = asyncio.Event()
        self._ready_msg: dict[str, Any] | None = None

    @property
    def delay_in_tokens(self) -> int:
        if self._ready_msg is not None:
            return int(self._ready_msg.get("delay_in_tokens", 6))
        return 6

    @property
    def frame_size(self) -> int:
        if self._ready_msg is not None:
            return int(self._ready_msg.get("frame_size", 1920))
        return 1920

    def update_options(
        self,
        *,
        buffer_size_seconds: NotGivenOr[float] = NOT_GIVEN,
    ) -> None:
        if is_given(buffer_size_seconds):
            self._opts.buffer_size_seconds = buffer_size_seconds

        self._reconnect_event.set()

    async def _run(self) -> None:
        """
        Run a single websocket connection to Gradium and make sure to reconnect
        when something went wrong.
        """

        closing_ws = False

        async def send_task(ws: aiohttp.ClientWebSocketResponse) -> None:
            samples_per_buffer = 1920

            audio_bstream = utils.audio.AudioByteStream(
                sample_rate=self._opts.sample_rate,
                num_channels=1,
                samples_per_channel=samples_per_buffer,
            )

            async for data in self._input_ch:
                if isinstance(data, self._FlushSentinel):
                    frames = audio_bstream.flush()
                else:
                    frames = audio_bstream.write(data.data.tobytes())

                for frame in frames:
                    if len(frame.data) % 2 != 0:
                        logger.warning("Frame data size not aligned to int16 (multiple of 2)")

                    audio_data = base64.b64encode(frame.data.tobytes()).decode("utf-8")
                    audio_msg = {
                        "type": "audio",
                        "audio": audio_data,
                    }
                    await ws.send_str(json.dumps(audio_msg))

        async def recv_task(ws: aiohttp.ClientWebSocketResponse) -> None:
            nonlocal closing_ws
            buffered_text: list[str] = []
            speaking = False
            remaining_vad_steps: int | None = None
            while True:
                try:
                    msg = await asyncio.wait_for(ws.receive(), timeout=5)
                except asyncio.TimeoutError:
                    if closing_ws:
                        break
                    continue

                if msg.type in (
                    aiohttp.WSMsgType.CLOSED,
                    aiohttp.WSMsgType.CLOSE,
                    aiohttp.WSMsgType.CLOSING,
                ):
                    if closing_ws:
                        return
                    raise APIStatusError(
                        "Gradium connection closed unexpectedly",
                        status_code=ws.close_code or -1,
                        body=f"{msg.data=} {msg.extra=}",
                    )

                if msg.type != aiohttp.WSMsgType.TEXT:
                    logger.error("Unexpected Gradium message type: %s", msg.type)
                    continue

                try:
                    data = json.loads(msg.data)

                    type_ = data.get("type", "")

                    if type_ == "text":
                        if speaking is False:
                            speaking = True
                            start_event = stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
                            self._event_ch.send_nowait(start_event)
                        buffered_text.append(data["text"])
                        event = stt.SpeechEvent(
                            type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
                            alternatives=[
                                stt.SpeechData(
                                    text=data["text"],
                                    language=self._opts.language,
                                    start_time=data["start_s"] + self.start_time_offset,
                                )
                            ],
                        )
                        self._event_ch.send_nowait(event)

                    elif type_ == "step":
                        if not speaking:
                            continue
                        if vad_bucket := self._opts.vad_bucket:
                            positive_vad = (
                                data["vad"][vad_bucket]["inactivity_prob"]
                                > self._opts.vad_threshold
                            )
                            if positive_vad:
                                if remaining_vad_steps is None:
                                    remaining_vad_steps = self.delay_in_tokens
                                    if self._opts.vad_flush:
                                        samples_per_channel = self.frame_size * self.delay_in_tokens
                                        zeros = AudioFrame.create(
                                            sample_rate=self._opts.sample_rate,
                                            num_channels=1,
                                            samples_per_channel=samples_per_channel,
                                        )
                                        await self._input_ch.send(zeros)
                                else:
                                    remaining_vad_steps -= 1
                                    if remaining_vad_steps <= 0:
                                        speaking = False
                                        remaining_vad_steps = None
                                        event = stt.SpeechEvent(
                                            type=stt.SpeechEventType.FINAL_TRANSCRIPT,
                                            alternatives=[
                                                stt.SpeechData(
                                                    text=" ".join(buffered_text),
                                                    language=self._opts.language,
                                                )
                                            ],
                                        )
                                        self._event_ch.send_nowait(event)

                                        buffered_text = []
                                        self._event_ch.send_nowait(
                                            stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH)
                                        )
                            else:
                                remaining_vad_steps = None

                    elif type_ == "ready":
                        self._ready_msg = data
                    elif type_ == "end_text":
                        # This message provides the end timestamp of the previous word in the stop_s field.
                        pass
                    else:
                        logger.warning(f"Unknown message type from Gradium {type_}")

                except Exception:
                    logger.exception("Failed to process message from Gradium")

        ws: aiohttp.ClientWebSocketResponse | None = None

        while True:
            try:
                ws = await self._connect_ws()
                tasks = [
                    asyncio.create_task(send_task(ws)),
                    asyncio.create_task(recv_task(ws)),
                ]
                wait_reconnect_task = asyncio.create_task(self._reconnect_event.wait())

                tasks_group: asyncio.Future[Any] = asyncio.gather(*tasks)
                try:
                    done, _ = await asyncio.wait(
                        [tasks_group, wait_reconnect_task],
                        return_when=asyncio.FIRST_COMPLETED,
                    )
                    for task in done:
                        if task != wait_reconnect_task:
                            task.result()

                    if wait_reconnect_task not in done:
                        break

                    self._reconnect_event.clear()
                finally:
                    await utils.aio.gracefully_cancel(*tasks, wait_reconnect_task)
                    tasks_group.cancel()
                    tasks_group.exception()
            finally:
                if ws is not None:
                    await ws.close()

    async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
        headers = {"x-api-key": self._api_key, "x-api-source": "livekit"}

        ws = await self._session.ws_connect(self._model_endpoint, headers=headers)

        # Build and send the setup payload as the first message
        setup_msg: dict[str, Any] = {
            "type": "setup",
            "model_name": self._model_name,
            "input_format": "pcm",
        }
        json_config: dict[str, Any] = {"language": self._opts.language.language}
        if self._opts.temperature is not None:
            json_config["temp"] = self._opts.temperature
        setup_msg["json_config"] = json_config

        await ws.send_str(json.dumps(setup_msg))
        return ws