livekit--agents/livekit-plugins/livekit-plugins-mistralai/livekit/plugins/mistralai/stt.py

from __future__ import annotations

import asyncio
import os
import weakref
from dataclasses import dataclass

from livekit import rtc
from livekit.agents import (
    APIConnectionError,
    APIConnectOptions,
    APIStatusError,
    APITimeoutError,
    LanguageCode,
    stt,
    utils,
    vad,
)
from livekit.agents.types import (
    DEFAULT_API_CONNECT_OPTIONS,
    NOT_GIVEN,
    NotGivenOr,
)
from livekit.agents.utils import AudioBuffer, is_given
from livekit.agents.voice.io import TimedString
from mistralai.client import Mistral
from mistralai.client.errors import SDKError
from mistralai.client.models import (
    RealtimeTranscriptionError,
    RealtimeTranscriptionSessionCreated,
    TranscriptionStreamDone,
    TranscriptionStreamLanguage,
    TranscriptionStreamTextDelta,
)
from mistralai.extra.realtime import RealtimeConnection, RealtimeTranscription

from .log import logger
from .models import STTModels


def _is_realtime(model: str) -> bool:
    return "realtime" in model


DEFAULT_MODEL: STTModels = "voxtral-mini-latest"

SAMPLE_RATE: int = 16000
NUM_CHANNELS: int = 1


@dataclass
class _STTOptions:
    model: STTModels | str
    language: LanguageCode | None
    context_bias: list[str] | None
    target_streaming_delay_ms: int | None


class STT(stt.STT):
    def __init__(
        self,
        client: Mistral | None = None,
        api_key: NotGivenOr[str] = NOT_GIVEN,
        model: NotGivenOr[str] = NOT_GIVEN,
        language: NotGivenOr[str] = NOT_GIVEN,
        context_bias: NotGivenOr[list[str]] = NOT_GIVEN,
        target_streaming_delay_ms: NotGivenOr[int] = NOT_GIVEN,
        vad: vad.VAD | None = None,
    ):
        """
        Create a new instance of MistralAI STT.

        Args:
            client: Optional pre-configured MistralAI client instance.
            api_key: Your Mistral AI API key. If not provided, will use the MISTRAL_API_KEY environment variable.
            model: The Mistral AI model to use for transcription, default is batch "voxtral-mini-latest".
            language: The optional language code to use for better transcription accuracy if language is already known (e.g., "fr" for French).
                Only used with batch models.
            context_bias: Up to 100 words or phrases to guide the model toward good spelling or names or domain-specific vocabulary.
                Only used with batch models.
            target_streaming_delay_ms: Target streaming delay in milliseconds for realtime mode. Only used with realtime models.
            vad: Voice Activity Detector used to trigger audio flush for realtime models (which lack server-side endpointing).
                When not provided, Silero VAD is auto-loaded with default settings. Only used with realtime models.
        """
        resolved_model = model if is_given(model) else DEFAULT_MODEL
        is_realtime = _is_realtime(resolved_model)
        super().__init__(
            capabilities=stt.STTCapabilities(
                streaming=is_realtime,
                interim_results=is_realtime,
                aligned_transcript=False,
                offline_recognize=not is_realtime,
            )
        )
        self._opts = _STTOptions(
            model=resolved_model,
            language=LanguageCode(language) if is_given(language) else None,
            context_bias=context_bias if is_given(context_bias) else None,
            target_streaming_delay_ms=target_streaming_delay_ms
            if is_given(target_streaming_delay_ms)
            else None,
        )

        if is_realtime and vad is None:
            try:
                from livekit.plugins.silero import VAD as SileroVAD

                vad = SileroVAD.load()
            except ImportError as e:
                raise ImportError(
                    "livekit-plugins-silero is required for Voxtral realtime models (no server-side endpointing)."
                ) from e
        self._vad = vad

        mistral_api_key = api_key if is_given(api_key) else os.environ.get("MISTRAL_API_KEY")
        if not client and not mistral_api_key:
            raise ValueError("Mistral AI API key is required. Set MISTRAL_API_KEY or pass api_key")
        self._client = client or Mistral(api_key=mistral_api_key)
        self._streams: weakref.WeakSet[SpeechStream] = weakref.WeakSet()
        self._pool = utils.ConnectionPool[RealtimeConnection](
            connect_cb=self._connect_ws,
            close_cb=self._close_ws,
        )

    async def _connect_ws(self, timeout: float) -> RealtimeConnection:
        rt = RealtimeTranscription(self._client.sdk_configuration)
        http_headers = None
        cfg = self._client.sdk_configuration
        client_headers = getattr(cfg.async_client, "headers", None) or getattr(
            cfg.client, "headers", None
        )
        if client_headers:
            http_headers = dict(client_headers)
        return await asyncio.wait_for(
            rt.connect(
                model=self._opts.model,
                target_streaming_delay_ms=self._opts.target_streaming_delay_ms,
                http_headers=http_headers,
            ),
            timeout=timeout,
        )

    async def _close_ws(self, conn: RealtimeConnection) -> None:
        await conn.close()
        ws = conn._websocket
        if ws.keepalive_task is not None:
            ws.keepalive_task.cancel()

    def prewarm(self) -> None:
        if _is_realtime(self._opts.model):
            self._pool.prewarm()

    async def aclose(self) -> None:
        await self._pool.aclose()

    @property
    def model(self) -> str:
        return self._opts.model

    @property
    def provider(self) -> str:
        return "MistralAI"

    def update_options(
        self,
        *,
        language: NotGivenOr[str] = NOT_GIVEN,
        context_bias: NotGivenOr[list[str]] = NOT_GIVEN,
        target_streaming_delay_ms: NotGivenOr[int] = NOT_GIVEN,
    ) -> None:
        """
        Update the STT options.

        Args:
            language: The optional language code to use for better transcription accuracy if language is already known (e.g., "fr" for French).
                Only used with batch models.
            context_bias: Up to 100 words or phrases to guide the model toward good spelling or names or domain-specific vocabulary.
                Only used with batch models.
            target_streaming_delay_ms: Target streaming delay in milliseconds for realtime mode. Only used with realtime models.
        """
        if is_given(language):
            self._opts.language = LanguageCode(language)
        if is_given(context_bias):
            self._opts.context_bias = context_bias
        if is_given(target_streaming_delay_ms):
            self._opts.target_streaming_delay_ms = target_streaming_delay_ms
            self._pool.invalidate()
            for stream in self._streams:
                stream._reconnect_event.set()

    async def _recognize_impl(
        self,
        buffer: AudioBuffer,
        *,
        language: NotGivenOr[str] = NOT_GIVEN,
        conn_options: APIConnectOptions,
    ) -> stt.SpeechEvent:
        try:
            if is_given(language):
                self._opts.language = LanguageCode(language)
            data = rtc.combine_audio_frames(buffer).to_wav_bytes()

            resp = await self._client.audio.transcriptions.complete_async(
                model=self._opts.model,
                file={"content": data, "file_name": "audio.wav"},
                language=self._opts.language if self._opts.language else None,
                context_bias=self._opts.context_bias if self._opts.context_bias else None,
                timestamp_granularities=["segment"] if not self._opts.language else None,
            )

            return stt.SpeechEvent(
                type=stt.SpeechEventType.FINAL_TRANSCRIPT,
                alternatives=[
                    stt.SpeechData(
                        text=resp.text,
                        language=LanguageCode(resp.language)
                        if resp.language
                        else self._opts.language or LanguageCode(""),
                        start_time=resp.segments[0].start if resp.segments else 0,
                        end_time=resp.segments[-1].end if resp.segments else 0,
                        words=[
                            TimedString(
                                text=segment.text,
                                start_time=segment.start,
                                end_time=segment.end,
                            )
                            for segment in resp.segments
                        ]
                        if resp.segments
                        else None,
                    ),
                ],
            )

        except SDKError as e:
            if e.status_code in (408, 504):
                raise APITimeoutError() from e
            raise APIStatusError(e.message, status_code=e.status_code, body=e.body) from e
        except Exception as e:
            raise APIConnectionError() from e

    def stream(
        self,
        *,
        language: NotGivenOr[str] = NOT_GIVEN,
        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
    ) -> SpeechStream:
        stream = SpeechStream(
            stt=self,
            pool=self._pool,
            conn_options=conn_options,
            language=language,
            vad_instance=self._vad,
        )
        self._streams.add(stream)
        return stream


class SpeechStream(stt.RecognizeStream):
    """Realtime speech recognition stream for Voxtral Realtime."""

    def __init__(
        self,
        *,
        stt: STT,
        pool: utils.ConnectionPool[RealtimeConnection],
        conn_options: APIConnectOptions,
        language: NotGivenOr[str] = NOT_GIVEN,
        vad_instance: vad.VAD | None = None,
    ) -> None:
        super().__init__(
            stt=stt,
            conn_options=conn_options,
            sample_rate=SAMPLE_RATE,
        )
        self._pool = pool
        self._opts = stt._opts
        self._vad = vad_instance
        self._reconnect_event = asyncio.Event()
        self._request_id = ""
        self._audio_duration = 0.0
        self._speaking = False
        self._detected_language = LanguageCode(language) if is_given(language) else LanguageCode("")

    async def _send_task(
        self,
        connection: RealtimeConnection,
        vad_stream: vad.VADStream | None,
    ) -> None:
        samples_per_chunk = SAMPLE_RATE // 20  # 50ms chunks
        audio_bstream = utils.audio.AudioByteStream(
            sample_rate=SAMPLE_RATE,
            num_channels=NUM_CHANNELS,
            samples_per_channel=samples_per_chunk,
        )

        async for data in self._input_ch:
            if isinstance(data, self._FlushSentinel):
                for frame in audio_bstream.flush():
                    await connection.send_audio(frame.data.tobytes())
                await connection.flush_audio()
            else:
                if vad_stream is not None:
                    vad_stream.push_frame(data)

                data_bytes = data.data.tobytes()
                bytes_per_second = SAMPLE_RATE * NUM_CHANNELS * 2
                self._audio_duration += len(data_bytes) / bytes_per_second
                for frame in audio_bstream.write(data.data.tobytes()):
                    await connection.send_audio(frame.data.tobytes())

        if vad_stream is not None:
            vad_stream.end_input()
        await connection.end_audio()

    async def _vad_task(
        self,
        vad_stream: vad.VADStream,
        connection: RealtimeConnection,
    ) -> None:
        async for ev in vad_stream:
            if ev.type == vad.VADEventType.START_OF_SPEECH:
                if not self._speaking:
                    self._speaking = True
                    self._event_ch.send_nowait(
                        stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
                    )
            elif ev.type == vad.VADEventType.END_OF_SPEECH:
                # force Mistral to finalize the current speech segment
                await connection.flush_audio()
                if self._speaking:
                    self._speaking = False
                    self._event_ch.send_nowait(
                        stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH)
                    )

    async def _recv_task(self, connection: RealtimeConnection) -> None:
        current_text = ""

        async for event in connection.events():
            if isinstance(event, RealtimeTranscriptionSessionCreated):
                if event.session:
                    self._request_id = event.session.request_id

            elif isinstance(event, TranscriptionStreamLanguage):
                if event.audio_language:
                    self._detected_language = LanguageCode(event.audio_language)

            elif isinstance(event, TranscriptionStreamTextDelta):
                current_text += event.text
                self._event_ch.send_nowait(
                    stt.SpeechEvent(
                        type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
                        request_id=self._request_id,
                        alternatives=[
                            stt.SpeechData(
                                text=current_text,
                                language=self._detected_language,
                            )
                        ],
                    )
                )

            elif isinstance(event, TranscriptionStreamDone):
                current_text = ""
                words = (
                    [
                        TimedString(
                            text=seg.text,
                            start_time=seg.start + self.start_time_offset,
                            end_time=seg.end + self.start_time_offset,
                        )
                        for seg in event.segments
                    ]
                    if event.segments
                    else None
                )

                self._event_ch.send_nowait(
                    stt.SpeechEvent(
                        type=stt.SpeechEventType.FINAL_TRANSCRIPT,
                        request_id=self._request_id,
                        alternatives=[
                            stt.SpeechData(
                                text=event.text,
                                language=LanguageCode(event.language)
                                if event.language
                                else self._detected_language,
                                words=words,
                            )
                        ],
                    )
                )

                self._event_ch.send_nowait(
                    stt.SpeechEvent(
                        type=stt.SpeechEventType.RECOGNITION_USAGE,
                        request_id=self._request_id,
                        recognition_usage=stt.RecognitionUsage(
                            audio_duration=event.usage.prompt_audio_seconds or 0,
                            input_tokens=event.usage.prompt_tokens or 0,
                            output_tokens=event.usage.completion_tokens or 0,
                        ),
                    )
                )

            elif isinstance(event, RealtimeTranscriptionError):
                raise APIStatusError(
                    message=str(event.error.message), status_code=event.error.code, body=event.error
                )

    @utils.log_exceptions(logger=logger)
    async def _run(self) -> None:
        try:
            while True:
                async with self._pool.connection(timeout=self._conn_options.timeout) as connection:
                    vad_stream = self._vad.stream() if self._vad else None

                    tasks = [
                        asyncio.create_task(self._send_task(connection, vad_stream)),
                        asyncio.create_task(self._recv_task(connection)),
                    ]
                    if vad_stream is not None:
                        tasks.append(asyncio.create_task(self._vad_task(vad_stream, connection)))

                    tasks_group = asyncio.gather(*tasks)
                    wait_reconnect = asyncio.create_task(self._reconnect_event.wait())

                    try:
                        done, _ = await asyncio.wait(
                            (tasks_group, wait_reconnect),
                            return_when=asyncio.FIRST_COMPLETED,
                        )

                        for task in done:
                            if task != wait_reconnect:
                                task.result()

                        if wait_reconnect not in done:
                            break

                        self._reconnect_event.clear()
                    finally:
                        await utils.aio.gracefully_cancel(*tasks, wait_reconnect)
                        tasks_group.cancel()
                        tasks_group.exception()
                        if vad_stream is not None:
                            await vad_stream.aclose()

        except (APIStatusError, APITimeoutError, APIConnectionError):
            raise
        except SDKError as e:
            if e.status_code in (408, 504):
                raise APITimeoutError() from e
            raise APIStatusError(e.message, status_code=e.status_code, body=e.body) from e
        except Exception as e:
            raise APIConnectionError() from e