"""Transcript synchronizer pacing must ignore expressive markup.
The synchronizer forwards the raw LLM text (markup intact — the room output strips
it downstream) but paces the display against the *visible* words only. Markup tags
carry spaces in their attributes, so the word stream shreds them into fragments
(````); a per-token
strip can't recognize those, and each fragment was paced as if it were spoken — the
transcript drifted seconds behind the audio on every expressive sentence.
"""
from __future__ import annotations
import time
import pytest
from livekit import rtc
from livekit.agents import tokenize
from livekit.agents.voice import io
from livekit.agents.voice.transcription._speaking_rate import SpeakingRateDetector
from livekit.agents.voice.transcription.synchronizer import (
_SegmentSynchronizerImpl,
_TextSyncOptions,
)
pytestmark = pytest.mark.unit
SAMPLE_RATE = 16000
AUDIO_DURATION = 3.0
# ~10 visible hyphens of speech, but ~40 hyphens of markup fragments. With the bug
# the markup alone adds >10s of pacing; with the fix the whole transcript paces out
# in about the audio duration.
MARKED_UP_TURN = (
' '
"Hello there my friend! "
' '
' '
"How are you today?"
)
class _CollectorTextOutput(io.TextOutput):
def __init__(self) -> None:
super().__init__(label="test-collector", next_in_chain=None)
self.words: list[str] = []
async def capture_text(self, text: str) -> None:
self.words.append(str(text))
def flush(self) -> None:
pass
def _silent_frames(duration: float) -> list[rtc.AudioFrame]:
samples_per_frame = SAMPLE_RATE // 100 # 10ms
frame = rtc.AudioFrame(
data=b"\x00\x00" * samples_per_frame,
sample_rate=SAMPLE_RATE,
num_channels=1,
samples_per_channel=samples_per_frame,
)
return [frame] * int(duration * 100)
async def test_markup_fragments_add_no_pacing_delay() -> None:
opts = _TextSyncOptions(
speed=1.0,
hyphenate_word=tokenize.basic.hyphenate_word,
word_tokenizer=tokenize.basic.WordTokenizer(
retain_format=True, ignore_punctuation=False, split_character=True
),
speaking_rate_detector=SpeakingRateDetector(),
)
collector = _CollectorTextOutput()
impl = _SegmentSynchronizerImpl(opts, next_in_chain=collector)
try:
for frame in _silent_frames(AUDIO_DURATION):
impl.push_audio(frame)
impl.end_audio_input()
impl.push_text(MARKED_UP_TURN)
impl.end_text_input()
start = time.monotonic()
impl.on_playback_started(time.time())
# forwarding is done once the main task exhausts the word stream and the
# capture task drains the output channel
await impl._main_atask
await impl._capture_atask
elapsed = time.monotonic() - start
# every raw token is still forwarded (markup included — stripped downstream)
assert "".join(collector.words) == MARKED_UP_TURN
# the pacing budget must cover only the visible words, i.e. roughly the
# audio duration; with markup fragments paced as speech it exceeds 12s
assert elapsed < AUDIO_DURATION + 3.0, (
f"transcript took {elapsed:.1f}s — markup is being paced as spoken text"
)
finally:
await impl.aclose()