wehub-resource-sync
375 lines
12 KiB
Python
375 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import numpy as np
|
|
import numpy.typing as npt
|
|
import pytest
|
|
|
|
from tests.testing_processor import fetch_events
|
|
|
|
try:
|
|
from agents.voice import (
|
|
AudioInput,
|
|
StreamedAudioResult,
|
|
TTSModelSettings,
|
|
VoicePipeline,
|
|
VoicePipelineConfig,
|
|
VoiceStreamEvent,
|
|
VoiceStreamEventAudio,
|
|
VoiceStreamEventLifecycle,
|
|
)
|
|
|
|
from .fake_models import FakeStreamedAudioInput, FakeSTT, FakeTTS, FakeWorkflow
|
|
from .helpers import extract_events
|
|
except ImportError:
|
|
pass
|
|
|
|
|
|
def test_streamed_audio_result_odd_length_buffer_int16() -> None:
|
|
result = StreamedAudioResult(
|
|
FakeTTS(),
|
|
TTSModelSettings(dtype=np.int16),
|
|
VoicePipelineConfig(),
|
|
)
|
|
|
|
transformed = result._transform_audio_buffer([b"\x01"], np.int16)
|
|
|
|
assert transformed.dtype == np.int16
|
|
assert transformed.tolist() == [1]
|
|
|
|
|
|
def test_streamed_audio_result_odd_length_buffer_float32() -> None:
|
|
result = StreamedAudioResult(
|
|
FakeTTS(),
|
|
TTSModelSettings(dtype=np.float32),
|
|
VoicePipelineConfig(),
|
|
)
|
|
|
|
transformed = result._transform_audio_buffer([b"\x01"], np.float32)
|
|
|
|
assert transformed.dtype == np.float32
|
|
assert transformed.shape == (1, 1)
|
|
assert transformed[0, 0] == pytest.approx(1 / 32767.0)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_streamed_audio_result_preserves_cross_chunk_sample_boundaries() -> None:
|
|
class SplitSampleTTS(FakeTTS):
|
|
async def run(self, text: str, settings: TTSModelSettings):
|
|
del text, settings
|
|
yield b"\x01"
|
|
yield b"\x00"
|
|
|
|
result = StreamedAudioResult(
|
|
SplitSampleTTS(),
|
|
TTSModelSettings(buffer_size=1, dtype=np.int16),
|
|
VoicePipelineConfig(),
|
|
)
|
|
local_queue: asyncio.Queue[VoiceStreamEvent | None] = asyncio.Queue()
|
|
|
|
await result._stream_audio("hello", local_queue, finish_turn=True)
|
|
|
|
audio_chunks: list[bytes] = []
|
|
while True:
|
|
event = await local_queue.get()
|
|
assert event is not None
|
|
if isinstance(event, VoiceStreamEventAudio) and event.data is not None:
|
|
audio_chunks.append(event.data.tobytes())
|
|
if isinstance(event, VoiceStreamEventLifecycle) and event.event == "turn_ended":
|
|
break
|
|
|
|
assert audio_chunks == [np.array([1], dtype=np.int16).tobytes()]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_streamed_audio_result_synthesizes_short_custom_splitter_chunk() -> None:
|
|
texts: list[str] = []
|
|
|
|
class RecordingTTS(FakeTTS):
|
|
async def run(self, text: str, settings: TTSModelSettings):
|
|
texts.append(text)
|
|
yield np.zeros(2, dtype=np.int16).tobytes()
|
|
|
|
def split_immediately(text: str) -> tuple[str, str]:
|
|
return text, ""
|
|
|
|
result = StreamedAudioResult(
|
|
RecordingTTS(),
|
|
TTSModelSettings(buffer_size=1, text_splitter=split_immediately),
|
|
VoicePipelineConfig(),
|
|
)
|
|
|
|
await result._add_text("ok")
|
|
await result._turn_done()
|
|
await result._done()
|
|
|
|
events, audio_chunks = await extract_events(result)
|
|
|
|
assert texts == ["ok"]
|
|
assert events == ["turn_started", "audio", "turn_ended", "session_ended"]
|
|
assert audio_chunks == [np.zeros(2, dtype=np.int16).tobytes()]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_streamed_audio_result_ignores_empty_custom_splitter_chunk() -> None:
|
|
texts: list[str] = []
|
|
|
|
class RecordingTTS(FakeTTS):
|
|
async def run(self, text: str, settings: TTSModelSettings):
|
|
texts.append(text)
|
|
yield np.zeros(2, dtype=np.int16).tobytes()
|
|
|
|
def discard_text(_text: str) -> tuple[str, str]:
|
|
return "", ""
|
|
|
|
result = StreamedAudioResult(
|
|
RecordingTTS(),
|
|
TTSModelSettings(buffer_size=1, text_splitter=discard_text),
|
|
VoicePipelineConfig(),
|
|
)
|
|
|
|
await result._add_text("ok")
|
|
await result._turn_done()
|
|
await result._done()
|
|
|
|
events, audio_chunks = await extract_events(result)
|
|
|
|
assert texts == []
|
|
assert events == ["turn_started", "turn_ended", "session_ended"]
|
|
assert audio_chunks == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_run_single_turn() -> None:
|
|
# Single turn. Should produce a single audio output, which is the TTS output for "out_1".
|
|
|
|
fake_stt = FakeSTT(["first"])
|
|
workflow = FakeWorkflow([["out_1"]])
|
|
fake_tts = FakeTTS()
|
|
config = VoicePipelineConfig(tts_settings=TTSModelSettings(buffer_size=1))
|
|
pipeline = VoicePipeline(
|
|
workflow=workflow, stt_model=fake_stt, tts_model=fake_tts, config=config
|
|
)
|
|
audio_input = AudioInput(buffer=np.zeros(2, dtype=np.int16))
|
|
result = await pipeline.run(audio_input)
|
|
events, audio_chunks = await extract_events(result)
|
|
assert events == [
|
|
"turn_started",
|
|
"audio",
|
|
"turn_ended",
|
|
"session_ended",
|
|
]
|
|
await fake_tts.verify_audio("out_1", audio_chunks[0])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_streamed_audio_input() -> None:
|
|
# Multi turn. Should produce 2 audio outputs, which are the TTS outputs of "out_1" and "out_2"
|
|
|
|
fake_stt = FakeSTT(["first", "second"])
|
|
workflow = FakeWorkflow([["out_1"], ["out_2"]])
|
|
fake_tts = FakeTTS()
|
|
pipeline = VoicePipeline(workflow=workflow, stt_model=fake_stt, tts_model=fake_tts)
|
|
|
|
streamed_audio_input = await FakeStreamedAudioInput.get(count=2)
|
|
|
|
result = await pipeline.run(streamed_audio_input)
|
|
events, audio_chunks = await extract_events(result)
|
|
assert events == [
|
|
"turn_started",
|
|
"audio", # out_1
|
|
"turn_ended",
|
|
"turn_started",
|
|
"audio", # out_2
|
|
"turn_ended",
|
|
"session_ended",
|
|
]
|
|
assert len(audio_chunks) == 2
|
|
await fake_tts.verify_audio("out_1", audio_chunks[0])
|
|
await fake_tts.verify_audio("out_2", audio_chunks[1])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_run_single_turn_split_words() -> None:
|
|
# Single turn. Should produce multiple audio outputs, which are the TTS outputs of "foo bar baz"
|
|
# split into words and then "foo2 bar2 baz2" split into words.
|
|
|
|
fake_stt = FakeSTT(["first"])
|
|
workflow = FakeWorkflow([["foo bar baz"]])
|
|
fake_tts = FakeTTS(strategy="split_words")
|
|
config = VoicePipelineConfig(tts_settings=TTSModelSettings(buffer_size=1))
|
|
pipeline = VoicePipeline(
|
|
workflow=workflow, stt_model=fake_stt, tts_model=fake_tts, config=config
|
|
)
|
|
audio_input = AudioInput(buffer=np.zeros(2, dtype=np.int16))
|
|
result = await pipeline.run(audio_input)
|
|
events, audio_chunks = await extract_events(result)
|
|
assert events == [
|
|
"turn_started",
|
|
"audio", # foo
|
|
"audio", # bar
|
|
"audio", # baz
|
|
"turn_ended",
|
|
"session_ended",
|
|
]
|
|
await fake_tts.verify_audio_chunks("foo bar baz", audio_chunks)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_run_multi_turn_split_words() -> None:
|
|
# Multi turn. Should produce multiple audio outputs, which are the TTS outputs of "foo bar baz"
|
|
# split into words.
|
|
|
|
fake_stt = FakeSTT(["first", "second"])
|
|
workflow = FakeWorkflow([["foo bar baz"], ["foo2 bar2 baz2"]])
|
|
fake_tts = FakeTTS(strategy="split_words")
|
|
config = VoicePipelineConfig(tts_settings=TTSModelSettings(buffer_size=1))
|
|
pipeline = VoicePipeline(
|
|
workflow=workflow, stt_model=fake_stt, tts_model=fake_tts, config=config
|
|
)
|
|
streamed_audio_input = await FakeStreamedAudioInput.get(count=6)
|
|
result = await pipeline.run(streamed_audio_input)
|
|
events, audio_chunks = await extract_events(result)
|
|
assert events == [
|
|
"turn_started",
|
|
"audio", # foo
|
|
"audio", # bar
|
|
"audio", # baz
|
|
"turn_ended",
|
|
"turn_started",
|
|
"audio", # foo2
|
|
"audio", # bar2
|
|
"audio", # baz2
|
|
"turn_ended",
|
|
"session_ended",
|
|
]
|
|
assert len(audio_chunks) == 6
|
|
await fake_tts.verify_audio_chunks("foo bar baz", audio_chunks[:3])
|
|
await fake_tts.verify_audio_chunks("foo2 bar2 baz2", audio_chunks[3:])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_float32() -> None:
|
|
# Single turn. Should produce a single audio output, which is the TTS output for "out_1".
|
|
|
|
fake_stt = FakeSTT(["first"])
|
|
workflow = FakeWorkflow([["out_1"]])
|
|
fake_tts = FakeTTS()
|
|
config = VoicePipelineConfig(tts_settings=TTSModelSettings(buffer_size=1, dtype=np.float32))
|
|
pipeline = VoicePipeline(
|
|
workflow=workflow, stt_model=fake_stt, tts_model=fake_tts, config=config
|
|
)
|
|
audio_input = AudioInput(buffer=np.zeros(2, dtype=np.int16))
|
|
result = await pipeline.run(audio_input)
|
|
events, audio_chunks = await extract_events(result)
|
|
assert events == [
|
|
"turn_started",
|
|
"audio",
|
|
"turn_ended",
|
|
"session_ended",
|
|
]
|
|
await fake_tts.verify_audio("out_1", audio_chunks[0], dtype=np.float32)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_transform_data() -> None:
|
|
# Single turn. Should produce a single audio output, which is the TTS output for "out_1".
|
|
|
|
def _transform_data(
|
|
data_chunk: npt.NDArray[np.int16 | np.float32],
|
|
) -> npt.NDArray[np.int16]:
|
|
return data_chunk.astype(np.int16)
|
|
|
|
fake_stt = FakeSTT(["first"])
|
|
workflow = FakeWorkflow([["out_1"]])
|
|
fake_tts = FakeTTS()
|
|
config = VoicePipelineConfig(
|
|
tts_settings=TTSModelSettings(
|
|
buffer_size=1,
|
|
dtype=np.float32,
|
|
transform_data=_transform_data,
|
|
)
|
|
)
|
|
pipeline = VoicePipeline(
|
|
workflow=workflow, stt_model=fake_stt, tts_model=fake_tts, config=config
|
|
)
|
|
audio_input = AudioInput(buffer=np.zeros(2, dtype=np.int16))
|
|
result = await pipeline.run(audio_input)
|
|
events, audio_chunks = await extract_events(result)
|
|
assert events == [
|
|
"turn_started",
|
|
"audio",
|
|
"turn_ended",
|
|
"session_ended",
|
|
]
|
|
await fake_tts.verify_audio("out_1", audio_chunks[0], dtype=np.int16)
|
|
|
|
|
|
class _BlockingWorkflow(FakeWorkflow):
|
|
def __init__(self, gate: asyncio.Event):
|
|
super().__init__()
|
|
self._gate = gate
|
|
|
|
async def run(self, _: str):
|
|
await self._gate.wait()
|
|
yield "out_1"
|
|
|
|
|
|
class _OnStartYieldThenFailWorkflow(FakeWorkflow):
|
|
async def on_start(self):
|
|
yield "intro"
|
|
raise RuntimeError("boom")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_trace_not_finished_before_single_turn_completes() -> None:
|
|
fake_stt = FakeSTT(["first"])
|
|
fake_tts = FakeTTS()
|
|
gate = asyncio.Event()
|
|
workflow = _BlockingWorkflow(gate)
|
|
config = VoicePipelineConfig(tts_settings=TTSModelSettings(buffer_size=1))
|
|
pipeline = VoicePipeline(
|
|
workflow=workflow, stt_model=fake_stt, tts_model=fake_tts, config=config
|
|
)
|
|
|
|
audio_input = AudioInput(buffer=np.zeros(2, dtype=np.int16))
|
|
result = await pipeline.run(audio_input)
|
|
await asyncio.sleep(0)
|
|
|
|
events_before_unblock = fetch_events()
|
|
assert "trace_start" in events_before_unblock
|
|
assert "trace_end" not in events_before_unblock
|
|
|
|
gate.set()
|
|
await extract_events(result)
|
|
assert fetch_events()[-1] == "trace_end"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_trace_finishes_after_multi_turn_processing() -> None:
|
|
fake_stt = FakeSTT(["first", "second"])
|
|
workflow = FakeWorkflow([["out_1"], ["out_2"]])
|
|
fake_tts = FakeTTS()
|
|
pipeline = VoicePipeline(workflow=workflow, stt_model=fake_stt, tts_model=fake_tts)
|
|
|
|
streamed_audio_input = await FakeStreamedAudioInput.get(count=2)
|
|
result = await pipeline.run(streamed_audio_input)
|
|
await extract_events(result)
|
|
assert fetch_events()[-1] == "trace_end"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voicepipeline_multi_turn_on_start_exception_does_not_abort() -> None:
|
|
fake_stt = FakeSTT(["first"])
|
|
workflow = _OnStartYieldThenFailWorkflow([["out_1"]])
|
|
fake_tts = FakeTTS()
|
|
pipeline = VoicePipeline(workflow=workflow, stt_model=fake_stt, tts_model=fake_tts)
|
|
|
|
streamed_audio_input = await FakeStreamedAudioInput.get(count=1)
|
|
result = await pipeline.run(streamed_audio_input)
|
|
events, _ = await extract_events(result)
|
|
|
|
assert events[-1] == "session_ended"
|
|
assert "error" not in events
|