chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
|
||||
def add_attention_backend(server_args, attention_config):
|
||||
"""Append attention backend CLI arg if specified.
|
||||
|
||||
Args:
|
||||
server_args: List of server arguments to extend in-place.
|
||||
attention_config: Dict with 'backend' key, or None.
|
||||
"""
|
||||
if attention_config and "backend" in attention_config:
|
||||
server_args.extend(["--attention-backend", attention_config["backend"]])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rocm_aiter_fa_attention():
|
||||
"""Return attention config for transcription/translation tests on ROCm.
|
||||
|
||||
On ROCm, audio tests require ROCM_AITER_FA attention backend.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mary_had_lamb():
|
||||
path = AudioAsset("mary_had_lamb").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def winning_call():
|
||||
path = AudioAsset("winning_call").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def foscolo():
|
||||
# Test translation it->en
|
||||
path = AudioAsset("azacinto_foscolo").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
@@ -0,0 +1,400 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Evaluate Transcription API correctness by computing Word Error Rate (WER)
|
||||
on a given ASR dataset. When provided, it will also compare the WER against
|
||||
a baseline.
|
||||
This simulates real work usage of the API and makes sure that the frontend and
|
||||
AsyncLLMEngine are working correctly.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from statistics import mean, median
|
||||
|
||||
import pytest
|
||||
import soundfile
|
||||
import torch
|
||||
from datasets import Audio, load_dataset
|
||||
from evaluate import load
|
||||
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
|
||||
|
||||
from vllm.benchmarks.datasets.datasets import ASRDataset
|
||||
from vllm.multimodal.audio import get_audio_duration
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
from ....models.registry import HF_EXAMPLE_MODELS
|
||||
from ....utils import RemoteOpenAIServer
|
||||
|
||||
# Tuned to prevent OOM on 18GB GPUs in transcription correctness tests.
|
||||
MAX_SEQS_FOR_TRANSCRIPTION_TEST = 8
|
||||
GPU_UTIL_FOR_TRANSCRIPTION_TEST = 0.5
|
||||
|
||||
|
||||
def to_bytes(y, sr):
|
||||
buffer = io.BytesIO()
|
||||
soundfile.write(buffer, y, sr, format="WAV")
|
||||
buffer.seek(0)
|
||||
return buffer
|
||||
|
||||
|
||||
def load_audio_sample(audio):
|
||||
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
|
||||
if "array" in audio and "sampling_rate" in audio:
|
||||
return audio["array"], audio["sampling_rate"]
|
||||
|
||||
if audio.get("path"):
|
||||
return soundfile.read(audio["path"], dtype="float32")
|
||||
|
||||
if audio.get("bytes") is not None:
|
||||
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
|
||||
|
||||
raise ValueError("Audio sample did not contain array, path, or bytes data")
|
||||
|
||||
|
||||
# not all models have a normalizer so use the one from whisper as a standard option
|
||||
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
|
||||
normalizer_tokenizer = get_tokenizer(
|
||||
"openai/whisper-large-v3",
|
||||
tokenizer_mode=normalizer_model_info.tokenizer_mode,
|
||||
trust_remote_code=normalizer_model_info.trust_remote_code,
|
||||
)
|
||||
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
|
||||
|
||||
|
||||
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
|
||||
# Send loaded audio directly instead of loading from disk,
|
||||
# don't account for that time though
|
||||
with to_bytes(y, sr) as f:
|
||||
start_time = time.perf_counter()
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
# NOTE there's no streaming in transcriptions, can't measure ttft
|
||||
latency = end_time - start_time
|
||||
num_output_tokens = len(
|
||||
tokenizer(transcription.text, add_special_tokens=False).input_ids
|
||||
)
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, reference, extra_body=None
|
||||
):
|
||||
# Use semaphore to limit concurrent requests.
|
||||
async with sem:
|
||||
result = await transcribe_audio(
|
||||
client, tokenizer, audio, sr, extra_body=extra_body
|
||||
)
|
||||
# Normalize *english* output/reference for evaluation.
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
)
|
||||
|
||||
# Warmup call as the first `load_audio` server-side is quite slow.
|
||||
audio, sr = load_audio_sample(data[0]["audio"])
|
||||
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio, sr = load_audio_sample(sample["audio"])
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
def print_performance_metrics(results, total_time):
|
||||
latencies = [res[0] for res in results]
|
||||
total_tokens = sum([res[1] for res in results])
|
||||
|
||||
total = len(results)
|
||||
print(f"Total Requests: {total}")
|
||||
print(f"Successful Requests: {len(latencies)}")
|
||||
print(f"Average Latency: {mean(latencies):.4f} seconds")
|
||||
print(f"Median Latency: {median(latencies):.4f} seconds")
|
||||
perc = sorted(latencies)[int(len(latencies) * 0.95) - 1]
|
||||
print(f"95th Percentile Latency: {perc:.4f} seconds")
|
||||
# Throughput
|
||||
req_throughput = len(latencies) / total_time
|
||||
print(f"Estimated req_Throughput: {req_throughput:.2f} requests/s")
|
||||
throughput = total_tokens / total_time
|
||||
print(f"Estimated Throughput: {throughput:.2f} tok/s")
|
||||
|
||||
|
||||
def add_duration(sample):
|
||||
y, sr = load_audio_sample(sample["audio"])
|
||||
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
|
||||
return sample
|
||||
|
||||
|
||||
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
|
||||
asr_dataset_kwargs = {
|
||||
"dataset_path": dataset_repo,
|
||||
"dataset_split": split,
|
||||
"disable_shuffle": True,
|
||||
"no_stream": True,
|
||||
}
|
||||
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
|
||||
if key in hf_kwargs:
|
||||
asr_dataset_kwargs[key] = hf_kwargs[key]
|
||||
return ASRDataset(**asr_dataset_kwargs).data
|
||||
|
||||
return load_dataset(dataset_repo, split=split, **hf_kwargs)
|
||||
|
||||
|
||||
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
## Load and filter the dataset.
|
||||
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
|
||||
dataset = dataset.cast_column("audio", Audio(decode=False))
|
||||
if "duration_ms" not in dataset.column_names:
|
||||
# Compute duration to filter.
|
||||
dataset = dataset.map(add_duration)
|
||||
|
||||
# Whisper max supported duration.
|
||||
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
|
||||
return dataset
|
||||
|
||||
|
||||
def run_evaluation(
|
||||
model: str,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_reqs: int,
|
||||
n_examples: int = -1,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
if n_examples > 0:
|
||||
dataset = dataset.select(range(n_examples))
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(
|
||||
process_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
if print_metrics:
|
||||
print_performance_metrics(results, total_time)
|
||||
# Compute WER
|
||||
predictions = [res[2] for res in results]
|
||||
references = [res[3] for res in results]
|
||||
wer = load("wer")
|
||||
wer_score = 100 * wer.compute(references=references, predictions=predictions)
|
||||
print("WER:", wer_score)
|
||||
return wer_score
|
||||
|
||||
|
||||
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
|
||||
LONGFORM_DATASET_SPLIT = "test"
|
||||
LONGFORM_NUM_SAMPLES = 6
|
||||
|
||||
|
||||
def load_longform_dataset():
|
||||
dataset = load_asr_dataset_rows(
|
||||
LONGFORM_DATASET_REPO,
|
||||
split=LONGFORM_DATASET_SPLIT,
|
||||
)
|
||||
assert len(dataset) >= LONGFORM_NUM_SAMPLES
|
||||
return dataset.select(range(LONGFORM_NUM_SAMPLES))
|
||||
|
||||
|
||||
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
|
||||
with open(audio_path, "rb") as f:
|
||||
start_time = time.perf_counter()
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
|
||||
latency = end_time - start_time
|
||||
num_output_tokens = len(
|
||||
tokenizer(transcription.text, add_special_tokens=False).input_ids
|
||||
)
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, reference, extra_body=None
|
||||
):
|
||||
async with sem:
|
||||
result = await transcribe_audio_path(
|
||||
client, tokenizer, audio_path, extra_body=extra_body
|
||||
)
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_longform_dataset(
|
||||
model, client, data, concurrent_request, extra_body=None
|
||||
):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
)
|
||||
|
||||
warmup_path = data[0]["audio"]["path"]
|
||||
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio_path = sample["audio"]["path"]
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
def run_longform_evaluation(
|
||||
model: str,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_reqs: int,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(
|
||||
process_longform_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
if print_metrics:
|
||||
print_performance_metrics(results, total_time)
|
||||
|
||||
predictions = [res[2] for res in results]
|
||||
references = [res[3] for res in results]
|
||||
wer = load("wer")
|
||||
wer_score = 100 * wer.compute(references=references, predictions=predictions)
|
||||
print("WER:", wer_score)
|
||||
return wer_score
|
||||
|
||||
|
||||
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
|
||||
# NOTE: Expected WER measured with equivalent hf.transformers args:
|
||||
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
|
||||
@pytest.mark.parametrize(
|
||||
"model_config",
|
||||
[
|
||||
("openai/whisper-large-v3", 12.744980),
|
||||
# CohereASR is used to test the variable encoder length code paths
|
||||
("CohereLabs/cohere-transcribe-03-2026", 11.92),
|
||||
],
|
||||
)
|
||||
# Original dataset is 20GB+ in size, hence we use a pre-filtered slice.
|
||||
@pytest.mark.parametrize(
|
||||
"dataset_repo", ["D4nt3/esb-datasets-earnings22-validation-tiny-filtered"]
|
||||
)
|
||||
def test_wer_correctness(
|
||||
model_config, dataset_repo, n_examples=-1, max_concurrent_request=None
|
||||
):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
f"--max_num_seqs={MAX_SEQS_FOR_TRANSCRIPTION_TEST}",
|
||||
f"--gpu_memory_utilization={GPU_UTIL_FOR_TRANSCRIPTION_TEST}",
|
||||
]
|
||||
if model_info.trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
) as remote_server:
|
||||
dataset = load_shortform_eval_dataset(dataset_repo)
|
||||
|
||||
if not max_concurrent_request:
|
||||
# No max concurrency
|
||||
max_concurrent_request = n_examples if n_examples > 0 else len(dataset)
|
||||
|
||||
client = remote_server.get_async_client()
|
||||
wer = run_evaluation(
|
||||
model_name,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_request,
|
||||
n_examples,
|
||||
)
|
||||
|
||||
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
|
||||
|
||||
if expected_wer:
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
|
||||
|
||||
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
|
||||
# checks for long audio transcription correctness and RMS split.
|
||||
@pytest.mark.parametrize(
|
||||
"model_config",
|
||||
[("openai/whisper-large-v3", 9.5)],
|
||||
)
|
||||
def test_long_audio_wer_correctness(model_config):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
server_args = [
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
]
|
||||
|
||||
if model_info.trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
|
||||
# 1800 seconds is 30 minutes
|
||||
env_dict = {
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
dataset = load_longform_dataset()
|
||||
client = remote_server.get_async_client()
|
||||
wer = run_longform_evaluation(
|
||||
model=model_name,
|
||||
client=client,
|
||||
dataset=dataset,
|
||||
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
|
||||
)
|
||||
|
||||
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
@@ -0,0 +1,336 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
|
||||
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
# Increase engine iteration timeout for ROCm where first-use JIT compilation
|
||||
# can exceed the default 60s, causing a silent deadlock in feed_tokens.
|
||||
REALTIME_ENV_OVERRIDES = {
|
||||
**ROCM_ENV_OVERRIDES,
|
||||
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",
|
||||
}
|
||||
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
"mistral",
|
||||
"--config_format",
|
||||
"mistral",
|
||||
"--load_format",
|
||||
"mistral",
|
||||
] + ROCM_EXTRA_ARGS
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
|
||||
|
||||
|
||||
def _get_websocket_url(server: RemoteOpenAIServer) -> str:
|
||||
"""Convert HTTP URL to WebSocket URL for realtime endpoint."""
|
||||
http_url = server.url_root
|
||||
ws_url = http_url.replace("http://", "ws://")
|
||||
return f"{ws_url}/v1/realtime"
|
||||
|
||||
|
||||
async def receive_event(ws, timeout: float = 60.0) -> dict:
|
||||
"""Receive and parse JSON event from WebSocket."""
|
||||
message = await asyncio.wait_for(ws.recv(), timeout=timeout)
|
||||
return json.loads(message)
|
||||
|
||||
|
||||
async def send_event(ws, event: dict) -> None:
|
||||
"""Send JSON event to WebSocket."""
|
||||
await ws.send(json.dumps(event))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mary_had_lamb_audio_chunks() -> list[str]:
|
||||
"""Audio split into ~1 second chunks for streaming."""
|
||||
path = AudioAsset("mary_had_lamb").get_local_path()
|
||||
audio, _ = load_audio(str(path), sr=16000, mono=True)
|
||||
|
||||
# Split into ~0.1 second chunks (1600 samples at 16kHz)
|
||||
chunk_size = 1600
|
||||
chunks = []
|
||||
for i in range(0, len(audio), chunk_size):
|
||||
chunk = audio[i : i + chunk_size]
|
||||
chunk_int16 = (chunk * 32767).astype(np.int16)
|
||||
chunk_bytes = chunk_int16.tobytes()
|
||||
chunks.append(base64.b64encode(chunk_bytes).decode("utf-8"))
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_multi_chunk_streaming(
|
||||
model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
|
||||
):
|
||||
"""Test streaming multiple audio chunks before committing."""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
# Receive session.created
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
assert event["type"] == "session.created"
|
||||
|
||||
await send_event(ws, {"type": "session.update", "model": model_name})
|
||||
|
||||
# Wait for the server to acknowledge the session update.
|
||||
try:
|
||||
while True:
|
||||
event = await receive_event(ws, timeout=5.0)
|
||||
if event["type"] == "session.updated":
|
||||
break
|
||||
except TimeoutError:
|
||||
warnings.warn(
|
||||
f"session.updated not received within {5.0}s after "
|
||||
"session.update. The server may not implement this event.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# (ROCm) Warm-up: send a non-final commit (required to start
|
||||
# transcription) with a small audio chunk to trigger aiter
|
||||
# compilation on first use.
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit"})
|
||||
await send_event(
|
||||
ws,
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": mary_had_lamb_audio_chunks[0],
|
||||
},
|
||||
)
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
|
||||
|
||||
# (ROCm) Drain all warm-up responses with generous timeout for
|
||||
# JIT compilation
|
||||
warmup_done = False
|
||||
while not warmup_done:
|
||||
event = await receive_event(ws, timeout=600.0)
|
||||
if event["type"] in ("transcription.done", "error"):
|
||||
warmup_done = True
|
||||
|
||||
# Now send the real test audio
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit"})
|
||||
|
||||
# Send multiple audio chunks
|
||||
for chunk in mary_had_lamb_audio_chunks:
|
||||
await send_event(
|
||||
ws, {"type": "input_audio_buffer.append", "audio": chunk}
|
||||
)
|
||||
|
||||
# Send commit to end
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
|
||||
|
||||
# Collect transcription deltas
|
||||
full_text = ""
|
||||
done_received = False
|
||||
|
||||
while not done_received:
|
||||
event = await receive_event(ws, timeout=60.0)
|
||||
|
||||
if event["type"] == "transcription.delta":
|
||||
full_text += event["delta"]
|
||||
elif event["type"] == "transcription.done":
|
||||
done_received = True
|
||||
assert "text" in event
|
||||
elif event["type"] == "error":
|
||||
pytest.fail(f"Received error: {event}")
|
||||
|
||||
# Verify transcription contains expected content
|
||||
assert event["type"] == "transcription.done"
|
||||
assert event["text"] == full_text
|
||||
assert full_text == (
|
||||
" First words I spoke in the original phonograph."
|
||||
" A little piece of practical poetry. Mary had a little lamb,"
|
||||
" it sleeps with quite a flow, and everywhere that Mary went,"
|
||||
" the lamb was sure to go."
|
||||
) or full_text == (
|
||||
" First words I spoke in the original phonograph."
|
||||
" A little piece of practical poetry. Mary had a little lamb,"
|
||||
" it squeaked with quite a flow, and everywhere that Mary went,"
|
||||
" the lamb was sure to go."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_empty_commit_does_not_crash_engine(
|
||||
model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
|
||||
):
|
||||
"""Test that committing without audio does not crash the engine.
|
||||
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/34532.
|
||||
An empty commit (no prior input_audio_buffer.append) used to trigger
|
||||
``AssertionError: For realtime you must provide a multimodal_embedding
|
||||
at every step`` which killed the entire engine process, disconnecting
|
||||
every connected client.
|
||||
"""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
|
||||
# --- First connection: empty commit (no audio appended) ----------
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
assert event["type"] == "session.created"
|
||||
|
||||
await send_event(ws, {"type": "session.update", "model": model_name})
|
||||
|
||||
try:
|
||||
while True:
|
||||
event = await receive_event(ws, timeout=5.0)
|
||||
if event["type"] == "session.updated":
|
||||
break
|
||||
except TimeoutError:
|
||||
warnings.warn(
|
||||
f"session.updated not received within {5.0}s after "
|
||||
"session.update. The server may not implement this event.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# Start generation without sending any audio
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit"})
|
||||
|
||||
# Immediately signal end-of-audio
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
|
||||
|
||||
# We should get *some* response (error or empty transcription),
|
||||
# but the engine must NOT crash.
|
||||
# (ROCm) Use generous timeout for first request (aiter JIT compilation)
|
||||
event = await receive_event(ws, timeout=360.0)
|
||||
assert event["type"] in (
|
||||
"error",
|
||||
"transcription.done",
|
||||
"transcription.delta",
|
||||
)
|
||||
|
||||
# --- Second connection: normal transcription ---------------------
|
||||
# Verifies the engine is still alive after the empty commit above.
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
assert event["type"] == "session.created"
|
||||
|
||||
await send_event(ws, {"type": "session.update", "model": model_name})
|
||||
|
||||
try:
|
||||
while True:
|
||||
event = await receive_event(ws, timeout=5.0)
|
||||
if event["type"] == "session.updated":
|
||||
break
|
||||
except TimeoutError:
|
||||
warnings.warn(
|
||||
f"session.updated not received within {5.0}s after "
|
||||
"session.update. The server may not implement this event.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# Start transcription
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit"})
|
||||
|
||||
for chunk in mary_had_lamb_audio_chunks:
|
||||
await send_event(
|
||||
ws, {"type": "input_audio_buffer.append", "audio": chunk}
|
||||
)
|
||||
|
||||
await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
|
||||
|
||||
done_received = False
|
||||
while not done_received:
|
||||
event = await receive_event(ws, timeout=60.0)
|
||||
if event["type"] == "transcription.done":
|
||||
done_received = True
|
||||
elif event["type"] == "error":
|
||||
pytest.fail(f"Engine error after empty commit: {event}")
|
||||
assert done_received
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_session_update_invalid_model_returns_error(
|
||||
model_name, rocm_aiter_fa_attention
|
||||
):
|
||||
"""Test that session.update with an invalid model returns an error."""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
assert event["type"] == "session.created"
|
||||
|
||||
# Send session.update with a model that doesn't exist
|
||||
await send_event(
|
||||
ws,
|
||||
{"type": "session.update", "model": "nonexistent-model"},
|
||||
)
|
||||
|
||||
event = await receive_event(ws, timeout=10.0)
|
||||
assert event["type"] == "error"
|
||||
assert "nonexistent-model" in event["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_commit_without_session_update_returns_error(
|
||||
model_name, rocm_aiter_fa_attention
|
||||
):
|
||||
"""Test that committing before validating the model returns an error
|
||||
and does not fall through to processing."""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
assert event["type"] == "session.created"
|
||||
|
||||
# Send commit without sending session.update first
|
||||
await send_event(
|
||||
ws,
|
||||
{"type": "input_audio_buffer.commit", "final": True},
|
||||
)
|
||||
|
||||
event = await receive_event(ws, timeout=10.0)
|
||||
assert event["type"] == "error"
|
||||
assert "model_not_validated" in event.get("code", "")
|
||||
@@ -0,0 +1,191 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse
|
||||
|
||||
|
||||
async def _never_finishes():
|
||||
await asyncio.Event().wait()
|
||||
yield
|
||||
|
||||
|
||||
async def _records_start_then_never_finishes(started_request_ids, request_id):
|
||||
started_request_ids.append(request_id)
|
||||
await asyncio.Event().wait()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("engine_inputs", "expected_request_ids"),
|
||||
[
|
||||
([{"prompt": "chunk"}], ["transcribe-outer-request"]),
|
||||
(
|
||||
[{"prompt": "chunk-0"}, {"prompt": "chunk-1"}],
|
||||
["transcribe-outer-request-0", "transcribe-outer-request-1"],
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_non_streaming_cancel_aborts_engine_requests(
|
||||
engine_inputs, expected_request_ids
|
||||
):
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
|
||||
abort=AsyncMock(),
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
)
|
||||
|
||||
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
server.engine_client = engine_client
|
||||
server.task_type = "transcribe"
|
||||
server.models = SimpleNamespace(model_name=lambda: "audio")
|
||||
server.model_config = SimpleNamespace(max_model_len=1024)
|
||||
server.model_cls = SimpleNamespace(no_space_languages=set())
|
||||
server.default_sampling_params = {}
|
||||
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
|
||||
server._check_model = AsyncMock(return_value=None)
|
||||
server._maybe_get_adapters = Mock(return_value=None)
|
||||
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0))
|
||||
server._log_inputs = Mock()
|
||||
|
||||
request = SimpleNamespace(
|
||||
model="audio",
|
||||
response_format="json",
|
||||
stream=False,
|
||||
use_beam_search=False,
|
||||
max_completion_tokens=None,
|
||||
language="en",
|
||||
prompt="",
|
||||
to_sampling_params=Mock(return_value=object()),
|
||||
)
|
||||
raw_request = SimpleNamespace(
|
||||
headers={"X-Request-Id": "outer-request"},
|
||||
state=SimpleNamespace(),
|
||||
)
|
||||
|
||||
task = asyncio.create_task(
|
||||
server._create_speech_to_text(
|
||||
audio_data=b"audio",
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
response_class=TranscriptionResponse,
|
||||
stream_generator_method=Mock(),
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
generated_request_ids = [
|
||||
call.args[2] for call in engine_client.generate.call_args_list
|
||||
]
|
||||
assert generated_request_ids == expected_request_ids
|
||||
engine_client.abort.assert_awaited_once_with(expected_request_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
started_request_ids: list[str] = []
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
generate=Mock(
|
||||
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
|
||||
started_request_ids, _args[2]
|
||||
)
|
||||
),
|
||||
abort=AsyncMock(),
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
)
|
||||
|
||||
engine_inputs = [
|
||||
{"prompt": "chunk-0"},
|
||||
{"prompt": "chunk-1"},
|
||||
{"prompt": "chunk-2"},
|
||||
]
|
||||
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
server.engine_client = engine_client
|
||||
server.task_type = "transcribe"
|
||||
server.models = SimpleNamespace(model_name=lambda: "audio")
|
||||
server.model_config = SimpleNamespace(max_model_len=1024)
|
||||
server.model_cls = SimpleNamespace(no_space_languages=set())
|
||||
server.default_sampling_params = {}
|
||||
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
|
||||
server._check_model = AsyncMock(return_value=None)
|
||||
server._maybe_get_adapters = Mock(return_value=None)
|
||||
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0))
|
||||
server._log_inputs = Mock()
|
||||
|
||||
request = SimpleNamespace(
|
||||
model="audio",
|
||||
response_format="json",
|
||||
stream=False,
|
||||
use_beam_search=False,
|
||||
max_completion_tokens=None,
|
||||
language="en",
|
||||
prompt="",
|
||||
to_sampling_params=Mock(return_value=object()),
|
||||
)
|
||||
raw_request = SimpleNamespace(
|
||||
headers={"X-Request-Id": "outer-request"},
|
||||
state=SimpleNamespace(),
|
||||
)
|
||||
|
||||
task = asyncio.create_task(
|
||||
server._create_speech_to_text(
|
||||
audio_data=b"audio",
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
response_class=TranscriptionResponse,
|
||||
stream_generator_method=Mock(),
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
expected_request_ids = [
|
||||
"transcribe-outer-request-0",
|
||||
"transcribe-outer-request-1",
|
||||
"transcribe-outer-request-2",
|
||||
]
|
||||
assert set(started_request_ids) == set(expected_request_ids)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_language_detection_cancel_aborts_engine_request():
|
||||
engine_client = SimpleNamespace(
|
||||
generate=Mock(return_value=_never_finishes()),
|
||||
abort=AsyncMock(),
|
||||
)
|
||||
|
||||
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
server.engine_client = engine_client
|
||||
server.asr_config = SimpleNamespace()
|
||||
server.tokenizer = Mock()
|
||||
server.model_cls = SimpleNamespace(
|
||||
get_language_detection_prompt=Mock(return_value={"prompt": "detect"}),
|
||||
get_language_token_ids=Mock(return_value=[1]),
|
||||
parse_language_detection_output=Mock(),
|
||||
)
|
||||
|
||||
request_id = "transcribe-outer-request-lang_detect"
|
||||
task = asyncio.create_task(server._detect_language(Mock(), request_id))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
engine_client.abort.assert_awaited_once_with(request_id)
|
||||
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for the speech-to-text upload size pre-check.
|
||||
|
||||
These tests verify that over-limit audio uploads are rejected *before*
|
||||
the full file is materialized into memory, closing the vulnerability
|
||||
where vLLM would allocate memory proportional to an oversized upload
|
||||
before enforcing the VLLM_MAX_AUDIO_CLIP_FILESIZE_MB limit.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
|
||||
def _make_upload_file(data: bytes, *, size: int | None = None) -> AsyncMock:
|
||||
"""Create a mock UploadFile that yields data in chunks."""
|
||||
mock = AsyncMock()
|
||||
mock.size = size
|
||||
|
||||
offset = 0
|
||||
|
||||
async def _read(n: int = -1):
|
||||
nonlocal offset
|
||||
if n <= 0:
|
||||
chunk = data[offset:]
|
||||
offset = len(data)
|
||||
return chunk
|
||||
chunk = data[offset : offset + n]
|
||||
offset += len(chunk)
|
||||
return chunk
|
||||
|
||||
mock.read = AsyncMock(side_effect=_read)
|
||||
return mock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_oversized_upload_via_content_length():
|
||||
"""File is rejected early when file.size exceeds the limit."""
|
||||
max_mb = 1
|
||||
oversized_bytes = max_mb * 1024 * 1024 + 1
|
||||
|
||||
upload = _make_upload_file(b"", size=oversized_bytes)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
|
||||
await read_upload_with_limit(upload, max_size_mb=max_mb)
|
||||
|
||||
upload.read.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_oversized_upload_via_chunked_read():
|
||||
"""File is rejected mid-read without materializing the full content."""
|
||||
max_mb = 1
|
||||
max_bytes = max_mb * 1024 * 1024
|
||||
oversized_data = b"\x00" * (max_bytes + 1024)
|
||||
|
||||
upload = _make_upload_file(oversized_data, size=None)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
|
||||
await read_upload_with_limit(upload, max_size_mb=max_mb)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accepts_file_within_limit():
|
||||
"""File within the limit is read successfully."""
|
||||
max_mb = 1
|
||||
data = b"\x00" * (512 * 1024) # 512 KiB, well under 1 MB
|
||||
|
||||
upload = _make_upload_file(data, size=len(data))
|
||||
result = await read_upload_with_limit(upload, max_size_mb=max_mb)
|
||||
|
||||
assert result == data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accepts_file_at_exact_limit():
|
||||
"""File exactly at the limit boundary is accepted."""
|
||||
max_mb = 1
|
||||
max_bytes = max_mb * 1024 * 1024
|
||||
data = b"\x00" * max_bytes
|
||||
|
||||
upload = _make_upload_file(data, size=len(data))
|
||||
result = await read_upload_with_limit(upload, max_size_mb=max_mb)
|
||||
|
||||
assert result == data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_at_one_byte_over_limit():
|
||||
"""File one byte over the limit is rejected."""
|
||||
max_mb = 1
|
||||
max_bytes = max_mb * 1024 * 1024
|
||||
data = b"\x00" * (max_bytes + 1)
|
||||
|
||||
upload = _make_upload_file(data, size=None)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
|
||||
await read_upload_with_limit(upload, max_size_mb=max_mb)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uses_env_default_when_no_limit_specified():
|
||||
"""Uses VLLM_MAX_AUDIO_CLIP_FILESIZE_MB when max_size_mb is not given."""
|
||||
with patch("vllm.entrypoints.speech_to_text.base.utils.envs") as mock_envs:
|
||||
mock_envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB = 2
|
||||
max_bytes = 2 * 1024 * 1024
|
||||
oversized_data = b"\x00" * (max_bytes + 1)
|
||||
|
||||
upload = _make_upload_file(oversized_data, size=None)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
|
||||
await read_upload_with_limit(upload)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunked_read_does_not_fully_materialize():
|
||||
"""Verify that for large oversized files, we stop reading early.
|
||||
|
||||
The function reads in 64 KiB chunks and aborts once the accumulated
|
||||
size exceeds the limit. We confirm that far fewer read calls were made
|
||||
than would be required to fully materialize the file.
|
||||
"""
|
||||
max_mb = 1
|
||||
max_bytes = max_mb * 1024 * 1024
|
||||
large_size = max_bytes * 10 # 10x the limit
|
||||
data = b"\x00" * large_size
|
||||
|
||||
upload = _make_upload_file(data, size=None)
|
||||
|
||||
with pytest.raises(VLLMValidationError):
|
||||
await read_upload_with_limit(upload, max_size_mb=max_mb)
|
||||
|
||||
chunk_size = 64 * 1024
|
||||
calls_for_full_read = large_size // chunk_size + 1
|
||||
calls_to_exceed_limit = max_bytes // chunk_size + 1
|
||||
actual_calls = upload.read.call_count
|
||||
assert actual_calls <= calls_to_exceed_limit + 1
|
||||
assert actual_calls < calls_for_full_read
|
||||
@@ -0,0 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def transcription_server_with_force_include_usage():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--enable-force-include-usage",
|
||||
"--gpu-memory-utilization",
|
||||
"0.2",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def transcription_client_with_force_include_usage(
|
||||
transcription_server_with_force_include_usage,
|
||||
):
|
||||
async with (
|
||||
transcription_server_with_force_include_usage.get_async_client() as async_client
|
||||
):
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_with_enable_force_include_usage(
|
||||
transcription_client_with_force_include_usage, winning_call
|
||||
):
|
||||
res = (
|
||||
await transcription_client_with_force_include_usage.audio.transcriptions.create(
|
||||
model="openai/whisper-large-v3-turbo",
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
)
|
||||
|
||||
async for chunk in res:
|
||||
if not len(chunk.choices):
|
||||
# final usage sent
|
||||
usage = chunk.usage
|
||||
assert isinstance(usage, dict)
|
||||
assert usage["prompt_tokens"] > 0
|
||||
assert usage["completion_tokens"] > 0
|
||||
assert usage["total_tokens"] > 0
|
||||
else:
|
||||
assert not hasattr(chunk, "usage")
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``Qwen3ASR``'s user-text sanitizer.
|
||||
|
||||
The sanitizer is the security boundary between user-supplied transcription
|
||||
fields (``prompt`` / ``response_prefix``) and the structured ChatML prompt
|
||||
template. It must strip both ``<|...|>`` control tokens and the
|
||||
``<asr_text>`` assistant-prefix delimiter, and it must do so to a fixpoint
|
||||
so nested payloads cannot reconstruct a valid token after a single pass.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.qwen3_asr import _sanitize_transcription_user_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
# No-op cases
|
||||
("", ""),
|
||||
("plain text", "plain text"),
|
||||
("|piped|content", "|piped|content"),
|
||||
("contains < and > but not as a token", "contains < and > but not as a token"),
|
||||
# Single-pass strips
|
||||
("<|im_end|>", ""),
|
||||
("<|im_start|>assistant<|im_end|>", "assistant"),
|
||||
("a<|x|>b", "ab"),
|
||||
("foo<asr_text>bar", "foobar"),
|
||||
# Nested ChatML reconstruction attacks (would bypass a single re.sub)
|
||||
("<|im<|x|>_end|>", ""),
|
||||
("<|<|inner|>middle<|x|>_end|>", ""),
|
||||
# Nested <asr_text> reconstruction attack
|
||||
# (would bypass a single str.replace)
|
||||
("<asr_te<asr_text>xt>", ""),
|
||||
("<asr_te<asr_te<asr_text>xt>xt>", ""),
|
||||
# Combined attacks across both kinds of token
|
||||
("<|im_end|>foo<asr_text>bar<|<|x|>im_end|>", "foobar"),
|
||||
("foo<asr_te<|x|>xt>bar", "foobar"),
|
||||
],
|
||||
)
|
||||
def test_sanitize_strips_control_tokens(text: str, expected: str) -> None:
|
||||
assert _sanitize_transcription_user_text(text) == expected
|
||||
|
||||
|
||||
def test_sanitize_handles_falsy_inputs() -> None:
|
||||
assert _sanitize_transcription_user_text("") == ""
|
||||
# The dataclass default for ``response_prefix`` is the empty string;
|
||||
# the sanitizer must accept that without exception or extra work.
|
||||
assert _sanitize_transcription_user_text(None) == "" # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_sanitize_is_idempotent() -> None:
|
||||
"""Once sanitized, applying again must be a no-op (fixpoint property)."""
|
||||
cases = [
|
||||
"plain text",
|
||||
"<|im<|x|>_end|>",
|
||||
"<asr_te<asr_text>xt>",
|
||||
"<|im_end|>foo<asr_text>bar<|<|x|>im_end|>",
|
||||
]
|
||||
for raw in cases:
|
||||
once = _sanitize_transcription_user_text(raw)
|
||||
twice = _sanitize_transcription_user_text(once)
|
||||
assert once == twice, f"not idempotent for {raw!r}"
|
||||
+390
@@ -0,0 +1,390 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ASR inter-chunk spacing: ``asr_inter_chunk_separator`` and transcription
|
||||
serving (mocked).
|
||||
|
||||
Unit tests cover the helper and ``SupportsTranscription.no_space_languages``.
|
||||
Integration-style tests exercise ``OpenAIServingTranscription`` streaming and
|
||||
``create_transcription`` without loading a model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.speech_to_text.base.serving import (
|
||||
SpeechToTextBaseServing,
|
||||
asr_inter_chunk_separator,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
|
||||
from vllm.entrypoints.speech_to_text.transcription.serving import (
|
||||
OpenAIServingTranscription,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
StreamingTranscriptionPostProcessor,
|
||||
SupportsTranscription,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_asr import Qwen3ASRForConditionalGeneration
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
# --- Unit: helper + protocol -------------------------------------------------
|
||||
|
||||
|
||||
def test_default_no_space_languages_includes_zh_and_ja():
|
||||
assert SupportsTranscription.no_space_languages == {"ja", "zh"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("language", "expected_sep"),
|
||||
[
|
||||
("en", " "),
|
||||
("EN", " "),
|
||||
("zh", ""),
|
||||
("ZH", ""),
|
||||
("ja", ""),
|
||||
(None, " "),
|
||||
],
|
||||
)
|
||||
def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
|
||||
sep = asr_inter_chunk_separator(language, SupportsTranscription.no_space_languages)
|
||||
assert sep == expected_sep
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_passes_plain_text_without_prefix():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
|
||||
assert post_processor.process_delta("Hello", False) == "Hello"
|
||||
assert post_processor.process_delta(" world", True) == " world"
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_buffers_prefix_with_leading_space():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
|
||||
assert post_processor.process_delta(" language Eng", False) == ""
|
||||
assert post_processor.process_delta("lish<asr", False) == ""
|
||||
assert post_processor.process_delta("_text>Hello", True) == "Hello"
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_keeps_independent_state():
|
||||
processor_cls = Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
|
||||
first_processor = processor_cls()
|
||||
second_processor = processor_cls()
|
||||
|
||||
assert first_processor.process_delta(" language Eng", False) == ""
|
||||
assert second_processor.process_delta("plain text", True) == "plain text"
|
||||
assert first_processor.process_delta("lish<asr_text>Hello", True) == "Hello"
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_emits_finished_incomplete_prefix():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
|
||||
assert (
|
||||
post_processor.process_delta(" language English", True) == " language English"
|
||||
)
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_stops_buffering_long_plain_prefix():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
text = " language " + ("x" * 50)
|
||||
|
||||
assert post_processor.process_delta(text, False) == text
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_stops_buffering_prefix_with_newline():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
text = " language English\nhello"
|
||||
|
||||
assert post_processor.process_delta(text, False) == text
|
||||
|
||||
|
||||
def test_joined_chunks_english_has_space_between():
|
||||
sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
|
||||
assert sep.join(["hello", "world"]) == "hello world"
|
||||
|
||||
|
||||
def test_joined_chunks_chinese_has_no_space_between():
|
||||
sep = asr_inter_chunk_separator("zh", SupportsTranscription.no_space_languages)
|
||||
assert sep.join(["你好", "世界"]) == "你好世界"
|
||||
|
||||
|
||||
# --- Integration: serving (no model) -----------------------------------------
|
||||
|
||||
|
||||
class _StubTranscriptionModel:
|
||||
"""Minimal stand-in for a SupportsTranscription implementation (no torch)."""
|
||||
|
||||
no_space_languages: set[str] = {"ja", "zh"}
|
||||
supports_segment_timestamp = False
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
return SpeechToTextConfig(
|
||||
sample_rate=16000.0,
|
||||
max_audio_clip_s=5.0,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def get_streaming_post_processor_cls(
|
||||
cls,
|
||||
) -> type[StreamingTranscriptionPostProcessor]:
|
||||
return StreamingTranscriptionPostProcessor
|
||||
|
||||
|
||||
def _request_output(text: str, finish_reason: str | None = "stop") -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="rid",
|
||||
prompt=None,
|
||||
prompt_token_ids=None,
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=(1, 2, 3),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
def _sse_delta_contents(sse_body: str) -> list[str]:
|
||||
"""Extract ``choices[0].delta.content`` from each ``data:`` line (streaming API)."""
|
||||
contents: list[str] = []
|
||||
for line in sse_body.splitlines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload == "[DONE]":
|
||||
continue
|
||||
obj = json.loads(payload)
|
||||
for choice in obj.get("choices") or []:
|
||||
delta = choice.get("delta") or {}
|
||||
if "content" in delta:
|
||||
contents.append(delta["content"])
|
||||
return contents
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_english_inserts_space_between_chunks():
|
||||
"""Online streaming: first output per audio chunk is prefixed with *separator*."""
|
||||
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("hello")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("world")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = _StubTranscriptionModel
|
||||
serving.streaming_post_processor_cls = (
|
||||
_StubTranscriptionModel.get_streaming_post_processor_cls()
|
||||
)
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-model",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
sep = asr_inter_chunk_separator("en", _StubTranscriptionModel.no_space_languages)
|
||||
assert sep == " "
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_hello(), gen_world()],
|
||||
request_id="test-req",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-req"),
|
||||
audio_duration_s=1.0,
|
||||
separator=sep,
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
sse = "".join(out_lines)
|
||||
combined = "".join(_sse_delta_contents(sse))
|
||||
assert combined.strip() == "hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_chinese_no_space_between_chunks():
|
||||
async def gen_a() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("你好")
|
||||
|
||||
async def gen_b() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("世界")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = _StubTranscriptionModel
|
||||
serving.streaming_post_processor_cls = (
|
||||
_StubTranscriptionModel.get_streaming_post_processor_cls()
|
||||
)
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-model",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
sep = asr_inter_chunk_separator("zh", _StubTranscriptionModel.no_space_languages)
|
||||
assert sep == ""
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_a(), gen_b()],
|
||||
request_id="test-req-zh",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-req-zh"),
|
||||
audio_duration_s=1.0,
|
||||
separator=sep,
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
combined = "".join(_sse_delta_contents("".join(out_lines)))
|
||||
assert combined == "你好世界"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_strips_qwen3_asr_prefix_per_chunk():
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("language Eng", finish_reason=None)
|
||||
yield _request_output("lish<asr", finish_reason=None)
|
||||
yield _request_output("_text>Hello", finish_reason=None)
|
||||
yield _request_output("")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output(" language Eng", finish_reason=None)
|
||||
yield _request_output("lish<asr_text>world")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = Qwen3ASRForConditionalGeneration
|
||||
serving.streaming_post_processor_cls = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
|
||||
)
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-qwen3-asr",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_hello(), gen_world()],
|
||||
request_id="test-qwen3-asr",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-qwen3-asr"),
|
||||
audio_duration_s=1.0,
|
||||
separator=" ",
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
|
||||
combined = "".join(_sse_delta_contents("".join(out_lines)))
|
||||
assert combined == "Hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_transcription_non_streaming_joins_chunks_by_language():
|
||||
"""``create_transcription`` uses the same separator logic as the helper."""
|
||||
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("hello")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("world")
|
||||
|
||||
engine_client = MagicMock()
|
||||
engine_client.model_config = MagicMock()
|
||||
engine_client.model_config.get_diff_sampling_param.return_value = {
|
||||
"max_tokens": 256,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
engine_client.model_config.max_model_len = 8192
|
||||
engine_client.errored = False
|
||||
engine_client.generate.side_effect = [gen_hello(), gen_world()]
|
||||
|
||||
models = MagicMock(spec=OpenAIServingModels)
|
||||
models.lora_requests = {}
|
||||
models.is_base_model.return_value = True
|
||||
|
||||
preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.model_loader.get_model_cls",
|
||||
return_value=_StubTranscriptionModel,
|
||||
),
|
||||
patch.object(
|
||||
SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock
|
||||
),
|
||||
):
|
||||
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
|
||||
|
||||
req_en = TranscriptionRequest.model_construct(
|
||||
file=MagicMock(),
|
||||
model="stub-model",
|
||||
language="en",
|
||||
stream=False,
|
||||
response_format="json",
|
||||
)
|
||||
out_en = await serving.create_transcription(
|
||||
b"\x00\x00", req_en, raw_request=None
|
||||
)
|
||||
assert not isinstance(out_en, ErrorResponse)
|
||||
assert out_en.text == "hello world"
|
||||
|
||||
async def gen_nihao() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("你好")
|
||||
|
||||
async def gen_shijie() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("世界")
|
||||
|
||||
engine_client.generate.side_effect = [gen_nihao(), gen_shijie()]
|
||||
|
||||
req_zh = TranscriptionRequest.model_construct(
|
||||
file=MagicMock(),
|
||||
model="stub-model",
|
||||
language="zh",
|
||||
stream=False,
|
||||
response_format="json",
|
||||
)
|
||||
out_zh = await serving.create_transcription(
|
||||
b"\x00\x00", req_zh, raw_request=None
|
||||
)
|
||||
assert not isinstance(out_zh, ErrorResponse)
|
||||
assert out_zh.text == "你好世界"
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# imports for structured outputs tests
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
|
||||
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
"mistral",
|
||||
"--config_format",
|
||||
"mistral",
|
||||
"--load_format",
|
||||
"mistral",
|
||||
]
|
||||
|
||||
|
||||
async def transcribe_and_check(
|
||||
client,
|
||||
model_name: str,
|
||||
file,
|
||||
*,
|
||||
language: str,
|
||||
expected_text: str,
|
||||
expected_seconds: int | None = None,
|
||||
case_sensitive: bool = False,
|
||||
):
|
||||
"""Run a transcription request and assert the output contains
|
||||
*expected_text* and optionally that usage reports *expected_seconds*.
|
||||
|
||||
Provides detailed failure messages with the actual transcription output.
|
||||
"""
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
model=model_name,
|
||||
file=file,
|
||||
language=language,
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_usage = out["usage"]
|
||||
|
||||
if case_sensitive:
|
||||
assert expected_text in out_text, (
|
||||
f"Expected {expected_text!r} in transcription output, got: {out_text!r}"
|
||||
)
|
||||
else:
|
||||
assert expected_text.lower() in out_text.lower(), (
|
||||
f"Expected {expected_text!r} (case-insensitive) in transcription "
|
||||
f"output, got: {out_text!r}"
|
||||
)
|
||||
|
||||
if expected_seconds is not None:
|
||||
assert out_usage["seconds"] == expected_seconds, (
|
||||
f"Expected {expected_seconds}s of audio, "
|
||||
f"got {out_usage['seconds']}s. Full usage: {out_usage!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]
|
||||
)
|
||||
async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
|
||||
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
model_name,
|
||||
mary_had_lamb,
|
||||
language="en",
|
||||
expected_text="Mary had a little lamb",
|
||||
expected_seconds=16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
|
||||
"""Ensure STT (transcribe) requests can pass LoRA through to generate."""
|
||||
# ROCm SPECIFIC CONFIGURATION:
|
||||
# To ensure the test passes on ROCm, we modify the max model length to 512.
|
||||
# We DO NOT apply this to other platforms to maintain strict upstream parity.
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
model_name = "ibm-granite/granite-speech-3.3-2b"
|
||||
lora_model_name = "speech"
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
"--enable-lora",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--lora-modules",
|
||||
f"{lora_model_name}={model_name}",
|
||||
"--max-model-len",
|
||||
"512" if current_platform.is_rocm() else "2048",
|
||||
"--max-num-seqs",
|
||||
"1",
|
||||
]
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
lora_model_name,
|
||||
mary_had_lamb,
|
||||
language="en",
|
||||
expected_text="mary had a little lamb",
|
||||
expected_seconds=16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"]
|
||||
)
|
||||
async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):
|
||||
# Gemma accuracy on some of the audio samples we use is particularly bad,
|
||||
# hence we use a different one here. WER is evaluated separately.
|
||||
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
max_wait_seconds=480,
|
||||
env_dict=ROCM_ENV_OVERRIDES,
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
model_name,
|
||||
foscolo,
|
||||
language="it",
|
||||
expected_text="ove il mio corpo fanciulletto",
|
||||
)
|
||||
+438
@@ -0,0 +1,438 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# imports for structured outputs tests
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import soundfile as sf
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "openai/whisper-large-v3-turbo"
|
||||
|
||||
# Disable prefix caching on ROCm to reduce non-determinism in
|
||||
# streaming-vs-non-streaming comparisons.
|
||||
_ROCM_ARGS = ["--no-enable-prefix-caching"] if current_platform.is_rocm() else []
|
||||
|
||||
|
||||
def _get_attention_backend_params() -> list[str | None]:
|
||||
"""Return attention backends to parametrize the server fixture with.
|
||||
|
||||
On ROCm, we test multiple backends explicitly:
|
||||
- None: default auto-selection (ROCM_ATTN for decoder self-attention,
|
||||
falls back to ROCM_AITER_UNIFIED_ATTN or TRITON_ATTN for
|
||||
cross-attention since ROCM_ATTN doesn't support ENCODER_DECODER)
|
||||
- TRITON_ATTN: always available on ROCm
|
||||
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950
|
||||
|
||||
On non-ROCm platforms, we just run with the default backend.
|
||||
"""
|
||||
try:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
backends: list[str | None] = [None, "TRITON_ATTN"]
|
||||
from vllm.platforms.rocm import _ON_MI3XX
|
||||
|
||||
if _ON_MI3XX:
|
||||
backends.append("ROCM_AITER_UNIFIED_ATTN")
|
||||
return backends
|
||||
except Exception:
|
||||
pass
|
||||
return [None]
|
||||
|
||||
|
||||
# Aiter backends need VLLM_ROCM_USE_AITER=1 (and MHA=1 for ROCM_AITER_FA)
|
||||
# to be enabled in the server subprocess.
|
||||
_AITER_ENV = {
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_MHA": "1",
|
||||
}
|
||||
|
||||
_ATTN_BACKENDS = _get_attention_backend_params()
|
||||
_ATTN_IDS = [b or "default" for b in _ATTN_BACKENDS]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=_ATTN_BACKENDS, ids=_ATTN_IDS)
|
||||
def server(request):
|
||||
args = [*_ROCM_ARGS]
|
||||
env_dict = None
|
||||
if request.param is not None:
|
||||
args += ["--attention-backend", request.param]
|
||||
if "AITER" in request.param:
|
||||
# TODO: re-enable once AITER reenables fp16 unified attention.
|
||||
pytest.skip("ROCM_AITER_UNIFIED_ATTN does not support fp16")
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def whisper_client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio(whisper_client, mary_had_lamb):
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_usage = out["usage"]
|
||||
assert "Mary had a little lamb," in out_text
|
||||
assert out_usage["seconds"] == 16, out_usage["seconds"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio_batched(mary_had_lamb, winning_call, whisper_client):
|
||||
transcription = whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
transcription2 = whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
# Await both transcriptions by scheduling coroutines together
|
||||
transcription, transcription2 = await asyncio.gather(transcription, transcription2)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
assert "Mary had a little lamb," in out_text
|
||||
out2 = json.loads(transcription2)
|
||||
out_text2 = out2["text"]
|
||||
assert "Edgar Martinez" in out_text2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bad_requests(mary_had_lamb, whisper_client):
|
||||
# invalid language
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME, file=mary_had_lamb, language="hh", temperature=0.0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_audio_request(mary_had_lamb, whisper_client):
|
||||
mary_had_lamb.seek(0)
|
||||
audio, sr = load_audio(mary_had_lamb)
|
||||
# Add small silence after each audio for repeatability in the split process
|
||||
audio = np.pad(audio, (0, 1600))
|
||||
repeated_audio = np.tile(audio, 10)
|
||||
# Repeated audio to buffer
|
||||
buffer = io.BytesIO()
|
||||
sf.write(buffer, repeated_audio, sr, format="WAV")
|
||||
buffer.seek(0)
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=buffer,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_usage = out["usage"]
|
||||
counts = out_text.count("Mary had a little lamb")
|
||||
assert counts == 10, counts
|
||||
assert out_usage["seconds"] == 161, out_usage["seconds"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_audio_file(whisper_client):
|
||||
"""Corrupted audio should surface as HTTP 400."""
|
||||
invalid_audio = io.BytesIO(b"not a valid audio file")
|
||||
invalid_audio.name = "invalid.wav"
|
||||
|
||||
with pytest.raises(openai.BadRequestError) as exc_info:
|
||||
await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=invalid_audio,
|
||||
language="en",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Invalid or unsupported audio file" in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_endpoints(whisper_client):
|
||||
# text to text model
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await whisper_client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "system", "content": "You are a helpful assistant."}],
|
||||
)
|
||||
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await whisper_client.completions.create(model=MODEL_NAME, prompt="Hello")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_response(winning_call, whisper_client):
|
||||
transcription = ""
|
||||
res_no_stream = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
response_format="json",
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
)
|
||||
res = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
# Reconstruct from chunks and validate
|
||||
async for chunk in res:
|
||||
text = chunk.choices[0]["delta"]["content"]
|
||||
transcription += text
|
||||
|
||||
assert transcription == res_no_stream.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_options(winning_call, whisper_client):
|
||||
res = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
extra_body=dict(stream_include_usage=True, stream_continuous_usage_stats=True),
|
||||
timeout=30,
|
||||
)
|
||||
final = False
|
||||
continuous = True
|
||||
async for chunk in res:
|
||||
if not len(chunk.choices):
|
||||
# final usage sent
|
||||
final = True
|
||||
else:
|
||||
continuous = continuous and hasattr(chunk, "usage")
|
||||
assert final and continuous
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sampling_params(mary_had_lamb, whisper_client):
|
||||
"""
|
||||
Compare sampling with params and greedy sampling to assert results
|
||||
are different when extreme sampling parameters values are picked.
|
||||
"""
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
temperature=0.8,
|
||||
extra_body=dict(
|
||||
seed=42,
|
||||
repetition_penalty=1.9,
|
||||
top_k=12,
|
||||
top_p=0.4,
|
||||
min_p=0.5,
|
||||
frequency_penalty=1.8,
|
||||
presence_penalty=2.0,
|
||||
),
|
||||
)
|
||||
|
||||
greedy_transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=dict(seed=42),
|
||||
)
|
||||
|
||||
assert greedy_transcription.text != transcription.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_prompt(mary_had_lamb, whisper_client):
|
||||
prompt = "This is a speech, recorded in a phonograph."
|
||||
# Prompts should not omit the part of original prompt while transcribing.
|
||||
prefix = "The first words I spoke in the original phonograph"
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)["text"]
|
||||
assert prefix in out
|
||||
transcription_wprompt = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
prompt=prompt,
|
||||
temperature=0.0,
|
||||
)
|
||||
out_prompt = json.loads(transcription_wprompt)["text"]
|
||||
assert prefix in out_prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_with_timestamp(mary_had_lamb, whisper_client):
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="verbose_json",
|
||||
temperature=0.0,
|
||||
)
|
||||
assert transcription.segments is not None
|
||||
assert len(transcription.segments) > 0
|
||||
assert transcription.segments[0].avg_logprob is not None
|
||||
assert transcription.segments[0].compression_ratio is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_with_max_tokens(whisper_client, mary_had_lamb):
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body={"max_completion_tokens": 1},
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) == 1
|
||||
# max_completion_tokens > max_model_len
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body={"max_completion_tokens": int(1e6)},
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) < 450 # ~Whisper max output len
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("fixture_name", "expected_lang", "expected_text"),
|
||||
[
|
||||
("mary_had_lamb", "en", ["Mary had a little lamb"]),
|
||||
("foscolo", "it", ["zacinto", "sacre"]),
|
||||
],
|
||||
ids=["english", "italian"],
|
||||
)
|
||||
async def test_language_auto_detect(
|
||||
whisper_client, fixture_name, expected_lang, expected_text, request
|
||||
):
|
||||
"""Auto-detect language when no language param is provided."""
|
||||
audio_file = request.getfixturevalue(fixture_name)
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=audio_file,
|
||||
response_format="verbose_json",
|
||||
temperature=0.0,
|
||||
)
|
||||
assert transcription.language == expected_lang
|
||||
text_lower = transcription.text.lower()
|
||||
assert any(word.lower() in text_lower for word in expected_text), (
|
||||
f"Expected {expected_lang} text but got: {transcription.text}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whisper_beam_search_single_beam(mary_had_lamb, whisper_client):
|
||||
"""Test beam search with encoder-decoder model (Whisper) on transcriptions with
|
||||
one beam aligns with greedy decoding.
|
||||
"""
|
||||
beam_transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body=dict(
|
||||
use_beam_search=True,
|
||||
n=1,
|
||||
),
|
||||
)
|
||||
|
||||
greedy_transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
greedy_res = json.loads(greedy_transcription)["text"]
|
||||
beam_res = json.loads(beam_transcription)["text"]
|
||||
assert greedy_res == beam_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whisper_beam_search_multibeam(mary_had_lamb, whisper_client):
|
||||
"""Test n>1 for beam search returns one transcription (best beam)."""
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body=dict(
|
||||
use_beam_search=True,
|
||||
n=2,
|
||||
),
|
||||
)
|
||||
|
||||
result = json.loads(transcription)
|
||||
|
||||
text = result["text"]
|
||||
|
||||
assert text is not None
|
||||
assert len(text) > 0
|
||||
assert "mary had a little lamb" in text.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_with_beams_raises(winning_call, whisper_client):
|
||||
"""Test that stream=True + beam search raises bad request for now."""
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
stream=True,
|
||||
extra_body=dict(
|
||||
use_beam_search=True,
|
||||
n=2,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,318 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
|
||||
# imports for structured outputs tests
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import soundfile as sf
|
||||
|
||||
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
SERVER_ARGS = ["--enforce-eager"]
|
||||
|
||||
|
||||
def _get_rocm_attention_config(model_name):
|
||||
"""Return appropriate ROCm attention config for the given model.
|
||||
|
||||
Whisper uses cross-attention (ENCODER_DECODER) which ROCM_AITER_FA does
|
||||
not support. For Whisper we use ROCM_AITER_UNIFIED_ATTN (or TRITON_ATTN
|
||||
as fallback); other models can use ROCM_AITER_FA.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return None
|
||||
|
||||
if "whisper" in model_name.lower():
|
||||
try:
|
||||
from vllm.platforms.rocm import _ON_MI3XX
|
||||
|
||||
if _ON_MI3XX:
|
||||
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Could not import _ON_MI3XX from rocm platform, "
|
||||
"falling back to TRITON_ATTN for Whisper."
|
||||
)
|
||||
return {"backend": "TRITON_ATTN"}
|
||||
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
|
||||
|
||||
def _get_server_args(attention_config):
|
||||
"""Get server args with attention backend if specified."""
|
||||
args = SERVER_ARGS.copy()
|
||||
add_attention_backend(args, attention_config)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
scope="module", params=["openai/whisper-small", "google/gemma-3n-E2B-it"]
|
||||
)
|
||||
def server(request):
|
||||
# Parametrize over model name
|
||||
attention_config = _get_rocm_attention_config(request.param)
|
||||
with RemoteOpenAIServer(
|
||||
request.param, _get_server_args(attention_config)
|
||||
) as remote_server:
|
||||
yield remote_server, request.param
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client_and_model(server):
|
||||
server, model_name = server
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client, model_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_asr_model(foscolo):
|
||||
# text to text model
|
||||
model_name = "JackFram/llama-68m"
|
||||
attention_config = _get_rocm_attention_config(model_name)
|
||||
with RemoteOpenAIServer(
|
||||
model_name, _get_server_args(attention_config)
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await client.audio.translations.create(
|
||||
model=model_name, file=foscolo, temperature=0.0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio_with_lora(mary_had_lamb):
|
||||
"""Ensure STT (translate) requests can pass LoRA through to generate."""
|
||||
# ROCm SPECIFIC CONFIGURATION:
|
||||
# To ensure the test passes on ROCm, we modify the max model length to 512.
|
||||
# We DO NOT apply this to other platforms to maintain strict upstream parity.
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# NOTE - careful to call this test before the module scoped server
|
||||
# fixture, otherwise it'll OOMkill the CI
|
||||
model_name = "ibm-granite/granite-speech-3.3-2b"
|
||||
lora_model_name = "speech"
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
"--enable-lora",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--lora-modules",
|
||||
f"{lora_model_name}={model_name}",
|
||||
"--max-model-len",
|
||||
"512" if current_platform.is_rocm() else "2048",
|
||||
"--max-num-seqs",
|
||||
"1",
|
||||
]
|
||||
|
||||
add_attention_backend(server_args, _get_rocm_attention_config(model_name))
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
translation = await client.audio.translations.create(
|
||||
model=lora_model_name,
|
||||
file=mary_had_lamb,
|
||||
extra_body=dict(language="en", to_language="es"),
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(translation)["text"].strip().lower()
|
||||
assert "pequeño" in out.split(" ")
|
||||
|
||||
|
||||
# NOTE: (NickLucche) the large-v3-turbo model was not trained on translation!
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio(foscolo, client_and_model):
|
||||
client, model_name = client_and_model
|
||||
translation = await client.audio.translations.create(
|
||||
model=model_name,
|
||||
file=foscolo,
|
||||
response_format="text",
|
||||
# TODO remove `language="it"` once language detection is implemented
|
||||
extra_body=dict(language="it", to_language="en"),
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(translation)["text"].strip().lower()
|
||||
assert "greek sea" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_prompt(foscolo, client_and_model):
|
||||
client, model_name = client_and_model
|
||||
# Condition whisper on starting text
|
||||
prompt = "Nor have I ever"
|
||||
transcription = await client.audio.translations.create(
|
||||
model=model_name,
|
||||
file=foscolo,
|
||||
prompt=prompt,
|
||||
extra_body=dict(language="it", to_language="en"),
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)["text"]
|
||||
assert "Nor will I ever touch the sacred" not in out
|
||||
assert prompt not in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_response(foscolo, client_and_model, server):
|
||||
client, model_name = client_and_model
|
||||
translation = ""
|
||||
res_no_stream = await client.audio.translations.create(
|
||||
model=model_name,
|
||||
file=foscolo,
|
||||
response_format="json",
|
||||
extra_body=dict(language="it", to_language="en", seed=42),
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
# Stream via HTTPX since OpenAI translation client doesn't expose streaming
|
||||
server, model_name = server
|
||||
url = server.url_for("v1/audio/translations")
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
data = {
|
||||
"model": model_name,
|
||||
"language": "it",
|
||||
"to_language": "en",
|
||||
"stream": True,
|
||||
"temperature": 0.0,
|
||||
"seed": 42,
|
||||
}
|
||||
foscolo.seek(0)
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
files = {"file": foscolo}
|
||||
async with http_client.stream(
|
||||
"POST", url, headers=headers, data=data, files=files
|
||||
) as response:
|
||||
async for line in response.aiter_lines():
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("data: "):
|
||||
line = line[len("data: ") :]
|
||||
if line.strip() == "[DONE]":
|
||||
break
|
||||
chunk = json.loads(line)
|
||||
text = chunk["choices"][0].get("delta", {}).get("content")
|
||||
translation += text or ""
|
||||
|
||||
res_stream = translation.split()
|
||||
# NOTE There's a small non-deterministic issue here, likely in the attn
|
||||
# computation, which will cause a few tokens to be different, while still
|
||||
# being very close semantically.
|
||||
assert (
|
||||
sum([x == y for x, y in zip(res_stream, res_no_stream.text.split())])
|
||||
>= len(res_stream) * 0.87
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_options(foscolo, server):
|
||||
server, model_name = server
|
||||
url = server.url_for("v1/audio/translations")
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
data = {
|
||||
"model": model_name,
|
||||
"language": "it",
|
||||
"to_language": "en",
|
||||
"stream": True,
|
||||
"stream_include_usage": True,
|
||||
"stream_continuous_usage_stats": True,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
foscolo.seek(0)
|
||||
final = False
|
||||
continuous = True
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
files = {"file": foscolo}
|
||||
async with http_client.stream(
|
||||
"POST", url, headers=headers, data=data, files=files
|
||||
) as response:
|
||||
async for line in response.aiter_lines():
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("data: "):
|
||||
line = line[len("data: ") :]
|
||||
if line.strip() == "[DONE]":
|
||||
break
|
||||
chunk = json.loads(line)
|
||||
choices = chunk.get("choices", [])
|
||||
if not choices:
|
||||
# final usage sent
|
||||
final = True
|
||||
else:
|
||||
continuous = continuous and ("usage" in chunk)
|
||||
assert final and continuous
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_audio_request(foscolo, client_and_model):
|
||||
client, model_name = client_and_model
|
||||
if model_name == "google/gemma-3n-E2B-it":
|
||||
pytest.skip("Gemma3n does not support long audio requests")
|
||||
foscolo.seek(0)
|
||||
audio, sr = load_audio(foscolo)
|
||||
repeated_audio = np.tile(audio, 2)
|
||||
# Repeated audio to buffer
|
||||
buffer = io.BytesIO()
|
||||
sf.write(buffer, repeated_audio, sr, format="WAV")
|
||||
buffer.seek(0)
|
||||
translation = await client.audio.translations.create(
|
||||
model=model_name,
|
||||
file=buffer,
|
||||
extra_body=dict(language="it", to_language="en"),
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(translation)["text"].strip().lower()
|
||||
assert out.count("greek sea") == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_with_max_tokens(mary_had_lamb, client_and_model):
|
||||
client, model_name = client_and_model
|
||||
transcription = await client.audio.translations.create(
|
||||
model=model_name,
|
||||
file=mary_had_lamb,
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body={"max_completion_tokens": 1},
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
print(out_text)
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(model_name)
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) == 1
|
||||
# max_completion_tokens > max_model_len
|
||||
# max_model_len=32768 for Gemma-3n-E2B-it
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
model=model_name,
|
||||
file=mary_had_lamb,
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"max_completion_tokens": int(1e6),
|
||||
"repetition_penalty": 1.3,
|
||||
},
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
print(out_text)
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) < 450 # ~Whisper max output len
|
||||
Reference in New Issue
Block a user