wehub/vllm-project--vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
pre-commit / pre-run-check (push) Has been cancelled
Details
pre-commit / pre-commit (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:55:37 +08:00
commit 7ce4c8e27e
5900 changed files with 1668062 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/models/multimodal/processing/__init__.py
View File
tests/models/multimodal/processing/test_audio_in_video.py
+115
View File
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
caching.
Tests the use_audio_in_video feature where audio is extracted from video and
processed together with video frames in an interleaved manner.
Regression test: when use_audio_in_video=True and the multimodal processor
cache is warm, the second request goes through MultiModalProcessorSenderCache
which sets mm_kwargs["video"] items to None on a cache hit. The processor
must still detect use_audio_in_video=True (via token-count heuristic) and
produce the same prompt_token_ids as the first (cache-miss) request.
Without the fix the cache-hit path left use_audio_in_video=False, causing
audio placeholder tokens to be inserted separately instead of being derived
from the interleaved video placeholders – yielding a different (wrong) token
sequence on every subsequent request for the same video.
"""
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorSenderCache
from ....multimodal.utils import random_audio, random_video
from ...utils import build_model_context
MODELS = [
"Qwen/Qwen2.5-Omni-3B",
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
]
def create_mm_data(num_videos: int) -> dict[str, list]:
# Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
# stays fast even without a GPU.
mm_data = dict[str, list](video=[], audio=[])
for i in range(num_videos):
rng = np.random.RandomState(i)
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
mm_data["video"].append(video)
mm_data["audio"].append((audio, sr))
return mm_data
@pytest.mark.parametrize("model_id", MODELS)
@pytest.mark.parametrize("num_videos", [1, 2])
def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
"""
Regression test for https://github.com/vllm-project/vllm/pull/36800
MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
on a cache hit, so mm_kwargs["video"] items become None on the second call.
The Qwen processor override of _maybe_apply_prompt_updates must detect
use_audio_in_video=True via token-count heuristics and re-derive the audio
placeholders correctly.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": num_videos, "image": 0, "video": num_videos},
mm_processor_cache_gb=1,
)
# Baseline: no cache, always processes from scratch.
baseline_processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config, cache=None
)
# Sender cache: on a cache hit returns (None, prompt_updates) for each
# item, setting mm_kwargs["video"] = [None] – the exact condition that
# triggered the original bug.
sender_cache = MultiModalProcessorSenderCache(ctx.model_config)
cached_processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config, cache=sender_cache
)
video_token_id = baseline_processor.info.get_hf_config().video_token_id
mm_data = create_mm_data(num_videos)
hf_processor_mm_kwargs = {"use_audio_in_video": True}
def run(processor):
return processor(
[video_token_id] * num_videos,
mm_items=baseline_processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)["prompt_token_ids"]
baseline_ids = run(baseline_processor)
# First call on the sender-cache processor: cache miss.
# mm_kwargs["video"] items are real tensors; use_audio_in_video is
# detected normally from the item data.
first_ids = run(cached_processor)
assert first_ids == baseline_ids, (
"Cache-miss call produced different prompt_token_ids than baseline.\n"
f" baseline : {baseline_ids}\n"
f" cache-miss: {first_ids}"
)
# Second call on the sender-cache processor: cache hit.
# MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
# so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
# not detected, yielding wrong token ids.
second_ids = run(cached_processor)
assert second_ids == baseline_ids, (
"Cache-hit call produced different prompt_token_ids than baseline.\n"
"This is the regression introduced when use_audio_in_video detection\n"
"fails for None mm_kwargs items on a cache hit.\n"
f" baseline : {baseline_ids}\n"
f" cache-hit: {second_ids}"
)
tests/models/multimodal/processing/test_audioflamingo3.py
+153
View File
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The vLLM team.
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
# reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from transformers import PretrainedConfig
from tests.models.registry import HF_EXAMPLE_MODELS
class MockAudioFlamingo3Config(PretrainedConfig):
model_type = "audioflamingo3"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.audio_config = PretrainedConfig()
self.text_config = PretrainedConfig()
class MockAudioFlamingo3Processor:
def __init__(self):
self.audio_token = "<sound>"
self.audio_token_id = 12345
self.max_audio_len = 60
self.feature_extractor = MockFeatureExtractor()
self.tokenizer = self._tokenize
def __call__(self, text=None, audio=None, **kwargs):
return {
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
"input_features": torch.zeros((3, 80, 3000)),
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
}
def _tokenize(self, text, **kwargs):
return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
class MockFeatureExtractor:
def __init__(self):
self.sampling_rate = 16000
self.chunk_length = 30
self.hop_length = 160
def __call__(self, audios, **kwargs):
return {
"input_features": torch.zeros((len(audios), 80, 3000)),
"attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
}
@pytest.fixture
def mock_ctx():
config = MockAudioFlamingo3Config()
ctx = MagicMock()
ctx.get_hf_config.return_value = config
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
**data, **kwargs
)
ctx.model_config.hf_config = config
return ctx
@pytest.fixture(autouse=True)
def check_transformers_version():
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
model_info.check_transformers_version(on_fail="skip")
def test_audio_chunk_counting(mock_ctx):
from vllm.model_executor.models.audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3MultiModalProcessor,
AudioFlamingo3ProcessingInfo,
)
info = AudioFlamingo3ProcessingInfo(mock_ctx)
processor = AudioFlamingo3MultiModalProcessor(
info, AudioFlamingo3DummyInputsBuilder(info)
)
sr = 16000
audio_1 = np.zeros(30 * sr)
audio_2 = np.zeros(75 * sr)
mm_data = {"audio": [audio_1, audio_2]}
prompt = "<|user|>Listen.<|end|>"
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
chunk_counts = processed["chunk_counts"]
assert chunk_counts[0].item() == 1
assert chunk_counts[1].item() == 2
assert len(chunk_counts) == 2
assert processed["feature_attention_mask"].shape == (3, 3000)
def test_dummy_data_generation(mock_ctx):
from vllm.model_executor.models.audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3ProcessingInfo,
)
info = AudioFlamingo3ProcessingInfo(mock_ctx)
builder = AudioFlamingo3DummyInputsBuilder(info)
mm_counts = {"audio": 2}
dummy_data = builder.get_dummy_mm_data(100, mm_counts, {})
assert "audio" in dummy_data
assert len(dummy_data["audio"]) == 2
expected_len = 60 * 16000
assert len(dummy_data["audio"][0]) == expected_len
def test_audio_token_count_matches_hf_processor_math():
from vllm.model_executor.models.audioflamingo3 import (
_count_audio_tokens_from_mask,
)
feature_attention_mask = torch.zeros((3, 3000), dtype=torch.long)
feature_attention_mask[0, :2999] = 1
feature_attention_mask[1, :2999] = 1
feature_attention_mask[2, :1500] = 1
chunk_counts = torch.tensor([2, 1], dtype=torch.long)
assert (
_count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 0) == 1499
)
assert _count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 1) == 375
tests/models/multimodal/processing/test_common.py
+496
View File
@@ -0,0 +1,496 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Set as AbstractSet
from functools import partial
import numpy as np
import pytest
from PIL import Image
from vllm.config import ModelConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.platforms import current_platform
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.utils.mistral import is_mistral_tokenizer
from ....multimodal.utils import random_audio, random_image, random_video
from ...registry import (
_MULTIMODAL_EXAMPLE_MODELS,
_TRANSFORMERS_BACKEND_MODELS,
HF_EXAMPLE_MODELS,
)
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
Add metadata to video mm_data
"""
def create_metadata(frames: np.ndarray):
num_frames = len(frames)
return {
"total_num_frames": num_frames,
"fps": 2.0,
"duration": num_frames / 2.0,
"video_backend": "opencv",
"frames_indices": list(range(num_frames)),
"do_sample_frames": True,
}
# Ensure video metadata is included
if "video" in mm_data:
video = mm_data["video"]
if isinstance(video, list):
# multiple videos
mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
else:
# single video
mm_data["video"] = (video, create_metadata(video))
return mm_data
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
Patch the multimodal data for GLM-ASR model.
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
"""
if "audio" in mm_data:
audio = mm_data["audio"]
if isinstance(audio, list) and len(audio) > 1:
# Limit to single audio to match text requirement
mm_data["audio"] = [audio[0]]
return mm_data
_IGNORE_MM_KEYS = {
# In Ultravox, the audio_features can be different depending on padding
# The slight difference should not be a problem though, since
# attention_mask lets us ignore the difference.
"ultravox": {"audio_features"},
}
MM_DATA_PATCHES = {
"glmasr": glmasr_patch_mm_data,
}
_XPU_EXCLUDED_MODEL_IDS = {
"baidu/Unlimited-OCR",
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
"Qwen/Qwen2.5-Omni-7B-AWQ",
}
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
for model_arch in model_arch_list:
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
yield model_info.default
for extra_type, extra_model_id in model_info.extras.items():
if "fp" in extra_type:
continue # Redundant to test quantized models
yield extra_model_id
def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
model_ids = list(_iter_model_ids_to_test(model_arch_list))
if current_platform.is_xpu():
for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS:
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
return model_ids
def get_model_ids_to_test():
transformers_arch_ids = {
model_id
for info in _TRANSFORMERS_BACKEND_MODELS.values()
for model_id in (info.default, *info.extras.values())
}
vllm_only_archs = {
arch
for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
if not any(
model_id in transformers_arch_ids
for model_id in (info.default, *info.extras.values())
)
}
return _get_model_ids_to_test(vllm_only_archs)
def get_text_token_prompts(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
):
dummy_inputs = processor.dummy_inputs
tokenizer: TokenizerLike = processor.info.get_tokenizer()
model_config = processor.info.ctx.model_config
if processor.info.data_parser.video_needs_metadata:
mm_data = add_video_metadata(mm_data)
model_type = model_config.hf_config.model_type
if model_type in MM_DATA_PATCHES:
mm_data = MM_DATA_PATCHES[model_type](mm_data)
parsed_data = processor.info.parse_mm_data(mm_data)
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
if is_mistral_tokenizer(tokenizer):
inputs = dummy_inputs.get_dummy_processor_inputs(
model_config.max_model_len,
mm_counts,
mm_options={},
# Assume all Mistral models define this extra argument
mm_data=mm_data, # type: ignore[call-arg]
)
else:
inputs = dummy_inputs.get_dummy_processor_inputs(
model_config.max_model_len,
mm_counts,
mm_options={},
)
text_prompt: str | None
token_prompt: list[int]
if isinstance(inputs.prompt, list):
text_prompt = None
token_prompt = inputs.prompt
elif isinstance(inputs.prompt, str):
text_prompt = inputs.prompt
token_prompt = tokenizer.encode(
text_prompt,
**processor.info.get_default_tok_params().get_encode_kwargs(),
)
else:
raise TypeError(type(inputs.prompt))
return text_prompt, token_prompt
def random_vision_chunk(
rng: np.random.RandomState,
min_wh: int,
max_wh: int,
min_frames: int,
max_frames: int,
) -> dict:
num_frames = rng.randint(min_frames, max_frames + 1)
if num_frames == 1:
# Single image chunk
wh = rng.randint(min_wh, max_wh + 1)
image = random_image(rng, wh, wh + 1)
return {"type": "image", "image": image}
frames = []
for _ in range(num_frames):
wh = rng.randint(min_wh, max_wh + 1)
frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
frames.append(frame)
video_array = np.stack(frames, axis=0)
return {"type": "video_chunk", "video_chunk": video_array}
def _test_processing_correctness(
model_id_or_arch: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
# Use model architecture to get the default model id
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
model_id = model_info.default
else:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
model_id = model_id_or_arch
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_config = ModelConfig(
model_id,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
hf_overrides=model_info.hf_overrides,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=model_info.dtype,
)
# Ensure that the cache can fit all of the data
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
model_config.multimodal_config.mm_processor_cache_gb = 2048
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
cache = MultiModalProcessorOnlyCache(model_config)
processing_info = factories.info(ctx)
supported_mm_limits = processing_info.get_supported_mm_limits()
# Keep integer limits for local data generation
limit_mm_per_prompt_ints = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
if modality == "video":
return VideoDummyOptions(count=count)
if modality == "image":
return ImageDummyOptions(count=count)
if modality == "audio":
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
# Assign normalized DummyOptions to the model config
model_config.get_multimodal_config().limit_per_prompt = {
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt_ints.items()
}
baseline_processor = factories.build_processor(ctx, cache=None)
cached_processor = factories.build_processor(ctx, cache=cache)
rng = np.random.RandomState(0)
# GLM-ASR requires a minimum audio length of 70ms
min_audio_len = 512 if model_config.hf_config.model_type != "glmasr" else 1120
input_to_hit = {
"image": Image.new("RGB", size=(128, 128)),
"video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
"audio": (np.zeros((min_audio_len,)), 16000),
"vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
}
input_factory = {
"image": partial(random_image, rng, min_wh=128, max_wh=256),
"video": partial(
random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
),
"audio": partial(
random_audio,
rng,
min_len=min_audio_len,
max_len=min_audio_len + 512,
sr=16000,
),
"vision_chunk": partial(
random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
),
}
for batch_idx in range(num_batches):
mm_data = {
k: [
(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
for _ in range(rng.randint(limit + 1))
]
for k, limit in limit_mm_per_prompt_ints.items()
}
# Drop unnecessary keys and test single -> multi conversion
if rng.rand() < simplify_rate:
for k in list(mm_data.keys()):
if not mm_data[k]:
del mm_data[k]
elif len(mm_data[k]) == 1:
mm_data[k] = mm_data[k][0]
_test_processing_correctness_one(
model_config,
mm_data,
baseline_processor,
cached_processor,
batch_idx,
hit_rate,
num_batches,
simplify_rate,
)
def _test_processing_correctness_one(
model_config: ModelConfig,
mm_data: MultiModalDataDict,
baseline_processor: BaseMultiModalProcessor,
cached_processor: BaseMultiModalProcessor,
batch_idx: int,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
model_type = model_config.hf_config.model_type
text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
mm_items = baseline_processor.info.parse_mm_data(mm_data)
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
baseline_tokenized_result = baseline_processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_tokenized_result = cached_processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
baseline_tokenized_result,
cached_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
if text_prompt is not None:
baseline_text_result = baseline_processor(
text_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_text_result = cached_processor(
text_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
baseline_text_result,
cached_text_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
_assert_inputs_equal(
baseline_text_result,
baseline_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
_assert_inputs_equal(
cached_text_result,
cached_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
if model_id == "google/gemma-3n-E2B-it":
pytest.skip("Fix later")
if model_id == "OpenGVLab/InternVL2-2B":
pytest.skip("Fix later")
if model_id == "openvla/openvla-7b":
pytest.skip(
"OpenVLA uses a custom vLLM processor because its HF remote "
"processor is incompatible with current Transformers."
)
if model_id == "jinaai/jina-reranker-m0":
pytest.skip("Fix later")
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
pytest.skip(
"Voxtral Realtime doesn't make use of any place-holder "
"tokens and hence cannot pass the processing "
"correctness test as is. Let's revisit adapting this "
"test once more realtime models exist."
)
if model_id == "CohereLabs/cohere-transcribe-03-2026":
pytest.skip("Fix later")
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
pytest.skip(
"MOSS-Audio uses a custom processor that dynamically expands "
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
"video backends (qwen_vl_utils / codec) that require real encoded "
"video bytes and metadata. The synthetic numpy-array videos used by "
"this test yield empty video features, so the generic correctness "
"check cannot exercise the video path. Image processing is covered "
"by registration/inference tests."
)
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
)
def _assert_inputs_equal(
a: MultiModalInput,
b: MultiModalInput,
*,
ignore_mm_keys: set[str] | None = None,
msg: str = "",
):
if ignore_mm_keys is None:
ignore_mm_keys = set()
ignore_prompt_keys = ("prompt", "mm_kwargs")
a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}
b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}
assert a_rest == b_rest, msg
a_data = a["mm_kwargs"].get_data()
b_data = b["mm_kwargs"].get_data()
for key in ignore_mm_keys:
a_data.pop(key, None)
b_data.pop(key, None)
assert batched_tensors_equal(a_data, b_data), msg
tests/models/multimodal/processing/test_deepseek_ocr.py
+134
View File
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
images that are small enough to not require cropping produce an empty
images_crop tensor with shape (0, 3, 640, 640). The _parse_and_validate_image_input
method must correctly read image_size from this tensor's shape rather than
falling back to base_size, which would cause a TensorSchema mismatch.
Run with:
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
"""
import pytest
from PIL import Image
from transformers import AutoTokenizer
from vllm.model_executor.models.deepseek_ocr import DeepseekOCRImagePixelInputs
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
@pytest.fixture(scope="module")
def processor():
"""Load the DeepseekOCRProcessor with tokenizer from HuggingFace."""
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
return DeepseekOCRProcessor(tokenizer=tokenizer)
class TestDeepseekOCREmptyImagesCrop:
"""Verify TensorSchema validation handles empty images_crop correctly."""
def test_empty_images_crop_small_image(self, processor):
"""A small image (<=640px) produces empty images_crop and should
not crash the TensorSchema validation.
Previously, the code used ``numel() > 0`` to decide whether to read
image_size from the tensor shape. When numel()==0, it fell back to
base_size=1024, mismatching the actual tensor dim of 640.
"""
# Small image: both dims <= IMAGE_SIZE (640) → no crops
small_image = Image.new("RGB", (100, 100), color="red")
result = processor(
prompt="<image>\nDescribe this image.",
images=[small_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
# Processor must produce an empty crop tensor for a small image
assert images_crop.shape[0] == 0
base_size = pixel_values.shape[-1]
image_size = images_crop.shape[-1] if images_crop is not None else base_size
# This should NOT raise ValueError
schema = DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": base_size,
"image_size": image_size,
},
)
assert schema.data.shape == (1, 3, 1024, 1024)
assert schema.images_crop.shape == (0, 3, 640, 640)
def test_populated_images_crop_large_image(self, processor):
"""A large image (>640px) produces populated images_crop."""
# Large image: exceeds IMAGE_SIZE (640) → dynamic crop tiles
large_image = Image.new("RGB", (1200, 800), color="blue")
result = processor(
prompt="<image>\nDescribe this image.",
images=[large_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
assert images_crop.shape[0] > 0
base_size = pixel_values.shape[-1]
image_size = images_crop.shape[-1]
schema = DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": base_size,
"image_size": image_size,
},
)
assert schema.data.shape == (1, 3, 1024, 1024)
assert schema.images_crop.shape[-1] == 640
def test_mismatched_image_size_raises(self, processor):
"""Deliberately wrong image_size binding should still be caught
by TensorSchema validation."""
small_image = Image.new("RGB", (100, 100), color="green")
result = processor(
prompt="<image>\nDescribe this image.",
images=[small_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
with pytest.raises(ValueError, match="images_crop"):
DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": 1024,
"image_size": 1024, # Wrong! Tensor has 640
},
)
tests/models/multimodal/processing/test_gemma3.py
+189
View File
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.gemma3n_audio_utils import (
adjust_audio_features_to_expected_length,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# Gemma3 (image) model
GEMMA3_MODEL_ID = "google/gemma-3-4b-it"
# Gemma3n (multimodal with audio) model
GEMMA3N_MODEL_ID = "google/gemma-3n-E2B-it"
# Expected audio tokens for Gemma3n (audio_soft_tokens_per_image)
GEMMA3N_EXPECTED_AUDIO_TOKENS = 188
class TestGemma3nAudioTensorLogic:
"""CPU-based tests for Gemma3n audio feature tensor manipulation.
These tests validate the padding/truncation logic in
adjust_audio_features_to_expected_length() which fixes the
integer overflow in _process_audio_input when audio_seq_len > 188.
"""
def test_padding_when_audio_short(self):
"""Test that short audio is padded to expected length."""
batch_size, seq_len, embed_dim = 1, 100, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == 0
# First 100 tokens should be original, rest should be padding (zeros)
assert torch.allclose(result[:, :seq_len, :], audio_features)
assert torch.allclose(
result[:, seq_len:, :],
torch.zeros(batch_size, expected_tokens - seq_len, embed_dim),
)
def test_truncation_when_audio_long(self):
"""Test that long audio is truncated to expected length.
This is the key test for the overflow fix. Previously, when
audio_seq_len > expected_tokens, the code would compute a negative
padding value causing: RuntimeError: numel: integer multiplication overflow
"""
batch_size, seq_len, embed_dim = 1, 192, 256 # 192 > 188
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == seq_len - expected_tokens # 192 - 188 = 4
# Result should be first 188 tokens of original
assert torch.allclose(result, audio_features[:, :expected_tokens, :])
def test_no_change_when_exact_length(self):
"""Test that exact-length audio passes through unchanged."""
batch_size, embed_dim = 1, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, expected_tokens, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == audio_features.shape
assert tokens_truncated == 0
assert torch.allclose(result, audio_features)
def test_original_bug_would_fail(self):
"""Verify the original buggy implementation would cause overflow.
The original code always tried to pad, which fails when
audio_seq_len > expected_tokens because expand() gets negative size.
"""
batch_size, seq_len, embed_dim = 1, 192, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
padding_embs = torch.zeros(1, 1, embed_dim)
# Original buggy logic (always pads, never truncates)
extra_padding_tokens = expected_tokens - seq_len # = -4 (negative!)
with pytest.raises(RuntimeError):
# This should fail with negative size error
padding_embs.expand(batch_size, extra_padding_tokens, embed_dim)
@pytest.mark.parametrize(
"seq_len",
[50, 100, 150, 187, 188, 189, 192, 200, 300],
)
def test_various_audio_lengths(self, seq_len: int):
"""Test padding/truncation with various audio lengths."""
batch_size, embed_dim = 1, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
# Should not raise any errors
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
# Output should always be expected_tokens length
assert result.shape == (batch_size, expected_tokens, embed_dim)
# Verify truncation count is correct
if seq_len > expected_tokens:
assert tokens_truncated == seq_len - expected_tokens
else:
assert tokens_truncated == 0
def test_batch_processing(self):
"""Test that batch processing works correctly."""
batch_size, seq_len, embed_dim = 4, 192, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == seq_len - expected_tokens
@pytest.mark.parametrize("model_id", [GEMMA3_MODEL_ID])
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
def test_get_image_size_with_most_features(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
max_image_size = processor.info.get_image_size_with_most_features()
max_tokens = processor.info.get_num_image_tokens(
image_width=max_image_size.width,
image_height=max_image_size.height,
processor=hf_processor,
mm_kwargs=mm_processor_kwargs,
)
prompt = "<start_of_image>"
image_seq_length = hf_processor.image_seq_length
for asset in image_assets:
mm_data = {"image": [asset.pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
num_patches_tensor = mm_kwargs_data["num_patches"]
tokens = int(num_patches_tensor.item()) * image_seq_length
assert tokens <= max_tokens
tests/models/multimodal/processing/test_gemma4.py
+287
View File
@@ -0,0 +1,287 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
import pytest
import torch
from PIL import Image as PILImage
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4ImagePixelInputs,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
from vllm.utils.mem_constants import GiB_bytes
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# TODO: to be updated to "google/gemma-4-e2b-it" once the models are available
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
def test_gemma4_image_schema_accepts_variable_patch_counts():
Gemma4ImagePixelInputs(
pixel_values=[
torch.randn(10080, 768),
torch.randn(2520, 768),
],
pixel_position_ids=[
torch.zeros(10080, 2, dtype=torch.long),
torch.zeros(2520, 2, dtype=torch.long),
],
)
def test_gemma4_image_batching_keeps_variable_patch_counts_unstacked():
field = MultiModalFieldConfig.batched("image").field
elems = field.build_elems(
"image",
"pixel_values",
[torch.randn(10080, 768), torch.randn(2520, 768)],
)
reduced = field.reduce_data(list(elems))
assert isinstance(reduced, list)
assert [tensor.shape for tensor in reduced] == [
torch.Size([10080, 768]),
torch.Size([2520, 768]),
]
@pytest.mark.parametrize(
"image_width,image_height,max_soft_tokens",
[
# Production repro: a 3x900 image (extreme aspect ratio) made the
# prompt-side estimator return 289 while the HF Gemma 4 image
# processor's vision tower output capped at 280, producing the
# "Attempted to assign 280 multimodal tokens to 289 placeholders"
# mismatch that crashed EngineCore.
(900, 3, 280),
(3, 900, 280),
# Same pathology should hold for the video-frame budget (70 tokens).
(900, 3, 70),
# And for any other supported budget.
(4000, 2, 1120),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
model_id: str,
image_width: int,
image_height: int,
max_soft_tokens: int,
):
"""Regression for the Gemma 3/4 multimodal crash.
`_compute_num_soft_tokens` must never return a value larger than
`max_soft_tokens`. The HF Gemma 4 image processor clamps its vision
tower output to that value; if the prompt-side estimator returns more,
the prompt has more `image` placeholder tokens than the encoder will
fill, and `_merge_multimodal_embeddings` raises `ValueError` deep in
the model forward.
"""
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
image_height=image_height,
max_soft_tokens=max_soft_tokens,
)
assert num_soft_tokens <= max_soft_tokens, (
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
f"image_width={image_width}, image_height={image_height}, "
f"max_soft_tokens={max_soft_tokens} — exceeds the cap that the HF "
f"image processor enforces on its vision tower output. This is "
f"the placeholder/encoder count mismatch that crashes EngineCore."
)
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_image_tokens"),
[
({}, 280),
({"max_soft_tokens": 70}, 70),
({"max_soft_tokens": 280}, 280),
({"max_soft_tokens": 1120}, 1120),
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
({"images_kwargs": None}, 280),
({"images_kwargs": "not-a-dict"}, 280),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_image_tokens: int,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"image": 1, "video": 1},
)
assert tokens is not None
assert tokens["image"] == expected_image_tokens
assert tokens["video"] == 32 * (70 + 2 + 6)
@pytest.mark.parametrize(
("limit_mm_per_prompt", "expected_video_tokens"),
[
({"video": 1}, 32 * (70 + 2 + 6)),
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
model_id: str,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
expected_video_tokens: int,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt=limit_mm_per_prompt,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"video": 1},
)
assert tokens is not None
assert tokens["image"] == 280
assert tokens["video"] == expected_video_tokens
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
replacement = prompt_update.resolve(0).content.full
expected = processor.info.get_image_repl(
image_width=image_size[0],
image_height=image_size[1],
processor=processor.info.get_hf_processor(),
max_soft_tokens=560,
).full
assert replacement == expected
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_limit_mm_per_prompt(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that limit_mm_per_prompt accurately restricts multiple images."""
# We only allow 1 image
ctx = build_model_context(
model_id,
mm_processor_kwargs={},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Provide 2 images in the prompt
prompt = "<image><image>"
# image_assets usually has multiple images
images = [asset.pil_image for asset in image_assets][:2]
if len(images) < 2:
images = [images[0], images[0]]
mm_data = {"image": images}
# Expect ValueError when exceeding limit
with pytest.raises(ValueError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Regression guard for PR #43169 follow-up: the batched Gemma4 vision encoder
# admitted ``chunk ~= 53`` on a 22 GiB L4 with a 26B AWQ model loaded,
# allocating 2.43 GiB int64 inside
# ``F.one_hot(num_classes=position_embedding_size)`` and OOMing because only
# 2.41 GiB was actually free. The fix sizes ``chunk`` from currently-free GPU
# memory and counts the ``F.one_hot`` transient as the dominant cost.
_encoder_chunk = Gemma4ForConditionalGeneration._encoder_chunk
# Gemma4 vision config default (HF: configuration_gemma4.py).
_POSITION_EMBEDDING_SIZE = 10240
# Video frame: max_soft_tokens=70, pooling_kernel_size=2 -> 70 * 4 patches.
_VIDEO_PATCHES_PER_FRAME = 280
def test_encoder_chunk_tight_budget_fits_in_free():
free = 3 * GiB_bytes # L4 22 GiB after 26B AWQ load.
total = 22 * GiB_bytes
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, free, total, _POSITION_EMBEDDING_SIZE
)
one_hot_bytes = chunk * _VIDEO_PATCHES_PER_FRAME * 2 * _POSITION_EMBEDDING_SIZE * 8
assert one_hot_bytes <= free // 2
def test_encoder_chunk_roomy_gpu_keeps_batching():
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME,
60 * GiB_bytes,
80 * GiB_bytes,
_POSITION_EMBEDDING_SIZE,
)
assert chunk > 8
def test_encoder_chunk_zero_patches_is_safe():
assert (
_encoder_chunk(0, 60 * GiB_bytes, 80 * GiB_bytes, _POSITION_EMBEDDING_SIZE) == 1
)
def test_encoder_chunk_zero_position_embedding_size_is_safe():
# Degenerate config: must not raise ZeroDivisionError.
assert (
_encoder_chunk(_VIDEO_PATCHES_PER_FRAME, 60 * GiB_bytes, 80 * GiB_bytes, 0) == 1
)
def test_encoder_chunk_no_free_memory_falls_back_to_one():
assert (
_encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, 0, 22 * GiB_bytes, _POSITION_EMBEDDING_SIZE
)
== 1
)
tests/models/multimodal/processing/test_gemma4_unified.py
+205
View File
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
import pytest
import torch
from PIL import Image as PILImage
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# The Unified model ID for testing purposes
GEMMA4_UNIFIED_MODEL_ID = "google/gemma-4-12B-it"
def test_gemma4_unified_image_schema_accepts_variable_patch_counts():
Gemma4ImagePixelInputs(
pixel_values=[
torch.randn(10080, 768),
torch.randn(2520, 768),
],
pixel_position_ids=[
torch.zeros(10080, 2, dtype=torch.long),
torch.zeros(2520, 2, dtype=torch.long),
],
)
def test_gemma4_unified_image_batching_keeps_variable_patch_counts_unstacked():
field = MultiModalFieldConfig.batched("image").field
elems = field.build_elems(
"image",
"pixel_values",
[torch.randn(10080, 768), torch.randn(2520, 768)],
)
reduced = field.reduce_data(list(elems))
assert isinstance(reduced, list)
assert [tensor.shape for tensor in reduced] == [
torch.Size([10080, 768]),
torch.Size([2520, 768]),
]
@pytest.mark.parametrize(
"image_width,image_height,max_soft_tokens",
[
(900, 3, 280),
(3, 900, 280),
(900, 3, 70),
(4000, 2, 1120),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
model_id: str,
image_width: int,
image_height: int,
max_soft_tokens: int,
):
"""Verify ``_compute_num_soft_tokens`` caps output at ``max_soft_tokens``."""
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
image_height=image_height,
max_soft_tokens=max_soft_tokens,
)
assert num_soft_tokens <= max_soft_tokens, (
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
f"image_width={image_width}, image_height={image_height}, "
f"max_soft_tokens={max_soft_tokens} — exceeds the cap."
)
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_image_tokens"),
[
({}, 280),
({"max_soft_tokens": 70}, 70),
({"max_soft_tokens": 280}, 280),
({"max_soft_tokens": 1120}, 1120),
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
({"images_kwargs": None}, 280),
({"images_kwargs": "not-a-dict"}, 280),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_image_tokens: int,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"image": 1, "video": 1},
)
assert tokens is not None
assert tokens["image"] == expected_image_tokens
assert tokens["video"] == 32 * (70 + 2 + 6)
@pytest.mark.parametrize(
("limit_mm_per_prompt", "expected_video_tokens"),
[
({"video": 1}, 32 * (70 + 2 + 6)),
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
model_id: str,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
expected_video_tokens: int,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt=limit_mm_per_prompt,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"video": 1},
)
assert tokens is not None
assert tokens["image"] == 280
assert tokens["video"] == expected_video_tokens
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
replacement = prompt_update.resolve(0).content.full
expected = processor.info.get_image_repl(
image_width=image_size[0],
image_height=image_size[1],
processor=processor.info.get_hf_processor(),
max_soft_tokens=560,
).full
assert replacement == expected
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_limit_mm_per_prompt(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that limit_mm_per_prompt restricts multiple images correctly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs={},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<image><image>"
images = [asset.pil_image for asset in image_assets][:2]
if len(images) < 2:
images = [images[0], images[0]]
mm_data = {"image": images}
with pytest.raises(ValueError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
tests/models/multimodal/processing/test_glm4_1v.py
+126
View File
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.video import VideoAsset
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("expected_toks_per_frame", [299])
@pytest.mark.parametrize(
"num_frames, fps, expected_grid_t",
[
# pre-sampled fixed frames (unexpected behavior,
# but we still expect it to work without errors)
(32, 1, 16),
(32, 2, 16),
(128, 1, 64),
(128, 2, 64),
# post-sampled frames (expected behavior)
(-1, 1, 5),
(-1, 2, 10),
],
)
def test_processor_override(
model_id: str,
expected_toks_per_frame: int,
expected_grid_t: int,
fps: int,
num_frames: int,
):
"""Ensure GLM4vMultiModalProcessor can handle video frames properly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
hf_processor_mm_kwargs = {"fps": fps}
# Build the image str / prompt based on the number of images we pass
video_assets = VideoAsset(name="baby_reading", num_frames=num_frames)
prompt = "<|begin_of_video|><|video|><|end_of_video|>"
video, metadata = video_assets.np_ndarrays, video_assets.metadata
metadata["fps"] = fps
mm_data = {"video": [(video, metadata)]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
video_token_id = tokenizer.convert_tokens_to_ids(hf_processor.video_token)
video_tok_count = processed_inputs["prompt_token_ids"].count(video_token_id)
grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0]
assert grid_t == expected_grid_t
assert video_tok_count == expected_toks_per_frame * grid_t
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("fps", [2])
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
def test_video_loader_consistency(
model_id: str,
fps: int,
backend: str,
):
"""
Ensure dynamic video loader (pre-sampled by loader) and normal video
loader (post-sampled by processor) produce same video processing outputs.
"""
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {"fps": fps}
# Build the image str / prompt based on the number of images we pass
prompt = "<|begin_of_video|><|video|><|end_of_video|>"
video_path = VideoAsset(name="baby_reading", num_frames=-1).video_path
with open(video_path, "rb") as f:
video_bytes = f.read()
static_video, static_metadata = VideoBackend.load_bytes(
video_bytes, backend=backend
)
dynamic_video, dynamic_metadata = DynamicVideoBackend.load_bytes(
video_bytes, fps=fps, backend=backend
)
# pre-sampled loader shouldn't read all frames
assert len(dynamic_video) < len(static_video)
static_mm_data = {"video": [(static_video, static_metadata)]}
dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]}
static_outputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(static_mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
dynamic_outputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(dynamic_mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
assert static_outputs["prompt_token_ids"] == dynamic_outputs["prompt_token_ids"]
assert batched_tensors_equal(
static_outputs["mm_kwargs"].get_data(),
dynamic_outputs["mm_kwargs"].get_data(),
)
tests/models/multimodal/processing/test_h2ovl.py
+181
View File
@@ -0,0 +1,181 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for H2OVL's multimodal preprocessing kwargs."""
from collections.abc import Mapping
import pytest
from PIL import Image
from transformers import PretrainedConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.image import rescale_image_size
from vllm.multimodal.processing import BaseMultiModalProcessor
from ....conftest import ImageTestAssets
from ...utils import build_model_context
def _get_expected_num_patches(
config: PretrainedConfig,
image: Image.Image,
num_imgs: int,
min_num: int,
max_num: int,
):
from vllm.transformers_utils.processors.h2ovl import (
calculate_h2ovl_targets,
get_h2ovl_target_ratios,
)
width, height = image.size
# Calculate the expected number of blocks
if num_imgs == 1 and config.use_msac:
# First pass
blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_h2ovl_target_ratios(
min_num=1,
max_num=max_num,
prior_aspect_ratio=None,
),
image_size=config.vision_config.image_size,
use_thumbnail=False, # Thumbnail is handled separately
)
# Second pass
blocks2, _, _, _ = calculate_h2ovl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_h2ovl_target_ratios(
min_num=3,
max_num=max_num,
prior_aspect_ratio=aspect_ratio,
),
image_size=config.vision_config.image_size,
use_thumbnail=False,
)
# Add thumbnail if use_thumbnail is True and total_blocks > 1
if config.use_thumbnail:
blocks1 += 1 if blocks1 > 1 else 0
blocks2 += 1 if blocks2 > 1 else 0
# Total blocks is the sum of blocks from both passes minus
# overlapping
total_blocks = blocks1 + blocks2 - 1
return total_blocks
blocks, _, _, _ = calculate_h2ovl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_h2ovl_target_ratios(
min_num,
max_num,
prior_aspect_ratio=None,
),
image_size=config.vision_config.image_size,
use_thumbnail=False,
)
expected_num_patches = blocks
if config.use_thumbnail and expected_num_patches > 1:
expected_num_patches += 1
return expected_num_patches
def _run_check(
processor: BaseMultiModalProcessor,
images: list[Image.Image],
min_num: int,
max_num: int,
mm_processor_kwargs: Mapping[str, object],
):
tokenizer = processor.info.get_tokenizer()
config = processor.info.get_hf_config()
prompt = "<image>" * len(images)
mm_data = {"image": images}
total_expected_num_patches = sum(
_get_expected_num_patches(config, image, len(images), min_num, max_num)
for image in images
)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@pytest.mark.parametrize(
"model_id",
[
"h2oai/h2ovl-mississippi-800m",
"h2oai/h2ovl-mississippi-2b",
],
)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
[4.0, 2.0, 1.0],
],
)
@pytest.mark.parametrize(
("min_dynamic_patch", "max_dynamic_patch"),
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
)
@pytest.mark.parametrize("dynamic_image_size", [True, False])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
model_id: str,
image_assets: ImageTestAssets,
size_factors: list[int],
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool | None,
kwargs_on_init: bool,
):
mm_processor_kwargs = {
"min_dynamic_patch": min_dynamic_patch,
"max_dynamic_patch": max_dynamic_patch,
"dynamic_image_size": dynamic_image_size,
}
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": len(size_factors)},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
min_num = min_dynamic_patch if dynamic_image_size else 1
max_num = max_dynamic_patch if dynamic_image_size else 1
_run_check(
processor,
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
min_num,
max_num,
hf_processor_mm_kwargs,
)
tests/models/multimodal/processing/test_idefics3.py
+82
View File
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Idefics3's multimodal preprocessing kwargs."""
import pytest
from packaging.version import Version
from transformers import Idefics3Config
from transformers import __version__ as TRANSFORMERS_VERSION
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.skipif(
Version(TRANSFORMERS_VERSION) < Version("5.2.0"),
reason="See https://github.com/huggingface/transformers/pull/43948",
)
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"size": {"longest_edge": 364}}, 169),
({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
# Same as the previous test - don't initialize mm_processor_kwargs
# in this test and assume that the kwargs will be correctly expanded by
# the partial when calling the custom input processor.
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
placeholders = (
"<image>"
if num_imgs == 1
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
)
prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
# Build mm_data
image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
dummy_image_size = (image_size * 4, image_size * 4)
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
mm_data = {"image": [dummy_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure the placeholders format are correct
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processed_inputs = hf_processor(
text=prompt,
images=mm_data["image"],
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
)
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
# Ensure we have the right number of placeholders per num_crops size
image_token_id = ctx.get_hf_config().image_token_id
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/multimodal/processing/test_internvl.py
+135
View File
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for InternVL's multimodal preprocessing kwargs."""
from collections.abc import Mapping
import pytest
from PIL import Image
from transformers import PretrainedConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.image import rescale_image_size
from vllm.multimodal.processing import BaseMultiModalProcessor
from ....conftest import ImageTestAssets
from ...utils import build_model_context
def _get_expected_num_patches(
config: PretrainedConfig,
image: Image.Image,
num_imgs: int,
min_num: int,
max_num: int,
):
from vllm.transformers_utils.processors.internvl import (
calculate_internvl_targets,
get_internvl_target_ratios,
)
width, height = image.size
blocks, _, _ = calculate_internvl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_internvl_target_ratios(
min_num,
max_num,
),
image_size=config.vision_config.image_size,
use_thumbnail=False,
)
expected_num_patches = blocks
if config.use_thumbnail and expected_num_patches > 1:
expected_num_patches += 1
return expected_num_patches
def _run_check(
processor: BaseMultiModalProcessor,
images: list[Image.Image],
min_num: int,
max_num: int,
mm_processor_kwargs: Mapping[str, object],
):
tokenizer = processor.info.get_tokenizer()
config = processor.info.get_hf_config()
prompt = "<image>" * len(images)
mm_data = {"image": images}
total_expected_num_patches = sum(
_get_expected_num_patches(config, image, len(images), min_num, max_num)
for image in images
)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
[4.0, 2.0, 1.0],
],
)
@pytest.mark.parametrize(
("min_dynamic_patch", "max_dynamic_patch"),
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
)
@pytest.mark.parametrize("dynamic_image_size", [True, False])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
model_id: str,
image_assets: ImageTestAssets,
size_factors: list[int],
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool | None,
kwargs_on_init: bool,
):
mm_processor_kwargs = {
"min_dynamic_patch": min_dynamic_patch,
"max_dynamic_patch": max_dynamic_patch,
"dynamic_image_size": dynamic_image_size,
}
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": len(size_factors)},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
min_num = min_dynamic_patch if dynamic_image_size else 1
max_num = max_dynamic_patch if dynamic_image_size else 1
_run_check(
processor,
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
min_num,
max_num,
hf_processor_mm_kwargs,
)
tests/models/multimodal/processing/test_llama4.py
+89
View File
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Llama4's multimodal preprocessing kwargs."""
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-4-Scout-17B-16E-Instruct"])
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
@pytest.mark.parametrize("num_imgs", [1, 5])
@pytest.mark.parametrize("mm_processor_cache_gb", [0, 4])
@pytest.mark.parametrize("tokenized_prompt", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict,
num_imgs: int,
mm_processor_cache_gb: int,
tokenized_prompt: bool,
):
"""Ensure llama4 processor works properly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": num_imgs},
mm_processor_cache_gb=mm_processor_cache_gb,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
config = processor.info.get_hf_config()
tokenizer = processor.info.get_tokenizer()
hf_processor = processor.info.get_hf_processor()
vocab = tokenizer.get_vocab()
prompt = (
"<|begin_of_text|><|header_start|>user<|header_end|>"
+ "<|image|>" * num_imgs
+ "<|eot|><|header_start|>assistant<|header_end|>"
)
mm_data = {
"image": [
image_assets[(i % len(image_assets))].pil_image for i in range(num_imgs)
]
}
if tokenized_prompt:
prompt = tokenizer.encode(prompt)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
mm_data = processed_inputs["mm_kwargs"].get_data()
# place holder replacements
prompt_token_ids = processed_inputs["prompt_token_ids"]
assert prompt_token_ids.count(config.boi_token_index) == num_imgs
assert prompt_token_ids.count(config.eoi_token_index) == num_imgs
assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs
aspect_ratios = mm_data["aspect_ratios"]
num_x_separators = num_y_separators = 0
for tiles_y, tiles_x in aspect_ratios:
if tiles_x * tiles_y > 1:
num_x_separators += (tiles_x - 1) * tiles_y
num_y_separators += tiles_y
assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators
assert (
prompt_token_ids.count(vocab[hf_processor.tile_global_token])
== num_y_separators
)
# image token offsets
img_locs = processed_inputs["mm_placeholders"].get("image", [])
assert len(img_locs) == num_imgs
assert [img_loc.offset for img_loc in img_locs] == [
i for i, v in enumerate(prompt_token_ids) if v == config.boi_token_index
]
# patch sizes and masks
num_patches_per_chunk = processor.info.get_patch_per_chunk(config.vision_config)
assert (
prompt_token_ids.count(config.image_token_index)
== sum(mm_data["patches_per_image"]) * num_patches_per_chunk
)
assert len(mm_data["pixel_values"]) == sum(mm_data["patches_per_image"])
tests/models/multimodal/processing/test_llava_next.py
+198
View File
@@ -0,0 +1,198 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from functools import partial
import pytest
from PIL import Image
from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from ...utils import build_model_context
def _validate_image_max_tokens_one(
processor: BaseMultiModalProcessor,
max_tokens: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
info = processor.info
feature_size = info.get_num_image_tokens(
image_width=image_size.width, image_height=image_size.height
)
try:
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
except Exception as exc:
failed_size_excs.append((image_size, exc))
@pytest.mark.skip(
"This test takes around 5 minutes to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
def test_processor_max_tokens(model_id):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
info = processor.info
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 2
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(32, 4096), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_max_tokens_one,
processor,
info.get_max_image_tokens(), # type: ignore
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
def _validate_image_prompt_replacements_one(
processor: BaseMultiModalProcessor,
num_imgs: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
prompt = "<image>" * num_imgs
image = Image.new("RGB", size=image_size)
mm_data = {"image": [image] * num_imgs}
try:
# The processor will throw an error if there is a mismatch
# in the prompt replacements
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == num_imgs
first_placeholder = image_placeholders[0]
# NOTE: There is a BOS token
assert first_placeholder.offset == 1
assert (
first_placeholder.length
== (len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
)
except Exception as exc:
failed_size_excs.append((image_size, exc))
def _test_image_prompt_replacements(
processor,
*,
num_imgs: int,
image_sizes: list[ImageSize],
) -> None:
"""
Ensure LlavaNextMultiModalProcessor
handles prompt replacement properly for input images.
"""
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_prompt_replacements_one,
processor,
num_imgs,
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_processor_prompt_replacements_regression(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image_ratios = [
(171, 152),
(184, 161),
(198, 176),
(333, 296),
(369, 328),
(488, 183),
(2560, 1669),
]
image_sizes = [
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
]
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
@pytest.mark.skip(
"This test takes around 2 hours to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@pytest.mark.parametrize("num_imgs", [1])
def test_processor_prompt_replacements_all(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 2
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(64, 1024), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
tests/models/multimodal/processing/test_llava_onevision.py
+196
View File
@@ -0,0 +1,196 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from functools import partial
import pytest
from PIL import Image
from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from ...utils import build_model_context
def _validate_image_max_tokens_one(
processor: BaseMultiModalProcessor,
max_tokens: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
info = processor.info
feature_size = info.get_num_image_tokens(
image_width=image_size.width, image_height=image_size.height
)
try:
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
except Exception as exc:
failed_size_excs.append((image_size, exc))
@pytest.mark.skip(
"This test takes around 5 minutes to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_processor_max_tokens(model_id):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
info = processor.info
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 6
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(32, 4096), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_max_tokens_one,
processor,
info.get_max_image_tokens(), # type: ignore
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
def _validate_image_prompt_replacements_one(
processor: BaseMultiModalProcessor,
num_imgs: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
prompt = "<image>" * num_imgs
image = Image.new("RGB", size=image_size)
mm_data = {"image": [image] * num_imgs}
try:
# The processor will throw an error if there is a mismatch
# in the prompt replacements
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == num_imgs
first_placeholder = image_placeholders[0]
assert first_placeholder.offset == 0
assert (
first_placeholder.length
== len(processed_inputs["prompt_token_ids"]) // num_imgs
)
except Exception as exc:
failed_size_excs.append((image_size, exc))
def _test_image_prompt_replacements(
processor,
*,
num_imgs: int,
image_sizes: list[ImageSize],
) -> None:
"""
Ensure LlavaOnevisionMultiModalProcessor
handles prompt replacement properly for input images.
"""
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_prompt_replacements_one,
processor,
num_imgs,
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_processor_prompt_replacements_regression(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image_ratios = [
(171, 152),
(184, 161),
(198, 176),
(333, 296),
(369, 328),
(488, 183),
(2560, 1669),
]
image_sizes = [
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
]
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
@pytest.mark.skip(
"This test takes around 2 hours to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
@pytest.mark.parametrize("num_imgs", [1])
def test_processor_prompt_replacements_all(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 6
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(64, 1024), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
tests/models/multimodal/processing/test_llava_onevision2.py
+302
View File
@@ -0,0 +1,302 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism.
The codec video backend cannot be exercised end-to-end in CI because it
relies on the model's ``trust_remote_code`` package and real video bytes.
These tests cover the pure-Python marker plumbing that wraps a video *path*
for vLLM's ``MultiModalDataParser``:
* :func:`prepare_codec_video_input` must encode a per-path hash into its
dummy ndarray so distinct codec videos do not collide on ``mm_hash``
(otherwise ``EncoderCacheManager`` would skip the encoder for every video
after the first).
* :func:`_extract_codec_video_paths` must round-trip the marker back to the
original path(s) after the parser strips the metadata dict, and must return
``None`` for non-codec inputs.
"""
import types
import numpy as np
import pytest
from vllm.model_executor.models.llava_onevision2 import (
_CODEC_VIDEO_MARKER,
LlavaOnevision2VideoBackend,
_extract_codec_video_paths,
_frame_video_to_pil_and_timestamps,
_validate_video_source,
prepare_codec_video_input,
)
from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata
def _model_config(local: str = "", domains=None):
# Minimal stand-in for vLLM's ModelConfig: the validator only reads
# ``allowed_local_media_path`` and ``allowed_media_domains``.
return types.SimpleNamespace(
allowed_local_media_path=local,
allowed_media_domains=domains,
)
def test_prepare_codec_video_input_shape_and_marker():
dummy, meta = prepare_codec_video_input("/data/foo.mp4")
# 4-D ndarray satisfies MultiModalDataParser's video shape check.
assert isinstance(dummy, np.ndarray)
assert dummy.shape == (1, 1, 16, 3)
assert dummy.dtype == np.uint8
# Metadata carries the exact path under the codec marker key.
assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"}
def test_prepare_codec_video_input_is_deterministic():
# Same path must yield identical dummy bytes (stable mm_hash).
a, _ = prepare_codec_video_input("/data/foo.mp4")
b, _ = prepare_codec_video_input("/data/foo.mp4")
assert a.tobytes() == b.tobytes()
def test_prepare_codec_video_input_distinct_paths_distinct_bytes():
# Distinct paths must yield distinct dummy bytes so the parser-visible
# ndarray (the only part reaching MultiModalHasher) varies per video.
paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"]
payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths}
assert len(payloads) == len(paths)
def test_extract_codec_video_paths_parser_list_shape():
# Parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")]
assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"]
def test_extract_codec_video_paths_single_raw_tuple():
# Single raw (ndarray, dict) tuple (pre-parser path) is also accepted.
item = prepare_codec_video_input("/x/solo.mp4")
assert _extract_codec_video_paths(item) == ["/x/solo.mp4"]
def test_extract_codec_video_paths_non_codec_returns_none():
# Plain decoded-frame inputs (ndarray / list of ndarray) are not codec
# markers and must fall through to the frame backend.
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
assert _extract_codec_video_paths(plain) is None
assert _extract_codec_video_paths([plain, plain]) is None
assert _extract_codec_video_paths([]) is None
# Tuple without the marker key is ignored.
assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None
def test_extract_codec_video_paths_mixed_batch_returns_none():
# If any item in the batch lacks the marker, the whole batch is treated
# as non-codec (the backend does not mix codec and frame videos).
codec = prepare_codec_video_input("/x/1.mp4")
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
assert _extract_codec_video_paths([codec, plain]) is None
# ---------------------------------------------------------------------------
# Media access controls (_validate_video_source)
#
# The codec backend keeps the raw path string alive past vLLM's
# MultiModalDataParser and hands it to the trust-remote-code codec module,
# which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the
# codec backend is restricted to *local files* confined to
# --allowed-local-media-path; remote http(s)/data URLs are rejected (they must
# use the frame backend, which rides the connector). The validator also returns
# the *resolved* path so the codec module opens exactly what was validated
# (no symlink-retarget / validate-vs-open gap).
# ---------------------------------------------------------------------------
def test_validate_http_url_rejected():
# Codec backend is local-only: remote URLs bypass the connector's domain
# and redirect controls, so they are rejected regardless of allowlist.
with pytest.raises(ValueError):
_validate_video_source("http://example.com/v.mp4", _model_config())
def test_validate_https_url_rejected_even_when_host_allowlisted():
with pytest.raises(ValueError):
_validate_video_source(
"https://good.com/v.mp4",
_model_config(domains=["good.com"]),
)
def test_validate_data_url_rejected():
# data: URLs are a remote/inline source for the frame backend, not codec.
with pytest.raises(ValueError):
_validate_video_source("data:video/mp4;base64,AAAA", _model_config())
def test_validate_local_file_blocked_without_allowed_path():
# Local file access is opt-in: without --allowed-local-media-path the
# bare path is rejected.
with pytest.raises(ValueError):
_validate_video_source("/data/v.mp4", _model_config())
def test_validate_local_file_allowed_inside_allowed_dir(tmp_path):
f = tmp_path / "v.mp4"
f.touch()
assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f)
def test_validate_local_file_traversal_blocked():
# Path traversal escaping the allowed root is rejected after resolution.
with pytest.raises(ValueError):
_validate_video_source(
"/tmp/ov2/../../etc/passwd",
_model_config(local="/tmp/ov2"),
)
def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path):
f = tmp_path / "v.mp4"
f.touch()
assert _validate_video_source(
f.as_uri(), _model_config(local=str(tmp_path))
) == str(f)
def test_validate_file_scheme_percent_encoded_traversal_blocked():
# ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first
# (url2pathname) so the path cannot stay literally under the allowed root.
with pytest.raises(ValueError):
_validate_video_source(
"file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd",
_model_config(local="/tmp/ov2"),
)
def test_validate_unsupported_scheme_blocked():
with pytest.raises(ValueError):
_validate_video_source("ftp://example.com/v.mp4", _model_config())
def test_validate_returns_resolved_path_through_symlink(tmp_path):
# The validator resolves symlinks and returns the *real* path, so the codec
# module opens exactly what was validated (closes the validate-vs-open gap).
real = tmp_path / "real.mp4"
real.touch()
link = tmp_path / "link.mp4"
link.symlink_to(real)
assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str(
real
)
def test_validate_relative_bare_path_blocked():
# Bare (scheme=="") paths come only from the codec backend and must be
# absolute: resolving a relative path against an ambiguous CWD before the
# confinement check is brittle/unsafe, so it is rejected outright.
with pytest.raises(ValueError):
_validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2"))
# ---------------------------------------------------------------------------
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
#
# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
# (``video_needs_metadata=True``). The helper materialises PIL frames and
# per-frame timestamps (``frame_index / fps``), padding the frame count up to
# the even temporal-merge boundary.
# ---------------------------------------------------------------------------
def test_frame_video_to_pil_and_timestamps_basic():
frames = np.zeros((4, 8, 8, 3), dtype=np.uint8)
metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]}
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
assert len(pil_frames) == 4
assert all(f.size == (8, 8) for f in pil_frames)
# timestamps = frame_index / fps
assert timestamps == [0.0, 2.0, 4.0, 6.0]
def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count():
# Odd frame count -> last frame repeated to satisfy temporal merge=2.
frames = np.zeros((3, 8, 8, 3), dtype=np.uint8)
metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]}
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
assert len(pil_frames) == 4
assert len(timestamps) == 4
# The padded frame reuses the final index/timestamp.
assert timestamps == [0.0, 1.0, 2.0, 2.0]
def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse():
# Missing frames_indices -> sequential range; missing/zero fps -> default.
frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {}))
assert len(pil_frames) == 2
# Default fps is 1.0, indices fall back to range(T).
assert timestamps == [0.0, 1.0]
def test_frame_video_to_pil_and_timestamps_rejects_non_tuple():
# Bare arrays (no metadata) must be rejected: the frame backend requires
# the (frames, metadata) tuple produced by the registered loader.
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
with pytest.raises(ValueError):
_frame_video_to_pil_and_timestamps(plain)
# ---------------------------------------------------------------------------
# LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller
# supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can
# override the conservative defaults (fps=1.0, max_frames=32). Unset target
# fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants.
# ---------------------------------------------------------------------------
def _src(total_frames: int, fps: float) -> VideoSourceMetadata:
return VideoSourceMetadata(
total_frames_num=total_frames,
original_fps=fps,
duration=total_frames / fps if fps > 0 else 0.0,
)
def test_backend_defaults_cap_at_32_frames():
# 300 frames @ 1fps source, target unset -> capped at default max_frames=32.
src = _src(300, 1.0)
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0)
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
assert len(idx) == 32
assert idx[0] == 0
assert idx[-1] == 299
assert len(idx) % 2 == 0
def test_backend_target_num_frames_overrides_default_cap():
# VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap.
src = _src(300, 1.0)
target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0)
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
assert len(idx) == 128
assert idx[0] == 0
assert idx[-1] == 299
def test_backend_target_fps_controls_sampling_when_below_cap():
# 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded).
src = _src(60, 30.0)
target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0)
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
# fps-derived nframes (2) is below the 128 cap, so fps wins.
assert len(idx) <= 8
assert len(idx) % 2 == 0
tests/models/multimodal/processing/test_minimax_m3.py
+138
View File
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
These exercise the vendored processor directly (no checkpoint / GPU needed), so
they validate the long-side resize spec and the resulting prompt-token counts
deterministically.
"""
import pytest
import torch
from vllm.transformers_utils.processors.minimax_m3 import (
IMAGE_MAX_TOTAL_PIXELS,
MIN_SHORT_SIDE_PIXEL,
VIDEO_MAX_TOTAL_PIXELS,
MiniMaxM3VLImageProcessor,
MiniMaxM3VLVideoProcessor,
smart_resize,
)
# Long sides are multiples of patch_size*merge_size (28) so the rounding is
# exact and the expected token counts are unambiguous.
LONG_SIDES = [252, 504, 1008]
MERGE2 = 2**2 # merge_size ** 2
def _image_tokens(grid_thw) -> int:
g = list(grid_thw)
return int(g[0] * g[1] * g[2]) // MERGE2
# --------------------------------------------------------------------------- #
# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
# --------------------------------------------------------------------------- #
def test_smart_resize_long_side_shrink():
# (a) long side exceeds the cap -> shrink so the long side equals the cap.
h, w = smart_resize(
2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
)
assert max(h, w) == 1008
assert (h, w) == (1008, 504) # aspect ratio preserved
def test_smart_resize_short_side_enlarge():
# (b) long side within the cap but short side below the floor -> enlarge so
# the short side reaches min_short_side_pixel.
h, w = smart_resize(
200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
)
assert min(h, w) == MIN_SHORT_SIDE_PIXEL # 112
def test_smart_resize_total_pixels_raises():
# (c) still over the area cap after resizing -> raise instead of inferring.
with pytest.raises(ValueError, match="max_total_pixels"):
smart_resize(
5000,
5000,
factor=28,
max_long_side_pixel=4000,
max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
)
def test_smart_resize_backward_compatible_area_bound():
# Without max_long_side_pixel the original Qwen-style area bound is used.
assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
# --------------------------------------------------------------------------- #
# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
# --------------------------------------------------------------------------- #
def test_image_tokens_increase_with_max_long_side_pixel():
proc = MiniMaxM3VLImageProcessor()
counts = []
for long_side in LONG_SIDES:
patches = proc.get_number_of_image_patches(
2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
)
counts.append(patches // MERGE2)
assert counts == [81, 324, 1296]
assert counts[0] < counts[1] < counts[2]
def test_image_processor_defaults_match_spec():
proc = MiniMaxM3VLImageProcessor()
assert proc.max_long_side_pixel is None # opt-in
assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
def test_image_preprocess_pipeline_monotonic():
proc = MiniMaxM3VLImageProcessor()
image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
counts = []
for long_side in LONG_SIDES:
out = proc.preprocess(
[image],
do_resize=True,
max_long_side_pixel=long_side,
return_tensors="pt",
)
counts.append(_image_tokens(out["image_grid_thw"][0]))
assert counts == [81, 324, 1296]
# --------------------------------------------------------------------------- #
# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
# --------------------------------------------------------------------------- #
def test_video_tokens_increase_with_max_long_side_pixel():
proc = MiniMaxM3VLVideoProcessor()
assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
counts = []
for long_side in LONG_SIDES:
out = proc.preprocess(
videos=[video],
do_resize=True,
max_long_side_pixel=long_side,
return_tensors="pt",
)
counts.append(_image_tokens(out["video_grid_thw"][0]))
assert counts[0] < counts[1] < counts[2]
def test_video_volumetric_cap_raises():
proc = MiniMaxM3VLVideoProcessor()
# 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
with pytest.raises(ValueError, match="max_total_pixels"):
proc.preprocess(
videos=[video],
do_resize=True,
max_long_side_pixel=1008,
return_tensors="pt",
)
tests/models/multimodal/processing/test_mllama4.py
+55
View File
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for mllama's multimodal preprocessing and profiling."""
import pytest
from torch import prod
from transformers import Llama4Config
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-Guard-4-12B"])
@pytest.mark.parametrize("max_model_len", [4096, 8192, 25600, 131072])
def test_profiling(model_id: str, max_model_len: int):
model_config_kwargs = {
"max_model_len": max_model_len,
}
mm_counts = {"image": 1}
ctx = build_model_context(
model_id,
model_config_kwargs=model_config_kwargs,
limit_mm_per_prompt=mm_counts,
)
mm_inputs = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
ctx.model_config,
mm_counts=mm_counts,
)
hf_config = ctx.get_hf_config(Llama4Config)
image_size = hf_config.vision_config.image_size
patch_size = hf_config.vision_config.patch_size
downsample_ratio = int(
round(1.0 / (hf_config.vision_config.pixel_shuffle_ratio**2))
)
tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio
mm_data = mm_inputs["mm_kwargs"].get_data()
chunks_per_image = prod(mm_data["patches_per_image"])
total_num_patches = chunks_per_image * tokens_per_patch
num_tiles = (
mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][1]
) # x-y separator tokens
total_tokens = (
total_num_patches.item() + num_tiles.item() + 3
) # image start, image, image end
assert total_num_patches == sum(
item.get_num_embeds() for item in mm_inputs["mm_placeholders"]["image"]
)
assert total_tokens == sum(
placeholder.length for placeholder in mm_inputs["mm_placeholders"]["image"]
)
tests/models/multimodal/processing/test_molmo2.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import torch
from vllm.model_executor.models.molmo2 import build_flat_image_bool_length
def test_build_flat_image_bool_length_matches_molmoweb_processor_tokens():
hf_config = SimpleNamespace(
image_patch_id=151938,
low_res_image_start_token_id=151940,
image_start_token_id=151936,
image_col_id=151939,
image_end_token_id=151937,
)
image_grids = torch.tensor([[14, 14, 14, 23]], dtype=torch.long)
image_tokens, num_image_tokens = build_flat_image_bool_length(
image_grids,
hf_config,
image_use_col_tokens=True,
use_single_crop_col_tokens=None,
use_single_crop_start_token=False,
)
assert num_image_tokens.tolist() == [550]
assert len(image_tokens) == 550
assert image_tokens[0].item() == hf_config.image_start_token_id
assert (image_tokens == hf_config.image_col_id).sum().item() == 28
def test_build_flat_image_bool_length_respects_disabled_col_tokens():
hf_config = SimpleNamespace(
image_patch_id=151938,
low_res_image_start_token_id=151940,
image_start_token_id=151936,
image_col_id=151939,
image_end_token_id=151937,
)
image_grids = torch.tensor([[2, 3, 5, 7]], dtype=torch.long)
image_tokens, num_image_tokens = build_flat_image_bool_length(
image_grids,
hf_config,
image_use_col_tokens=False,
use_single_crop_col_tokens=False,
use_single_crop_start_token=True,
)
assert num_image_tokens.tolist() == [45]
assert len(image_tokens) == 45
assert image_tokens[0].item() == hf_config.low_res_image_start_token_id
assert (image_tokens == hf_config.image_col_id).sum().item() == 0
tests/models/multimodal/processing/test_moondream3.py
+553
View File
@@ -0,0 +1,553 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Moondream3 multimodal processing.
Includes:
- Processor creation and application tests
- Image tokenization and placeholder expansion tests
- Tiling and cropping logic tests (CPU-based)
- Pixel normalization tests
"""
import numpy as np
import pytest
import torch
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
# Expected multimodal prefix: BOS + 729 image tokens.
EXPECTED_IMAGE_TOKENS = 730
# Vision encoder constants
CROP_SIZE = 378
PATCH_SIZE = 14
MAX_CROPS = 12
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_creation(model_id: str):
"""Test that Moondream3 processor can be created."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
assert processor is not None
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_apply(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that Moondream3 processor can process inputs.
NOTE: The prompt includes the leading BOS token because Moondream3
pre-fills BOS and image embeddings together.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
assert "prompt_token_ids" in processed_inputs
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_pixel_values(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that pixel values are correctly produced."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Check mm_kwargs contains pixel_values
mm_kwargs = processed_inputs.get("mm_kwargs")
assert mm_kwargs is not None
mm_data_result = mm_kwargs.get_data()
assert "pixel_values" in mm_data_result
# Verify pixel_values shape
pixel_values = mm_data_result["pixel_values"]
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
assert pixel_values.shape[2] == 3 # RGB channels
assert pixel_values.shape[3] == 378 # crop height
assert pixel_values.shape[4] == 378 # crop width
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_image_token_expansion(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that <image> placeholder is expanded to correct number of tokens."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_multi_crop_tiling(
model_id: str,
):
"""Test that large images produce correct multi-crop tiling."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Create a large image that requires multiple crops
large_image = Image.new("RGB", (1000, 1000), color="blue")
pixel_values, tiling = processor.preprocess_image(large_image)
# Large images should produce more than 1x1 tiling
assert tiling[0] >= 1 and tiling[1] >= 1
# Check that we have global crop + local crops
expected_crops = tiling[0] * tiling[1] + 1
assert pixel_values.shape[0] == expected_crops
@pytest.mark.parametrize(
"image_size",
[
(500, 500),
(800, 600),
(1920, 1080),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_tiling_various_sizes(
image_size: tuple[int, int],
model_id: str,
):
"""Test tiling with various image sizes."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
width, height = image_size
image = Image.new("RGB", (width, height), color="red")
pixel_values, tiling = processor.preprocess_image(image)
# Basic shape checks
assert pixel_values.dim() == 4 # [num_crops, C, H, W]
assert pixel_values.shape[1] == 3 # RGB
assert pixel_values.shape[2] == 378 # crop height
assert pixel_values.shape[3] == 378 # crop width
# Tiling should respect max_crops (12)
assert tiling[0] * tiling[1] <= 12
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_pixel_normalization(
model_id: str,
):
"""Test that pixel values are normalized to [-1, 1] range."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Create test image
image = Image.new("RGB", (378, 378), color="green")
pixel_values, _ = processor.preprocess_image(image)
# Normalization: (x - 0.5) / 0.5 = 2*x - 1
# For input [0, 1], output should be [-1, 1]
assert pixel_values.min() >= -1.0
assert pixel_values.max() <= 1.0
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_chat_template_with_image(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that chat template correctly formats BOS + image + prompt."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = ctx.tokenizer
# Use the chat template format
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
token_ids = processed_inputs["prompt_token_ids"]
# BOS token (<|endoftext|>) should be token ID 0
bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
assert bos_token_id == 0
# First token should be BOS
assert token_ids[0] == bos_token_id
@pytest.mark.parametrize(
"content",
[
pytest.param(
[
{
"type": "image_url",
"image_url": {"url": "https://example.invalid/image.png"},
},
{"type": "text", "text": "What is in this image?"},
],
id="image-first",
),
pytest.param(
[
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {"url": "https://example.invalid/image.png"},
},
],
id="text-first",
),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_chat_template_content_list_uses_moondream_image_prefix(
image_assets: ImageTestAssets,
content: list[dict[str, object]],
model_id: str,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
prompt = hf_processor.tokenizer.apply_chat_template(
[{"role": "user", "content": content}],
chat_template=hf_processor.chat_template,
tokenize=False,
)
expected_prompt = (
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
"What is in this image?<|md_reserved_2|>"
)
assert prompt == expected_prompt
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_bos_token_always_first(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that BOS token (ID 0) is always at position 0."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Start with BOS token explicitly
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
token_ids = processed_inputs["prompt_token_ids"]
# Token ID 0 (<|endoftext|>) should be the first token
assert token_ids[0] == 0, (
f"Expected BOS token (0) at position 0, got {token_ids[0]}"
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_with_small_image(
model_id: str,
):
"""Test processor with image smaller than crop size."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Small image (smaller than crop size)
small_image = Image.new("RGB", (100, 100), color="yellow")
pixel_values, tiling = processor.preprocess_image(small_image)
# Small images should use 1x1 tiling
assert tiling == (1, 1)
# Should have 2 crops (global + 1 local)
assert pixel_values.shape[0] == 2
@pytest.mark.parametrize(
"image_kind",
[
pytest.param("numpy_hwc", id="numpy-hwc"),
pytest.param("numpy_chw", id="numpy-chw"),
pytest.param("torch_chw", id="torch-chw"),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_preprocess_image_accepts_non_pil_inputs(
image_assets: ImageTestAssets,
image_kind: str,
model_id: str,
):
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
pil_image = image_assets[0].pil_image.convert("RGB")
hwc_array = np.asarray(pil_image)
expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
if image_kind == "numpy_hwc":
image = hwc_array
elif image_kind == "numpy_chw":
image = np.transpose(hwc_array, (2, 0, 1))
else:
image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
pixel_values, tiling = processor.preprocess_image(image)
assert tiling == expected_tiling
assert pixel_values.shape == expected_pixel_values.shape
assert pixel_values.dtype == torch.bfloat16
assert torch.equal(pixel_values, expected_pixel_values)
@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_apply_accepts_non_pil_image_inputs(
image_assets: ImageTestAssets,
image_kind: str,
model_id: str,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data({"image": [image]}),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
class TestMoondream3TilingLogic:
"""CPU-based tests for Moondream3 tiling selection logic.
These tests validate the select_tiling() function which determines
how images are divided into crops for the vision encoder.
"""
def test_small_image_no_tiling(self):
"""Small images should use 1x1 tiling."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
assert tiling == (1, 1)
def test_exact_crop_size(self):
"""Image exactly at crop size should use 1x1."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
assert tiling == (1, 1)
def test_large_square_image(self):
"""Large square image should use multiple tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert h_tiles >= 2
assert w_tiles >= 2
assert h_tiles * w_tiles <= MAX_CROPS
def test_wide_image(self):
"""Wide image should have more width tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert w_tiles >= h_tiles
def test_tall_image(self):
"""Tall image should have more height tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert h_tiles >= w_tiles
def test_respects_max_crops(self):
"""Tiling should not exceed max_crops."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
)
h_tiles, w_tiles = tiling
assert h_tiles * w_tiles <= 4
class TestMoondream3VisionShapes:
"""CPU-based tests for vision encoder expected shapes.
These tests verify the mathematical relationships between
crop size, patch size, and token counts.
"""
def test_expected_patch_count(self):
"""Test 378/14 = 27 patches per side, 729 total."""
patches_per_side = CROP_SIZE // PATCH_SIZE
total_patches = patches_per_side**2
assert patches_per_side == 27
assert total_patches == EXPECTED_IMAGE_TOKENS - 1
def test_patch_embedding_input_dim(self):
"""Test patch embedding input dimension."""
channels = 3
input_dim = PATCH_SIZE * PATCH_SIZE * channels
assert input_dim == 14 * 14 * 3
assert input_dim == 588
class TestMoondream3TauAttention:
"""CPU-based tests for tau attention scaling components.
These tests validate the tau attention formula used in Moondream3:
- Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
- Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
"""
def test_tau_position_range(self):
"""Test tau position scaling produces values in valid range."""
num_heads = 32
seq_len = 100
tau_alpha = torch.randn(num_heads)
positions = torch.arange(seq_len)
pos_float = (positions.float() + 1.0).clamp(min=1e-6)
pos_log = pos_float.log()
tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
assert tau_pos.shape == (num_heads, seq_len)
# tau_pos should be between 0.5 and 1.5
assert tau_pos.min() >= 0.5
assert tau_pos.max() <= 1.5
def test_tau_token_output_range(self):
"""Test tau token scaling output is bounded by tanh."""
import torch.nn.functional as F
seq_len = 100
qkv_dim = 6144 # 2048 * 3
num_heads = 32
qkv = torch.randn(seq_len, qkv_dim)
tau_wq = torch.randn(num_heads, qkv_dim)
tok_feat = F.gelu(qkv)
tok_q = torch.tanh(tok_feat @ tau_wq.t())
assert tok_q.shape == (seq_len, num_heads)
# tanh output is bounded by [-1, 1]
assert tok_q.min() >= -1.0
assert tok_q.max() <= 1.0
tests/models/multimodal/processing/test_moss_audio.py
+646
View File
@@ -0,0 +1,646 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from transformers import Qwen3Config
from vllm.model_executor.models.interfaces import SupportsLoRA, supports_lora
from vllm.model_executor.models.moss_audio import (
MOSS_AUDIO_BOS_TOKEN,
MOSS_AUDIO_BOS_TOKEN_ID,
MOSS_AUDIO_EOS_TOKEN,
MOSS_AUDIO_EOS_TOKEN_ID,
MOSS_AUDIO_PLACEHOLDER,
MOSS_AUDIO_TOKEN,
MOSS_AUDIO_TOKEN_ID,
MossAudioConfig,
MossAudioDummyInputsBuilder,
MossAudioEncoder,
MossAudioEncoderConfig,
MossAudioModel,
MossAudioMultiModalProcessor,
MossAudioProcessingInfo,
MossAudioProcessor,
MossQwen3ForCausalLM,
MossQwen3Model,
)
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.sequence import IntermediateTensors
class _Tokenizer:
def encode(self, text, add_special_tokens=False):
del add_special_tokens
return [ord(char) for char in text]
def decode(self, token_ids, **kwargs):
del kwargs
return "".join(chr(token_id) for token_id in token_ids)
def batch_decode(self, batch_token_ids, **kwargs):
return [self.decode(token_ids, **kwargs) for token_ids in batch_token_ids]
class _MMConfig:
enable_mm_embeds = False
mm_processor_cache_gb = 1
def merge_mm_processor_kwargs(self, kwargs):
return dict(kwargs)
def get_limit_per_prompt(self, modality):
del modality
return 3
class _ModelConfig:
def __init__(self):
self.model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
self.revision = None
self.max_model_len = 4096
self.encoder_config = {}
self.dtype = torch.float32
self.hf_config = MossAudioConfig(language_config=Qwen3Config())
self.multimodal_config = _MMConfig()
def get_multimodal_config(self):
return self.multimodal_config
def get_inputs_embeds_size(self):
return None
class _ProcessingContext:
def __init__(self):
self.model_config = _ModelConfig()
self.tokenizer = _Tokenizer()
def get_tokenizer(self):
return self.tokenizer
def get_hf_config(self):
return self.model_config.hf_config
def get_mm_config(self):
return self.model_config.get_multimodal_config()
def get_merged_mm_kwargs(self, kwargs):
return self.get_mm_config().merge_mm_processor_kwargs(kwargs)
def call_hf_processor(self, hf_processor, data, kwargs):
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
return hf_processor(**data, **merged_kwargs)
class _TestMossAudioProcessingInfo(MossAudioProcessingInfo):
def _get_processor_config_defaults(self):
return {}
def _vllm_config(tensor_parallel_size=1, pipeline_parallel_size=1, hf_config=None):
if hf_config is None:
hf_config = MossAudioConfig(language_config=Qwen3Config())
return SimpleNamespace(
model_config=SimpleNamespace(
hf_config=hf_config,
multimodal_config=None,
),
quant_config=None,
parallel_config=SimpleNamespace(
tensor_parallel_size=tensor_parallel_size,
pipeline_parallel_size=pipeline_parallel_size,
),
)
class _FakeAudioEncoder:
dtype = torch.float32
def __init__(self, deepstack_layers=0):
self.deepstack_layers = deepstack_layers
self.output_deepstack_hidden_states = None
self.input_shape = None
self.feature_lens = None
def __call__(self, audio_data, *, feature_lens, output_deepstack_hidden_states):
self.input_shape = tuple(audio_data.shape)
self.feature_lens = feature_lens.detach().cpu().clone()
self.output_deepstack_hidden_states = output_deepstack_hidden_states
lengths = MossAudioEncoder._compute_downsampled_length(feature_lens)
hidden_states = torch.ones(1, int(lengths.sum().item()), 8)
if not output_deepstack_hidden_states:
return hidden_states, None
return hidden_states, [
hidden_states * scale for scale in range(2, 2 + self.deepstack_layers)
]
def _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=1, tp_rank=0):
import vllm.model_executor.layers.linear as linear_layers
import vllm.model_executor.models.moss_audio as moss_audio_module
import vllm.model_executor.parameter as parameter_module
for module in (moss_audio_module, linear_layers, parameter_module):
monkeypatch.setattr(
module, "get_tensor_model_parallel_world_size", lambda: tp_size
)
monkeypatch.setattr(
linear_layers, "get_tensor_model_parallel_rank", lambda: tp_rank
)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_rank", lambda: tp_rank
)
monkeypatch.setattr(
linear_layers, "tensor_model_parallel_all_reduce", lambda tensor: tensor
)
def _build_moss_audio_processor(cache=None):
ctx = _ProcessingContext()
info = _TestMossAudioProcessingInfo(ctx)
return (
MossAudioMultiModalProcessor(
info,
MossAudioDummyInputsBuilder(info),
cache=cache,
),
ctx,
)
def _assert_mm_inputs_equal(left, right):
assert left["prompt_token_ids"] == right["prompt_token_ids"]
assert left["mm_hashes"] == right["mm_hashes"]
left_placeholder = left["mm_placeholders"]["audio"][0]
right_placeholder = right["mm_placeholders"]["audio"][0]
assert left_placeholder.offset == right_placeholder.offset
assert left_placeholder.length == right_placeholder.length
assert left_placeholder.is_embed.tolist() == right_placeholder.is_embed.tolist()
assert batched_tensors_equal(
left["mm_kwargs"].get_data(),
right["mm_kwargs"].get_data(),
)
@pytest.mark.parametrize(
("prompt", "prefix"),
[
(
f"before {MOSS_AUDIO_PLACEHOLDER} after",
[*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
),
(
f"before {MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}"
f"{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN} after",
[*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
),
("Describe this audio.", [MOSS_AUDIO_BOS_TOKEN_ID]),
],
)
def test_moss_audio_processor_expands_audio_placeholders(prompt, prefix):
raw_mel_len = 17
processed = MossAudioProcessor(_Tokenizer())(
text=prompt, audio=[torch.zeros(160 * raw_mel_len)]
)
input_ids = processed["input_ids"][0].tolist()
assert input_ids[: len(prefix)] == prefix
assert input_ids.count(MOSS_AUDIO_BOS_TOKEN_ID) == 1
assert input_ids.count(MOSS_AUDIO_EOS_TOKEN_ID) == 1
assert input_ids.count(MOSS_AUDIO_TOKEN_ID) == (
MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
)
assert processed["audio_data"].shape == (1, 128, raw_mel_len)
assert processed["audio_data_seqlens"].tolist() == [raw_mel_len]
def test_moss_audio_processor_preserves_placeholder_without_audio():
processed = MossAudioProcessor(_Tokenizer())(
text=f"before {MOSS_AUDIO_PLACEHOLDER} after"
)
assert processed["input_ids"][0].tolist() == [
*[ord(char) for char in "before "],
MOSS_AUDIO_BOS_TOKEN_ID,
MOSS_AUDIO_TOKEN_ID,
MOSS_AUDIO_EOS_TOKEN_ID,
*[ord(char) for char in " after"],
]
assert "audio_data" not in processed
assert "audio_data_seqlens" not in processed
def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
raw_mel_len = 17
audio = np.zeros(160 * raw_mel_len, dtype=np.float32)
prompt = f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."
baseline_processor, ctx = _build_moss_audio_processor()
mm_items = baseline_processor.info.parse_mm_data({"audio": [audio]})
token_prompt = ctx.get_tokenizer().encode(prompt, add_special_tokens=False)
baseline_text = baseline_processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
baseline_token = baseline_processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
cached_processor, _ = _build_moss_audio_processor(cache=cache)
cached_text_miss = cached_processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_text_hit = cached_processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_token_hit = cached_processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
prompt_token_ids = baseline_text["prompt_token_ids"]
assert prompt_token_ids.count(MOSS_AUDIO_TOKEN_ID) == expected_audio_tokens
assert baseline_text["mm_placeholders"]["audio"][0].length == (
expected_audio_tokens + 2
)
_assert_mm_inputs_equal(baseline_text, baseline_token)
_assert_mm_inputs_equal(baseline_text, cached_text_miss)
_assert_mm_inputs_equal(baseline_text, cached_text_hit)
_assert_mm_inputs_equal(baseline_text, cached_token_hit)
def test_moss_audio_supports_language_model_lora_only():
assert supports_lora(MossAudioModel)
model = object.__new__(MossAudioModel)
assert isinstance(model, SupportsLoRA)
mapping = model.get_mm_mapping()
assert mapping.language_model == ["language_model."]
assert mapping.tower_model == []
assert mapping.connector == []
def test_moss_audio_error_paths():
model = object.__new__(MossAudioModel)
with pytest.raises(ValueError, match="DeepStack audio token count mismatch"):
model._cache_deepstack_input_embeds(
inputs_embeds=torch.zeros(4, 8),
deepstack_embeddings=((torch.ones(1, 8),),),
is_multimodal=torch.tensor([False, True, True, False]),
)
with pytest.raises(ValueError, match="too short"):
MossAudioProcessor(_Tokenizer())(
text=MOSS_AUDIO_PLACEHOLDER, audio=[torch.empty(0)]
)
with pytest.raises(ValueError, match="too short"):
model._parse_and_validate_audio_input(
audio_data=torch.zeros(1, 128, 1),
audio_data_seqlens=torch.tensor([0], dtype=torch.long),
)
def test_moss_audio_validates_tp_config():
vllm_config = _vllm_config(tensor_parallel_size=2)
vllm_config.model_config.hf_config.adapter_hidden_size = 7
with pytest.raises(ValueError, match="adapter_hidden_size"):
MossAudioModel(vllm_config=vllm_config)
vllm_config = _vllm_config(tensor_parallel_size=2)
vllm_config.model_config.hf_config.audio_config.d_model = 6
vllm_config.model_config.hf_config.audio_config.encoder_attention_heads = 3
with pytest.raises(ValueError, match="encoder_attention_heads"):
MossAudioModel(vllm_config=vllm_config)
def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch():
model = object.__new__(MossAudioModel)
with pytest.raises(ValueError, match="audio_data batch size"):
model._parse_and_validate_audio_input(
audio_data=[torch.zeros(128, 8), torch.zeros(128, 11)],
audio_data_seqlens=torch.tensor([8], dtype=torch.long),
)
@pytest.mark.parametrize("deepstack_scales", [(), (7, 11)])
def test_moss_audio_embed_multimodal_packs_by_audio(deepstack_scales):
model = object.__new__(MossAudioModel)
model.audio_encoder = _FakeAudioEncoder(len(deepstack_scales))
model.audio_adapter = lambda hidden_states: hidden_states * 5
model.deepstack_audio_merger_list = [
lambda hidden_states, scale=scale: hidden_states * scale
for scale in deepstack_scales
]
model.deepstack_input_embeds = None
embeddings = model.embed_multimodal(
audio_data=torch.zeros(2, 128, 9),
audio_data_seqlens=torch.tensor([8, 9], dtype=torch.long),
)
assert model.audio_encoder.output_deepstack_hidden_states is bool(deepstack_scales)
assert [embeds.shape for embeds in embeddings] == [
torch.Size([1, 8 * (1 + len(deepstack_scales))]),
torch.Size([2, 8 * (1 + len(deepstack_scales))]),
]
if not deepstack_scales:
assert model.deepstack_input_embeds is None
return
main_embeddings, deepstack_embeddings = model._split_multimodal_embeddings(
embeddings, hidden_size=8
)
assert [embeds.shape for embeds in main_embeddings] == [
torch.Size([1, 8]),
torch.Size([2, 8]),
]
assert [[e.shape for e in layer] for layer in deepstack_embeddings] == [
[torch.Size([1, 8]), torch.Size([2, 8])] for _ in deepstack_scales
]
assert torch.equal(main_embeddings[0], torch.full((1, 8), 5.0))
for idx, scale in enumerate(deepstack_scales):
assert torch.equal(
deepstack_embeddings[idx][0],
torch.full((1, 8), float((idx + 2) * scale)),
)
def test_moss_audio_embed_input_ids_caches_packed_deepstack():
class _FakeLanguageModel:
def embed_input_ids(self, input_ids):
return torch.zeros(input_ids.shape[0], 8)
model = object.__new__(MossAudioModel)
model.language_model = _FakeLanguageModel()
model.deepstack_audio_merger_list = [object(), object()]
model.deepstack_input_embeds = None
multimodal_embeddings = (
torch.cat([torch.full((1, 8), x) for x in (5.0, 14.0, 33.0)], dim=-1),
torch.cat([torch.full((2, 8), x) for x in (7.0, 22.0, 44.0)], dim=-1),
)
is_multimodal = torch.tensor([False, True, True, True, False])
inputs_embeds = model.embed_input_ids(
input_ids=torch.arange(5),
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
assert torch.equal(inputs_embeds[1], torch.full((8,), 5.0))
assert torch.equal(inputs_embeds[2], torch.full((8,), 7.0))
assert torch.equal(inputs_embeds[3], torch.full((8,), 7.0))
assert model.deepstack_input_embeds is not None
tensors = model.deepstack_input_embeds.tensors
assert set(tensors) == {"deepstack_input_embeds_0", "deepstack_input_embeds_1"}
for tensor in tensors.values():
assert tensor[is_multimodal].abs().sum() > 0
assert torch.equal(tensor[~is_multimodal], torch.zeros(2, 8))
def _patch_pp_group(monkeypatch, *, first=True, last=True):
import vllm.model_executor.models.moss_audio as moss_audio_module
monkeypatch.setattr(
moss_audio_module,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=first, is_last_rank=last),
)
def test_moss_audio_pp_forward_routes_deepstack(monkeypatch):
for first in (True, False):
calls: list[dict[str, object]] = []
def fake_lm(*args, _calls=calls, **kwargs):
del args
_calls.append(kwargs)
return torch.ones(1, 1)
_patch_pp_group(monkeypatch, first=first)
model = object.__new__(MossAudioModel)
torch.nn.Module.__init__(model)
model.language_model = fake_lm
cached = IntermediateTensors({"deepstack_input_embeds_0": torch.ones(3, 8)})
inter = IntermediateTensors(
{
"hidden_states": torch.ones(3, 8),
"residual": torch.zeros(3, 8),
"deepstack_input_embeds_0": torch.full((3, 8), 5.0),
}
)
inputs_embeds = torch.full((3, 8), 9.0)
model.deepstack_input_embeds = cached
model.forward(
input_ids=None,
positions=torch.arange(3),
intermediate_tensors=None if first else inter,
inputs_embeds=inputs_embeds if first else None,
)
kwargs = calls[0]
assert kwargs["inputs_embeds"] is (inputs_embeds if first else None)
assert kwargs["deepstack_input_embeds"] is (cached if first else inter)
assert model.deepstack_input_embeds is None
calls = []
def fake_lm_non_first_rank(*args, **kwargs):
del args
calls.append(kwargs)
return torch.ones(1, 1)
_patch_pp_group(monkeypatch, first=False)
model = object.__new__(MossAudioModel)
torch.nn.Module.__init__(model)
model.language_model = fake_lm_non_first_rank
model.deepstack_input_embeds = IntermediateTensors({})
inter = IntermediateTensors(
{
"hidden_states": torch.ones(3, 8),
"residual": torch.zeros(3, 8),
}
)
model.forward(
input_ids=None,
positions=torch.arange(3),
intermediate_tensors=inter,
inputs_embeds=torch.ones(3, 8),
)
assert calls[0]["inputs_embeds"] is None
assert calls[0]["deepstack_input_embeds"] is inter
def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch):
class AddOne(torch.nn.Module):
def forward(self, positions, hidden_states, residual):
del positions, residual
return hidden_states + 1, torch.zeros_like(hidden_states)
def make_model(num_layers, deepstack_layers=None):
model = object.__new__(MossQwen3Model)
torch.nn.Module.__init__(model)
model.start_layer, model.end_layer = 0, num_layers
model.layers = torch.nn.ModuleList([AddOne() for _ in range(num_layers)])
model.norm = lambda hidden_states, residual: (hidden_states, residual)
model._maybe_add_hidden_state = lambda aux, *args: aux
model.deepstack_inject_layer_indices = (
range(0) if deepstack_layers is None else deepstack_layers
)
return model
_patch_pp_group(monkeypatch, first=True, last=True)
output = make_model(3).forward(
input_ids=None,
positions=torch.arange(2),
inputs_embeds=torch.zeros(2, 4),
deepstack_input_embeds=IntermediateTensors(
{
"deepstack_input_embeds_2": torch.full((2, 4), 5.0),
}
),
)
assert torch.equal(output, torch.full((2, 4), 8.0))
_patch_pp_group(monkeypatch, first=True, last=False)
deepstack = IntermediateTensors(
{
"deepstack_input_embeds_0": torch.full((2, 4), 7.0),
"deepstack_input_embeds_3": torch.full((2, 4), 11.0),
}
)
output = make_model(2, range(4)).forward(
input_ids=None,
positions=torch.arange(2),
inputs_embeds=torch.zeros(2, 4),
deepstack_input_embeds=deepstack,
)
assert isinstance(output, IntermediateTensors)
assert set(output.tensors) == {
"hidden_states",
"residual",
"deepstack_input_embeds_2",
"deepstack_input_embeds_3",
}
assert torch.equal(output["hidden_states"], torch.full((2, 4), 9.0))
assert torch.equal(output["deepstack_input_embeds_2"], torch.zeros(2, 4))
assert output["deepstack_input_embeds_3"] is deepstack["deepstack_input_embeds_3"]
inner_model = make_model(0, range(2))
inner_model.make_empty_intermediate_tensors = lambda batch, dtype, device: (
IntermediateTensors(
{
"hidden_states": torch.zeros(batch, 4, dtype=dtype, device=device),
"residual": torch.zeros(batch, 4, dtype=dtype, device=device),
}
)
)
language_model = object.__new__(MossQwen3ForCausalLM)
torch.nn.Module.__init__(language_model)
language_model.model = inner_model
language_model.config = SimpleNamespace(hidden_size=4)
language_model.deepstack_inject_layer_indices = range(2)
tensors = MossQwen3ForCausalLM.make_empty_intermediate_tensors(
language_model,
batch_size=3,
dtype=torch.float16,
device=torch.device("cpu"),
)
assert set(tensors.tensors) == {
"hidden_states",
"residual",
"deepstack_input_embeds_0",
"deepstack_input_embeds_1",
}
assert tensors["deepstack_input_embeds_0"].shape == (3, 4)
assert tensors["deepstack_input_embeds_0"].dtype == torch.float16
_patch_pp_group(monkeypatch, first=True, last=False)
forward_tensors = inner_model.forward(
input_ids=None,
positions=torch.arange(3),
inputs_embeds=torch.ones(3, 4, dtype=torch.float16),
deepstack_input_embeds=None,
)
assert isinstance(forward_tensors, IntermediateTensors)
assert set(forward_tensors.tensors) == set(tensors.tensors)
def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch):
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.device import DeviceConfig
_patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=2)
config = MossAudioEncoderConfig(
d_model=8,
output_dim=8,
num_mel_bins=8,
encoder_layers=1,
encoder_attention_heads=2,
encoder_ffn_dim=16,
downsample_hidden_size=2,
deepstack_encoder_layer_indexes=[],
)
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
encoder = MossAudioEncoder(config)
attention = encoder.layers[0].self_attn
assert all(hasattr(attention, name) for name in ("q_proj", "k_proj", "v_proj"))
assert hasattr(attention, "out_proj")
assert not hasattr(attention, "qkv")
assert attention.k_proj.bias is None
weight_names = [
"layers.0.self_attn.q_proj.weight",
"layers.0.self_attn.q_proj.bias",
"layers.0.self_attn.k_proj.weight",
"layers.0.self_attn.v_proj.weight",
"layers.0.self_attn.v_proj.bias",
"layers.0.self_attn.out_proj.weight",
"layers.0.self_attn.out_proj.bias",
"conv1.weight",
"conv1.bias",
]
params = dict(encoder.named_parameters(remove_duplicate=False))
assert "layers.0.self_attn.k_proj.bias" not in params
weights = {
name: torch.full_like(params[name], fill_value=float(i + 1))
for i, name in enumerate(weight_names)
}
loaded = AutoWeightsLoader(encoder).load_weights(weights.items())
assert "load_weights" not in MossAudioEncoder.__dict__
assert loaded == set(weight_names)
assert not any(".qkv." in name for name in loaded)
assert torch.equal(
params["layers.0.self_attn.q_proj.weight"],
weights["layers.0.self_attn.q_proj.weight"],
)
tests/models/multimodal/processing/test_nemotron_vl.py
+137
View File
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs."""
from collections.abc import Mapping
import pytest
from PIL import Image
from transformers import PretrainedConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.image import rescale_image_size
from vllm.multimodal.processing import BaseMultiModalProcessor
from ....conftest import ImageTestAssets
from ...utils import build_model_context
def _get_expected_num_patches(
config: PretrainedConfig,
image: Image.Image,
num_imgs: int,
min_num: int,
max_num: int,
):
from vllm.transformers_utils.processors.nemotron_vl import (
calculate_nemotron_vl_targets,
get_nemotron_vl_target_ratios,
)
width, height = image.size
blocks, _, _ = calculate_nemotron_vl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_nemotron_vl_target_ratios(
min_num,
max_num,
),
image_size=config.force_image_size,
use_thumbnail=False,
)
expected_num_patches = blocks
if config.use_thumbnail and expected_num_patches > 1:
expected_num_patches += 1
return expected_num_patches
def _run_check(
processor: BaseMultiModalProcessor,
images: list[Image.Image],
min_num: int,
max_num: int,
mm_processor_kwargs: Mapping[str, object],
):
tokenizer = processor.info.get_tokenizer()
config = processor.info.get_hf_config()
image_processor = processor.info.get_image_processor()
config.use_thumbnail = image_processor.use_thumbnail
prompt = "<image>" * len(images)
mm_data = {"image": images}
total_expected_num_patches = sum(
_get_expected_num_patches(config, image, len(images), min_num, max_num)
for image in images
)
print(total_expected_num_patches)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("<image>")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@pytest.mark.parametrize("model_id", ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"])
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
[4.0, 2.0, 1.0],
],
)
@pytest.mark.parametrize(
("min_dynamic_patch", "max_dynamic_patch"),
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
)
@pytest.mark.parametrize("dynamic_image_size", [True, False])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
model_id: str,
image_assets: ImageTestAssets,
size_factors: list[int],
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool | None,
kwargs_on_init: bool,
):
mm_processor_kwargs = {
"min_dynamic_patch": min_dynamic_patch,
"max_dynamic_patch": max_dynamic_patch,
"dynamic_image_size": dynamic_image_size,
}
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": len(size_factors)},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
min_num = min_dynamic_patch if dynamic_image_size else 1
max_num = max_dynamic_patch if dynamic_image_size else 1
_run_check(
processor,
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
min_num,
max_num,
hf_processor_mm_kwargs,
)
tests/models/multimodal/processing/test_openvla.py
+210
View File
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for OpenVLA multimodal preprocessing."""
import numpy as np
import pytest
import torch
from PIL import Image
from transformers import LlamaConfig
from vllm.model_executor.models.openvla import (
OpenVLAForActionPrediction,
OpenVLAMultiModalProcessor,
OpenVLAProcessingInfo,
)
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
from vllm.transformers_utils.configs.openvla import OpenVLAConfig
from vllm.transformers_utils.processors.openvla import (
IMAGENET_MEAN,
IMAGENET_STD,
SIGLIP_MEAN,
SIGLIP_STD,
OpenVLAImageProcessor,
OpenVLAProcessor,
preprocess_openvla_image,
to_rgb_image,
)
pytestmark = pytest.mark.cpu_test
class _FakeTokenizer:
bos_token_id = 1
init_kwargs: dict[str, object] = {}
def encode(self, prompt: str, **kwargs: object) -> list[int]:
assert prompt == "In: test\nOut:"
if kwargs == {"add_special_tokens": True}:
return [self.bos_token_id, 10, 11]
assert kwargs == {"add_special_tokens": False}
return [10, 11]
def __call__(self, text: str, **kwargs: object) -> dict[str, list[list[int]]]:
return {"input_ids": [self.encode(text, **kwargs)]}
class _FakeProcessingInfo:
def __init__(self) -> None:
self.config = OpenVLAConfig()
def get_hf_config(self) -> OpenVLAConfig:
return self.config
def get_tokenizer(self) -> _FakeTokenizer:
return _FakeTokenizer()
def get_num_image_tokens(self, *, image_width: int, image_height: int) -> int:
assert image_width > 0
assert image_height > 0
return 256
class _FakeOpenVLAProcessingInfo(OpenVLAProcessingInfo):
def get_hf_config(self) -> OpenVLAConfig:
return OpenVLAConfig()
def _make_processor() -> OpenVLAMultiModalProcessor:
processor = OpenVLAMultiModalProcessor.__new__(OpenVLAMultiModalProcessor)
processor.info = _FakeProcessingInfo()
return processor
def test_openvla_config_converts_text_config_dict() -> None:
config = OpenVLAConfig(
text_config={
"vocab_size": 123,
"hidden_size": 64,
"intermediate_size": 128,
"num_hidden_layers": 2,
"num_attention_heads": 4,
},
)
assert isinstance(config.text_config, LlamaConfig)
assert config.text_config.vocab_size == 123
assert config.text_config.hidden_size == 64
assert config.text_config.architectures == ["LlamaForCausalLM"]
@pytest.mark.parametrize(
("image", "expected_size", "expected_pixel"),
[
(
torch.tensor(
[
[[1.0, 1.0], [1.0, 1.0]],
[[0.0, 0.0], [0.0, 0.0]],
[[0.0, 0.0], [0.0, 0.0]],
]
),
(2, 2),
(255, 0, 0),
),
(
np.full((4, 5, 1), 128, dtype=np.uint8),
(5, 4),
(128, 128, 128),
),
],
)
def test_openvla_to_rgb_image(
image: torch.Tensor | np.ndarray,
expected_size: tuple[int, int],
expected_pixel: tuple[int, int, int],
) -> None:
rgb_image = to_rgb_image(image)
assert rgb_image.mode == "RGB"
assert rgb_image.size == expected_size
assert rgb_image.getpixel((0, 0)) == expected_pixel
def test_openvla_preprocess_image_matches_expected_normalization() -> None:
image = Image.fromarray(
np.arange(12 * 10 * 3, dtype=np.uint8).reshape(10, 12, 3),
mode="RGB",
)
pixel_values = preprocess_openvla_image(image, image_size=224)
resized = image.resize((224, 224), Image.Resampling.BICUBIC)
raw = np.asarray(resized, dtype=np.float32) / 255.0
expected_dinov2 = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1)
expected_siglip = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1)
expected = np.concatenate([expected_dinov2, expected_siglip], axis=0)
assert pixel_values.shape == (6, 224, 224)
assert pixel_values.dtype == torch.float32
torch.testing.assert_close(pixel_values, torch.from_numpy(expected))
def test_openvla_processor_outputs_pixel_values() -> None:
processor = OpenVLAProcessor(
image_processor=OpenVLAImageProcessor(image_size=224),
tokenizer=_FakeTokenizer(),
)
image = Image.new("RGB", (8, 8), color=(255, 0, 0))
batch = processor(
text="In: test\nOut:",
images=image,
text_kwargs={"add_special_tokens": True},
)
assert batch["input_ids"] == [[1, 10, 11]]
assert batch["pixel_values"].shape == (1, 6, 224, 224)
assert batch["pixel_values"].dtype == torch.float32
def test_openvla_image_processor_outputs_pixel_values() -> None:
processor = OpenVLAImageProcessor(image_size=224)
image = Image.new("RGB", (8, 8), color=(255, 0, 0))
output = processor(images=image)
assert output["pixel_values"].shape == (1, 6, 224, 224)
assert output["pixel_values"].dtype == torch.float32
def test_openvla_processing_info_token_counts() -> None:
info = _FakeOpenVLAProcessingInfo.__new__(_FakeOpenVLAProcessingInfo)
assert info.get_supported_mm_limits() == {"image": 1}
assert info.get_num_image_tokens(image_width=640, image_height=480) == 256
assert info.get_image_size_with_most_features().width == 224
assert info.get_image_size_with_most_features().height == 224
assert info.get_mm_max_tokens_per_item(seq_len=2048, mm_counts={"image": 1}) == {
"image": 256
}
def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
processor = _make_processor()
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
assert (
processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})
is False
)
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
resolved = prompt_update.resolve(0)
content = resolved.content
assert resolved.modality == "image"
assert [
(match.start_idx, match.end_idx)
for match in resolved.iter_matches([1, 10, 11], _FakeTokenizer())
] == [(1, 1)]
assert content.full == [32000] * 256
is_embed = content.is_embed(None, content.full)
assert is_embed.dtype == torch.bool
assert is_embed.tolist() == [True] * 256
def test_openvla_placeholder_string() -> None:
assert OpenVLAForActionPrediction.get_placeholder_str("image", 0) is None
tests/models/multimodal/processing/test_phi3v.py
+58
View File
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for phi3v's multimodal preprocessing kwargs."""
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"num_crops": 4}, 757),
({"num_crops": 16}, 1921),
# the default num_crops of phi-3.5-vision is 4
({}, 757),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, int],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Phi3VMultiModalProcessor handles num_crops properly."""
# Avoid initializing CUDA early
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/multimodal/processing/test_phi4mm.py
+64
View File
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for phi4mm's multimodal preprocessing kwargs."""
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["microsoft/Phi-4-multimodal-instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"dynamic_hd": 4}, 1329),
({"dynamic_hd": 16}, 4433),
# the default num_crops of phi-4-multimodal is 36
({}, 9585),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, int],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Phi4MMMultiModalProcessor handles dynamic_hd properly."""
# Avoid initializing CUDA early
from vllm.model_executor.models.phi4mm import _IMAGE_PLACEHOLDER_TOKEN_ID
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
image_size = ctx.get_hf_config().embd_layer["image_embd_layer"]["crop_size"]
dummy_image_size = (image_size * 7, image_size * 7)
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
mm_data = {"image": [dummy_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
img_tok_count = processed_inputs["prompt_token_ids"].count(
_IMAGE_PLACEHOLDER_TOKEN_ID
)
assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/multimodal/processing/test_qwen2_5_omni_embed.py
+386
View File
@@ -0,0 +1,386 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
correctly assigned to audio/image/video token positions.
Regression test for: https://github.com/vllm-project/vllm/issues/34506
- Non-interleaved mixed modalities (audio + image + video) should correctly
assign audio embeddings to audio positions, image to image, video to video.
- Interleaved (use_audio_in_video) should also work correctly.
"""
from unittest.mock import Mock
import pytest
import torch
from vllm.model_executor.models.qwen2_5_omni_thinker import (
check_interleaved_audio_video,
merge_interleaved_embeddings,
)
# Fake token IDs
AUDIO_TOKEN_ID = 1001
IMAGE_TOKEN_ID = 1002
VIDEO_TOKEN_ID = 1003
TEXT_TOKEN_ID = 0
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def make_token_seq(
audio_n: int, image_n: int, video_n: int, text_prefix: int = 3, text_sep: int = 2
):
"""
Build a flat token sequence:
[text_prefix] [AUDIO * audio_n] [text_sep] [IMAGE * image_n]
[text_sep] [VIDEO * video_n] [text_sep]
Returns (input_ids tensor, is_multimodal mask, positions dict).
"""
tokens = (
[TEXT_TOKEN_ID] * text_prefix
+ [AUDIO_TOKEN_ID] * audio_n
+ [TEXT_TOKEN_ID] * text_sep
+ [IMAGE_TOKEN_ID] * image_n
+ [TEXT_TOKEN_ID] * text_sep
+ [VIDEO_TOKEN_ID] * video_n
+ [TEXT_TOKEN_ID] * text_sep
)
input_ids = torch.tensor(tokens)
is_multimodal = (
(input_ids == AUDIO_TOKEN_ID)
| (input_ids == IMAGE_TOKEN_ID)
| (input_ids == VIDEO_TOKEN_ID)
)
return input_ids, is_multimodal
def make_interleaved_seq(
video_chunks: list[int], audio_chunks: list[int], text_prefix: int = 2
):
"""
Build an interleaved sequence like use_audio_in_video:
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
"""
tokens = [TEXT_TOKEN_ID] * text_prefix
for v, a in zip(video_chunks, audio_chunks):
tokens += [VIDEO_TOKEN_ID] * v + [AUDIO_TOKEN_ID] * a
input_ids = torch.tensor(tokens)
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
return input_ids, is_multimodal
# ---------------------------------------------------------------------------
# Tests for check_interleaved_audio_video
# ---------------------------------------------------------------------------
class TestCheckInterleavedAudioVideo:
def test_non_interleaved_audio_then_video(self):
"""Audio entirely before video → not interleaved."""
input_ids, is_multimodal = make_token_seq(5, 0, 4)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_non_interleaved_with_image(self):
"""Audio + image + video (the mixed_modalities case) → not interleaved."""
input_ids, is_multimodal = make_token_seq(5, 4, 6)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_no_audio(self):
"""Video only → not interleaved."""
input_ids, is_multimodal = make_token_seq(0, 0, 6)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_interleaved(self):
"""V A V A interleaved → True."""
input_ids, is_multimodal = make_interleaved_seq([4, 4], [3, 3])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_batched_non_interleaved_no_false_positive(self):
"""
Regression test for https://github.com/vllm-project/vllm/issues/35394.
5 identical non-interleaved mixed-modality requests batched together:
each has [audio][image][video] in separate blocks with text between them.
Across the batch, audio from request N falls between video blocks of
request N and request N+1, causing the global ranges to overlap.
check_interleaved_audio_video must return False (not a false positive).
"""
# Build one request: [text][audio*5][text][image*4][text][video*6][text]
single_ids, _ = make_token_seq(5, 4, 6)
# Batch 5 identical requests (separated by text tokens to simulate padding)
sep = torch.tensor([TEXT_TOKEN_ID] * 3)
batched_ids = torch.cat([single_ids, sep] * 5)
is_multimodal = (
(batched_ids == AUDIO_TOKEN_ID)
| (batched_ids == IMAGE_TOKEN_ID)
| (batched_ids == VIDEO_TOKEN_ID)
)
is_video = is_multimodal & (batched_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (batched_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
), "Batched non-interleaved requests should not be detected as interleaved"
# ---------------------------------------------------------------------------
# Tests for embed_input_ids via a minimal mock
# ---------------------------------------------------------------------------
def make_mock_model(hidden: int = 8):
"""
Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
that has enough structure to run embed_input_ids.
"""
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniThinkerForConditionalGeneration,
)
model = Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
# Config with token IDs
cfg = Mock()
cfg.video_token_index = VIDEO_TOKEN_ID
cfg.audio_token_index = AUDIO_TOKEN_ID
model.config = cfg
# embed_input_ids: simply embed each token as a one-hot-like vector
# token_id * ones so we can verify which embedding ends up where.
def fake_lm_embed(ids: torch.Tensor) -> torch.Tensor:
# Use .clone() so the tensor is contiguous (expand() creates a strided
# view with shared memory, which masked_scatter_ cannot handle).
return ids.float().unsqueeze(-1).expand(-1, hidden).clone()
lang_model = Mock()
lang_model.embed_input_ids = fake_lm_embed
model.get_language_model = Mock(return_value=lang_model)
# _embed_text_input_ids: delegate to SupportsMultiModal's implementation
from vllm.model_executor.models.interfaces import SupportsMultiModal
model._embed_text_input_ids = (
lambda *a, **kw: SupportsMultiModal._embed_text_input_ids(model, *a, **kw)
)
# super().embed_input_ids → use SupportsMultiModal.embed_input_ids
def fake_super_embed(
ids,
mm_embs=None,
*,
is_multimodal=None,
):
return SupportsMultiModal.embed_input_ids(
model,
ids,
mm_embs,
is_multimodal=is_multimodal,
)
# Bind embed_input_ids as the real method
model.embed_input_ids = (
lambda *a, **kw: Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids(
model, *a, **kw
)
)
# Store super-embed for use inside the method
model._super_embed_input_ids = fake_super_embed
return model, hidden
def build_mm_embeds(
audio_n, image_n, video_n, hidden, audio_val=10.0, image_val=20.0, video_val=30.0
):
"""
Build multimodal_embeddings list in position order (audio, image, video).
Each embedding is filled with a distinct constant so we can verify placement.
"""
embs = []
if audio_n:
embs.append(torch.full((audio_n, hidden), audio_val))
if image_n:
embs.append(torch.full((image_n, hidden), image_val))
if video_n:
embs.append(torch.full((video_n, hidden), video_val))
return embs
class TestEmbedInputIds:
def _run(self, audio_n, image_n, video_n, hidden=8):
"""
Run embed_input_ids for a non-interleaved mixed-modality sequence.
Returns (result_embeds, input_ids, is_multimodal).
"""
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
mm_embeds = build_mm_embeds(audio_n, image_n, video_n, hidden)
model, _ = make_mock_model(hidden)
result = model.embed_input_ids(
input_ids, mm_embeds, is_multimodal=is_multimodal
)
return result, input_ids, is_multimodal
def test_audio_only(self):
"""Audio-only: audio positions get audio embeddings."""
audio_n, hidden = 5, 8
audio_val = 10.0
result, input_ids, is_multimodal = self._run(audio_n, 0, 0, hidden)
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
"Audio positions should get audio embeddings"
)
def test_video_only(self):
"""Video-only: video positions get video embeddings."""
video_n, hidden = 6, 8
video_val = 30.0
result, input_ids, is_multimodal = self._run(0, 0, video_n, hidden)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
"Video positions should get video embeddings"
)
def test_mixed_modalities_audio_goes_to_audio_pos(self):
"""
Regression test for GitHub issue #34506:
With audio + image + video (non-interleaved), audio positions must
receive audio embeddings (not image or video embeddings).
"""
audio_n, image_n, video_n, hidden = 5, 4, 6, 8
audio_val, image_val, video_val = 10.0, 20.0, 30.0
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
image_pos = (input_ids == IMAGE_TOKEN_ID).nonzero(as_tuple=True)[0]
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
mean_a = result[audio_pos].mean().item()
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
f"Audio emb wrong: expected {audio_val}, got mean={mean_a:.1f}"
)
mean_i = result[image_pos].mean().item()
assert result[image_pos].allclose(torch.full((image_n, hidden), image_val)), (
f"Image emb wrong: expected {image_val}, got mean={mean_i:.1f}"
)
mean_v = result[video_pos].mean().item()
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
f"Video emb wrong: expected {video_val}, got mean={mean_v:.1f}"
)
def test_text_positions_unchanged(self):
"""Text positions should keep their text embeddings."""
audio_n, image_n, video_n, hidden = 3, 2, 4, 8
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
text_pos = (~is_multimodal).nonzero(as_tuple=True)[0]
# Text tokens have value TEXT_TOKEN_ID=0, so embed → 0.0
assert result[text_pos].allclose(torch.zeros(len(text_pos), hidden)), (
"Text positions should keep text embeddings"
)
def test_interleaved_use_audio_in_video(self):
"""
Interleaved (use_audio_in_video): video chunks interleaved with audio.
Video embeddings must go to video positions, audio to audio positions.
"""
hidden = 8
audio_val, video_val = 10.0, 30.0
# Two video chunks of 4, two audio chunks of 3
video_chunks = [4, 4]
audio_chunks = [3, 3]
input_ids, is_multimodal = make_interleaved_seq(video_chunks, audio_chunks)
video_n = sum(video_chunks) # 8
audio_n = sum(audio_chunks) # 6
# mm_embeds come in [video, audio] order (video feature first in
# mm_features when positions are the same for use_audio_in_video)
mm_embeds = [
torch.full((video_n, hidden), video_val),
torch.full((audio_n, hidden), audio_val),
]
model, _ = make_mock_model(hidden)
result = model.embed_input_ids(
input_ids, mm_embeds, is_multimodal=is_multimodal
)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
"Interleaved: video positions should get video embeddings"
)
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
"Interleaved: audio positions should get audio embeddings"
)
# ---------------------------------------------------------------------------
# Tests for merge_interleaved_embeddings helper
# ---------------------------------------------------------------------------
class TestMergeInterleavedEmbeddings:
def test_basic_interleaved(self):
"""Video chunks + audio chunks scattered to correct positions."""
hidden = 4
input_ids, is_multimodal = make_interleaved_seq([3, 3], [2, 2])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
num_video = is_video.sum().item() # 6
num_audio = is_audio.sum().item() # 4
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
torch.full((num_video, hidden), 30.0),
torch.full((num_audio, hidden), 10.0),
]
result = merge_interleaved_embeddings(
inputs_embeds,
mm_embeds,
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
video_pos = is_video.nonzero(as_tuple=True)[0]
audio_pos = is_audio.nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
if __name__ == "__main__":
pytest.main([__file__, "-v"])
tests/models/multimodal/processing/test_qwen2_vl.py
+130
View File
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from packaging.version import Version
from transformers import __version__ as TRANSFORMERS_VERSION
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
[
({}, 1426, (5704, 1176)),
({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
(
{
"size": {
"shortest_edge": 64**2,
"longest_edge": 512**2,
},
},
330,
(1320, 1176),
),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_toks_per_img: int,
expected_pixels_shape: tuple[int, int],
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
if (
Version(TRANSFORMERS_VERSION) < Version("5.2.0")
and "size" in mm_processor_kwargs
):
pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`")
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values"].shape
assert img_tok_count == expected_toks_per_img * num_imgs
assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
assert pixel_shape[1] == expected_pixels_shape[1]
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
@pytest.mark.parametrize(
"mm_processor_kwargs",
[
{"min_pixels": 28 * 28, "max_pixels": 1280 * 28 * 28},
{"min_pixels": 28 * 28, "max_pixels": 1283 * 28 * 28},
{"size": {"shortest_edge": 28 * 28, "longest_edge": 1280 * 28 * 28}},
{"size": {"shortest_edge": 28 * 28, "longest_edge": 1283 * 28 * 28}},
],
)
def test_get_image_size_with_most_features(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
):
if (
Version(TRANSFORMERS_VERSION) < Version("5.2.0")
and "size" in mm_processor_kwargs
):
pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`")
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
max_image_size = processor.info.get_image_size_with_most_features()
max_tokens = processor.info.get_num_image_tokens(
image_width=max_image_size.width,
image_height=max_image_size.height,
image_processor=hf_processor.image_processor,
mm_kwargs=mm_processor_kwargs,
)
prompt = "<|vision_start|><|image_pad|><|vision_end|>"
for asset in image_assets:
mm_data = {"image": [asset.pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
t, h, w = grid_thw[0]
tokens = (t * h * w) // (merge_size**2)
assert tokens < max_tokens
tests/models/multimodal/processing/test_qwen3_omni.py
+112
View File
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Qwen3 Omni audio processing and sample rate handling."""
from typing import Any
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
@pytest.mark.parametrize(
("audio_sample_rate", "audio_duration_sec"),
[
(16000, 1.0), # Native Whisper sample rate, 1 second
(16000, 2.0), # Native Whisper sample rate, 2 seconds
],
)
def test_processor_with_audio_sample_rate(
model_id: str,
audio_sample_rate: int,
audio_duration_sec: float,
) -> None:
"""
Test that vLLM's processor generates expected outputs with audio_sample_rate.
This validates that the processor correctly handles audio_sample_rate
passed via hf_processor_mm_kwargs and generates audio tokens.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
# Create audio data at the specified sample rate
audio_length = int(audio_sample_rate * audio_duration_sec)
rng = np.random.RandomState(42)
audio_data = rng.rand(audio_length).astype(np.float32)
# Build prompt with audio placeholder
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
# Apply processor with audio_sample_rate in mm_kwargs
hf_processor_mm_kwargs: dict[str, Any] = {
"audio_sample_rate": audio_sample_rate,
}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Verify audio tokens are generated
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token)
aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id)
assert aud_tok_count >= 1, (
f"Expected at least 1 audio token but got {aud_tok_count}. "
f"sample_rate: {audio_sample_rate}Hz, duration: {audio_duration_sec}s"
)
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
def test_longer_audio_generates_more_tokens(model_id: str) -> None:
"""
Test that longer audio generates more tokens than shorter audio.
This validates that audio_sample_rate is being used correctly by checking
that audio duration affects token count as expected.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
audio_sample_rate = 16000
rng = np.random.RandomState(42)
def get_token_count(duration: float) -> int:
audio_length = int(audio_sample_rate * duration)
audio_data = rng.rand(audio_length).astype(np.float32)
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
hf_processor_mm_kwargs: dict[str, Any] = {
"audio_sample_rate": audio_sample_rate,
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token)
return processed["prompt_token_ids"].count(audio_token_id)
short_tokens = get_token_count(1.0)
long_tokens = get_token_count(2.0)
assert long_tokens > short_tokens, (
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
f"Got short={short_tokens}, long={long_tokens}"
)
tests/models/multimodal/processing/test_qwen3_vl.py
+186
View File
@@ -0,0 +1,186 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL processor.
Covers the fix for num_frames-based timestamp calculation
(issue vllm-project/vllm#35909).
"""
from typing import Any
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def _build_video_mm_data(
num_frames: int,
width: int = 128,
height: int = 128,
original_fps: float = 30.0,
) -> dict[str, Any]:
"""Create synthetic video data with metadata indicating that
HF processor should re-sample frames (do_sample_frames=True).
``total_num_frames`` is set equal to the ndarray frame count so
that HF's ``sample_frames`` indices stay within bounds of the
actual tensor that is passed."""
video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
metadata = {
"fps": original_fps,
"duration": num_frames / original_fps,
"total_num_frames": num_frames,
"frames_indices": list(range(num_frames)),
"video_backend": "opencv",
"do_sample_frames": True,
}
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
[8, 16],
)
def test_processor_num_frames_timestamp(
model_id: str,
num_frames: int,
) -> None:
"""Regression test: using ``num_frames`` (without ``fps``) must not
cause a timestamp / token-count mismatch.
Before the fix, ``_get_video_second_idx`` ignored the explicit
``num_frames`` and fell back to an fps-based calculation, which
produced a different number of timestamp entries and ultimately led
to shape mismatches in downstream token construction.
We deliberately choose ``num_frames`` values (8, 16) that differ
from what the default fps-based path would compute (which clamps
to ``min_frames=4`` for a short video at 30 fps), so this test
would fail without the fix.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
mm_data = _build_video_mm_data(num_frames=num_frames)
# Process with explicit num_frames (no fps) -- this is the path
# that was broken before the fix.
hf_mm_kwargs: dict[str, Any] = {"num_frames": num_frames}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
# Basic sanity: the processor must produce video tokens.
token_ids = processed["prompt_token_ids"]
assert len(token_ids) > 0, "Processor produced empty token list"
# Verify that video placeholders were actually inserted.
assert "mm_placeholders" in processed
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 1, (
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize("num_videos", [2, 4])
def test_processor_multi_video(
model_id: str,
num_videos: int,
) -> None:
"""Verify that multi-video processing produces correct placeholders.
This exercises the token-level replacement path in
``_call_hf_processor`` which avoids the quadratic text-level
prompt expansion.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": num_videos},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos
mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"num_frames": 8},
)
token_ids = processed["prompt_token_ids"]
assert len(token_ids) > 0
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == num_videos, (
f"Expected {num_videos} video placeholders, got {len(video_phs)}"
)
# All placeholders should have the same length (same video params)
# and must not overlap.
lengths = {ph.length for ph in video_phs}
assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}"
for i in range(1, len(video_phs)):
prev_end = video_phs[i - 1].offset + video_phs[i - 1].length
assert video_phs[i].offset >= prev_end, (
f"Placeholder {i} overlaps with placeholder {i - 1}"
)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
)
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
) -> None:
"""Regression test: a multi-video request with list-valued per-video
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
Before the fix, ``_call_hf_processor`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 2},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = (
"<|vision_start|><|video_pad|><|vision_end|>"
"<|vision_start|><|video_pad|><|vision_end|>"
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=16)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
)
tests/models/multimodal/processing/test_smolvlm.py
+82
View File
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for smolvlm's multimodal preprocessing kwargs."""
import pytest
from packaging.version import Version
from transformers import SmolVLMConfig
from transformers import __version__ as TRANSFORMERS_VERSION
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.skipif(
Version(TRANSFORMERS_VERSION) < Version("5.2.0"),
reason="See https://github.com/huggingface/transformers/pull/43948",
)
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"max_image_size": {"longest_edge": 384}}, 1377),
({"max_image_size": {"longest_edge": 768}}, 405),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
# Same as the previous test - don't initialize mm_processor_kwargs
# in this test and assume that the kwargs will be correctly expanded by
# the partial when calling the custom input processor.
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
placeholders = (
"<image>"
if num_imgs == 1
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
)
prompt = f"<|im_start|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
# Build mm_data
image_size = ctx.get_hf_config(SmolVLMConfig).vision_config.image_size
dummy_image_size = (image_size * 4, image_size * 4)
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
mm_data = {"image": [dummy_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure the placeholders format are correct
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processed_inputs = hf_processor(
text=prompt,
images=mm_data["image"],
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
)
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
# Ensure we have the right number of placeholders per num_crops size
image_token_id = ctx.get_hf_config().image_token_id
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/multimodal/processing/test_step3_vl_image_embeds.py
+58
View File
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Step3-VL precomputed image embedding inputs."""
import pytest
import torch
from vllm.model_executor.models.step3_vl import (
Step3VLForConditionalGeneration,
Step3VLImageEmbeddingInputs,
)
class _FakeStep3VL:
@staticmethod
def _process_image_features(image_features: torch.Tensor) -> torch.Tensor:
return image_features
def test_image_embedding_inputs_construction():
"""Step3VLImageEmbeddingInputs should store embeddings in the data field."""
image_embeds = torch.randn(2, 16, 64)
inputs = Step3VLImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
assert inputs["type"] == "image_embeds"
assert torch.equal(inputs["data"], image_embeds)
assert torch.equal(inputs.data, image_embeds)
def test_image_embedding_inputs_validation_rejects_wrong_rank():
"""Validation should reject tensors with wrong rank."""
with pytest.raises(ValueError, match="rank"):
Step3VLImageEmbeddingInputs(
type="image_embeds",
data=torch.randn(16, 64),
)
def test_process_image_embeds_does_not_require_pixel_input_fields():
"""The image_embeds branch should not reference patch pixel metadata."""
image_embeds = torch.randn(2, 4, 8)
image_input = Step3VLImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
outputs = Step3VLForConditionalGeneration._process_image_input(
_FakeStep3VL(),
image_input,
)
assert len(outputs) == 2
assert torch.equal(outputs[0], image_embeds[0])
assert torch.equal(outputs[1], image_embeds[1])
tests/models/multimodal/processing/test_tensor_schema.py
+255
View File
@@ -0,0 +1,255 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections.abc import Iterable
from contextlib import contextmanager
from functools import partial
from typing import Any, TypeAlias
import numpy as np
import pytest
import torch
import torch.nn as nn
from PIL import Image
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.config.cache import CacheConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.models.interfaces import supports_multimodal
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.multimodal.utils import group_and_batch_mm_kwargs
from vllm.platforms import current_platform
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.utils.collection_utils import is_list_of
from vllm.utils.torch_utils import set_default_torch_dtype
from ....utils import create_new_process_for_each_test
from ...registry import HF_EXAMPLE_MODELS
from ...utils import dummy_hf_overrides
from .test_common import get_model_ids_to_test, get_text_token_prompts
ImageInput = list[Image.Image]
VideoInput: TypeAlias = (
list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]]
)
AudioInput = list[tuple[np.ndarray, int]]
def _resize_data(
_data: Image.Image | np.ndarray, size_factor: float
) -> Image.Image | np.ndarray:
assert size_factor <= 1, "Size factor must be less than 1"
# Image input
if isinstance(_data, Image.Image):
W, H = _data.width, _data.height
W, H = map(lambda x: int(x * size_factor), (W, H))
return _data.resize((W, H))
# Video input with PIL Images
elif is_list_of(_data, Image.Image):
W, H = next(iter(_data)).width, next(iter(_data)).height
T = len(_data)
T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H))
return [d.resize((W, H)) for d in _data[:T]]
# Video input with numpy arrays
elif isinstance(_data, np.ndarray) and _data.ndim >= 4:
T, H, W, C = _data.shape[-4:]
T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W))
return _data[..., :T, :H, :W, :C]
# Audio input
elif isinstance(_data, np.ndarray) and _data.ndim == 1:
return _data[: int(len(_data) * size_factor)]
raise AssertionError("This line should be unreachable.")
def resize_mm_data(
data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...]
) -> ImageInput | VideoInput | AudioInput:
size_factors = size_factors[: len(data)]
if is_list_of(data, (Image.Image, np.ndarray, list)):
return [_resize_data(d, s) for d, s in zip(data, size_factors)]
elif is_list_of(data, tuple):
return [_resize_data(d, s) for (d, _), s in zip(data, size_factors)]
raise ValueError("Unsupported multimodal data type.")
def create_batched_mm_kwargs(
model_config: ModelConfig,
processor: BaseMultiModalProcessor,
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
processing_info = processor.info
dummy_inputs = processor.dummy_inputs
supported_mm_limits = processing_info.get_supported_mm_limits()
mm_counts = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
processor_inputs = dummy_inputs.get_dummy_processor_inputs(
seq_len=model_config.max_model_len,
mm_counts=mm_counts,
mm_options={},
)
mm_items = processor_inputs.mm_data_items
resized_mm_data = {
modality: resize_mm_data(items.data, size_factors)
for modality, items in mm_items.items()
}
# video metadata will be added back to the resized video data here.
text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
mm_kwargs = processor(
prompt=token_prompt if text_prompt is None else text_prompt,
mm_items=processor.info.parse_mm_data(resized_mm_data),
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
)["mm_kwargs"].require_data()
return group_and_batch_mm_kwargs(
[
(modality, item)
for modality in supported_mm_limits
for item in mm_kwargs[modality]
]
)
# TODO(Isotr0py): Don't initialize model during test
@contextmanager
def initialize_dummy_model(
model_cls: type[nn.Module],
model_config: ModelConfig,
):
temp_file = tempfile.mkstemp()[1]
current_device = torch.get_default_device()
vllm_config = VllmConfig(
model_config=model_config, cache_config=CacheConfig(block_size=16)
)
with set_current_vllm_config(vllm_config=vllm_config):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=1)
with set_default_torch_dtype(model_config.dtype):
torch.set_default_device(current_platform.device_type)
model = model_cls(vllm_config=vllm_config)
torch.set_default_device(current_device)
yield model
del model
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
def test_model_tensor_schema(model_id: str):
if model_id == "moonshotai/Kimi-K2.5":
# FIXME(Isotr0py): Fix Kimi-K2.5's offline inference about vision chunks.
pytest.skip(
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_arch = next(
arch for arch, info in HF_EXAMPLE_MODELS.hf_models.items() if info == model_info
)
hf_overrides_fn = partial(
dummy_hf_overrides,
model_arch=model_arch,
exist_overrides=model_info.hf_overrides,
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
)
# ROCm: Detect if model uses AWQ quantization and set appropriate dtype
if "awq" in model_id.lower() and current_platform.is_rocm():
dtype = "float16"
else:
dtype = model_info.dtype
model_config = ModelConfig(
model_id,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
hf_overrides=hf_overrides_fn,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=dtype,
)
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
assert supports_multimodal(model_cls)
factories = model_cls._processor_factory
inputs_parse_methods = []
for attr_name in dir(model_cls):
attr = getattr(model_cls, attr_name)
if hasattr(attr, "__annotations__"):
return_type = attr.__annotations__.get("return", None)
if return_type is not None and "Input" in str(return_type):
inputs_parse_methods.append(attr_name)
if not any(inputs_parse_methods):
pytest.skip(f"{model_arch} does not support tensor schema validation.")
ctx = InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
processing_info = factories.info(ctx)
supported_mm_limits = processing_info.get_supported_mm_limits()
limit_mm_per_prompt = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
if modality == "video":
return VideoDummyOptions(count=count)
if modality == "image":
return ImageDummyOptions(count=count)
if modality == "audio":
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
model_config.get_multimodal_config().limit_per_prompt = {
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt.items()
}
processor = factories.build_processor(ctx, cache=None)
with initialize_dummy_model(model_cls, model_config) as model:
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
for method_name in inputs_parse_methods:
print(
f"Testing `{method_name}` with modality={modality} "
f"and mm_kwargs{list(mm_kwargs.keys())}"
)
getattr(model, method_name)(modality=modality, **mm_kwargs)
tests/models/multimodal/processing/test_transformers.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.image import ImageAsset
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_multimodal_processor(model_id):
model_config = ModelConfig(
model=model_id,
model_impl="transformers",
)
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
image_pil = ImageAsset("cherry_blossom").pil_image
mm_data = {"image": image_pil}
str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
str_processed_inputs = mm_processor(
prompt=str_prompt,
mm_items=mm_processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
ids_prompt = [
151644,
872,
220,
151646,
198,
3838,
374,
279,
2213,
315,
419,
2168,
30,
151645,
151644,
77091,
198,
]
ids_processed_inputs = mm_processor(
prompt=ids_prompt,
mm_items=mm_processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
assert (
str_processed_inputs["prompt_token_ids"]
== ids_processed_inputs["prompt_token_ids"]
)