wehub-resource-sync
303 lines
12 KiB
Python
303 lines
12 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism.
|
|
|
|
The codec video backend cannot be exercised end-to-end in CI because it
|
|
relies on the model's ``trust_remote_code`` package and real video bytes.
|
|
These tests cover the pure-Python marker plumbing that wraps a video *path*
|
|
for vLLM's ``MultiModalDataParser``:
|
|
|
|
* :func:`prepare_codec_video_input` must encode a per-path hash into its
|
|
dummy ndarray so distinct codec videos do not collide on ``mm_hash``
|
|
(otherwise ``EncoderCacheManager`` would skip the encoder for every video
|
|
after the first).
|
|
* :func:`_extract_codec_video_paths` must round-trip the marker back to the
|
|
original path(s) after the parser strips the metadata dict, and must return
|
|
``None`` for non-codec inputs.
|
|
"""
|
|
|
|
import types
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from vllm.model_executor.models.llava_onevision2 import (
|
|
_CODEC_VIDEO_MARKER,
|
|
LlavaOnevision2VideoBackend,
|
|
_extract_codec_video_paths,
|
|
_frame_video_to_pil_and_timestamps,
|
|
_validate_video_source,
|
|
prepare_codec_video_input,
|
|
)
|
|
from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata
|
|
|
|
|
|
def _model_config(local: str = "", domains=None):
|
|
# Minimal stand-in for vLLM's ModelConfig: the validator only reads
|
|
# ``allowed_local_media_path`` and ``allowed_media_domains``.
|
|
return types.SimpleNamespace(
|
|
allowed_local_media_path=local,
|
|
allowed_media_domains=domains,
|
|
)
|
|
|
|
|
|
def test_prepare_codec_video_input_shape_and_marker():
|
|
dummy, meta = prepare_codec_video_input("/data/foo.mp4")
|
|
|
|
# 4-D ndarray satisfies MultiModalDataParser's video shape check.
|
|
assert isinstance(dummy, np.ndarray)
|
|
assert dummy.shape == (1, 1, 16, 3)
|
|
assert dummy.dtype == np.uint8
|
|
|
|
# Metadata carries the exact path under the codec marker key.
|
|
assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"}
|
|
|
|
|
|
def test_prepare_codec_video_input_is_deterministic():
|
|
# Same path must yield identical dummy bytes (stable mm_hash).
|
|
a, _ = prepare_codec_video_input("/data/foo.mp4")
|
|
b, _ = prepare_codec_video_input("/data/foo.mp4")
|
|
assert a.tobytes() == b.tobytes()
|
|
|
|
|
|
def test_prepare_codec_video_input_distinct_paths_distinct_bytes():
|
|
# Distinct paths must yield distinct dummy bytes so the parser-visible
|
|
# ndarray (the only part reaching MultiModalHasher) varies per video.
|
|
paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"]
|
|
payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths}
|
|
assert len(payloads) == len(paths)
|
|
|
|
|
|
def test_extract_codec_video_paths_parser_list_shape():
|
|
# Parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
|
|
items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")]
|
|
assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"]
|
|
|
|
|
|
def test_extract_codec_video_paths_single_raw_tuple():
|
|
# Single raw (ndarray, dict) tuple (pre-parser path) is also accepted.
|
|
item = prepare_codec_video_input("/x/solo.mp4")
|
|
assert _extract_codec_video_paths(item) == ["/x/solo.mp4"]
|
|
|
|
|
|
def test_extract_codec_video_paths_non_codec_returns_none():
|
|
# Plain decoded-frame inputs (ndarray / list of ndarray) are not codec
|
|
# markers and must fall through to the frame backend.
|
|
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
|
assert _extract_codec_video_paths(plain) is None
|
|
assert _extract_codec_video_paths([plain, plain]) is None
|
|
assert _extract_codec_video_paths([]) is None
|
|
# Tuple without the marker key is ignored.
|
|
assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None
|
|
|
|
|
|
def test_extract_codec_video_paths_mixed_batch_returns_none():
|
|
# If any item in the batch lacks the marker, the whole batch is treated
|
|
# as non-codec (the backend does not mix codec and frame videos).
|
|
codec = prepare_codec_video_input("/x/1.mp4")
|
|
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
|
assert _extract_codec_video_paths([codec, plain]) is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Media access controls (_validate_video_source)
|
|
#
|
|
# The codec backend keeps the raw path string alive past vLLM's
|
|
# MultiModalDataParser and hands it to the trust-remote-code codec module,
|
|
# which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the
|
|
# codec backend is restricted to *local files* confined to
|
|
# --allowed-local-media-path; remote http(s)/data URLs are rejected (they must
|
|
# use the frame backend, which rides the connector). The validator also returns
|
|
# the *resolved* path so the codec module opens exactly what was validated
|
|
# (no symlink-retarget / validate-vs-open gap).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_validate_http_url_rejected():
|
|
# Codec backend is local-only: remote URLs bypass the connector's domain
|
|
# and redirect controls, so they are rejected regardless of allowlist.
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source("http://example.com/v.mp4", _model_config())
|
|
|
|
|
|
def test_validate_https_url_rejected_even_when_host_allowlisted():
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source(
|
|
"https://good.com/v.mp4",
|
|
_model_config(domains=["good.com"]),
|
|
)
|
|
|
|
|
|
def test_validate_data_url_rejected():
|
|
# data: URLs are a remote/inline source for the frame backend, not codec.
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source("data:video/mp4;base64,AAAA", _model_config())
|
|
|
|
|
|
def test_validate_local_file_blocked_without_allowed_path():
|
|
# Local file access is opt-in: without --allowed-local-media-path the
|
|
# bare path is rejected.
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source("/data/v.mp4", _model_config())
|
|
|
|
|
|
def test_validate_local_file_allowed_inside_allowed_dir(tmp_path):
|
|
f = tmp_path / "v.mp4"
|
|
f.touch()
|
|
assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f)
|
|
|
|
|
|
def test_validate_local_file_traversal_blocked():
|
|
# Path traversal escaping the allowed root is rejected after resolution.
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source(
|
|
"/tmp/ov2/../../etc/passwd",
|
|
_model_config(local="/tmp/ov2"),
|
|
)
|
|
|
|
|
|
def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path):
|
|
f = tmp_path / "v.mp4"
|
|
f.touch()
|
|
assert _validate_video_source(
|
|
f.as_uri(), _model_config(local=str(tmp_path))
|
|
) == str(f)
|
|
|
|
|
|
def test_validate_file_scheme_percent_encoded_traversal_blocked():
|
|
# ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first
|
|
# (url2pathname) so the path cannot stay literally under the allowed root.
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source(
|
|
"file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd",
|
|
_model_config(local="/tmp/ov2"),
|
|
)
|
|
|
|
|
|
def test_validate_unsupported_scheme_blocked():
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source("ftp://example.com/v.mp4", _model_config())
|
|
|
|
|
|
def test_validate_returns_resolved_path_through_symlink(tmp_path):
|
|
# The validator resolves symlinks and returns the *real* path, so the codec
|
|
# module opens exactly what was validated (closes the validate-vs-open gap).
|
|
real = tmp_path / "real.mp4"
|
|
real.touch()
|
|
link = tmp_path / "link.mp4"
|
|
link.symlink_to(real)
|
|
assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str(
|
|
real
|
|
)
|
|
|
|
|
|
def test_validate_relative_bare_path_blocked():
|
|
# Bare (scheme=="") paths come only from the codec backend and must be
|
|
# absolute: resolving a relative path against an ambiguous CWD before the
|
|
# confinement check is brittle/unsafe, so it is rejected outright.
|
|
with pytest.raises(ValueError):
|
|
_validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
|
|
#
|
|
# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
|
|
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
|
|
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
|
|
# (``video_needs_metadata=True``). The helper materialises PIL frames and
|
|
# per-frame timestamps (``frame_index / fps``), padding the frame count up to
|
|
# the even temporal-merge boundary.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_frame_video_to_pil_and_timestamps_basic():
|
|
frames = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
|
metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]}
|
|
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
|
|
|
|
assert len(pil_frames) == 4
|
|
assert all(f.size == (8, 8) for f in pil_frames)
|
|
# timestamps = frame_index / fps
|
|
assert timestamps == [0.0, 2.0, 4.0, 6.0]
|
|
|
|
|
|
def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count():
|
|
# Odd frame count -> last frame repeated to satisfy temporal merge=2.
|
|
frames = np.zeros((3, 8, 8, 3), dtype=np.uint8)
|
|
metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]}
|
|
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
|
|
|
|
assert len(pil_frames) == 4
|
|
assert len(timestamps) == 4
|
|
# The padded frame reuses the final index/timestamp.
|
|
assert timestamps == [0.0, 1.0, 2.0, 2.0]
|
|
|
|
|
|
def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse():
|
|
# Missing frames_indices -> sequential range; missing/zero fps -> default.
|
|
frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
|
|
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {}))
|
|
|
|
assert len(pil_frames) == 2
|
|
# Default fps is 1.0, indices fall back to range(T).
|
|
assert timestamps == [0.0, 1.0]
|
|
|
|
|
|
def test_frame_video_to_pil_and_timestamps_rejects_non_tuple():
|
|
# Bare arrays (no metadata) must be rejected: the frame backend requires
|
|
# the (frames, metadata) tuple produced by the registered loader.
|
|
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
|
with pytest.raises(ValueError):
|
|
_frame_video_to_pil_and_timestamps(plain)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller
|
|
# supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can
|
|
# override the conservative defaults (fps=1.0, max_frames=32). Unset target
|
|
# fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _src(total_frames: int, fps: float) -> VideoSourceMetadata:
|
|
return VideoSourceMetadata(
|
|
total_frames_num=total_frames,
|
|
original_fps=fps,
|
|
duration=total_frames / fps if fps > 0 else 0.0,
|
|
)
|
|
|
|
|
|
def test_backend_defaults_cap_at_32_frames():
|
|
# 300 frames @ 1fps source, target unset -> capped at default max_frames=32.
|
|
src = _src(300, 1.0)
|
|
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0)
|
|
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
|
|
|
|
assert len(idx) == 32
|
|
assert idx[0] == 0
|
|
assert idx[-1] == 299
|
|
assert len(idx) % 2 == 0
|
|
|
|
|
|
def test_backend_target_num_frames_overrides_default_cap():
|
|
# VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap.
|
|
src = _src(300, 1.0)
|
|
target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0)
|
|
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
|
|
|
|
assert len(idx) == 128
|
|
assert idx[0] == 0
|
|
assert idx[-1] == 299
|
|
|
|
|
|
def test_backend_target_fps_controls_sampling_when_below_cap():
|
|
# 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded).
|
|
src = _src(60, 30.0)
|
|
target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0)
|
|
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
|
|
|
|
# fps-derived nframes (2) is below the 128 cap, so fps wins.
|
|
assert len(idx) <= 8
|
|
assert len(idx) % 2 == 0
|