vllm-project--vllm/tests/model_executor/test_paddleocr_vl_mrope.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field

import pytest
import torch

from vllm.model_executor.models.paddleocr_vl import (
    PaddleOCRVLForConditionalGeneration,
)
from vllm.multimodal.inputs import (
    MultiModalFeatureSpec,
    MultiModalFieldElem,
    MultiModalKwargsItem,
    PlaceholderRange,
)

pytestmark = pytest.mark.skip_global_cleanup


@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
    original = torch.get_default_device()
    torch.set_default_device("cpu")
    yield
    torch.set_default_device(original)


@dataclass
class DummyVisionConfig:
    spatial_merge_size: int = 2
    patch_size: int = 14


@dataclass
class DummyConfig:
    image_token_id: int = 151655
    video_token_id: int = 151654
    vision_start_token_id: int = 151652
    vision_end_token_id: int = 151653
    vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)


def make_model(config: DummyConfig) -> PaddleOCRVLForConditionalGeneration:
    model = object.__new__(PaddleOCRVLForConditionalGeneration)
    model.config = config
    return model


def make_mm_feature(
    *,
    offset: int,
    length: int,
    image_grid_thw: tuple[int, int, int],
) -> MultiModalFeatureSpec:
    return MultiModalFeatureSpec(
        data=MultiModalKwargsItem(
            {
                "image_grid_thw": MultiModalFieldElem(
                    data=torch.tensor(image_grid_thw),
                    field=None,
                ),
            }
        ),
        modality="image",
        identifier="DUMMY",
        mm_position=PlaceholderRange(offset=offset, length=length),
    )


def test_get_mrope_input_positions_text_only():
    model = make_model(DummyConfig())
    input_tokens = [11, 12, 13, 14, 15]
    positions, delta = model.get_mrope_input_positions(
        input_tokens=input_tokens,
        mm_features=[],
    )
    expected = torch.tensor(
        [
            [0, 1, 2, 3, 4],
            [0, 1, 2, 3, 4],
            [0, 1, 2, 3, 4],
        ]
    )
    assert torch.equal(positions, expected)
    assert delta == 0


def test_get_mrope_input_positions_single_image():
    model = make_model(DummyConfig())
    spatial_merge_size = model.config.vision_config.spatial_merge_size

    t, h, w = 1, 2, 2
    num_image_tokens = t * h * w

    input_tokens = (
        [10]
        + [model.config.vision_start_token_id]
        + [model.config.image_token_id] * num_image_tokens
        + [model.config.vision_end_token_id]
        + [30, 31]
    )

    mm_features = [
        make_mm_feature(
            offset=2,  # 1 (text) + 1 (vision_start)
            length=num_image_tokens,
            image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
        )
    ]

    positions, delta = model.get_mrope_input_positions(
        input_tokens=input_tokens,
        mm_features=mm_features,
    )

    expected = torch.tensor(
        [
            [0, 1, 2, 2, 2, 2, 4, 5, 6],
            [0, 1, 2, 2, 3, 3, 4, 5, 6],
            [0, 1, 2, 3, 2, 3, 4, 5, 6],
        ]
    )

    assert torch.equal(positions, expected)
    expected_delta = (positions.max().item() + 1) - len(input_tokens)
    assert delta == expected_delta


def test_get_mrope_input_positions_multiple_images():
    model = make_model(DummyConfig())
    spatial_merge_size = model.config.vision_config.spatial_merge_size

    t1, h1, w1 = 1, 2, 2
    num1 = t1 * h1 * w1

    t2, h2, w2 = 1, 1, 3
    num2 = t2 * h2 * w2

    input_tokens = (
        [10]
        + [model.config.vision_start_token_id]
        + [model.config.image_token_id] * num1
        + [model.config.vision_end_token_id]
        + [20, 21]
        + [model.config.vision_start_token_id]
        + [model.config.image_token_id] * num2
        + [model.config.vision_end_token_id]
        + [30]
    )

    mm_features = [
        make_mm_feature(
            offset=2,
            length=num1,
            image_grid_thw=(t1, h1 * spatial_merge_size, w1 * spatial_merge_size),
        ),
        make_mm_feature(
            offset=2 + num1 + 1 + 2 + 1,
            length=num2,
            image_grid_thw=(t2, h2 * spatial_merge_size, w2 * spatial_merge_size),
        ),
    ]

    positions, delta = model.get_mrope_input_positions(
        input_tokens=input_tokens,
        mm_features=mm_features,
    )

    assert positions.shape == (3, 15)
    assert not torch.equal(positions[:, 2:6], torch.arange(4).expand(3, 4) + 2)
    assert not torch.equal(positions[:, 10:13], torch.arange(3).expand(3, 3) + 10)


def test_get_mrope_input_positions_image_at_start():
    model = make_model(DummyConfig())
    spatial_merge_size = model.config.vision_config.spatial_merge_size

    t, h, w = 1, 2, 2
    num_tokens = t * h * w

    input_tokens = (
        [model.config.vision_start_token_id]
        + [model.config.image_token_id] * num_tokens
        + [model.config.vision_end_token_id]
        + [10, 11]
    )

    mm_features = [
        make_mm_feature(
            offset=1,  # start token at index 0
            length=num_tokens,
            image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
        )
    ]

    positions, delta = model.get_mrope_input_positions(
        input_tokens=input_tokens,
        mm_features=mm_features,
    )

    expected = torch.tensor(
        [
            [0, 1, 1, 1, 1, 3, 4, 5],
            [0, 1, 1, 2, 2, 3, 4, 5],
            [0, 1, 2, 1, 2, 3, 4, 5],
        ]
    )

    assert torch.equal(positions, expected)