chore: import upstream snapshot with attribution

tests/models/multimodal/processing/test_moondream3.py

@@ -0,0 +1,553 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for Moondream3 multimodal processing.
+
+Includes:
+- Processor creation and application tests
+- Image tokenization and placeholder expansion tests
+- Tiling and cropping logic tests (CPU-based)
+- Pixel normalization tests
+"""
+
+import numpy as np
+import pytest
+import torch
+
+from vllm.multimodal import MULTIMODAL_REGISTRY
+
+from ....conftest import ImageTestAssets
+from ...utils import build_model_context
+
+MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
+# Expected multimodal prefix: BOS + 729 image tokens.
+EXPECTED_IMAGE_TOKENS = 730
+# Vision encoder constants
+CROP_SIZE = 378
+PATCH_SIZE = 14
+MAX_CROPS = 12
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_processor_creation(model_id: str):
+    """Test that Moondream3 processor can be created."""
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+    assert processor is not None
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_processor_apply(
+    image_assets: ImageTestAssets,
+    model_id: str,
+):
+    """Test that Moondream3 processor can process inputs.
+
+    NOTE: The prompt includes the leading BOS token because Moondream3
+    pre-fills BOS and image embeddings together.
+    """
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>"  # noqa: E501
+    mm_data = {"image": [image_assets[0].pil_image]}
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={},
+    )
+
+    assert "prompt_token_ids" in processed_inputs
+    image_placeholders = processed_inputs["mm_placeholders"]["image"]
+    assert len(image_placeholders) == 1
+    assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_processor_pixel_values(
+    image_assets: ImageTestAssets,
+    model_id: str,
+):
+    """Test that pixel values are correctly produced."""
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>"  # noqa: E501
+    mm_data = {"image": [image_assets[0].pil_image]}
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={},
+    )
+
+    # Check mm_kwargs contains pixel_values
+    mm_kwargs = processed_inputs.get("mm_kwargs")
+    assert mm_kwargs is not None
+    mm_data_result = mm_kwargs.get_data()
+    assert "pixel_values" in mm_data_result
+
+    # Verify pixel_values shape
+    pixel_values = mm_data_result["pixel_values"]
+    assert pixel_values.dim() == 5  # [batch, num_crops, C, H, W]
+    assert pixel_values.shape[2] == 3  # RGB channels
+    assert pixel_values.shape[3] == 378  # crop height
+    assert pixel_values.shape[4] == 378  # crop width
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_processor_image_token_expansion(
+    image_assets: ImageTestAssets,
+    model_id: str,
+):
+    """Test that <image> placeholder is expanded to correct number of tokens."""
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>"  # noqa: E501
+    mm_data = {"image": [image_assets[0].pil_image]}
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={},
+    )
+    image_placeholders = processed_inputs["mm_placeholders"]["image"]
+    assert len(image_placeholders) == 1
+    assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_multi_crop_tiling(
+    model_id: str,
+):
+    """Test that large images produce correct multi-crop tiling."""
+    from PIL import Image
+
+    from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
+
+    processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
+
+    # Create a large image that requires multiple crops
+    large_image = Image.new("RGB", (1000, 1000), color="blue")
+    pixel_values, tiling = processor.preprocess_image(large_image)
+
+    # Large images should produce more than 1x1 tiling
+    assert tiling[0] >= 1 and tiling[1] >= 1
+    # Check that we have global crop + local crops
+    expected_crops = tiling[0] * tiling[1] + 1
+    assert pixel_values.shape[0] == expected_crops
+
+
+@pytest.mark.parametrize(
+    "image_size",
+    [
+        (500, 500),
+        (800, 600),
+        (1920, 1080),
+    ],
+)
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_tiling_various_sizes(
+    image_size: tuple[int, int],
+    model_id: str,
+):
+    """Test tiling with various image sizes."""
+    from PIL import Image
+
+    from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
+
+    processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
+
+    width, height = image_size
+    image = Image.new("RGB", (width, height), color="red")
+    pixel_values, tiling = processor.preprocess_image(image)
+
+    # Basic shape checks
+    assert pixel_values.dim() == 4  # [num_crops, C, H, W]
+    assert pixel_values.shape[1] == 3  # RGB
+    assert pixel_values.shape[2] == 378  # crop height
+    assert pixel_values.shape[3] == 378  # crop width
+
+    # Tiling should respect max_crops (12)
+    assert tiling[0] * tiling[1] <= 12
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_pixel_normalization(
+    model_id: str,
+):
+    """Test that pixel values are normalized to [-1, 1] range."""
+    from PIL import Image
+
+    from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
+
+    processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
+
+    # Create test image
+    image = Image.new("RGB", (378, 378), color="green")
+    pixel_values, _ = processor.preprocess_image(image)
+
+    # Normalization: (x - 0.5) / 0.5 = 2*x - 1
+    # For input [0, 1], output should be [-1, 1]
+    assert pixel_values.min() >= -1.0
+    assert pixel_values.max() <= 1.0
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_chat_template_with_image(
+    image_assets: ImageTestAssets,
+    model_id: str,
+):
+    """Test that chat template correctly formats BOS + image + prompt."""
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+    tokenizer = ctx.tokenizer
+
+    # Use the chat template format
+    prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>"  # noqa: E501
+    mm_data = {"image": [image_assets[0].pil_image]}
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={},
+    )
+    token_ids = processed_inputs["prompt_token_ids"]
+
+    # BOS token (<|endoftext|>) should be token ID 0
+    bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
+    assert bos_token_id == 0
+
+    # First token should be BOS
+    assert token_ids[0] == bos_token_id
+
+
+@pytest.mark.parametrize(
+    "content",
+    [
+        pytest.param(
+            [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.invalid/image.png"},
+                },
+                {"type": "text", "text": "What is in this image?"},
+            ],
+            id="image-first",
+        ),
+        pytest.param(
+            [
+                {"type": "text", "text": "What is in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.invalid/image.png"},
+                },
+            ],
+            id="text-first",
+        ),
+    ],
+)
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_chat_template_content_list_uses_moondream_image_prefix(
+    image_assets: ImageTestAssets,
+    content: list[dict[str, object]],
+    model_id: str,
+):
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+    hf_processor = processor.info.get_hf_processor()
+
+    prompt = hf_processor.tokenizer.apply_chat_template(
+        [{"role": "user", "content": content}],
+        chat_template=hf_processor.chat_template,
+        tokenize=False,
+    )
+
+    expected_prompt = (
+        "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
+        "What is in this image?<|md_reserved_2|>"
+    )
+    assert prompt == expected_prompt
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
+        hf_processor_mm_kwargs={},
+    )
+    image_placeholders = processed_inputs["mm_placeholders"]["image"]
+    assert len(image_placeholders) == 1
+    assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_bos_token_always_first(
+    image_assets: ImageTestAssets,
+    model_id: str,
+):
+    """Test that BOS token (ID 0) is always at position 0."""
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    # Start with BOS token explicitly
+    prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>"  # noqa: E501
+    mm_data = {"image": [image_assets[0].pil_image]}
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={},
+    )
+    token_ids = processed_inputs["prompt_token_ids"]
+
+    # Token ID 0 (<|endoftext|>) should be the first token
+    assert token_ids[0] == 0, (
+        f"Expected BOS token (0) at position 0, got {token_ids[0]}"
+    )
+
+
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_processor_with_small_image(
+    model_id: str,
+):
+    """Test processor with image smaller than crop size."""
+    from PIL import Image
+
+    from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
+
+    processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
+
+    # Small image (smaller than crop size)
+    small_image = Image.new("RGB", (100, 100), color="yellow")
+    pixel_values, tiling = processor.preprocess_image(small_image)
+
+    # Small images should use 1x1 tiling
+    assert tiling == (1, 1)
+    # Should have 2 crops (global + 1 local)
+    assert pixel_values.shape[0] == 2
+
+
+@pytest.mark.parametrize(
+    "image_kind",
+    [
+        pytest.param("numpy_hwc", id="numpy-hwc"),
+        pytest.param("numpy_chw", id="numpy-chw"),
+        pytest.param("torch_chw", id="torch-chw"),
+    ],
+)
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_preprocess_image_accepts_non_pil_inputs(
+    image_assets: ImageTestAssets,
+    image_kind: str,
+    model_id: str,
+):
+    from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
+
+    processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
+    pil_image = image_assets[0].pil_image.convert("RGB")
+    hwc_array = np.asarray(pil_image)
+    expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
+
+    if image_kind == "numpy_hwc":
+        image = hwc_array
+    elif image_kind == "numpy_chw":
+        image = np.transpose(hwc_array, (2, 0, 1))
+    else:
+        image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
+
+    pixel_values, tiling = processor.preprocess_image(image)
+
+    assert tiling == expected_tiling
+    assert pixel_values.shape == expected_pixel_values.shape
+    assert pixel_values.dtype == torch.bfloat16
+    assert torch.equal(pixel_values, expected_pixel_values)
+
+
+@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
+@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
+def test_processor_apply_accepts_non_pil_image_inputs(
+    image_assets: ImageTestAssets,
+    image_kind: str,
+    model_id: str,
+):
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>"  # noqa: E501
+    hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
+    chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
+    image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
+
+    processed_inputs = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data({"image": [image]}),
+        hf_processor_mm_kwargs={},
+    )
+
+    image_placeholders = processed_inputs["mm_placeholders"]["image"]
+    assert len(image_placeholders) == 1
+    assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
+
+    mm_kwargs = processed_inputs["mm_kwargs"].get_data()
+    assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
+
+
+class TestMoondream3TilingLogic:
+    """CPU-based tests for Moondream3 tiling selection logic.
+
+    These tests validate the select_tiling() function which determines
+    how images are divided into crops for the vision encoder.
+    """
+
+    def test_small_image_no_tiling(self):
+        """Small images should use 1x1 tiling."""
+        from vllm.transformers_utils.processors.moondream3 import select_tiling
+
+        tiling = select_tiling(
+            height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
+        )
+        assert tiling == (1, 1)
+
+    def test_exact_crop_size(self):
+        """Image exactly at crop size should use 1x1."""
+        from vllm.transformers_utils.processors.moondream3 import select_tiling
+
+        tiling = select_tiling(
+            height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
+        )
+        assert tiling == (1, 1)
+
+    def test_large_square_image(self):
+        """Large square image should use multiple tiles."""
+        from vllm.transformers_utils.processors.moondream3 import select_tiling
+
+        tiling = select_tiling(
+            height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
+        )
+        h_tiles, w_tiles = tiling
+        assert h_tiles >= 2
+        assert w_tiles >= 2
+        assert h_tiles * w_tiles <= MAX_CROPS
+
+    def test_wide_image(self):
+        """Wide image should have more width tiles."""
+        from vllm.transformers_utils.processors.moondream3 import select_tiling
+
+        tiling = select_tiling(
+            height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
+        )
+        h_tiles, w_tiles = tiling
+        assert w_tiles >= h_tiles
+
+    def test_tall_image(self):
+        """Tall image should have more height tiles."""
+        from vllm.transformers_utils.processors.moondream3 import select_tiling
+
+        tiling = select_tiling(
+            height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
+        )
+        h_tiles, w_tiles = tiling
+        assert h_tiles >= w_tiles
+
+    def test_respects_max_crops(self):
+        """Tiling should not exceed max_crops."""
+        from vllm.transformers_utils.processors.moondream3 import select_tiling
+
+        tiling = select_tiling(
+            height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
+        )
+        h_tiles, w_tiles = tiling
+        assert h_tiles * w_tiles <= 4
+
+
+class TestMoondream3VisionShapes:
+    """CPU-based tests for vision encoder expected shapes.
+
+    These tests verify the mathematical relationships between
+    crop size, patch size, and token counts.
+    """
+
+    def test_expected_patch_count(self):
+        """Test 378/14 = 27 patches per side, 729 total."""
+        patches_per_side = CROP_SIZE // PATCH_SIZE
+        total_patches = patches_per_side**2
+
+        assert patches_per_side == 27
+        assert total_patches == EXPECTED_IMAGE_TOKENS - 1
+
+    def test_patch_embedding_input_dim(self):
+        """Test patch embedding input dimension."""
+        channels = 3
+        input_dim = PATCH_SIZE * PATCH_SIZE * channels
+
+        assert input_dim == 14 * 14 * 3
+        assert input_dim == 588
+
+
+class TestMoondream3TauAttention:
+    """CPU-based tests for tau attention scaling components.
+
+    These tests validate the tau attention formula used in Moondream3:
+    - Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
+    - Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
+    """
+
+    def test_tau_position_range(self):
+        """Test tau position scaling produces values in valid range."""
+        num_heads = 32
+        seq_len = 100
+
+        tau_alpha = torch.randn(num_heads)
+        positions = torch.arange(seq_len)
+
+        pos_float = (positions.float() + 1.0).clamp(min=1e-6)
+        pos_log = pos_float.log()
+        tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
+
+        assert tau_pos.shape == (num_heads, seq_len)
+        # tau_pos should be between 0.5 and 1.5
+        assert tau_pos.min() >= 0.5
+        assert tau_pos.max() <= 1.5
+
+    def test_tau_token_output_range(self):
+        """Test tau token scaling output is bounded by tanh."""
+        import torch.nn.functional as F
+
+        seq_len = 100
+        qkv_dim = 6144  # 2048 * 3
+        num_heads = 32
+
+        qkv = torch.randn(seq_len, qkv_dim)
+        tau_wq = torch.randn(num_heads, qkv_dim)
+
+        tok_feat = F.gelu(qkv)
+        tok_q = torch.tanh(tok_feat @ tau_wq.t())
+
+        assert tok_q.shape == (seq_len, num_heads)
+        # tanh output is bounded by [-1, 1]
+        assert tok_q.min() >= -1.0
+        assert tok_q.max() <= 1.0