chore: import upstream snapshot with attribution

python/sglang/srt/multimodal/processors/clip.py

@@ -0,0 +1,36 @@
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.clip import CLIPModel
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class ClipImageProcessor(BaseMultimodalProcessor):
+    models = [CLIPModel]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(image_token="<image>").build(
+            _processor
+        )
+
+    async def process_mm_data_async(
+        self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=self.mm_tokens,
+            image_data=image_data,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+        )

python/sglang/srt/multimodal/processors/cohere2_vision.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 SGLang Team
+"""SGLang multimodal processor for Cohere2Vision (Command-A-Vision)."""
+
+from typing import Dict, List, Union
+
+from sglang.srt.managers.multimodal_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.cohere2_vision import Cohere2VisionForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
+
+
+class Cohere2VisionSGLangImageProcessor(SGLangBaseProcessor):
+    models = [Cohere2VisionForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        # Cohere2Vision wraps each image as:
+        #   <|START_OF_IMG|> [<|IMG_PATCH|> * P^2 + <|IMG_LINE_BREAK|>] * N <|END_OF_IMG|>
+        # (N = patch count, P = patch_size). The HF processor expands the single
+        # <|IMG_PATCH|> placeholder into that block.
+        proc = _processor
+        boi_token = proc.boi_token
+        eoi_token = proc.eoi_token
+        image_token = proc.image_token  # "<|IMG_PATCH|>"
+        line_break_token = proc.img_line_break_token
+
+        self.image_token_id = proc.image_token_id
+        self.boi_token_id = proc.tokenizer.convert_tokens_to_ids(boi_token)
+        self.eoi_token_id = proc.tokenizer.convert_tokens_to_ids(eoi_token)
+        self.img_line_break_token_id = proc.tokenizer.convert_tokens_to_ids(
+            line_break_token
+        )
+
+        # Match the unexpanded <|IMG_PATCH|> placeholder so SGLang pairs each
+        # one with its image_data entry before the HF processor expands it.
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=image_token,
+            image_token_id=self.image_token_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+            discard_alpha_channel=True,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.image_token_id,
+            im_start_id=self.boi_token_id,
+            im_end_id=self.eoi_token_id,
+        )

python/sglang/srt/multimodal/processors/deepseek_ocr.py

@@ -0,0 +1,46 @@
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.deepseek_ocr import DeepseekOCRForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class DeepseekOCRProcessor(BaseMultimodalProcessor):
+    models = [DeepseekOCRForCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        _processor.image_size = 640
+        _processor.ocr2_mode = (
+            str(
+                getattr(getattr(hf_config, "vision_config", None), "model_name", "")
+            ).lower()
+            == "deepencoderv2"
+            or getattr(getattr(hf_config, "projector_config", None), "input_dim", None)
+            == 896
+        )
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<image>", image_token_id=self._processor.image_token_id
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=self.mm_tokens,
+            image_data=image_data,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/deepseek_vl_v2.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2023-2024 DeepSeek.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.deepseek_vl2 import DeepseekVL2ForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class DeepseekVL2ImageProcessor(BaseMultimodalProcessor):
+    models = [DeepseekVL2ForCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<image>", image_token_id=self._processor.image_token_id
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        max_req_input_len,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output,
+            self.mm_tokens,
+            max_req_input_len=max_req_input_len,
+            conversations=base_output.input_text,
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_token_id=self._processor.image_token_id,
+        )

python/sglang/srt/multimodal/processors/dots_vlm.py

@@ -0,0 +1,90 @@
+import re
+from typing import Dict, List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.dots_ocr import DotsOCRForCausalLM
+from sglang.srt.models.dots_vlm import DotsVLMForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class DotsVLMImageProcessor(BaseMultimodalProcessor):
+    models = [DotsVLMForCausalLM, DotsOCRForCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        # The single, pre-expanded image token.
+        self.IMAGE_TOKEN = "<|img|><|imgpad|><|endofimg|>"
+        # The regex that matches expanded image tokens.
+        self.IMAGE_TOKEN_REGEX = re.compile(r"<\|img\|>(?:<\|imgpad\|>)+<\|endofimg\|>")
+
+        assert len(_processor.tokenizer.encode("<|img|>")) == 1
+        self.im_start_id = _processor.tokenizer.encode("<|img|>")[0]
+        self.im_end_id = _processor.tokenizer.encode("<|endofimg|>")[0]
+        self.image_token_id = _processor.tokenizer.encode("<|imgpad|>")[0]
+        self.IM_TOKEN_ID = self.image_token_id
+        self.IM_START_TOKEN_ID = self.im_start_id
+        self.IM_END_TOKEN_ID = self.im_end_id
+
+        vision_config = hf_config.vision_config
+        patch_size = vision_config.patch_size
+        merge_size = vision_config.spatial_merge_size
+
+        self.IMAGE_FACTOR = patch_size * merge_size
+        self.MIN_PIXELS = getattr(
+            _processor.image_processor,
+            "min_pixels",
+            getattr(_processor.image_processor, "size", {}).get("shortest_edge"),
+        )
+        self.MAX_PIXELS = getattr(
+            _processor.image_processor,
+            "max_pixels",
+            getattr(_processor.image_processor, "size", {}).get("longest_edge"),
+        )
+        self.MAX_RATIO = 200
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.image_token_id,
+            image_token_regex=self.IMAGE_TOKEN_REGEX,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        max_req_input_len,
+        *args,
+        **kwargs,
+    ):
+        if isinstance(image_data, str):
+            image_data = [image_data]
+
+        if (
+            isinstance(image_data, list)
+            and image_data
+            and isinstance(image_data[0], list)
+        ):
+            image_data = sum(image_data, [])
+
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        combined_mm_item, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+        if combined_mm_item is None:
+            return None
+
+        return MultimodalProcessorOutput(
+            mm_items=combined_mm_item,
+            input_ids=input_ids.tolist(),
+            im_token_id=self.image_token_id,
+            im_start_id=self.im_start_id,
+            im_end_id=self.im_end_id,
+        )

python/sglang/srt/multimodal/processors/ernie45_vl.py

@@ -0,0 +1,440 @@
+import math
+import os
+from typing import List, Union
+
+import numpy as np
+import torch
+import torchvision
+from PIL import Image
+from torchvision.transforms import InterpolationMode
+from transformers import BaseImageProcessor
+
+from sglang.srt.environ import envs
+from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.ernie45_vl import Ernie4_5_VLMoeForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils import get_bool_env_var, is_npu, logger
+
+_is_npu = is_npu()
+
+SGL_USE_CUDA_IPC = get_bool_env_var("SGLANG_USE_CUDA_IPC_TRANSPORT")
+
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+# MAX_PIXELS = envs.SGLANG_IMAGE_MAX_PIXELS.get()
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+RESIZE_RESAMPLE = getattr(Image, envs.SGLANG_RESIZE_RESAMPLE.get(), None)
+if envs.SGLANG_RESIZE_RESAMPLE.is_set() and RESIZE_RESAMPLE is None:
+    logger.warning(
+        f"Invalid RESIZE_RESAMPLE value: '{envs.SGLANG_RESIZE_RESAMPLE.get()}'. "
+        f"Ignoring and using default."
+    )
+VIDEO_TOTAL_PIXELS = int(
+    float(os.environ.get("VIDEO_MAX_PIXELS", 128000 * 28 * 28 * 0.9))
+)
+
+VIDEO_MIN_PIXELS = 299 * 28 * 28
+VIDEO_MAX_PIXELS = 1196 * 28 * 28
+FRAME_FACTOR = 2
+FPS = 2.0
+FPS_MIN_FRAMES = 16
+FPS_MAX_FRAMES = 180
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = IMAGE_FACTOR,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+):
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        if height > width:
+            new_width = max(factor, round_by_factor(width, factor))
+            new_height = floor_by_factor(new_width * MAX_RATIO, factor)
+        else:
+            new_height = max(factor, round_by_factor(height, factor))
+            new_width = floor_by_factor(new_height * MAX_RATIO, factor)
+
+        height = new_height
+        width = new_width
+
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+
+    if min_pixels > h_bar * w_bar or h_bar * w_bar > max_pixels:
+        raise ValueError(f"encounter invalid h_bar: {h_bar}, w_bar: {w_bar}")
+
+    return h_bar, w_bar
+
+
+def resize_image(
+    image,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+    size_factor: int = IMAGE_FACTOR,
+) -> Image.Image:
+    width, height = image.size
+    min_pixels = min_pixels
+    max_pixels = max_pixels
+    resized_height, resized_width = smart_resize(
+        height,
+        width,
+        factor=size_factor,
+        min_pixels=min_pixels,
+        max_pixels=max_pixels,
+    )
+    image = image.resize((resized_width, resized_height), resample=RESIZE_RESAMPLE)
+    return image
+
+
+def round_by_factor(number: int | float, factor: int) -> int:
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int | float, factor: int) -> int:
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int | float, factor: int) -> int:
+    return math.floor(number / factor) * factor
+
+
+async def resize_image_async(
+    image,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+    size_factor: int = IMAGE_FACTOR,
+):
+    return resize_image(image, min_pixels, max_pixels, size_factor)
+
+
+def smart_nframes(
+    ele: dict,
+    total_frames: int,
+    video_fps: int | float,
+) -> int:
+    """calculate the number of frames for video used for model inputs.
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+            support either `fps` or `nframes`:
+                - nframes: the number of frames to extract for model inputs.
+                - fps: the fps to extract frames for model inputs.
+                    - min_frames: the minimum number of frames of the video, only used when fps is provided.
+                    - max_frames: the maximum number of frames of the video, only used when fps is provided.
+        total_frames (int): the original total number of frames of the video.
+        video_fps (int | float): the original fps of the video.
+
+    Raises:
+        ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
+
+    Returns:
+        int: the number of frames for video used for model inputs.
+    """
+    assert not (
+        "fps" in ele and "nframes" in ele
+    ), "Only accept either `fps` or `nframes`"
+    if "nframes" in ele:
+        nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
+    else:
+        fps = ele.get("fps", FPS)
+        min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
+        max_frames = floor_by_factor(
+            ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR
+        )
+        nframes = total_frames / video_fps * fps
+        if nframes > total_frames:
+            logger.warning(
+                f"smart_nframes: nframes[{nframes}] > total_frames[{total_frames}]"
+            )
+        nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+        nframes = floor_by_factor(nframes, FRAME_FACTOR)
+    if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
+        raise ValueError(
+            f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}."
+        )
+    return nframes
+
+
+# process video, qwen-specific
+async def preprocess_video(
+    vr,
+    image_factor: int = IMAGE_FACTOR,
+) -> torch.Tensor:
+
+    total_frames, video_fps = len(vr), vr.get_avg_fps()
+    nframes = smart_nframes({}, total_frames=total_frames, video_fps=video_fps)
+    idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
+    idx = np.unique(idx)
+    video_np = vr.get_batch(idx).asnumpy()
+    video = torch.from_numpy(video_np).pin_memory()
+    video = video.permute(0, 3, 1, 2)  # Convert to TCHW format
+    nframes, _, height, width = video.shape
+    min_pixels = VIDEO_MIN_PIXELS
+    total_pixels = VIDEO_TOTAL_PIXELS
+    max_pixels = max(
+        min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
+        int(min_pixels * 1.05),
+    )
+
+    resized_height, resized_width = smart_resize(
+        height,
+        width,
+        factor=image_factor,
+        min_pixels=min_pixels,
+        max_pixels=max_pixels,
+    )
+    video = torchvision.transforms.functional.resize(
+        video,
+        [resized_height, resized_width],
+        interpolation=InterpolationMode.BILINEAR,
+    )
+
+    video = video.permute(0, 2, 3, 1)
+    video = video.pin_memory()
+    video_metadata = {
+        "fps": video_fps,
+        "duration": total_frames / video_fps,
+        "total_num_frames": total_frames,
+        "frames_indices": idx,
+        "video_backend": "torchvision",
+    }
+
+    return video, video_metadata
+
+
+# Compatible with Ernie-VL Series
+class Ernie4_5_VLImageProcessor(SGLangBaseProcessor):
+    models = [Ernie4_5_VLMoeForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.hf_config = hf_config
+        self.model_type = hf_config.model_type
+        self.image_start_token_id = hf_config.image_start_token_id
+        self.image_end_token_id = hf_config.image_end_token_id
+        self.video_start_token_id = hf_config.video_start_token_id
+        self.video_end_token_id = hf_config.video_end_token_id
+
+        self.IMAGE_FACTOR = 28
+        self.MIN_PIXELS = 4 * 28 * 28
+        self.MAX_PIXELS = 16384 * 28 * 28
+        self.MAX_RATIO = 200
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>",
+            video_token="<|VIDEO_START|><|video@placeholder|><|VIDEO_END|>",
+            image_token_id=hf_config.im_patch_id,
+            video_token_id=hf_config.im_patch_id,  # image and video use the same token_id
+        ).build(_processor)
+
+        self.tokenizer = self._processor.tokenizer
+        self.image_processor = self._processor.image_processor
+
+    def _pixel_values_norm(
+        self,
+        pixel_values: torch.Tensor,
+        mm_kwargs: object,
+    ) -> torch.Tensor:
+        hf_config = self.hf_config
+        vision_config = hf_config.vision_config
+        image_processor = self.image_processor
+        image_mean_tensor = torch.tensor(
+            image_processor.image_mean, dtype=torch.float32
+        ).reshape([1, 3, 1, 1])
+        image_std_tensor = torch.tensor(
+            image_processor.image_std, dtype=torch.float32
+        ).reshape([1, 3, 1, 1])
+        rescale_factor = torch.tensor(
+            image_processor.rescale_factor, dtype=torch.float32
+        )
+        patch_size_squared = vision_config.patch_size**2
+
+        image_mean_tensor = image_mean_tensor.squeeze([-2, -1]).repeat_interleave(
+            patch_size_squared, -1
+        )
+        image_std_tensor = image_std_tensor.squeeze([-2, -1]).repeat_interleave(
+            patch_size_squared, -1
+        )
+
+        if not image_mean_tensor.is_contiguous():
+            image_mean_tensor = image_mean_tensor.contiguous()
+        if not image_std_tensor.is_contiguous():
+            image_std_tensor = image_std_tensor.contiguous()
+
+        pixel_values = (
+            rescale_factor * pixel_values.to(torch.float32) - image_mean_tensor
+        ) / image_std_tensor
+        pixel_values = pixel_values.to(hf_config.dtype)
+        return pixel_values
+
+    def process_mm_data(
+        self, input_text, images=None, videos=None, audios=None, **kwargs
+    ) -> dict:
+        """
+        process multimodal data with transformers AutoProcessor
+        """
+        if images:
+            kwargs["images"] = images
+            if self.image_config:
+                kwargs.setdefault("images_kwargs", {}).update(self.image_config)
+        if videos:
+            kwargs["videos"] = videos
+            if self.video_config:
+                kwargs.setdefault("videos_kwargs", {}).update(self.video_config)
+
+        processor = self._processor
+        if (
+            hasattr(processor, "image_processor")
+            and isinstance(processor.image_processor, BaseImageProcessor)
+            and not self.disable_fast_image_processor
+        ):
+            if not _is_npu:
+                kwargs["device"] = "cuda"
+
+        result = processor.__call__(
+            text=[input_text],
+            padding=True,
+            return_tensors="pt",
+            **kwargs,
+        )
+
+        # Divide the processor_output into two modalities: image and video.
+        if result is not None:
+            pixel_values = result["images"]
+            if pixel_values is not None:
+                result["images"] = self._pixel_values_norm(pixel_values, kwargs)
+            for key in list(result.keys()):
+                if result[key] is None:
+                    del result[key]
+                    continue
+                if key == "grid_thw":
+                    grid_thw = result["grid_thw"]
+                    pixel_values_all = result["images"]
+                    # Identify elements where the first
+                    # dimension is greater than 1 and
+                    # treat them as the video modality
+                    mask = grid_thw[:, 0] > 1
+                    result["video_grid_thw"] = grid_thw[mask]
+                    result["image_grid_thw"] = grid_thw[~mask]
+                    image_patch_num = result["image_grid_thw"].prod(dim=1).sum()
+                    result["pixel_values"] = pixel_values_all[:image_patch_num]
+                    result["pixel_values_videos"] = pixel_values_all[image_patch_num:]
+                    del result["images"]
+                    del result["grid_thw"]
+
+                    # del empty result
+                    if result["image_grid_thw"].numel() == 0:
+                        del result["image_grid_thw"]
+                    if result["pixel_values"].numel() == 0:
+                        del result["pixel_values"]
+                    if result["video_grid_thw"].numel() == 0:
+                        del result["video_grid_thw"]
+                    if result["pixel_values_videos"].numel() == 0:
+                        del result["pixel_values_videos"]
+
+        if not self.keep_mm_feature_on_device:
+            # move feature tensors to cpu
+            for feature_name in self.FEATURE_NAMES:
+                if SGL_USE_CUDA_IPC:
+                    pass
+                else:
+                    if feature_name in result and isinstance(
+                        result[feature_name], torch.Tensor
+                    ):
+                        result[feature_name] = result[feature_name].to("cpu")
+
+        return result
+
+    def compute_mrope_positions(self, input_ids, mm_items):
+        image_grid_thw = None
+        video_grid_thw = None
+        for item in mm_items:
+            if "image_grid_thw" in item.model_specific_data:
+                image_grid_thw = item.model_specific_data["image_grid_thw"]
+            if "video_grid_thw" in item.model_specific_data:
+                video_grid_thw = item.model_specific_data["video_grid_thw"]
+
+        input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_ernie45(
+            input_ids=input_ids_tensor,
+            hf_config=self.hf_config,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+        )
+        return mrope_positions.squeeze(1), mrope_position_delta
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            audio_data=request_obj.audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        # resize images if they are raw Image objects
+        resized_images = []
+        if base_output.images and isinstance(base_output.images[0], Image.Image):
+            for image in base_output.images:
+                resized_image = resize_image(image)
+                resized_images.append(resized_image)
+            base_output.images = resized_images
+
+        if base_output.videos:
+            videos_processed = [
+                await preprocess_video(video) for video in base_output.videos
+            ]
+            base_output.videos, _ = map(list, zip(*videos_processed))
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        input_ids = input_ids.flatten()
+
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_ernie45(
+            input_ids=input_ids.unsqueeze(0),
+            hf_config=self.hf_config,
+            image_grid_thw=getattr(ret, "image_grid_thw", None),
+            video_grid_thw=getattr(ret, "video_grid_thw", None),
+        )
+        mrope_positions = mrope_positions.squeeze(1)
+
+        assert (
+            input_ids.shape[0] == mrope_positions.shape[-1]
+        ), "input_ids and mrope_positions should have the same length"
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_start_id=self.image_start_token_id,
+            im_end_id=self.image_end_token_id,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            mrope_positions=mrope_positions,
+            mrope_position_delta=mrope_position_delta,
+        )

python/sglang/srt/multimodal/processors/gemma3.py

@@ -0,0 +1,55 @@
+import re
+from typing import Dict, List, Union
+
+from sglang.srt.managers.multimodal_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.gemma3_mm import Gemma3ForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
+
+# Copied from: https://github.com/huggingface/transformers/blob/main/src/transformers/models/gemma3/image_processing_gemma3_fast.py
+# will be removed in the future
+
+
+class Gemma3SGLangImageProcessor(SGLangBaseProcessor):
+    models = [Gemma3ForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.IM_START_TOKEN_ID = hf_config.boi_token_index
+        self.IM_END_TOKEN_ID = hf_config.eoi_token_index
+        self.mm_tokens = MultimodalSpecialTokens(
+            # The single, pre-expanded image token.
+            image_token="<start_of_image>",
+            image_token_id=hf_config.image_token_index,
+            # The regex that matches expanded image tokens.
+            image_token_regex=re.compile(
+                r"<start_of_image>(?:(?:<image_soft_token>)*<end_of_image>)?"
+            ),
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+            discard_alpha_channel=True,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_start_id=self.IM_START_TOKEN_ID,
+            im_end_id=self.IM_END_TOKEN_ID,
+        )

python/sglang/srt/multimodal/processors/gemma3n.py

@@ -0,0 +1,71 @@
+# Copyright 2025 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from typing import Dict, List, Optional, Union
+
+from sglang.srt.managers.multimodal_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.gemma3n_mm import Gemma3nForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
+
+
+class Gemma3nSGLangProcessor(SGLangBaseProcessor):
+    """Multimodal processor for Gemma3n supporting image and audio inputs."""
+
+    models = [Gemma3nForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.IM_START_TOKEN_ID = hf_config.boi_token_id
+        self.IM_END_TOKEN_ID = hf_config.eoi_token_id
+
+        self.AUDIO_START_TOKEN_ID = hf_config.boa_token_id
+        self.AUDIO_END_TOKEN_ID = hf_config.eoa_token_id
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<image_soft_token>",
+            image_token_id=hf_config.image_token_id,
+            audio_token="<audio_soft_token>",
+            audio_token_id=hf_config.audio_token_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: Optional[List[Union[str, bytes, Dict]]] = None,
+        audio_data: Optional[List[Union[str, bytes, Dict]]] = None,
+        input_text: str = "",
+        request_obj=None,
+        *args,
+        **kwargs,
+    ):
+        """Process multimodal data including images and audio."""
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+        )

python/sglang/srt/multimodal/processors/gemma4.py

@@ -0,0 +1,158 @@
+# Copyright 2025 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import re
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import torch
+
+from sglang.srt.managers.multimodal_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
+from sglang.srt.models.gemma4_audio import _SSCP_CONV_STRIDE_SIZES
+from sglang.srt.models.gemma4_mm import Gemma4ForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
+
+
+class Gemma4SGLangProcessor(SGLangBaseProcessor):
+    """Multimodal processor for Gemma4 supporting image, video, and audio inputs."""
+
+    models = [Gemma4ForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.IM_START_TOKEN_ID = hf_config.boi_token_id
+        self.IM_END_TOKEN_ID = hf_config.eoi_token_id
+
+        self.AUDIO_START_TOKEN_ID = hf_config.boa_token_id
+        self.AUDIO_END_TOKEN_ID = hf_config.eoa_token_id
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|image|>",
+            image_token_id=hf_config.image_token_id,
+            image_token_regex=re.compile(
+                r"<\|image>(?:<\|image\|>)+<image\|>|<\|image\|>"
+            ),
+            video_token="<|video|>",
+            video_token_id=hf_config.video_token_id,
+            video_token_regex=re.compile(
+                r"<\|image>(?:<\|video\|>)+<image\|>|<\|video\|>"
+            ),
+            audio_token="<|audio|>",
+            audio_token_id=hf_config.audio_token_id,
+            audio_token_regex=re.compile(
+                r"<\|audio>(?:<\|audio\|>)+<audio\|>|<\|audio\|>"
+            ),
+        ).build(_processor)
+
+        # Register image-processor and video-processor outputs so they are stored on
+        # MultimodalDataItem via collect_mm_items_from_processor_output.
+        self.ATTR_NAME_TO_MODALITY["image_position_ids"] = Modality.IMAGE
+        self.ATTR_NAME_TO_MODALITY["video_position_ids"] = Modality.VIDEO
+
+    def _get_audio_pad_multiple(self) -> int:
+        """Derive the waveform padding alignment from processor config.
+
+        The HF processor's ceil(duration_ms / audio_ms_per_token) formula can
+        overshoot by 1 token relative to what the SSCP convolutions produce.
+        Padding waveforms to a multiple of (hop_length * first_conv_stride)
+        aligns the two calculations.
+        See: gemma-4-eap-extras/examples/gemma-4-audio-examples.ipynb
+        """
+        fe = getattr(self._processor, "feature_extractor", None)
+        hop = getattr(fe, "hop_length", 160)
+        first_stride = _SSCP_CONV_STRIDE_SIZES[0][0]
+        return hop * first_stride
+
+    def _video_decoder_to_tensor(self, vdw: VideoDecoderWrapper) -> torch.Tensor:
+        """Convert a VideoDecoderWrapper to a (sampled_frames, C, H, W) uint8 tensor.
+
+        SGLang's load_video returns VideoDecoderWrapper which the HF
+        Gemma4VideoProcessor does not recognise (expects torch.Tensor or
+        np.ndarray).  We replicate HF's uniform frame sampling here to
+        avoid materialising the entire video in memory, then delegate the
+        rest (resize, patchify, position IDs) to the HF video processor.
+        """
+        total = len(vdw)
+        num_frames = getattr(
+            getattr(self._processor, "video_processor", None),
+            "num_frames",
+            32,
+        )
+        if total <= num_frames:
+            indices = list(range(total))
+        else:
+            indices = torch.arange(0, total, total / num_frames).int().tolist()
+        frames_np = vdw.get_frames_at(indices)  # (N, H, W, C)
+        return torch.from_numpy(frames_np).permute(0, 3, 1, 2).contiguous()
+
+    def process_mm_data(
+        self, input_text, images=None, videos=None, audios=None, **kwargs
+    ):
+        if audios:
+            pad_multiple = self._get_audio_pad_multiple()
+            padded = []
+            for a in audios:
+                a = np.asarray(a)
+                remainder = len(a) % pad_multiple
+                if remainder != 0:
+                    a = np.pad(a, (0, pad_multiple - remainder), mode="constant")
+                padded.append(a)
+            audios = padded
+        if videos:
+            videos = [
+                (
+                    self._video_decoder_to_tensor(v)
+                    if isinstance(v, VideoDecoderWrapper)
+                    else v
+                )
+                for v in videos
+            ]
+            kwargs.setdefault("do_sample_frames", False)
+        return super().process_mm_data(
+            input_text, images=images, videos=videos, audios=audios, **kwargs
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: Optional[List[Union[str, bytes, Dict]]] = None,
+        audio_data: Optional[List[Union[str, bytes, Dict]]] = None,
+        input_text: str = "",
+        request_obj=None,
+        *args,
+        **kwargs,
+    ):
+        """Process multimodal data including images, video, and audio."""
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data if request_obj else None,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+        )

python/sglang/srt/multimodal/processors/gemma4_unified.py

@@ -0,0 +1,33 @@
+# Copyright 2026 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from sglang.srt.models.gemma4_unified import Gemma4UnifiedForConditionalGeneration
+from sglang.srt.multimodal.processors.gemma4 import Gemma4SGLangProcessor
+
+
+class Gemma4UnifiedSGLangProcessor(Gemma4SGLangProcessor):
+    """Multimodal processor for the encoder-free unified Gemma4 (image/video/audio).
+
+    Identical to :class:`Gemma4SGLangProcessor` except for audio padding: the
+    unified model has no SSCP conformer, so the waveform is simply chunked into
+    fixed ``audio_samples_per_token`` (640) frames.  Padding the waveform up to a
+    multiple of that frame size keeps ``ceil(num_samples / spt)`` consistent with
+    the number of valid frames the feature extractor emits.
+    """
+
+    models = [Gemma4UnifiedForConditionalGeneration]
+
+    def _get_audio_pad_multiple(self) -> int:
+        fe = getattr(self._processor, "feature_extractor", None)
+        return getattr(fe, "audio_samples_per_token", 640)

python/sglang/srt/multimodal/processors/glm4v.py

@@ -0,0 +1,123 @@
+from typing import List, Union
+
+from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.glm4v import Glm4vForConditionalGeneration
+from sglang.srt.models.glm4v_moe import Glm4vMoeForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+
+try:
+    from sglang.srt.models.glm_ocr import GlmOcrForConditionalGeneration
+except ImportError:
+    GlmOcrForConditionalGeneration = None
+
+
+class Glm4vImageProcessor(SGLangBaseProcessor):
+    models = [
+        m
+        for m in [
+            Glm4vForConditionalGeneration,
+            Glm4vMoeForConditionalGeneration,
+            GlmOcrForConditionalGeneration,
+        ]
+        if m is not None
+    ]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        # GLM-V specific tokens
+        self.IMAGE_TOKEN = "<|image|>"
+        self.VIDEO_TOKEN = "<|video|>"
+        self.IMAGE_START_TOKEN = "<|begin_of_image|>"
+        self.IMAGE_END_TOKEN = "<|end_of_image|>"
+        self.VIDEO_START_TOKEN = "<|begin_of_video|>"
+        self.VIDEO_END_TOKEN = "<|end_of_video|>"
+
+        # Token IDs
+        self.IM_TOKEN_ID = hf_config.image_token_id
+        self.VIDEO_TOKEN_ID = hf_config.video_token_id
+        self.IMAGE_START_TOKEN_ID = hf_config.image_start_token_id
+        self.IMAGE_END_TOKEN_ID = hf_config.image_end_token_id
+        self.VIDEO_START_TOKEN_ID = hf_config.video_start_token_id
+        self.VIDEO_END_TOKEN_ID = hf_config.video_end_token_id
+
+        # Vision config
+        self.IMAGE_FACTOR = 28
+        self.MIN_PIXELS = 112 * 112
+        self.MAX_PIXELS = 30000 * 28 * 28 * 2
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+            video_token=self.VIDEO_TOKEN,
+            # Note: For GLM4v videos, it uses the video token before tokenization but uses image token after tokenization
+            video_token_id=self.IM_TOKEN_ID,
+        ).build(_processor)
+
+    def compute_mrope_positions(self, input_ids, mm_items):
+        image_grid_thw = None
+        video_grid_thw = None
+        for item in mm_items:
+            if "image_grid_thw" in item.model_specific_data:
+                image_grid_thw = item.model_specific_data["image_grid_thw"]
+            if "video_grid_thw" in item.model_specific_data:
+                video_grid_thw = item.model_specific_data["video_grid_thw"]
+
+        import torch
+
+        input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
+        attention_mask = torch.ones_like(input_ids_tensor)
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_glm4v(
+            input_ids=input_ids_tensor,
+            hf_config=self.hf_config,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            attention_mask=attention_mask,
+        )
+        return mrope_positions.squeeze(1), mrope_position_delta
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        if base_output.videos:
+            base_output.videos = request_obj.video_data
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        input_ids = input_ids.flatten()
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_glm4v(
+            input_ids=input_ids.unsqueeze(0),
+            hf_config=self.hf_config,
+            image_grid_thw=getattr(ret, "image_grid_thw", None),
+            video_grid_thw=getattr(ret, "video_grid_thw", None),
+            attention_mask=getattr(ret, "attention_mask", None),
+        )
+        mrope_positions = mrope_positions.squeeze(1)
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            mrope_positions=mrope_positions,
+            mrope_position_delta=mrope_position_delta,
+        )

python/sglang/srt/multimodal/processors/glm_image.py

@@ -0,0 +1,316 @@
+import logging
+from typing import List, Union
+
+import torch
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.glm_image_vl import GlmImageForConditionalGeneration
+
+logger = logging.getLogger(__name__)
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+
+
+class GlmImageProcessor(SGLangBaseProcessor):
+    models = [GlmImageForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.IMAGE_TOKEN = "<|image|>"
+        self.IMAGE_START_TOKEN = "<|begin_of_image|>"
+        self.IMAGE_END_TOKEN = "<|end_of_image|>"
+
+        self.IM_TOKEN_ID = hf_config.image_token_id
+        self.IMAGE_START_TOKEN_ID = hf_config.image_start_token_id
+        self.IMAGE_END_TOKEN_ID = hf_config.image_end_token_id
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+        ).build(_processor)
+
+    def _compute_glm_image_mrope_positions(
+        self,
+        input_ids: torch.Tensor,
+        image_grid_thw: torch.Tensor,
+    ):
+        """Compute MRoPE positions for GlmImage (image generation model).
+
+        For source images (prefill), creates 2D spatial encoding.
+        For target image grids (decode), pre-computes 2D spatial positions
+        so each generated token gets proper (temporal, height, width) coordinates.
+        For text tokens, uses sequential positions across all 3 dims.
+
+        The returned position_ids has shape (3, prefill_len + decode_len) where
+        decode_len covers the target grid tokens. During decode, the model looks
+        up positions by index (seq_len - 1) to get proper 2D spatial encoding.
+        """
+        seq_len = input_ids.shape[0]
+        device = input_ids.device
+
+        image_start_token_id = self.IMAGE_START_TOKEN_ID
+        image_end_token_id = self.IMAGE_END_TOKEN_ID
+
+        text_positions = torch.arange(seq_len, device=device).unsqueeze(0).repeat(3, 1)
+
+        # Find image boundaries
+        image_end_positions = torch.where(input_ids == image_end_token_id)[0]
+        image_start_positions = torch.where(input_ids == image_start_token_id)[0] + 1
+
+        current_pos = 0
+        prev_image_end = 0
+        position_id_parts = []
+
+        num_complete_images = len(image_end_positions)
+
+        for img_idx in range(min(num_complete_images, len(image_start_positions))):
+            start = image_start_positions[img_idx].item()
+            end = image_end_positions[img_idx].item()
+
+            if image_grid_thw is None or img_idx >= len(image_grid_thw):
+                break
+
+            _, height, width = image_grid_thw[img_idx].tolist()
+            height = int(height)
+            width = int(width)
+
+            # Text tokens before this image
+            llm_pos_length = start - prev_image_end
+            llm_position_ids = text_positions[
+                :, current_pos : current_pos + llm_pos_length
+            ]
+            current_pos += llm_pos_length
+
+            # Image tokens with 2D spatial encoding
+            image_seq_length = height * width
+            position_width = torch.arange(
+                current_pos, current_pos + width, device=device
+            ).repeat(height)
+            position_height = torch.arange(
+                current_pos, current_pos + height, device=device
+            ).repeat_interleave(width)
+            position_temporal = torch.full(
+                (image_seq_length,), current_pos, device=device, dtype=torch.long
+            )
+            vision_position_ids = torch.stack(
+                [position_temporal, position_height, position_width], dim=0
+            )
+            current_pos += max(height, width)
+
+            prev_image_end = end
+            position_id_parts.append(
+                torch.cat([llm_position_ids, vision_position_ids], dim=-1)
+            )
+
+        # Remaining text tokens
+        end_length = seq_len - prev_image_end
+        llm_position_ids = text_positions[:, current_pos : current_pos + end_length]
+        current_pos += end_length
+        position_id_parts.append(llm_position_ids)
+
+        # Prefill positions
+        position_ids = torch.cat(position_id_parts, dim=-1)
+
+        # --- Decode positions for target (incomplete) image grids ---
+        # Target grids are those in image_grid_thw beyond the complete images.
+        # These correspond to the image tokens the model will generate autoregressively.
+        # Each generated token needs a 2D spatial position based on its row/col
+        # in the target grid, matching HF's _cached_decode_position_ids logic.
+        if image_grid_thw is not None:
+            total_grids = len(image_grid_thw)
+            num_decode_grids = total_grids - num_complete_images
+
+            if num_decode_grids > 0:
+                decode_pos = current_pos
+                decode_parts = []
+
+                # Iterate in reverse order to match HF's get_rope_index:
+                # for i in range(1, num_decode_grids + 1): grid_idx = -i
+                for i in range(1, num_decode_grids + 1):
+                    grid_idx = -i
+                    _, h, w = image_grid_thw[grid_idx].tolist()
+                    h, w = int(h), int(w)
+                    total_tokens = h * w
+
+                    h_indices = (
+                        torch.arange(h, device=device)
+                        .unsqueeze(1)
+                        .expand(h, w)
+                        .flatten()
+                    )
+                    w_indices = (
+                        torch.arange(w, device=device)
+                        .unsqueeze(0)
+                        .expand(h, w)
+                        .flatten()
+                    )
+
+                    decode_temporal = torch.full(
+                        (total_tokens,), decode_pos, device=device, dtype=torch.long
+                    )
+                    decode_height = decode_pos + h_indices
+                    decode_width = decode_pos + w_indices
+
+                    decode_parts.append(
+                        torch.stack(
+                            [decode_temporal, decode_height, decode_width], dim=0
+                        )
+                    )
+                    decode_pos += max(h, w)
+
+                # End marker for tokens after target grid
+                end_marker = torch.full(
+                    (3, 1), decode_pos, device=device, dtype=torch.long
+                )
+                decode_parts.append(end_marker)
+
+                decode_positions = torch.cat(decode_parts, dim=1)
+                position_ids = torch.cat([position_ids, decode_positions], dim=1)
+
+        mrope_position_delta = torch.zeros([1], dtype=torch.long, device=device)
+        return position_ids, mrope_position_delta
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        image_grid_thw = None
+
+        # When input_text is a list of ints (pre-tokenized input_ids passed
+        # directly via engine.generate(input_ids=...)), preserve them as-is
+        # to avoid lossy decode→re-tokenize roundtrip.
+        if (
+            isinstance(input_text, list)
+            and len(input_text)
+            and isinstance(input_text[0], int)
+        ):
+            input_ids = torch.tensor(input_text, dtype=torch.long)
+            mm_items = []
+            if image_data:
+                for img in image_data:
+                    if not isinstance(img, dict):
+                        continue
+                    # Create proper mm_items from processor_output dicts
+                    # so pixel_values reach the vision encoder.
+                    # Only create items when actual pixel features are present.
+                    if "pixel_values" in img:
+                        items = self.collect_mm_items_from_processor_output(img)
+                        for item in items:
+                            if img.get("format") == "processor_output":
+                                from sglang.srt.managers.schedule_batch import (
+                                    MultimodalInputFormat,
+                                )
+
+                                item.format = MultimodalInputFormat.PROCESSOR_OUTPUT
+
+                            # Filter image_grid_thw on mm_item to only include
+                            # source grids that have corresponding pixel_values.
+                            # Target generation grids (no pixels) must NOT go to
+                            # vision encoder — they are only for MRoPE positions.
+                            pv = getattr(item, "feature", None)
+                            grid = getattr(item, "image_grid_thw", None)
+                            if pv is not None and grid is not None:
+                                total_pixels = pv.shape[0]
+                                source_patches = 0
+                                source_grid_count = 0
+                                for gi in range(len(grid)):
+                                    patches = int(grid[gi].prod().item())
+                                    if source_patches + patches <= total_pixels:
+                                        source_patches += patches
+                                        source_grid_count += 1
+                                    else:
+                                        break
+                                if source_grid_count < len(grid):
+                                    item.image_grid_thw = grid[:source_grid_count]
+
+                        mm_items.extend(items)
+                    # Extract full image_grid_thw for MRoPE position computation
+                    # (includes both source and target grids)
+                    if "image_grid_thw" in img:
+                        grid = img["image_grid_thw"]
+                        if isinstance(grid, torch.Tensor):
+                            image_grid_thw = grid
+                        if isinstance(grid, list):
+                            image_grid_thw = torch.tensor(grid)
+
+            # Add offsets to all mm_items (matching base_processor behavior).
+            # Offsets tell the chunked prefill where image tokens are in input_ids.
+            for mm_item in mm_items:
+                mm_token_id = self.mm_tokens.get_token_id_by_modality(mm_item.modality)
+                if mm_token_id is not None:
+                    mm_item.offsets = self.get_mm_items_offset(
+                        input_ids=input_ids,
+                        mm_token_id=mm_token_id,
+                    )
+        else:
+            base_output = await self.load_mm_data(
+                prompt=input_text,
+                image_data=image_data,
+                multimodal_tokens=self.mm_tokens,
+            )
+
+            mm_items, input_ids, ret = self.process_and_combine_mm_data(
+                base_output, self.mm_tokens
+            )
+
+            input_ids = input_ids.flatten()
+
+            # Get full image_grid_thw for MRoPE (includes target grids)
+            image_grid_thw = getattr(ret, "image_grid_thw", None)
+
+            # Filter mm_item grids to only source grids (with pixel_values).
+            # Target generation grids must NOT go to vision encoder.
+            for item in mm_items:
+                pv = getattr(item, "feature", None)
+                grid = getattr(item, "image_grid_thw", None)
+                if pv is not None and grid is not None:
+                    total_pixels = pv.shape[0]
+                    source_patches = 0
+                    source_grid_count = 0
+                    for gi in range(len(grid)):
+                        patches = int(grid[gi].prod().item())
+                        if source_patches + patches <= total_pixels:
+                            source_patches += patches
+                            source_grid_count += 1
+                        else:
+                            break
+                    if source_grid_count < len(grid):
+                        item.image_grid_thw = grid[:source_grid_count]
+
+        # Fallback: get image_grid_thw from mm_items or image_data dicts
+        if image_grid_thw is None:
+            grids = []
+            for item in mm_items:
+                g = getattr(item, "image_grid_thw", None)
+                if g is not None:
+                    grids.append(g if g.dim() == 2 else g.unsqueeze(0))
+            if grids:
+                image_grid_thw = torch.cat(grids, dim=0)
+        if image_grid_thw is None and image_data:
+            for img in image_data:
+                if isinstance(img, dict) and "image_grid_thw" in img:
+                    image_grid_thw = img["image_grid_thw"]
+                    if isinstance(image_grid_thw, torch.Tensor):
+                        break
+
+        mrope_positions, mrope_position_delta = self._compute_glm_image_mrope_positions(
+            input_ids=input_ids,
+            image_grid_thw=image_grid_thw,
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+            mrope_positions=mrope_positions,
+            mrope_position_delta=mrope_position_delta,
+        )

python/sglang/srt/multimodal/processors/glmasr.py

@@ -0,0 +1,54 @@
+import re
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.glmasr import GlmAsrForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class GlmAsrProcessor(BaseMultimodalProcessor):
+    models = [GlmAsrForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.AUDIO_TOKEN = "<|begin_of_audio|><|pad|><|end_of_audio|>"
+        self.AUDIO_TOKEN_REGEX = re.compile(
+            r"<\|begin_of_audio\|><\|pad\|><\|end_of_audio\|>"
+        )
+        # Collect special token ids
+        tokenizer = self._processor.tokenizer
+        self.audio_start_id = tokenizer.convert_tokens_to_ids("<|begin_of_audio|>")
+        self.audio_token_id = tokenizer.convert_tokens_to_ids("<|pad|>")
+        self.audio_end_id = tokenizer.convert_tokens_to_ids("<|end_of_audio|>")
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=self.AUDIO_TOKEN,
+            audio_token_regex=self.AUDIO_TOKEN_REGEX,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        audio_data,
+        input_text,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base_output is None:
+            return None
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            audio_start_id=self.audio_start_id,
+            audio_token_id=self.audio_token_id,
+            audio_end_id=self.audio_end_id,
+        )

python/sglang/srt/multimodal/processors/interns1pro.py

@@ -0,0 +1,122 @@
+import time
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.interns1pro import InternS1ProForConditionalGeneration
+from sglang.srt.multimodal.processors.qwen_vl import (
+    QwenVLImageProcessor,
+    preprocess_video,
+)
+from sglang.utils import logger
+
+
+class InternS1_1ImageProcessor(QwenVLImageProcessor):
+    models = [
+        InternS1ProForConditionalGeneration,
+    ]
+
+    def get_mm_data(self, prompt, embeddings, img_grid_thw):
+        input_ids, offsets = self.build_input_ids(prompt, img_grid_thw)
+
+        mm_items = [
+            MultimodalDataItem(
+                modality=Modality.IMAGE,
+                offsets=offsets,
+                precomputed_embeddings=embeddings,
+            )
+        ]
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=mm_items,
+            im_start_id=self.IM_START_TOKEN_ID,
+            im_end_id=self.IM_END_TOKEN_ID,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        entry_time = time.perf_counter()
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            audio_data=request_obj.audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        load_time = time.perf_counter()
+        rid = getattr(request_obj, "rid", "anonymous_rid")
+
+        video_metadata = None
+        if base_output.videos:
+            videos_processed = [
+                await preprocess_video(video, video_config=self.video_config)
+                for video in base_output.videos
+            ]
+            base_output.videos, video_metadata = map(list, zip(*videos_processed))
+
+        preprocess_time = time.perf_counter()
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output,
+            self.mm_tokens,
+            video_metadata=video_metadata,
+            do_sample_frames=False,
+        )
+
+        second_per_grid_ts = getattr(ret, "second_per_grid_ts", None)
+        if second_per_grid_ts is None:
+            second_per_grid_ts = getattr(ret, "video_second_per_grid", None)
+
+        process_time = time.perf_counter()
+
+        input_ids = input_ids.flatten()
+
+        image_grid_thw = None
+        if hasattr(ret, "image_grid_thw"):
+            image_grid_thw = ret.image_grid_thw
+
+        if image_grid_thw is None and image_data and isinstance(image_data[0], dict):
+            image_grid_thw = image_data[0].get("image_grid_thw")
+
+        video_grid_thw = None
+        if hasattr(ret, "video_grid_thw"):
+            video_grid_thw = ret.video_grid_thw
+
+        if video_grid_thw is None and request_obj.video_data:
+            first_video = request_obj.video_data[0]
+            if isinstance(first_video, dict):
+                video_grid_thw = first_video.get("video_grid_thw")
+
+        get_rope_index_time = time.perf_counter()
+
+        logger.debug(
+            f"[QwenVLProcessor Perf] {rid=}, "
+            f"load_time: {(load_time - entry_time) * 1000:.2f} ms, "
+            f"preprocess_time: {(preprocess_time - load_time) * 1000:.2f} ms, "
+            f"process_time: {(process_time - preprocess_time) * 1000:.2f} ms, "
+            f"get_rope_index_time: {(get_rope_index_time - process_time) * 1000:.2f} ms, "
+            f"total_time: {(get_rope_index_time - entry_time) * 1000:.2f} ms"
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_start_id=self.vision_start_token_id,
+            im_end_id=self.vision_end_token_id,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+        )

python/sglang/srt/multimodal/processors/internvl.py

@@ -0,0 +1,739 @@
+# Adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
+
+import logging
+from functools import lru_cache
+from typing import List
+
+import numpy as np
+import torch
+from PIL import Image
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.interns1 import InternS1ForConditionalGeneration
+from sglang.srt.models.internvl import InternVLChatModel
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    BaseMultiModalProcessorOutput,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils import get_device
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
+
+logger = logging.getLogger(__name__)
+
+
+class InternVLProcessor(BaseMultimodalProcessor):
+    models = [InternVLChatModel, InternS1ForConditionalGeneration]
+    gpu_image_decode = False  # InternVL HF processor does not support tensor inputs
+
+    IMAGENET_MEAN = [0.485, 0.456, 0.406]
+    IMAGENET_STD = [0.229, 0.224, 0.225]
+    IMAGE_MAX_NUM = 12
+
+    DEFAULT_VIDEO_NUM_FRAMES = 32
+    VIDEO_MAX_NUM = 1
+    VIDEO_USE_THUMBNAIL = False
+
+    CONTEXT_FALLBACK = 40960
+    CONTEXT_RESERVED = 256
+
+    # OpenAI multimodal placeholder tokens
+    IMAGE_PLACEHOLDER_TOKEN = "<image>"
+    VIDEO_PLACEHOLDER_TOKEN = "<video>"
+
+    IMG_START = "<img>"
+    IMG_END = "</img>"
+    IMG_CONTEXT = "<IMG_CONTEXT>"
+
+    @staticmethod
+    @lru_cache(maxsize=1)
+    def _get_normalize_tensors(device="cuda", dtype=torch.float32):
+        mean = torch.tensor(
+            InternVLProcessor.IMAGENET_MEAN, device=device, dtype=dtype
+        ).view(-1, 1, 1)
+        std = torch.tensor(
+            InternVLProcessor.IMAGENET_STD, device=device, dtype=dtype
+        ).view(-1, 1, 1)
+        return mean, std
+
+    def __init__(self, hf_config, server_args, _image_processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _image_processor, *args, **kwargs)
+
+        image_size = (
+            getattr(hf_config, "force_image_size", None)
+            or hf_config.vision_config.image_size
+        )
+        patch_size = hf_config.vision_config.patch_size
+        if isinstance(image_size, list):
+            image_size = image_size[0]
+        if isinstance(patch_size, list):
+            patch_size = patch_size[0]
+
+        if hasattr(self._processor, "tokenizer"):
+            tokenizer = self._processor.tokenizer
+        else:
+            tokenizer = self._processor
+        self.tokenizer = tokenizer
+
+        # Support both InternVL (llm_config) and InternS1 (text_config).
+        # Different multimodal models use different field names for the text backbone:
+        # - InternVL uses: hf_config.llm_config
+        # - InternS1 uses: hf_config.text_config
+        # - Some store architectures at top-level
+        text_cfg = (
+            getattr(hf_config, "llm_config", None)
+            or getattr(hf_config, "text_config", None)
+            or hf_config
+        )
+        llm_arch = (getattr(text_cfg, "architectures", []) or [None])[0]
+        self.llm_arch = llm_arch
+        video_token_map = {
+            "Qwen2ForCausalLM": "<|video_pad|>",
+            "Qwen3ForCausalLM": "<|video_pad|>",
+            "Qwen3MoeForCausalLM": "<|video_pad|>",
+            "GptOssForCausalLM": "<|reserved_200000|>",
+        }
+        self.VIDEO_CONTEXT_TOKEN = video_token_map.get(llm_arch, None)
+        self.video_token_id = (
+            tokenizer.convert_tokens_to_ids(self.VIDEO_CONTEXT_TOKEN)
+            if self.VIDEO_CONTEXT_TOKEN
+            else None
+        )
+
+        self.image_token_id = (
+            tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT)
+            if self.IMG_CONTEXT
+            else None
+        )
+        self.num_image_token = int(
+            (image_size // patch_size) ** 2 * (hf_config.downsample_ratio**2)
+        )
+
+        self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START)
+        self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END)
+
+        # Placeholder token use <image>/<video>
+        # Offset token id use IMG_CONTEXT / VIDEO_CONTEXT
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_PLACEHOLDER_TOKEN,
+            image_token_id=self.image_token_id,
+            video_token=self.VIDEO_PLACEHOLDER_TOKEN,
+            video_token_id=self.video_token_id,
+        ).build(_image_processor)
+
+        # Cache token id for IMG_CONTEXT (used by both branches)
+        self.img_context_token_id = tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT)
+
+        # InternLM2 legacy multimodal tokens: use <IMG_CONTEXT> as placeholder
+        self.mm_tokens_internlm2 = MultimodalSpecialTokens(
+            image_token=self.IMG_CONTEXT,
+            image_token_id=self.img_context_token_id,
+        ).build(_image_processor)
+
+        self.max_context_len = (
+            getattr(server_args, "context_length", None)
+            or getattr(server_args, "max_context_len", None)
+            or getattr(hf_config, "max_position_embeddings", None)
+            or getattr(text_cfg, "max_position_embeddings", None)
+            or self.CONTEXT_FALLBACK
+        )
+
+    @staticmethod
+    def dynamic_preprocess(
+        tensor, image_size=448, max_num=IMAGE_MAX_NUM, use_thumbnail=False
+    ):
+        # Tensor: (C,H,W) float on GPU
+        C, H, W = tensor.shape
+        aspect_ratio = W / H
+
+        # Generate all possible aspect ratios
+        target_ratios = set(
+            (i, j)
+            for n in range(1, max_num + 1)
+            for i in range(1, n + 1)
+            for j in range(1, n + 1)
+            if i * j <= max_num
+        )
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        # Find closest ratio
+        best_ratio_diff = float("inf")
+        best_ratio = (1, 1)
+
+        for x, y in target_ratios:
+            target_ar = x / y
+            diff = abs(aspect_ratio - target_ar)
+            blocks = x * y
+            best_blocks = best_ratio[0] * best_ratio[1]
+
+            if diff < best_ratio_diff:
+                best_ratio_diff = diff
+                best_ratio = (x, y)
+            elif diff == best_ratio_diff and blocks > best_blocks:
+                best_ratio = (x, y)
+
+        target_w, target_h = image_size * best_ratio[0], image_size * best_ratio[1]
+        blocks = best_ratio[0] * best_ratio[1]
+
+        # Resize on GPU
+        resized = torch.nn.functional.interpolate(
+            tensor.unsqueeze(0),
+            size=(target_h, target_w),
+            mode="bicubic",
+            align_corners=False,
+        ).squeeze(0)
+
+        # Split into tiles
+        tiles = []
+        for i in range(blocks):
+            x = (i % best_ratio[0]) * image_size
+            y = (i // best_ratio[0]) * image_size
+            tile = resized[:, y : y + image_size, x : x + image_size]
+            tiles.append(tile)
+
+        # Add thumbnail if needed
+        if use_thumbnail and len(tiles) > 1:
+            thumb = torch.nn.functional.interpolate(
+                tensor.unsqueeze(0),
+                size=(image_size, image_size),
+                mode="bicubic",
+                align_corners=False,
+            ).squeeze(0)
+            tiles.append(thumb)
+
+        return torch.stack(tiles).to(torch.bfloat16)
+
+    @staticmethod
+    def _open_video_reader(path: str):
+        return VideoDecoderWrapper(path)
+
+    def _ensure_placeholders_before_assistant(
+        self, prompt: str, placeholder: str, want: int
+    ) -> str:
+        if want <= 0:
+            return prompt
+        have = (prompt or "").count(placeholder)
+        missing = want - have
+        if missing <= 0:
+            return prompt
+
+        insert = "\n" + "\n".join([placeholder] * missing) + "\n"
+
+        marker = "<|im_start|>assistant"
+        idx = (prompt or "").rfind(marker)
+        if idx != -1:
+            return (prompt or "")[:idx] + insert + (prompt or "")[idx:]
+        return (prompt or "") + insert
+
+    def _token_len(self, text: str) -> int:
+        try:
+            ids = self.tokenizer(text, return_tensors="pt")["input_ids"].flatten()
+            return int(ids.numel())
+        except Exception:
+            return 0
+
+    def _resolve_video_num_frames(
+        self, *, requested: int, num_videos: int, text_len: int, image_tile_cnt: int
+    ) -> int:
+        if num_videos <= 0:
+            return 0
+        if not self.VIDEO_CONTEXT_TOKEN or not self.video_token_id:
+            return 0
+        image_tokens = image_tile_cnt * self.num_image_token
+        budget = (
+            int(self.max_context_len)
+            - int(text_len)
+            - int(image_tokens)
+            - int(self.CONTEXT_RESERVED)
+        )
+        if budget <= 0:
+            return 1
+        max_total_frames = max(1, budget // self.num_image_token)
+        frames_per_video = max(1, max_total_frames // max(num_videos, 1))
+        return max(1, min(int(requested), int(frames_per_video)))
+
+    @staticmethod
+    def _has_special_format(image_data, video_data):
+        """Check if any input items use processor_output or precomputed_embedding format."""
+        for data in list(image_data or []) + list(video_data or []):
+            if isinstance(data, dict) and data.get("format") in (
+                "processor_output",
+                "precomputed_embedding",
+            ):
+                return True
+        return False
+
+    async def _process_special_format(
+        self, image_data, video_data, input_text, request_obj, **kwargs
+    ):
+        """Handle processor_output and precomputed_embedding input formats.
+
+        Delegates to the base class process_and_combine_mm_data which has
+        built-in support for these formats.
+        """
+        # When user provides input_ids directly, input_text may be a list of ints
+        if isinstance(input_text, list):
+            user_input_ids = input_text
+            prompt = ""
+        else:
+            user_input_ids = None
+            prompt = input_text or ""
+
+        # When the prompt is empty (user provided input_ids directly),
+        # load_mm_data can't match multimodal tokens to data items.
+        # Build BaseMultiModalProcessorOutput directly from the dict items.
+        if not prompt and (image_data or video_data):
+            images = [d for d in (image_data or []) if isinstance(d, dict)]
+            videos = [d for d in (video_data or []) if isinstance(d, dict)]
+
+            # Raise if raw (non-dict) images/videos were silently filtered out.
+            # InternVL cannot process raw images without a text prompt because
+            # dynamic tiling and placeholder expansion require the prompt string.
+            raw_img_dropped = len(image_data or []) - len(images)
+            raw_vid_dropped = len(video_data or []) - len(videos)
+            if raw_img_dropped > 0 or raw_vid_dropped > 0:
+                raise ValueError(
+                    f"[internvl] Cannot process raw images/videos with pre-tokenized "
+                    f"input_ids. Provide multimodal data in 'processor_output' or "
+                    f"'precomputed_embedding' format, or use a text prompt instead. "
+                    f"(raw images dropped: {raw_img_dropped}, "
+                    f"raw videos dropped: {raw_vid_dropped})"
+                )
+
+            base_output = BaseMultiModalProcessorOutput(
+                input_text=prompt,
+                images=images,
+                videos=videos,
+            )
+        else:
+            base_output = await self.load_mm_data(
+                prompt=prompt,
+                image_data=image_data,
+                video_data=video_data,
+                multimodal_tokens=self.mm_tokens,
+                discard_alpha_channel=True,
+            )
+
+        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        # If user provided input_ids directly, use those and recompute offsets
+        if user_input_ids is not None:
+            input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
+            for mm_item in mm_items:
+                if (
+                    mm_item.modality == Modality.VIDEO
+                    and self.video_token_id is not None
+                ):
+                    mm_token_id = self.video_token_id
+                else:
+                    mm_token_id = self.img_context_token_id
+                mm_item.offsets = self.get_mm_items_offset(
+                    input_ids=input_ids_tensor,
+                    mm_token_id=mm_token_id,
+                )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids_tensor.flatten().tolist(),
+            mm_items=mm_items,
+            im_start_id=self.img_start_token_id,
+            im_end_id=self.img_end_token_id,
+            im_token_id=self.img_context_token_id,
+            video_token_id=self.video_token_id,
+        )
+
+    async def process_mm_data_async(
+        self, image_data, input_text, request_obj, **kwargs
+    ):
+        video_data = getattr(request_obj, "video_data", None) or []
+
+        # Handle processor_output and precomputed_embedding formats
+        if isinstance(input_text, list) or self._has_special_format(
+            image_data, video_data
+        ):
+            return await self._process_special_format(
+                image_data=image_data,
+                video_data=video_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+
+        is_internlm2 = self.llm_arch == "InternLM2ForCausalLM"
+
+        if is_internlm2:
+            return await self.process_internlm2_mm_data_async(
+                image_data=image_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+        else:
+            # Default branch uses OpenAI-style placeholders
+            return await self.process_qwen_mm_data_async(
+                image_data=image_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+
+    async def process_qwen_mm_data_async(
+        self, image_data, input_text, request_obj, **kwargs
+    ):
+
+        img_max_num = (
+            getattr(request_obj, "image_max_dynamic_patch", None)
+            or getattr(request_obj, "max_dynamic_patch", None)
+            or kwargs.get("image_max_dynamic_patch")
+            or kwargs.get("max_dynamic_patch")
+            or self.IMAGE_MAX_NUM
+        )
+        img_max_num = max(1, int(img_max_num))
+
+        vid_max_num = (
+            getattr(request_obj, "video_max_dynamic_patch", None)
+            or getattr(request_obj, "max_dynamic_patch", None)
+            or kwargs.get("video_max_dynamic_patch")
+            or kwargs.get("max_dynamic_patch")
+            or self.VIDEO_MAX_NUM
+        )
+        vid_max_num = max(1, int(vid_max_num))
+
+        # Qwen/Qwen3 branch: OpenAI-style placeholders <image>/<video>
+        prompt = input_text or ""
+        video_data = getattr(request_obj, "video_data", None) or []
+
+        if image_data:
+            prompt = self._ensure_placeholders_before_assistant(
+                prompt, self.IMAGE_PLACEHOLDER_TOKEN, len(image_data)
+            )
+        if video_data:
+            prompt = self._ensure_placeholders_before_assistant(
+                prompt, self.VIDEO_PLACEHOLDER_TOKEN, len(video_data)
+            )
+
+        logger.info(
+            "[internvl][qwen] placeholders image=%d video=%d",
+            prompt.count(self.IMAGE_PLACEHOLDER_TOKEN),
+            prompt.count(self.VIDEO_PLACEHOLDER_TOKEN),
+        )
+
+        base_output = await self.load_mm_data(
+            prompt=prompt,
+            image_data=image_data,
+            video_data=video_data,
+            multimodal_tokens=self.mm_tokens,  # expects <image>/<video>
+            discard_alpha_channel=True,
+        )
+
+        logger.info(
+            "[internvl][qwen] loaded images=%d videos=%d",
+            len(base_output.images),
+            len(base_output.videos),
+        )
+
+        mean, std = self._get_normalize_tensors(device=get_device())
+
+        # ----- Images -> tiles -----
+        num_patches_list: List[int] = []
+        pixel_values_list: List[torch.Tensor] = []
+
+        for image in base_output.images:
+            if isinstance(image, Image.Image):
+                img_np = np.array(image.convert("RGB"))
+                tensor = (
+                    torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float()
+                    / 255.0
+                )
+            else:
+                tensor = image.to(get_device())
+
+            tensor = (tensor - mean) / std
+            tiles = self.dynamic_preprocess(
+                tensor, image_size=448, max_num=img_max_num, use_thumbnail=True
+            )
+            pixel_values_list.append(tiles)
+            num_patches_list.append(int(tiles.shape[0]))
+
+        if image_data and not pixel_values_list:
+            raise ValueError(
+                "[internvl][qwen] image_data provided but no images parsed from prompt placeholders"
+            )
+
+        image_tensor = (
+            torch.cat(pixel_values_list, dim=0) if pixel_values_list else None
+        )
+
+        # ----- Videos -> frame tiles (optional) -----
+        video_tensor = None
+        video_patch_lists = []
+        video_pixel_values = []
+
+        requested_frames = int(
+            kwargs.get("video_num_frames", self.DEFAULT_VIDEO_NUM_FRAMES)
+        )
+        num_frames = self._resolve_video_num_frames(
+            requested=requested_frames,
+            num_videos=len(base_output.videos),
+            text_len=self._token_len(base_output.input_text or prompt),
+            image_tile_cnt=int(sum(num_patches_list)) if num_patches_list else 0,
+        )
+
+        if base_output.videos and num_frames > 0 and self.video_token_id is not None:
+            for video in base_output.videos:
+                is_video_obj = isinstance(video, VideoDecoderWrapper)
+                vr = video if is_video_obj else self._open_video_reader(str(video))
+                max_frame = len(vr) - 1
+                frame_indices = (
+                    [0]
+                    if num_frames == 1
+                    else np.linspace(0, max_frame, num=num_frames, dtype=int).tolist()
+                )
+
+                per_video_tiles = []
+                per_video_patch_cnt = []
+                for fi in frame_indices:
+                    img_np = vr[int(fi)]
+                    frame_t = (
+                        torch.from_numpy(img_np)
+                        .permute(2, 0, 1)
+                        .to(get_device())
+                        .float()
+                        / 255.0
+                    )
+                    frame_t = (frame_t - mean) / std
+
+                    tiles = self.dynamic_preprocess(
+                        frame_t,
+                        image_size=448,
+                        max_num=vid_max_num,
+                        use_thumbnail=self.VIDEO_USE_THUMBNAIL,
+                    )
+                    per_video_tiles.append(tiles)
+                    per_video_patch_cnt.append(int(tiles.shape[0]))
+
+                pv = torch.cat(per_video_tiles, dim=0)
+                video_pixel_values.append(pv)
+                video_patch_lists.append(per_video_patch_cnt)
+
+            video_tensor = (
+                torch.cat(video_pixel_values, dim=0) if video_pixel_values else None
+            )
+
+        # ----- Build prompt text with <img> + CONTEXT*n + </img> -----
+        img_ph = "<<<__IMG_PLACEHOLDER__>>>"
+        vid_ph = "<<<__VID_PLACEHOLDER__>>>"
+
+        input_text_mid = base_output.input_text or prompt
+        input_text_mid = input_text_mid.replace(self.IMAGE_PLACEHOLDER_TOKEN, img_ph)
+        input_text_mid = input_text_mid.replace(self.IMG_CONTEXT, img_ph)
+
+        if self.VIDEO_CONTEXT_TOKEN and self.video_token_id is not None:
+            input_text_mid = input_text_mid.replace(
+                self.VIDEO_PLACEHOLDER_TOKEN, vid_ph
+            )
+        else:
+            input_text_mid = input_text_mid.replace(self.VIDEO_PLACEHOLDER_TOKEN, "")
+
+        input_text_updated = input_text_mid
+
+        # Expand images
+        for num_patches in num_patches_list:
+            image_tokens = (
+                self.IMG_START
+                + (self.IMG_CONTEXT * (self.num_image_token * int(num_patches)))
+                + self.IMG_END
+            )
+            input_text_updated = input_text_updated.replace(img_ph, image_tokens, 1)
+
+        # Expand videos (each frame is one <img>...</img>)
+        if video_patch_lists and self.VIDEO_CONTEXT_TOKEN:
+            for frame_patch_list in video_patch_lists:
+                frame_lines = []
+                for i, patch_cnt in enumerate(frame_patch_list):
+                    ctx_cnt = int(self.num_image_token) * int(patch_cnt)
+                    frame_tokens = (
+                        self.IMG_START
+                        + (self.VIDEO_CONTEXT_TOKEN * ctx_cnt)
+                        + self.IMG_END
+                    )
+                    frame_lines.append(f"Frame {i+1}: {frame_tokens}")
+                video_tokens = "\n".join(frame_lines) + "\n"
+                input_text_updated = input_text_updated.replace(vid_ph, video_tokens, 1)
+
+        # Tokenize
+        input_ids_tensor = self.tokenizer(input_text_updated, return_tensors="pt")[
+            "input_ids"
+        ].flatten()
+        input_ids = input_ids_tensor.tolist()
+
+        # Offsets
+        image_offsets = []
+        if image_tensor is not None:
+            image_offsets = self.get_mm_items_offset(
+                input_ids=input_ids_tensor.to(get_device()),
+                mm_token_id=self.img_context_token_id,
+            )
+
+        video_offsets = []
+        if video_tensor is not None and self.video_token_id is not None:
+            video_offsets = self.get_mm_items_offset(
+                input_ids=input_ids_tensor.to(get_device()),
+                mm_token_id=self.video_token_id,
+            )
+
+        items = []
+        if image_tensor is not None:
+            # Split per-image for better cache granularity
+            assert len(num_patches_list) == len(image_offsets), (
+                f"InternVL: num_patches_list ({len(num_patches_list)}) != "
+                f"image_offsets ({len(image_offsets)})"
+            )
+            cumulative = 0
+            for i, num_patches in enumerate(num_patches_list):
+                items.append(
+                    MultimodalDataItem(
+                        feature=image_tensor[cumulative : cumulative + num_patches],
+                        modality=Modality.IMAGE,
+                        offsets=[image_offsets[i]],
+                    )
+                )
+                cumulative += num_patches
+        if video_tensor is not None:
+            items.append(
+                MultimodalDataItem(
+                    feature=video_tensor, modality=Modality.VIDEO, offsets=video_offsets
+                )
+            )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=items,
+            im_start_id=self.img_start_token_id,
+            im_end_id=self.img_end_token_id,
+            im_token_id=self.img_context_token_id,
+            video_token_id=self.video_token_id,
+        )
+
+    async def process_internlm2_mm_data_async(
+        self, image_data, input_text, request_obj, **kwargs
+    ):
+        # InternLM2 branch: legacy placeholder <IMG_CONTEXT> (stable for InternLM2 prompt behavior)
+        prompt = input_text or ""
+        video_data = getattr(request_obj, "video_data", None) or []
+        if video_data:
+            logger.warning(
+                "[internvl][internlm2] video input ignored for InternLM2 branch"
+            )
+
+        # Convert any OpenAI-style <image> into <IMG_CONTEXT>
+        prompt = prompt.replace(self.IMAGE_PLACEHOLDER_TOKEN, self.IMG_CONTEXT)
+
+        if image_data:
+            prompt = self._ensure_placeholders_before_assistant(
+                prompt, self.IMG_CONTEXT, len(image_data)
+            )
+
+        logger.info(
+            "[internvl][internlm2] placeholders img_context=%d",
+            prompt.count(self.IMG_CONTEXT),
+        )
+
+        base_output = await self.load_mm_data(
+            prompt=prompt,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens_internlm2,  # expects <IMG_CONTEXT>
+            discard_alpha_channel=True,
+        )
+
+        mean, std = self._get_normalize_tensors(device=get_device())
+
+        num_patches_list: List[int] = []
+        pixel_values_list: List[torch.Tensor] = []
+
+        for image in base_output.images:
+            if isinstance(image, Image.Image):
+                img_np = np.array(image.convert("RGB"))
+                tensor = (
+                    torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float()
+                    / 255.0
+                )
+            else:
+                tensor = image.to(get_device())
+
+            tensor = (tensor - mean) / std
+            tiles = self.dynamic_preprocess(
+                tensor, image_size=448, max_num=12, use_thumbnail=True
+            )
+            pixel_values_list.append(tiles)
+            num_patches_list.append(int(tiles.shape[0]))
+
+        if image_data and not pixel_values_list:
+            raise ValueError(
+                "[internvl][internlm2] image_data provided but no images parsed from prompt placeholders"
+            )
+
+        pixel_values = (
+            torch.cat(pixel_values_list, dim=0) if pixel_values_list else None
+        )
+
+        # Expand each <IMG_CONTEXT> into <img> + <IMG_CONTEXT>*N + </img>
+        ph = "<<<__IMG_CONTEXT_PLACEHOLDER__>>>"
+        input_text_base = (base_output.input_text or prompt).replace(
+            self.IMG_CONTEXT, ph
+        )
+
+        input_text_updated = input_text_base
+        for num_patches in num_patches_list:
+            image_tokens = (
+                self.IMG_START
+                + (self.IMG_CONTEXT * (self.num_image_token * int(num_patches)))
+                + self.IMG_END
+            )
+            input_text_updated = input_text_updated.replace(ph, image_tokens, 1)
+
+        # Tokenize
+        input_ids_tensor = self.tokenizer(input_text_updated, return_tensors="pt")[
+            "input_ids"
+        ].flatten()
+        input_ids = input_ids_tensor.tolist()
+
+        # Offsets
+        image_offsets = []
+        if pixel_values is not None:
+            image_offsets = self.get_mm_items_offset(
+                input_ids=input_ids_tensor.to(get_device()),
+                mm_token_id=self.img_context_token_id,
+            )
+
+        items = []
+        if pixel_values is not None:
+            # Split per-image for better cache granularity
+            assert len(num_patches_list) == len(image_offsets), (
+                f"InternVL: num_patches_list ({len(num_patches_list)}) != "
+                f"image_offsets ({len(image_offsets)})"
+            )
+            cumulative = 0
+            for i, num_patches in enumerate(num_patches_list):
+                items.append(
+                    MultimodalDataItem(
+                        feature=pixel_values[cumulative : cumulative + num_patches],
+                        modality=Modality.IMAGE,
+                        offsets=[image_offsets[i]],
+                    )
+                )
+                cumulative += num_patches
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=items,
+            im_start_id=self.img_start_token_id,
+            im_end_id=self.img_end_token_id,
+            im_token_id=self.img_context_token_id,
+            video_token_id=self.video_token_id,
+        )

python/sglang/srt/multimodal/processors/janus_pro.py

@@ -0,0 +1,45 @@
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.deepseek_janus_pro import MultiModalityCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class JanusProImageProcessor(BaseMultimodalProcessor):
+    models = [MultiModalityCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=_processor.image_token,
+            image_token_id=_processor.image_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        **kwargs,
+    ):
+        base_out = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_out, self.mm_tokens, prompt=base_out.input_text
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_start_id=self._processor.image_start_id,
+            im_end_id=self._processor.image_end_id,
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/kimi_common.py

@@ -0,0 +1,128 @@
+"""Kimi-specific grid-based multimodal data helpers.
+
+Shared by KimiVLImageProcessor and KimiK2_5VLImageProcessor.
+"""
+
+from typing import Union
+
+import numpy as np
+import torch
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+
+
+class KimiGridMMDataMixin:
+    """Mixin providing Kimi-specific grid-based multimodal data helpers.
+
+    Expects the concrete class to supply:
+      - self.hf_config  (with vision_config.merge_kernel_size)
+      - self._tokenizer (with .encode())
+    """
+
+    def resolve_image_token_counts(self, images):
+        """Kimi's processor is remote-code and does not implement the
+        transformers ``_get_num_multimodal_tokens`` convention; use its
+        ``media_tokens_calculator`` instead.
+
+        """
+        assert images is not None
+        media_tokens_calculator = (
+            self._processor.media_processor.media_tokens_calculator
+        )
+        return [
+            int(media_tokens_calculator({"type": "image", "image": image}))
+            for image in images
+        ]
+
+    def _num_image_tokens_from_grid(
+        self, grid_thw: Union[torch.Tensor, np.ndarray, list, tuple]
+    ) -> int:
+        """Compute Kimi-style image token count from 2D/3D grid metadata."""
+        merge_h, merge_w = self.hf_config.vision_config.merge_kernel_size
+
+        if isinstance(grid_thw, torch.Tensor):
+            vals = grid_thw.flatten().tolist()
+        elif isinstance(grid_thw, np.ndarray):
+            vals = grid_thw.reshape(-1).tolist()
+        elif isinstance(grid_thw, (list, tuple)):
+            vals = list(np.array(grid_thw).reshape(-1).tolist())
+        else:
+            raise TypeError(
+                f"Unsupported grid type for kimi image tokens: {type(grid_thw)}"
+            )
+
+        if len(vals) >= 3:
+            _t, h, w = vals[-3], vals[-2], vals[-1]
+        elif len(vals) == 2:
+            _t, h, w = 1, vals[0], vals[1]
+        else:
+            raise ValueError(
+                f"Invalid grid metadata for kimi image tokens: {vals} "
+                "(expected [t,h,w] or [h,w])"
+            )
+
+        h, w = int(h), int(w)
+        return (h * w) // (merge_h * merge_w)
+
+    def _build_kimi_mm_data_from_grids(
+        self, prompt, embeddings, **kwargs
+    ) -> MultimodalProcessorOutput:
+        image_token_id = kwargs.get("image_token_id", 0)
+        img_grid_thw = kwargs.get("img_grid_thw", None)
+
+        if not isinstance(prompt, list):
+            prompt = self._tokenizer.encode(prompt)
+
+        image_token_counts = [
+            self._num_image_tokens_from_grid(grid) for grid in img_grid_thw
+        ]
+
+        input_ids = []
+        offsets = []
+        img_idx = 0
+
+        for token in prompt:
+            if token != image_token_id:
+                input_ids.append(token)
+                continue
+
+            if img_idx >= len(image_token_counts):
+                raise ValueError(
+                    "The number of image placeholders exceeds img_grid_thw entries."
+                )
+
+            num_tokens = image_token_counts[img_idx]
+            start = len(input_ids)
+            input_ids.extend([image_token_id] * num_tokens)
+            offsets.append((start, len(input_ids) - 1))
+            img_idx += 1
+
+        if img_idx != len(image_token_counts):
+            raise ValueError(
+                "The number of image placeholders does not match img_grid_thw entries."
+            )
+
+        image_embeddings = embeddings[Modality.IMAGE]
+        mm_items = []
+        consumed = 0
+        for start, end in offsets:
+            num_tokens = end - start + 1
+            embedding_slice = image_embeddings[consumed : consumed + num_tokens]
+            consumed += num_tokens
+            mm_items.append(
+                MultimodalDataItem(
+                    modality=Modality.IMAGE,
+                    offsets=[(start, end)],
+                    precomputed_embeddings=embedding_slice,
+                )
+            )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=mm_items,
+            im_token_id=image_token_id,
+        )

python/sglang/srt/multimodal/processors/kimi_k25.py

@@ -0,0 +1,429 @@
+import math
+import re
+from collections import defaultdict
+from typing import Dict, List, Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+
+from sglang.srt.managers.schedule_batch import (
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin
+
+# ---------------------------------------------------------------------------
+# GPU image preprocessing utilities (resize, pad, normalize, patchify on CUDA)
+# ---------------------------------------------------------------------------
+
+
+def navit_resize_config(
+    width: int,
+    height: int,
+    patch_size: int,
+    merge_kernel_size: int,
+    in_patch_limit: int,
+    patch_limit_on_one_side: int,
+    fixed_output_tokens: int | None = None,
+) -> dict:
+    """Compute NaViT resize target dimensions and token count.
+
+    Pure math -- no image data needed, only (width, height).
+    """
+    s1 = math.sqrt(
+        in_patch_limit
+        / (max(1.0, width // patch_size) * max(1.0, height // patch_size))
+    )
+    s2 = patch_limit_on_one_side * patch_size / width
+    s3 = patch_limit_on_one_side * patch_size / height
+    scale = min(1.0, s1, s2, s3)
+    new_w = min(max(1, int(width * scale)), patch_limit_on_one_side * patch_size)
+    new_h = min(max(1, int(height * scale)), patch_limit_on_one_side * patch_size)
+
+    factor = merge_kernel_size * patch_size
+    pad_height = (factor - new_h % factor) % factor
+    pad_width = (factor - new_w % factor) % factor
+
+    if fixed_output_tokens is not None:
+        num_tokens = fixed_output_tokens
+    else:
+        token_height = (new_h + pad_height) // factor
+        token_width = (new_w + pad_width) // factor
+        num_tokens = token_height * token_width
+
+    return {
+        "num_tokens": num_tokens,
+        "new_width": new_w,
+        "new_height": new_h,
+        "pad_width": pad_width,
+        "pad_height": pad_height,
+    }
+
+
+def _get_image_dimensions(image: Union[torch.Tensor, Image.Image]) -> tuple[int, int]:
+    """Get (width, height) from a CUDA tensor or PIL Image."""
+    if isinstance(image, torch.Tensor):
+        # nvJPEG returns (C, H, W) uint8
+        return image.shape[2], image.shape[1]
+    return image.size  # PIL returns (width, height)
+
+
+def _pil_to_cuda_chw(image: Image.Image) -> torch.Tensor:
+    """Convert PIL Image to (C, H, W) uint8 CUDA tensor."""
+    arr = np.asarray(image.convert("RGB"))
+    return torch.from_numpy(arr).permute(2, 0, 1).cuda()
+
+
+def _ensure_chw_rgb(image: torch.Tensor) -> torch.Tensor:
+    """Coerce an already-decoded (C, H, W) image tensor to 3-channel RGB.
+
+    PIL inputs are RGB-normalized by _pil_to_cuda_chw, but pre-decoded
+    tensor inputs (e.g. nvJPEG / cached CUDA tensors) keep their native
+    channel count. Grayscale (1ch) or RGBA (4ch) images then break the
+    downstream torch.cat over a batch of images, which requires a
+    consistent channel dimension. Normalize every tensor to 3 channels.
+
+    Also move the tensor to the GPU (matching _pil_to_cuda_chw) so a CPU
+    input does not trip a device mismatch against the CUDA image_mean /
+    image_std_inv normalization constants downstream. No-op if already on
+    the device.
+    """
+    image = image.cuda()
+    if image.dim() == 2:  # (H, W) grayscale -> (1, H, W)
+        image = image.unsqueeze(0)
+    c = image.shape[0]
+    if c == 3:
+        return image
+    if c == 1:
+        return image.repeat(3, 1, 1)
+    # RGBA or other multi-channel layouts: keep the first 3 channels.
+    return image[:3]
+
+
+def _process_single_image(
+    image: Union[torch.Tensor, Image.Image],
+    config: dict,
+    image_mean: torch.Tensor,
+    image_std_inv: torch.Tensor,
+    patch_size: int,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Process a single image on GPU: resize -> pad -> normalize -> patchify."""
+    if isinstance(image, Image.Image):
+        image = _pil_to_cuda_chw(image)
+    else:
+        image = _ensure_chw_rgb(image)
+
+    new_h, new_w = config["new_height"], config["new_width"]
+    pad_h, pad_w = config["pad_height"], config["pad_width"]
+
+    x = image.unsqueeze(0).float()
+    x = F.interpolate(x, size=(new_h, new_w), mode="bicubic", align_corners=False)
+
+    if pad_h > 0 or pad_w > 0:
+        x = F.pad(x, (0, pad_w, 0, pad_h), value=0.0)
+
+    x = x / 255.0
+    x = (x - image_mean) * image_std_inv
+
+    _, C, H, W = x.shape
+    T = 1
+    gh, gw = H // patch_size, W // patch_size
+    x = x.view(T, C, gh, patch_size, gw, patch_size)
+    x = x.permute(0, 2, 4, 1, 3, 5).reshape(-1, C, patch_size, patch_size)
+
+    grid_thw = torch.tensor([T, gh, gw], dtype=torch.int64, device=x.device)
+    return x, grid_thw
+
+
+def _gpu_preprocess_images(
+    images: list[Union[torch.Tensor, Image.Image]],
+    resize_configs: list[dict],
+    image_mean: torch.Tensor,
+    image_std_inv: torch.Tensor,
+    patch_size: int,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """GPU preprocessing pipeline for a batch of images.
+
+    Groups images with the same target padded size for batch processing.
+    """
+    n = len(images)
+    if n == 0:
+        device = image_mean.device
+        return (
+            torch.empty(0, 3, patch_size, patch_size, device=device),
+            torch.empty(0, 3, dtype=torch.int64, device=device),
+        )
+
+    groups = defaultdict(list)
+    for idx, (image, config) in enumerate(zip(images, resize_configs)):
+        padded_h = config["new_height"] + config["pad_height"]
+        padded_w = config["new_width"] + config["pad_width"]
+        target_h = config["new_height"]
+        target_w = config["new_width"]
+        groups[(target_h, target_w, padded_h, padded_w)].append((idx, image, config))
+
+    all_patches = [None] * n
+    all_grids = [None] * n
+
+    for (target_h, target_w, padded_h, padded_w), group in groups.items():
+        if len(group) == 1:
+            idx, image, config = group[0]
+            patches, grid = _process_single_image(
+                image, config, image_mean, image_std_inv, patch_size
+            )
+            all_patches[idx] = patches
+            all_grids[idx] = grid
+        else:
+            tensors = []
+            for _, image, _ in group:
+                if isinstance(image, Image.Image):
+                    image = _pil_to_cuda_chw(image)
+                else:
+                    image = _ensure_chw_rgb(image)
+                tensors.append(image.unsqueeze(0).float())
+
+            resized = []
+            for t in tensors:
+                r = F.interpolate(
+                    t, size=(target_h, target_w), mode="bicubic", align_corners=False
+                )
+                resized.append(r)
+            batch = torch.cat(resized, dim=0)
+
+            pad_h = padded_h - target_h
+            pad_w = padded_w - target_w
+            if pad_h > 0 or pad_w > 0:
+                batch = F.pad(batch, (0, pad_w, 0, pad_h), value=0.0)
+
+            batch = batch / 255.0
+            batch = (batch - image_mean) * image_std_inv
+
+            B, C, H, W = batch.shape
+            T = 1
+            gh, gw = H // patch_size, W // patch_size
+            batch = batch.view(B, C, gh, patch_size, gw, patch_size)
+            batch = batch.permute(0, 2, 4, 1, 3, 5).reshape(
+                B, -1, C, patch_size, patch_size
+            )
+
+            grid = torch.tensor([T, gh, gw], dtype=torch.int64, device=batch.device)
+            for i, (idx, _, _) in enumerate(group):
+                all_patches[idx] = batch[i]
+                all_grids[idx] = grid
+
+    pixel_values = torch.cat(all_patches, dim=0)
+    grid_thws = torch.stack(all_grids, dim=0)
+    return pixel_values, grid_thws
+
+
+# ---------------------------------------------------------------------------
+# Kimi K2.5 GPU processor wrapper
+# ---------------------------------------------------------------------------
+
+
+class KimiGPUProcessorWrapper:
+    """Wraps Kimi's HF processor to do GPU image preprocessing.
+
+    GPU path: nvJPEG CUDA tensor / PIL -> _gpu_preprocess_images()
+    CPU fallback: PIL -> medias kwarg -> original HF KimiK25Processor.__call__
+
+    Exposes attributes that base class's process_mm_data needs so it behaves
+    like a normal HF processor from the outside.
+    """
+
+    def __init__(
+        self,
+        hf_processor,
+        image_token,
+        patch_size,
+        merge_kernel_size,
+        in_patch_limit,
+        patch_limit_on_one_side,
+        fixed_output_tokens,
+        image_mean,
+        image_std,
+    ):
+        self._hf_processor = hf_processor
+        self._image_token = image_token
+        self._patch_size = patch_size
+        self._merge_kernel_size = merge_kernel_size
+        self._in_patch_limit = in_patch_limit
+        self._patch_limit_on_one_side = patch_limit_on_one_side
+        self._fixed_output_tokens = fixed_output_tokens
+        self._image_mean = image_mean
+        self._image_std = image_std
+        self._gpu_norm_tensors = None
+
+        # Explicitly expose attributes that base class process_mm_data needs:
+        # - image_processor: checked via isinstance(..., BaseImageProcessor)
+        # - tokenizer: used for tokenization
+        # - media_processor: used by CPU fallback path
+        self.image_processor = hf_processor.image_processor
+        self.tokenizer = hf_processor.tokenizer
+        self.media_processor = hf_processor.media_processor
+
+    def __call__(self, text=None, images=None, **kwargs):
+        # process_mm_data passes images via kwargs["images"]
+        images = images or kwargs.pop("images", None)
+
+        if images and torch.cuda.is_available():
+            return self._gpu_call(text, images)
+        return self._cpu_call(text, images, **kwargs)
+
+    def _gpu_call(self, text, images):
+        """Bypass HF KimiK25VisionProcessor.preprocess entirely -- use GPU ops."""
+        input_text = text[0] if isinstance(text, list) else text
+
+        # 1. Compute resize configs (CPU math)
+        resize_configs = []
+        for image in images:
+            w, h = _get_image_dimensions(image)
+            resize_configs.append(
+                navit_resize_config(
+                    w,
+                    h,
+                    self._patch_size,
+                    self._merge_kernel_size,
+                    self._in_patch_limit,
+                    self._patch_limit_on_one_side,
+                    self._fixed_output_tokens,
+                )
+            )
+
+        # 2. Expand image tokens
+        parts = input_text.split(self._image_token)
+        result = [parts[0]]
+        for config, part in zip(resize_configs, parts[1:]):
+            result.append(self._image_token * config["num_tokens"] + part)
+        input_text = "".join(result)
+
+        # 3. Tokenize
+        text_inputs = self._hf_processor.tokenizer(input_text, return_tensors="pt")
+
+        # 4. GPU image preprocessing
+        image_mean, image_std_inv = self._get_gpu_norm_tensors()
+        pixel_values, grid_thws = _gpu_preprocess_images(
+            images, resize_configs, image_mean, image_std_inv, self._patch_size
+        )
+
+        grid_thws = grid_thws.cpu()
+
+        return {
+            "input_ids": text_inputs["input_ids"],
+            "pixel_values": pixel_values,
+            # Use SGL-standard key so get_new_expanded_mm_items() can split
+            # per-image for cache granularity (it looks up 'image_grid_thw').
+            "image_grid_thw": grid_thws,
+        }
+
+    def _cpu_call(self, text, images, **kwargs):
+        """Fallback: token expansion + medias kwarg -> original HF processor."""
+        input_text = text[0] if isinstance(text, list) else text
+
+        if images:
+            # Token expansion via media_tokens_calculator
+            parts = input_text.split(self._image_token)
+            result = [parts[0]]
+            for image, part in zip(images, parts[1:]):
+                num_tokens = self._hf_processor.media_processor.media_tokens_calculator(
+                    {"type": "image", "image": image}
+                )
+                result.append(self._image_token * num_tokens + part)
+            input_text = "".join(result)
+
+            # Convert to medias format for Kimi's HF processor
+            kwargs["medias"] = [{"type": "image", "image": img} for img in images]
+
+        out = self._hf_processor(text=[input_text], **kwargs)
+        grid_thws = out.pop("grid_thws", None)
+        if grid_thws is not None:
+            out["image_grid_thw"] = grid_thws
+        return out
+
+    def _get_gpu_norm_tensors(self, device="cuda"):
+        if self._gpu_norm_tensors is None:
+            image_mean = torch.tensor(
+                self._image_mean, device=device, dtype=torch.float32
+            ).view(1, 3, 1, 1)
+            image_std_inv = (
+                1.0 / torch.tensor(self._image_std, device=device, dtype=torch.float32)
+            ).view(1, 3, 1, 1)
+            self._gpu_norm_tensors = (image_mean, image_std_inv)
+        return self._gpu_norm_tensors
+
+
+# ---------------------------------------------------------------------------
+# Kimi K2.5 SGLang multimodal processor
+# ---------------------------------------------------------------------------
+
+
+# Compatible with KimiVLForConditionalGeneration
+class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
+    models = [KimiK25ForConditionalGeneration]
+    gpu_image_decode = True  # nvJPEG for JPEG, PIL fallback for others
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|media_pad|>",
+            # TODO: could we convert in MultimodalSpecialTokens?
+            image_token_id=hf_config.media_placeholder_token_id,
+            image_token_regex=re.compile(r"(?:<\|media_pad\|>)+"),
+        ).build(_processor)
+
+        # Extract media processing config from HF processor
+        media_proc_cfg = _processor.media_processor.media_proc_cfg
+
+        # Replace with GPU-capable wrapper
+        self._processor = KimiGPUProcessorWrapper(
+            _processor,
+            image_token=self.mm_tokens.image_token,
+            patch_size=media_proc_cfg["patch_size"],
+            merge_kernel_size=media_proc_cfg["merge_kernel_size"],
+            in_patch_limit=media_proc_cfg["in_patch_limit"],
+            patch_limit_on_one_side=media_proc_cfg["patch_limit_on_one_side"],
+            fixed_output_tokens=media_proc_cfg.get("fixed_output_tokens"),
+            image_mean=media_proc_cfg["image_mean"],
+            image_std=media_proc_cfg["image_std"],
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+        )
+
+    def get_mm_data(self, prompt, embeddings, **kwargs):
+        img_grid_thw = kwargs.get("img_grid_thw", None)
+        return self._build_kimi_mm_data_from_grids(
+            prompt=prompt,
+            embeddings=embeddings,
+            image_token_id=self.mm_tokens.image_token_id,
+            img_grid_thw=img_grid_thw,
+        )

python/sglang/srt/multimodal/processors/kimi_vl.py

@@ -0,0 +1,60 @@
+import re
+from typing import Dict, List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.kimi_vl import KimiVLForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin
+
+
+# Compatible with KimiVLForConditionalGeneration
+class KimiVLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
+    models = [KimiVLForConditionalGeneration]
+    gpu_image_decode = False  # KimiVL HF processor does not support tensor inputs
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|media_pad|>",
+            # TODO: could we convert in MultimodalSpecialTokens?
+            image_token_id=hf_config.media_placeholder_token_id,
+            image_token_regex=re.compile(r"(?:<\|media_pad\|>)+"),
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+        )
+
+    def get_mm_data(self, prompt, embeddings, **kwargs):
+        img_grid_thw = kwargs.get("img_grid_thw", None)
+        return self._build_kimi_mm_data_from_grids(
+            prompt=prompt,
+            embeddings=embeddings,
+            image_token_id=self.mm_tokens.image_token_id,
+            img_grid_thw=img_grid_thw,
+        )

python/sglang/srt/multimodal/processors/lfm2_vl.py

@@ -0,0 +1,85 @@
+# Copyright 2026 Liquid AI. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Multimodal processor for LFM2-VL models with SigLip2 NaFlex support."""
+
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
+from sglang.srt.models.lfm2_vl import Lfm2VlForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+
+
+class Lfm2VlImageProcessor(SGLangBaseProcessor):
+    """Multimodal processor for LFM2-VL vision-language models.
+
+    Uses the base class load_mm_data + process_and_combine_mm_data flow.
+    The HF processor handles NaFlex variable-resolution tiling internally.
+    """
+
+    models = [Lfm2VlForConditionalGeneration]
+    gpu_image_decode = False
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.IMAGE_TOKEN_ID = hf_config.image_token_id
+        self.IMAGE_TOKEN = "<image>"
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=hf_config.image_token_id,
+        ).build(_processor)
+
+        # Register NaFlex-specific HF processor outputs so
+        # collect_mm_items_from_processor_output picks them up
+        self.ATTR_NAME_TO_MODALITY["pixel_attention_mask"] = Modality.IMAGE
+        self.ATTR_NAME_TO_MODALITY["spatial_shapes"] = Modality.IMAGE
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        audio_data,
+        input_text: str,
+        request_obj,
+        **kwargs,
+    ):
+        if not image_data:
+            input_ids = self._tokenizer(
+                input_text, return_tensors="pt", add_special_tokens=False
+            ).input_ids
+            return {
+                "input_ids": input_ids.squeeze(0).tolist(),
+                "mm_items": [],
+                "im_token_id": self.IMAGE_TOKEN_ID,
+            }
+
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.IMAGE_TOKEN_ID,
+        )

python/sglang/srt/multimodal/processors/lightonocr.py

@@ -0,0 +1,110 @@
+# Copyright 2025 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""
+Multimodal processor for lightonai/LightOnOCR-2-1B.
+
+Key difference from Pixtral: LightOnOCR does NOT use image break/end tokens.
+The parent PixtralProcessor inserts row-break and image-end tokens between
+image patch rows. This processor removes them after the parent processing
+to produce a single contiguous range of image tokens per image.
+"""
+
+from typing import List, Union
+
+from sglang.srt.models.lightonocr import LightOnOCRForConditionalGeneration
+from sglang.srt.multimodal.processors.pixtral import PixtralProcessor
+
+
+class LightOnOCRProcessor(PixtralProcessor):
+    """Processor for LightOnOCR model."""
+
+    models = [LightOnOCRForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        # LightOnOCR uses image_token_id instead of image_token_index
+        if not hasattr(hf_config, "image_token_index"):
+            hf_config.image_token_index = getattr(hf_config, "image_token_id", 151655)
+
+        # Propagate spatial_merge_size from root config to vision_config
+        spatial_merge_size = getattr(hf_config, "spatial_merge_size", 2)
+        if hasattr(hf_config, "vision_config"):
+            vc = hf_config.vision_config
+            if not hasattr(vc, "spatial_merge_size") or vc.spatial_merge_size is None:
+                vc.spatial_merge_size = spatial_merge_size
+
+        if hasattr(_processor, "patch_size"):
+            _processor.spatial_merge_size = spatial_merge_size
+
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        # Identify break/end token IDs for removal
+        self._break_token_ids = set()
+        for attr in ("image_break_token_id", "image_break_id"):
+            tid = getattr(_processor, attr, None)
+            if tid is not None:
+                self._break_token_ids.add(tid)
+        for attr in ("image_end_token_id", "image_end_id"):
+            tid = getattr(_processor, attr, None)
+            if tid is not None:
+                self._break_token_ids.add(tid)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        result = await super().process_mm_data_async(
+            image_data=image_data,
+            input_text=input_text,
+            request_obj=request_obj,
+            *args,
+            **kwargs,
+        )
+
+        if not result or not self._break_token_ids:
+            return result
+
+        # Remove break/end tokens and fix multimodal item offsets
+        input_ids = result.input_ids or []
+        mm_items = result.mm_items or []
+
+        new_input_ids = []
+        old_to_new = {}
+        for old_idx, token_id in enumerate(input_ids):
+            if token_id not in self._break_token_ids:
+                old_to_new[old_idx] = len(new_input_ids)
+                new_input_ids.append(token_id)
+
+        if len(new_input_ids) == len(input_ids):
+            return result
+
+        # Remap multimodal item offsets to account for removed tokens
+        for mm_item in mm_items:
+            if not mm_item.offsets:
+                continue
+            new_indices = sorted(
+                old_to_new[idx]
+                for start, end in mm_item.offsets
+                for idx in range(start, end + 1)
+                if idx in old_to_new
+            )
+            if new_indices:
+                mm_item.offsets = [(new_indices[0], new_indices[-1])]
+
+        result.input_ids = new_input_ids
+        return result

python/sglang/srt/multimodal/processors/llava.py

@@ -0,0 +1,268 @@
+import asyncio
+import os
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+from transformers.models.auto.processing_auto import (
+    PROCESSOR_MAPPING_NAMES as HF_MAPPING_NAMES,
+)
+
+import sglang.srt.managers.multimodal_processor as sgl_mm_processor_utils
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.llava import (
+    LlavaForConditionalGeneration,
+    LlavaLlamaForCausalLM,
+    LlavaMistralForCausalLM,
+    LlavaQwenForCausalLM,
+)
+from sglang.srt.models.llavavid import LlavaVidForCausalLM
+from sglang.srt.models.mistral import Mistral3ForConditionalGeneration
+from sglang.srt.multimodal.mm_utils import (
+    ensure_numpy,
+    expand2square,
+    process_anyres_image,
+)
+from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor
+from sglang.srt.utils import ImageData, load_image, logger
+from sglang.utils import get_exception_traceback
+
+
+class LlavaImageProcessor(BaseMultimodalProcessor):
+    models = [
+        LlavaLlamaForCausalLM,
+        LlavaVidForCausalLM,
+        LlavaQwenForCausalLM,
+        LlavaMistralForCausalLM,
+    ]
+    gpu_image_decode = False  # Llava processes loaded image as PIL image explicitly
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+    @staticmethod
+    def _process_single_image_task(
+        image_data: Union[str, bytes, ImageData],
+        image_aspect_ratio: Optional[str] = None,
+        image_grid_pinpoints: Optional[str] = None,
+        processor=None,
+    ):
+
+        image_processor = processor.image_processor
+
+        try:
+            url = image_data.url if isinstance(image_data, ImageData) else image_data
+            image, image_size = load_image(url, False)
+            if image_size is not None:
+                # It is a video with multiple images
+                image_hash = hash(url)
+                pixel_values = image_processor(image)["pixel_values"]
+                for i in range(len(pixel_values)):
+                    pixel_values[i] = ensure_numpy(pixel_values[i]).astype(np.float16)
+                pixel_values = np.stack(pixel_values, axis=0)
+                return pixel_values, image_hash, image_size
+            else:
+                # It is an image
+                image_hash = hash(url)
+                if image_aspect_ratio == "pad":
+                    image = expand2square(
+                        image,
+                        tuple(int(x * 255) for x in image_processor.image_mean),
+                    )
+                    pixel_values = image_processor(image.convert("RGB"))[
+                        "pixel_values"
+                    ][0]
+                elif image_aspect_ratio == "anyres" or (
+                    image_aspect_ratio is not None
+                    and "anyres_max" in image_aspect_ratio
+                ):
+                    pixel_values = process_anyres_image(
+                        image, image_processor, image_grid_pinpoints
+                    )
+                else:
+                    pixel_values = image_processor(image)["pixel_values"][0]
+
+                pixel_values = ensure_numpy(pixel_values)
+                if isinstance(pixel_values, np.ndarray):
+                    pixel_values = pixel_values.astype(np.float16)
+
+                return pixel_values, image_hash, image.size
+        except Exception:
+            logger.error("Exception in TokenizerManager:\n" + get_exception_traceback())
+
+    async def _process_single_image(
+        self,
+        image_data: Union[bytes, str, ImageData],
+        aspect_ratio: str,
+        grid_pinpoints: str,
+    ):
+        if self.cpu_executor is not None:
+            loop = asyncio.get_running_loop()
+            fut = loop.run_in_executor(
+                self.cpu_executor,
+                LlavaImageProcessor._process_single_image_task,
+                image_data,
+                aspect_ratio,
+                grid_pinpoints,
+                self._processor,
+            )
+            timeout = int(os.environ.get("REQUEST_TIMEOUT", "10"))
+            return await asyncio.wait_for(fut, timeout=timeout)
+        else:
+            return self._process_single_image_task(
+                image_data,
+                aspect_ratio,
+                grid_pinpoints,
+                self._processor.image_processor,
+            )
+
+    def _process_precomputed_image_data(self, image_data: List[Dict]) -> Dict:
+        mm_items = []
+        for item in image_data:
+            # Infer size logic...
+            if "image_sizes" not in item:
+                if "pixel_values" in item:
+                    pv = item["pixel_values"]
+                    # Handle simplified if/else
+                    h, w = (
+                        (pv.shape[2], pv.shape[3])
+                        if len(pv.shape) == 4
+                        else (pv.shape[1], pv.shape[2])
+                    )
+                    item["image_sizes"] = [(w, h)]
+                else:
+                    item["image_sizes"] = [(336, 336)]
+
+            mm_items.append(
+                MultimodalDataItem(
+                    feature=item["feature"],
+                    modality=Modality.IMAGE,
+                    model_specific_data=item,
+                )
+            )
+        return MultimodalProcessorOutput(mm_items=mm_items)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, ImageData]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        # FIX: Handle precomputed embeddings (dictionaries)
+        # If the input is already a dictionary, we skip the CPU image processor.
+        # We also need to infer 'image_sizes' from 'pixel_values' if missing,
+        # because pad_input_ids requires it.
+        if (
+            isinstance(image_data, list)
+            and len(image_data) > 0
+            and isinstance(image_data[0], dict)
+        ):
+            return self._process_precomputed_image_data(image_data)
+
+        modalities = request_obj.modalities or ["image"]
+        aspect_ratio = getattr(self.hf_config, "image_aspect_ratio", None)
+        grid_pinpoints = (
+            self.hf_config.image_grid_pinpoints
+            if hasattr(self.hf_config, "image_grid_pinpoints")
+            and "anyres" in aspect_ratio
+            else None
+        )
+
+        if isinstance(image_data, list) and len(image_data) > 0:
+            if "multi-images" in modalities or "video" in modalities:
+                # Multiple images
+                aspect_ratio = "pad"  # LLaVA OneVision Handling: more than one image --> interleaved image mode or video mode. We do not use anyres
+                pixel_values, data_hashes, image_sizes = [], [], []
+                res = []
+                for img_data in image_data:
+                    res.append(
+                        self._process_single_image(
+                            img_data, aspect_ratio, grid_pinpoints
+                        )
+                    )
+
+                res = await asyncio.gather(*res)
+                for pixel_v, image_h, image_s in res:
+                    pixel_values.append(pixel_v)
+                    data_hashes.append(image_h)
+                    image_sizes.append(image_s)
+            else:
+                # A single image
+                pixel_values, image_hash, image_size = await self._process_single_image(
+                    image_data[0], aspect_ratio, grid_pinpoints
+                )
+                pixel_values = [pixel_values]
+                image_sizes = [image_size]
+        else:
+            raise ValueError(f"Invalid image data: {image_data}")
+        modality = Modality.IMAGE
+        if isinstance(request_obj.modalities, list):
+            if request_obj.modalities[0] == "video":
+                modality = Modality.VIDEO
+
+        # Create one item per image for better cache granularity
+        mm_items = []
+        for pixel_v, image_s in zip(pixel_values, image_sizes):
+            # Ensure ndim=4 so the model forward takes the correct encode branch
+            if isinstance(pixel_v, np.ndarray) and pixel_v.ndim == 3:
+                pixel_v = np.expand_dims(pixel_v, 0)
+            mm_items.append(
+                MultimodalDataItem(
+                    feature=pixel_v,
+                    model_specific_data={
+                        "image_sizes": [image_s],
+                        "image_aspect_ratio": aspect_ratio,
+                    },
+                    modality=modality,
+                )
+            )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+        )
+
+
+class LlavaMultimodalProcessor(BaseMultimodalProcessor):
+    """
+    This is a wrapper class used to identify the multimodal processor for Llava architectures' vision model.
+    """
+
+    models = [LlavaForConditionalGeneration, Mistral3ForConditionalGeneration]
+
+    def _get_sgl_processor_cls(self, model_type: str):
+        if model_type == "clip_vision_model":
+            return LlavaImageProcessor
+        if hf_name := HF_MAPPING_NAMES.get(model_type):
+            sgl_mm_processor_set = sgl_mm_processor_utils.PROCESSOR_MAPPING.values()
+            sgl_processor_cls = list(
+                filter(lambda p: p.__name__ == hf_name, sgl_mm_processor_set)
+            )
+            if sgl_processor_cls:
+                return sgl_processor_cls[0]
+        raise ValueError(
+            f"Cannot find corresponding multimodal processor registered in sglang for model type `{model_type}`"
+        )
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        assert hasattr(hf_config, "vision_config")
+        assert hasattr(hf_config, "text_config")
+        self.vision_config = hf_config.vision_config
+        self.text_config = hf_config.text_config
+        self.hf_config = hf_config
+
+        if vision_type := getattr(self.vision_config, "model_type"):
+            self.inner = self._get_sgl_processor_cls(vision_type)(
+                hf_config, server_args, _processor, *args, **kwargs
+            )
+        else:
+            raise ValueError(
+                f"Required `vision_config.model_type` is not found in hf_config: `{hf_config}`"
+            )
+
+    async def process_mm_data_async(self, *args, **kwargs):
+        return await self.inner.process_mm_data_async(*args, **kwargs)

python/sglang/srt/multimodal/processors/locate_anything.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+import re
+from typing import Dict, List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.locate_anything import LocateAnythingForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+
+
+# Compatible with LocateAnythingForConditionalGeneration
+class LocateAnythingImageProcessor(SGLangBaseProcessor):
+    models = [LocateAnythingForConditionalGeneration]
+    # The LocateAnything HF processor is remote-code and does not support tensor inputs.
+    gpu_image_decode = False
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        # The model's chat template emits numbered ``<image-N>`` placeholders.
+        # The HF LocateAnythingProcessor expands each into
+        # ``<img>`` + N×``<IMG_CONTEXT>`` + ``</img>`` and only the
+        # ``<IMG_CONTEXT>`` (id 151665) run carries vision embeddings, so the
+        # offset/embedding token id is image_token_index while the prompt-level
+        # placeholder we split on is ``<image-N>``.
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token_id=hf_config.image_token_index,
+            image_token_regex=re.compile(r"<image-\d+>"),
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/midashenglm.py

@@ -0,0 +1,167 @@
+import logging
+import re
+
+import torch
+
+from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
+from sglang.srt.models.midashenglm import MiDashengLMModel
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor):
+    """Multimodal processor for MiDashengLM audio-language model."""
+
+    models = [MiDashengLMModel]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.AUDIO_TOKEN = "<|audio_bos|><|AUDIO|><|audio_eos|>"
+        self.AUDIO_TOKEN_REGEX = re.compile(
+            r"<\|audio_bos\|>(?:<\|AUDIO\|>)+<\|audio_eos\|>"
+        )
+
+        tokenizer = self._processor.tokenizer
+        self.audio_start_id = tokenizer.convert_tokens_to_ids("<|audio_bos|>")
+        self.audio_token_id = tokenizer.convert_tokens_to_ids("<|AUDIO|>")
+        self.audio_end_id = tokenizer.convert_tokens_to_ids("<|audio_eos|>")
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=self.AUDIO_TOKEN,
+            audio_token_regex=self.AUDIO_TOKEN_REGEX,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+        self.ATTR_NAME_TO_MODALITY.update(
+            {
+                "input_values": Modality.AUDIO,
+                "audio_length": Modality.AUDIO,
+            }
+        )
+
+        if "input_values" not in self.FEATURE_NAMES:
+            self.FEATURE_NAMES.append("input_values")
+
+    def process_mm_data(
+        self, input_text, images=None, videos=None, audios=None, **kwargs
+    ):
+        """Override to use correct audio parameter name for MiDashengLM processor."""
+        if images:
+            kwargs["images"] = images
+        if videos:
+            kwargs["videos"] = videos
+        if audios:
+            kwargs["audio"] = audios
+            kwargs.setdefault("audio_kwargs", {})
+            kwargs["audio_kwargs"].setdefault("truncation", False)
+            if self.audio_config:
+                kwargs["audio_kwargs"].update(self.audio_config)
+
+        processor = self._processor
+        result = processor.__call__(
+            text=[input_text],
+            padding=True,
+            return_tensors="pt",
+            **kwargs,
+        )
+
+        if not getattr(self.server_args, "keep_mm_feature_on_device", False):
+            for feature_name in ["input_values"]:
+                if feature_name in result:
+                    result[feature_name] = result[feature_name].cpu()
+
+        return result
+
+    async def process_mm_data_async(
+        self,
+        audio_data,
+        input_text,
+        **kwargs,
+    ):
+        """Process audio data for MiDashengLM model.
+
+        Args:
+            audio_data: Audio input data
+            input_text: Text prompt
+            **kwargs: Additional arguments
+
+        Returns:
+            Dictionary containing processed multimodal data
+        """
+        logger.info("=" * 80)
+        logger.info("process_mm_data_async called")
+        logger.info(f"audio_data is not None: {audio_data is not None}")
+        logger.info(f"input_text: {input_text}")
+        logger.info("=" * 80)
+
+        if audio_data and not self.AUDIO_TOKEN_REGEX.search(input_text):
+            input_text = f"{self.AUDIO_TOKEN}{input_text}"
+            logger.info("Auto-prepended audio token")
+
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base_output is None:
+            logger.info("base_output is None")
+            return None
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+        logger.info(f"mm_items count: {len(mm_items)}")
+        logger.info(f"ret keys: {list(ret.keys())}")
+        logger.info(f"input_ids shape: {input_ids.shape}")
+        logger.info(
+            f"audio_token_id={self.audio_token_id}, audio_start_id={self.audio_start_id}, audio_end_id={self.audio_end_id}"
+        )
+        logger.info(
+            f"Count of audio_token_id in input_ids: {(input_ids == self.audio_token_id).sum().item()}"
+        )
+        for i, item in enumerate(mm_items):
+            logger.info(f"mm_item[{i}] modality: {item.modality}")
+            logger.info(
+                f"mm_item[{i}] pad_value: {getattr(item, 'pad_value', 'NOT SET')}"
+            )
+            logger.info(f"mm_item[{i}] offsets: {getattr(item, 'offsets', 'NOT SET')}")
+            logger.info(f"mm_item[{i}] has feature: {hasattr(item, 'feature')}")
+            if hasattr(item, "feature") and item.feature is not None:
+                logger.info(f"mm_item[{i}] feature shape: {item.feature.shape}")
+
+        if "audio_length" in ret and len(mm_items) > 0:
+            audio_length = ret["audio_length"]
+            if isinstance(audio_length, torch.Tensor):
+                audio_length = (
+                    audio_length.item()
+                    if audio_length.numel() == 1
+                    else audio_length[0].item()
+                )
+            mm_items[0].audio_length = audio_length
+            logger.info(
+                f"Set audio_length={audio_length} (from processor, mel frame count)"
+            )
+        elif "input_values" in ret and len(mm_items) > 0:
+            input_values = ret["input_values"]
+            audio_length = (
+                input_values.shape[-1]
+                if input_values.ndim >= 2
+                else input_values.shape[0]
+            )
+            mm_items[0].audio_length = audio_length
+            logger.info(f"Set audio_length={audio_length} (fallback, waveform length)")
+
+        result = MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            audio_start_id=self.audio_start_id,
+            audio_token_id=self.audio_token_id,
+            audio_end_id=self.audio_end_id,
+        )
+        logger.info(f"Returning {len(result.mm_items)} mm_items")
+        return result

python/sglang/srt/multimodal/processors/mimo_audio.py

@@ -0,0 +1,316 @@
+"""Stateful audio preprocessing pipeline shared by MiMo multimodal and ASR processors."""
+
+import io
+import math
+import os
+import time
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Optional
+
+import numpy as np
+import pybase64
+import torch
+
+from sglang.srt.utils import common
+from sglang.utils import logger
+
+try:
+    from torchcodec.decoders import AudioDecoder
+except ImportError:
+    logger.warning(
+        "torchcodec is not installed; audio inputs will fail at request time"
+    )
+    AudioDecoder = None
+
+try:
+    import torchaudio
+    from torchaudio.transforms import MelSpectrogram
+except ImportError:
+    logger.warning(
+        "torchaudio is not installed; audio inputs will fail at request time"
+    )
+    torchaudio = None
+    MelSpectrogram = None
+
+
+@dataclass
+class AudioInput:
+    """
+    if audio is str or bytes, only load it as mel spectrogram.
+    if audio is tuple, it is (waveform, original_sr)
+    if audio is torch.Tensor, it is tokenized input ids with shape (T, n_vq+).
+    if audio is np.ndarray, it is a pre-loaded waveform (1D, already resampled).
+    """
+
+    audio: str | bytes | tuple | torch.Tensor | np.ndarray
+
+    def __post_init__(self):
+        if not isinstance(self.audio, (str, bytes, tuple, torch.Tensor, np.ndarray)):
+            raise ValueError(
+                f"audio must be a str, bytes, tuple, torch.Tensor, or np.ndarray, but got {type(self.audio)}"
+            )
+        if isinstance(self.audio, tuple):
+            if (
+                len(self.audio) != 2
+                or not isinstance(self.audio[0], torch.Tensor)
+                or not isinstance(self.audio[1], (int, float))
+            ):
+                raise ValueError(
+                    f"audio must be a tuple of (waveform-T, original_sr-int/float), but got {len(self.audio)} elements and {type(self.audio[0])} and {type(self.audio[1])}"
+                )
+            if self.audio[0].ndim != 1:
+                raise ValueError(
+                    f"waveform must be a 1D tensor, but got {self.audio[0].ndim}D tensor"
+                )
+            if self.audio[1] <= 0:
+                raise ValueError(
+                    f"original_sr must be a positive number, but got {self.audio[1]}"
+                )
+        if isinstance(self.audio, torch.Tensor) and self.audio.ndim != 2:
+            raise ValueError(
+                f"audio must be a 2D tensor, but got {self.audio.ndim}D tensor"
+            )
+
+
+class MiMoAudioPipeline:
+    """Stateful audio preprocessing pipeline.
+
+    Composable: held by both MiMoProcessor (multimodal) and MiMoV2ASRProcessor.
+    Owns the mel spectrogram, resampler cache, http session, and the special
+    token ids for ``<|sosp|> <|empty|>* <|eosp|>`` placeholders.
+    """
+
+    def __init__(
+        self,
+        *,
+        audio_token_id: int,
+        audio_start_token_id: int,
+        audio_end_token_id: int,
+        audio_kernel_size: int = 3,
+        audio_stride_size: int = 2,
+        audio_avg_pooler: int = 2,
+        audio_group_size: int = 4,
+        audio_channels: int = 8,
+        audio_sampling_rate: int = 24000,
+        audio_nfft: int = 960,
+        audio_hop_length: int = 240,
+        audio_window_size: int = 960,
+        audio_fmin: int = 0,
+        audio_fmax: Optional[int] = None,
+        audio_n_mels: int = 128,
+        audio_input_id_per_second: int = 25,
+        max_resamplers: int = 16,
+    ) -> None:
+        self.audio_token_id = audio_token_id
+        self.audio_start_token_id = audio_start_token_id
+        self.audio_end_token_id = audio_end_token_id
+
+        self.audio_kernel_size = audio_kernel_size
+        self.audio_stride_size = audio_stride_size
+        self.audio_avg_pooler = audio_avg_pooler
+        self.audio_group_size = audio_group_size
+        self.audio_channels = audio_channels
+
+        self.audio_sampling_rate = audio_sampling_rate
+        self.audio_nfft = audio_nfft
+        self.audio_hop_length = audio_hop_length
+        self.audio_window_size = audio_window_size
+        self.audio_fmin = audio_fmin
+        self.audio_fmax = audio_fmax
+        self.audio_n_mels = audio_n_mels
+        self.audio_input_id_per_second = audio_input_id_per_second
+
+        self.mel_spectrogram_kwargs = dict(
+            sample_rate=audio_sampling_rate,
+            n_fft=audio_nfft,
+            hop_length=audio_hop_length,
+            win_length=audio_window_size,
+            f_min=audio_fmin,
+            f_max=audio_fmax,
+            n_mels=audio_n_mels,
+            power=1.0,
+            center=True,
+        )
+        self._mel_spectrogram = None
+        self._resamplers: OrderedDict[int, torchaudio.transforms.Resample] = (
+            OrderedDict()
+        )
+        self._resamplers_max = max_resamplers
+
+    @property
+    def audio_token_per_second(self) -> float:
+        return self.audio_input_id_per_second / self.audio_group_size
+
+    @staticmethod
+    def _ensure_audio_dependencies() -> None:
+        if torchaudio is None or MelSpectrogram is None:
+            raise RuntimeError(
+                "torchaudio is required for audio inputs; install torchaudio"
+            )
+
+    @property
+    def mel_spectrogram(self):
+        self._ensure_audio_dependencies()
+        if self._mel_spectrogram is None:
+            self._mel_spectrogram = MelSpectrogram(**self.mel_spectrogram_kwargs)
+        return self._mel_spectrogram
+
+    def compute_audio_token_len(self, mel_len: int) -> int:
+        n = mel_len + 3 - self.audio_kernel_size
+        n = (n + 2 - self.audio_kernel_size) // self.audio_stride_size + 1
+        n = n // self.audio_avg_pooler + int(n % self.audio_avg_pooler != 0)
+        return math.ceil(n / self.audio_group_size)
+
+    def preprocess_audio(self, audio):
+        """Load audio source → log-mel spectrogram + token length.
+
+        Input: filename string, bytes, or tuple of (waveform, original_sr).
+        Output: (mel-spectrogram tensor [T, n_mels], audio_token_len int).
+        """
+        self._ensure_audio_dependencies()
+        assert isinstance(
+            audio, (str, bytes, tuple)
+        ), f"audio must be a str, bytes or tuple, but got {type(audio)}"
+        if isinstance(audio, tuple):
+            waveform, original_sr = audio
+        else:
+            if isinstance(audio, bytes):
+                file = io.BytesIO(audio)
+            elif isinstance(audio, str):
+                if audio.startswith("data:"):
+                    file = io.BytesIO(
+                        pybase64.b64decode(audio.split(",")[1], validate=True)
+                    )
+                elif audio.startswith("http://") or audio.startswith("https://"):
+                    dl_start = time.perf_counter()
+                    timeout = int(os.getenv("REQUEST_TIMEOUT", "5"))
+                    try:
+                        with common.get_mm_http_session().get(
+                            audio, stream=True, timeout=timeout
+                        ) as response:
+                            response.raise_for_status()
+                            dl_elapsed_ms = (time.perf_counter() - dl_start) * 1000
+                            if dl_elapsed_ms > 1000.0:
+                                content_len = len(response.content)
+                                logger.warning(
+                                    f"Slow audio download: {dl_elapsed_ms:.2f}ms, "
+                                    f"size={content_len / 1024:.1f}KB, url={audio}"
+                                )
+                            file = io.BytesIO(response.content)
+                    except Exception as e:
+                        dl_elapsed_ms = (time.perf_counter() - dl_start) * 1000
+                        logger.error(
+                            f"Failed to download audio: {dl_elapsed_ms:.2f}ms, "
+                            f"error={type(e).__name__}: {e}, url={audio}"
+                        )
+                        raise
+                else:
+                    file = audio
+            if AudioDecoder is None:
+                raise RuntimeError(
+                    "torchcodec is required for audio decoding; install with `pip install torchcodec`."
+                )
+            try:
+                samples = AudioDecoder(file).get_all_samples()
+            except RuntimeError as e:
+                audio_source = (
+                    audio
+                    if isinstance(audio, str)
+                    and (audio.startswith("http://") or audio.startswith("https://"))
+                    else "<bytes or base64>"
+                )
+                logger.error(f"Failed to decode audio: {e}, source={audio_source}")
+                raise ValueError(
+                    f"Invalid audio format: source={audio_source}, detail={e}"
+                ) from e
+            waveform = samples.data
+            original_sr = samples.sample_rate
+
+        if original_sr != self.audio_sampling_rate:
+            if original_sr in self._resamplers:
+                self._resamplers.move_to_end(original_sr)
+            else:
+                if len(self._resamplers) >= self._resamplers_max:
+                    self._resamplers.popitem(last=False)
+                self._resamplers[original_sr] = torchaudio.transforms.Resample(
+                    orig_freq=original_sr, new_freq=self.audio_sampling_rate
+                )
+            waveform = self._resamplers[original_sr](waveform)
+        if waveform.ndim == 2:
+            waveform = waveform.mean(dim=0)
+        spec = self.mel_spectrogram(waveform[None, :])
+        spec = torch.log(torch.clip(spec, min=1e-7)).squeeze()
+        spec = spec.transpose(0, 1)
+
+        audio_token_len = self.compute_audio_token_len(spec.shape[0])
+        return spec, audio_token_len
+
+    def process_audio(self, audio_input: AudioInput):
+        """Dispatch on the underlying audio payload.
+
+        - str/bytes/tuple/np.ndarray waveform → returns (mel-spec, token_len) tuple
+        - 2D tensor of pre-tokenized audio codes → returns padded codes tensor
+          shaped [T//group, group, channels]
+        """
+        audio = audio_input.audio
+        if isinstance(audio, np.ndarray):
+            waveform = torch.from_numpy(audio).float()
+            audio = (waveform, self.audio_sampling_rate)
+        if isinstance(audio, (str, bytes, tuple)):
+            return self.preprocess_audio(audio)
+
+        assert (
+            audio.shape[1] >= self.audio_channels
+        ), f"audio must have at least {self.audio_channels} channels, but got {audio.shape[1]}"
+        T = audio.shape[0]
+        audio = audio[:, : self.audio_channels].to(torch.long)
+        padded_T = (
+            (T + self.audio_group_size - 1)
+            // self.audio_group_size
+            * self.audio_group_size
+        )
+        padded_audio = torch.cat(
+            [
+                audio,
+                torch.zeros(padded_T - T, self.audio_channels, dtype=torch.long)
+                + audio[-1, :],
+            ],
+            dim=0,
+        )
+        padded_audio = padded_audio.reshape(
+            padded_T // self.audio_group_size,
+            self.audio_group_size,
+            self.audio_channels,
+        )
+        return padded_audio
+
+    def build_audio_placeholder_ids(self, audio_token_len: int) -> list[int]:
+        return (
+            [self.audio_start_token_id]
+            + [self.audio_token_id] * audio_token_len
+            + [self.audio_end_token_id]
+        )
+
+    def process_audio_input(self, audio_input: AudioInput) -> dict:
+        """Run process_audio and produce the placeholder input_ids.
+
+        Replaces the duplicated _process_audio_content bodies in both processors.
+        Returns dict with input_ids, audio_input (mel or codes), and is_tokenized.
+        """
+        processed = self.process_audio(audio_input)
+        if isinstance(processed, tuple):
+            is_tokenized = False
+            audio_spec, audio_token_len = processed
+            payload = audio_spec
+        else:
+            is_tokenized = True
+            audio_token_len = processed.shape[0]
+            payload = processed
+
+        return {
+            "input_ids": self.build_audio_placeholder_ids(audio_token_len),
+            "audio_input": payload,
+            "audio_token_len": audio_token_len,
+            "is_tokenized": is_tokenized,
+        }

python/sglang/srt/multimodal/processors/mimo_v2_asr.py

@@ -0,0 +1,286 @@
+"""MiMo-V2-ASR multimodal processor.
+
+Audio preprocessing is delegated to :class:`MiMoAudioPipeline`; this
+processor only handles the special-token contract and content interleaving.
+"""
+
+import asyncio
+import re
+from dataclasses import dataclass
+from typing import List, Literal, Union
+
+import numpy as np
+import torch
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.mimo_v2_asr import MiMoV2ASRForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.multimodal.processors.mimo_audio import (
+    AudioInput,
+    MiMoAudioPipeline,
+)
+from sglang.utils import logger
+
+TextInput = str | list[int]
+
+
+@dataclass
+class _Content:
+    type: Literal["text", "audio"]
+    content: TextInput | AudioInput
+
+
+class MiMoV2ASRProcessor(BaseMultimodalProcessor):
+    """ASR-only MiMo processor.
+
+    Wires three special tokens into the input id stream around each audio
+    span: ``<|sosp|> <|empty|>* <|eosp|>``. The actual mel/codec preparation
+    is owned by :class:`MiMoAudioPipeline`, which is shared with the
+    multimodal MiMo-V2 processor.
+    """
+
+    models = [MiMoV2ASRForCausalLM]
+
+    AUDIO_PAD_TOKEN = "<|empty|>"
+    AUDIO_START_TOKEN = "<|sosp|>"
+    AUDIO_END_TOKEN = "<|eosp|>"
+
+    AUDIO_REGEX = re.compile(r"<\|sosp\|>(?:<\|empty\|>)+<\|eosp\|>")
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.tokenizer = _processor
+
+        self.audio_pipeline = MiMoAudioPipeline(
+            audio_token_id=self._resolve_special_token_id(self.AUDIO_PAD_TOKEN),
+            audio_start_token_id=self._resolve_special_token_id(self.AUDIO_START_TOKEN),
+            audio_end_token_id=self._resolve_special_token_id(self.AUDIO_END_TOKEN),
+            audio_sampling_rate=24000,
+        )
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=f"{self.AUDIO_START_TOKEN}{self.AUDIO_PAD_TOKEN}{self.AUDIO_END_TOKEN}",
+            audio_token_id=self.audio_token_id,
+            audio_token_regex=self.AUDIO_REGEX,
+        ).build(_processor)
+
+    def __getattr__(self, name):
+        # Delegate audio_pipeline fields so callers can use self.audio_token_id
+        # etc. directly. Only triggers when normal attribute lookup fails;
+        # __dict__.get avoids recursion before audio_pipeline is assigned.
+        pipeline = self.__dict__.get("audio_pipeline")
+        if pipeline is not None and hasattr(pipeline, name):
+            return getattr(pipeline, name)
+        raise AttributeError(name)
+
+    def _resolve_special_token_id(self, name: str) -> int:
+        tid = self.tokenizer.convert_tokens_to_ids(name)
+        if tid is None or tid == self.tokenizer.unk_token_id:
+            raise ValueError(
+                f"tokenizer missing required special token {name!r}; "
+                "checkpoint vocab does not match MiMo-V2-ASR"
+            )
+        return int(tid)
+
+    def _process_contents(self, contents: List[_Content]):
+        """Run pipeline + tokenizer over an interleaved content list.
+
+        Returns ``(input_ids: Tensor[L], audio_inputs: list[Tensor],
+        position_ids: Tensor[3,L], rope_deltas: Tensor[1,1])``.
+        """
+        input_ids: List[int] = []
+        audio_inputs: List[torch.Tensor] = []
+
+        for content in contents:
+            if content.type == "text":
+                if isinstance(content.content, str):
+                    input_ids.extend(self.tokenizer.encode(content.content))
+                else:
+                    input_ids.extend(content.content)
+            elif content.type == "audio":
+                result = self.audio_pipeline.process_audio_input(content.content)
+                audio_inputs.append(result["audio_input"])
+                input_ids.extend(result["input_ids"])
+
+        ids = torch.as_tensor(input_ids)
+        position_ids = torch.arange(ids.shape[0]).expand(3, -1)
+        rope_deltas = torch.zeros((1, 1), dtype=torch.int32)
+        return ids, audio_inputs, position_ids, rope_deltas
+
+    def process_mm_data(
+        self, input_text, images=None, videos=None, audios=None, **kwargs
+    ) -> dict:
+        if audios and not self.AUDIO_REGEX.search(input_text or ""):
+            input_text = f"{self.mm_tokens.audio_token}{input_text or ''}"
+
+        processed_audios: List[Union[tuple, torch.Tensor]] = []
+        if audios:
+            for audio in audios:
+                if isinstance(audio, np.ndarray):
+                    audio_tensor = torch.from_numpy(audio).float()
+                elif isinstance(audio, torch.Tensor):
+                    audio_tensor = audio.float()
+                else:
+                    processed_audios.append(audio)
+                    continue
+                if audio_tensor.ndim == 1:
+                    processed_audios.append(
+                        (audio_tensor.cpu().contiguous(), self.audio_sampling_rate)
+                    )
+                else:
+                    processed_audios.append(audio_tensor.cpu().contiguous())
+
+        contents: List[_Content] = []
+        if input_text and processed_audios:
+            multimodal_tokens_pattern = self.mm_tokens.get_combined_regex()
+            text_parts = re.split(multimodal_tokens_pattern, input_text)
+            audio_iter = iter(processed_audios)
+
+            for text_part in text_parts:
+                if multimodal_tokens_pattern.match(text_part):
+                    modality = self.mm_tokens.get_modality_of_token(text_part)
+                    if modality == Modality.AUDIO:
+                        try:
+                            audio = next(audio_iter)
+                            contents.append(
+                                _Content(type="audio", content=AudioInput(audio=audio))
+                            )
+                        except StopIteration:
+                            pass
+                else:
+                    if text_part:
+                        contents.append(_Content(type="text", content=text_part))
+        else:
+            contents.extend(
+                _Content(type="audio", content=AudioInput(audio=audio))
+                for audio in processed_audios
+            )
+
+        if not contents:
+            ids = self.tokenizer(
+                input_text or "",
+                return_tensors="pt",
+                add_special_tokens=True,
+            ).input_ids
+            return {"input_ids": ids}
+
+        input_ids, audio_inputs, position_ids, rope_deltas = self._process_contents(
+            contents
+        )
+
+        ret: dict = {
+            "input_ids": input_ids,
+            "mrope_positions": position_ids,
+            "mrope_position_delta": rope_deltas,
+        }
+        if audio_inputs:
+            ret["audio_features"] = audio_inputs
+        return ret
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        audio_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        if audio_data is None:
+            audio_data = getattr(request_obj, "audio_data", [])
+        if not audio_data:
+            return None
+        if not self.AUDIO_REGEX.search(input_text):
+            input_text = f"{self.mm_tokens.audio_token}{input_text}"
+
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=[],
+            video_data=[],
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+            audio_sample_rate=self.audio_sampling_rate,
+        )
+        multimodal_tokens_pattern = self.mm_tokens.get_combined_regex()
+
+        raw_audio_data = audio_data or []
+        loaded_audio_iter = iter(base_output.audios)
+        raw_audio_iter = iter(raw_audio_data)
+
+        text_parts = re.split(multimodal_tokens_pattern, base_output.input_text)
+        contents: List[_Content] = []
+
+        for text_part in text_parts:
+            if multimodal_tokens_pattern.match(text_part):
+                modality = self.mm_tokens.get_modality_of_token(text_part)
+                assert modality is not None
+
+                if modality == Modality.AUDIO:
+                    loaded_audio = next(loaded_audio_iter)
+                    raw_audio_item = next(raw_audio_iter)
+
+                    if isinstance(loaded_audio, np.ndarray):
+                        audio_source = loaded_audio
+                    elif isinstance(raw_audio_item, dict):
+                        audio_source = raw_audio_item.get("url", loaded_audio)
+                    elif isinstance(raw_audio_item, (str, bytes, torch.Tensor)):
+                        audio_source = raw_audio_item
+                    else:
+                        raise ValueError(
+                            f"unsupported audio item: loaded={type(loaded_audio).__name__}, "
+                            f"raw={type(raw_audio_item).__name__}"
+                        )
+
+                    contents.append(
+                        _Content(
+                            type="audio",
+                            content=AudioInput(audio=audio_source),
+                        )
+                    )
+            else:
+                if text_part:
+                    contents.append(_Content(type="text", content=text_part))
+
+        loop = asyncio.get_running_loop()
+        try:
+            input_ids, audio_inputs, position_ids, rope_deltas = (
+                await loop.run_in_executor(
+                    self.io_executor,
+                    lambda: self._process_contents(contents),
+                )
+            )
+        except RuntimeError as e:
+            logger.error(f"MiMo ASR processor failed in process_mm_data_async: {e}")
+            raise ValueError(f"Multimodal data is corrupted or cannot be decoded: {e}")
+
+        input_ids_flat = input_ids.flatten()
+        if audio_inputs:
+            mm_items = [
+                MultimodalDataItem(
+                    modality=Modality.AUDIO,
+                    feature=audio_inputs,
+                    offsets=self.get_mm_items_offset(
+                        input_ids=input_ids_flat,
+                        mm_token_id=self.audio_token_id,
+                    ),
+                )
+            ]
+        else:
+            mm_items = []
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids_flat.tolist(),
+            audio_token_id=self.audio_token_id,
+            audio_start_id=self.audio_start_token_id,
+            audio_end_id=self.audio_end_token_id,
+            mrope_positions=position_ids,
+            mrope_position_delta=rope_deltas,
+        )

python/sglang/srt/multimodal/processors/minicpm.py

@@ -0,0 +1,305 @@
+from typing import List, Union
+
+import torch
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.minicpmo import MiniCPMO
+from sglang.srt.models.minicpmv import MiniCPMV
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    BaseMultiModalProcessorOutput,
+    MultimodalSpecialTokens,
+)
+
+
+# Compatible with both 'O' and 'V'
+class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
+    models = [MiniCPMV, MiniCPMO]
+    support_dynamic_frame_expansion = True
+    gpu_image_decode = False  # MiniCPM HF processor does not support tensor inputs
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        # Collect special token ids
+        tokenizer = self._processor.tokenizer
+        self.slice_start_id = getattr(tokenizer, "slice_start_id", None)
+        self.slice_end_id = getattr(tokenizer, "slice_end_id", None)
+        self.audio_start_id = getattr(tokenizer, "audio_start_id", None)
+        self.audio_end_id = getattr(tokenizer, "audio_end_id", None)
+        self.im_start_id = getattr(tokenizer, "im_start_id", None)
+        self.im_end_id = getattr(tokenizer, "im_end_id", None)
+        self.im_token_id = getattr(tokenizer, "unk_id", None)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="(<image>./</image>)",
+            audio_token="(<audio>./</audio>)",
+            video_token="(<video>./</video>)",
+            image_token_id=self.im_token_id,
+        ).build(_processor)
+
+    @staticmethod
+    def _has_special_format(image_data, audio_data):
+        """Check if any input items use processor_output or precomputed_embedding format."""
+        for data in list(image_data or []) + list(audio_data or []):
+            if isinstance(data, dict) and data.get("format") in (
+                "processor_output",
+                "precomputed_embedding",
+            ):
+                return True
+        return False
+
+    async def _process_special_format(
+        self, image_data, audio_data, input_text, request_obj, **kwargs
+    ):
+        """Handle processor_output and precomputed_embedding input formats.
+
+        Delegates to the base class process_and_combine_mm_data which has
+        built-in support for these formats.
+        """
+        if isinstance(input_text, list):
+            user_input_ids = input_text
+            prompt = ""
+        else:
+            user_input_ids = None
+            prompt = input_text or ""
+
+        # Normalize dicts: the HF MiniCPM processor returns "tgt_sizes" (plural)
+        # but the base class ATTR_NAME_TO_MODALITY maps "tgt_size" (singular).
+        # Also flatten the nested batch dimension so the structure matches
+        # what the NORMAL path produces (flat list of per-patch tensors).
+        normalized_images = []
+        for d in image_data or []:
+            if isinstance(d, dict):
+                d = dict(d)
+                if "tgt_sizes" in d and "tgt_size" not in d:
+                    d["tgt_size"] = d.pop("tgt_sizes")
+                if d.get("format") == "processor_output":
+                    pixel_values = d.get("pixel_values")
+                    tgt_size = d.get("tgt_size")
+                    if pixel_values is not None and tgt_size is not None:
+                        pv_flat, ts_flat = [], []
+                        for pixel_b, tgt_b in zip(pixel_values, tgt_size):
+                            if isinstance(pixel_b, (list, tuple)):
+                                for pixel_n, tgt_n in zip(pixel_b, tgt_b):
+                                    pv_flat.append(pixel_n)
+                                    ts_flat.append(tgt_n)
+                            else:
+                                pv_flat.append(pixel_b)
+                                ts_flat.append(tgt_b)
+                        d["pixel_values"] = pv_flat
+                        d["tgt_size"] = ts_flat
+                normalized_images.append(d)
+            else:
+                normalized_images.append(d)
+
+        normalized_audios = list(audio_data or [])
+
+        if not prompt and (normalized_images or normalized_audios):
+            images = [d for d in normalized_images if isinstance(d, dict)]
+            audios = [d for d in normalized_audios if isinstance(d, dict)]
+
+            raw_img_dropped = len(normalized_images) - len(images)
+            raw_aud_dropped = len(normalized_audios) - len(audios)
+            if raw_img_dropped > 0 or raw_aud_dropped > 0:
+                raise ValueError(
+                    f"[minicpm] Cannot process raw media with pre-tokenized "
+                    f"input_ids. Provide multimodal data in 'processor_output' or "
+                    f"'precomputed_embedding' format, or use a text prompt instead. "
+                    f"(raw images dropped: {raw_img_dropped}, "
+                    f"raw audios dropped: {raw_aud_dropped})"
+                )
+
+            base_output = BaseMultiModalProcessorOutput(
+                input_text=prompt,
+                images=images,
+                audios=audios,
+            )
+        else:
+            base_output = await self.load_mm_data(
+                prompt=prompt,
+                image_data=normalized_images,
+                audio_data=audio_data,
+                multimodal_tokens=self.mm_tokens,
+            )
+
+        if base_output is None:
+            return None
+
+        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        if user_input_ids is not None:
+            input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
+            for mm_item in mm_items:
+                if mm_item.modality == Modality.IMAGE:
+                    image_offsets = self.get_mm_items_offset_by_pair(
+                        input_ids=input_ids_tensor,
+                        mm_start_id=self.im_start_id,
+                        mm_end_id=self.im_end_id,
+                    )
+                    slice_offsets = self.get_mm_items_offset_by_pair(
+                        input_ids=input_ids_tensor,
+                        mm_start_id=self.slice_start_id,
+                        mm_end_id=self.slice_end_id,
+                    )
+                    image_offsets.extend(slice_offsets)
+                    mm_item.offsets = sorted(image_offsets)
+                elif mm_item.modality == Modality.AUDIO:
+                    if (
+                        self.audio_start_id is not None
+                        and self.audio_end_id is not None
+                    ):
+                        mm_item.offsets = self.get_mm_items_offset_by_pair(
+                            input_ids=input_ids_tensor,
+                            mm_start_id=self.audio_start_id,
+                            mm_end_id=self.audio_end_id,
+                        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids_tensor.flatten().tolist(),
+            audio_start_id=self.audio_start_id,
+            audio_end_id=self.audio_end_id,
+            im_token_id=self.im_token_id,
+            im_start_id=self.im_start_id,
+            im_end_id=self.im_end_id,
+            slice_start_id=self.slice_start_id,
+            slice_end_id=self.slice_end_id,
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        audio_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        **kwargs,
+    ):
+        if isinstance(input_text, list) or self._has_special_format(
+            image_data, audio_data
+        ):
+            return await self._process_special_format(
+                image_data=image_data,
+                audio_data=audio_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base_output is None:
+            return None
+
+        res = self.process_mm_data(
+            input_text=base_output.input_text,
+            images=base_output.images,
+            audios=base_output.audios,
+        )
+
+        pixel_values = res["pixel_values"]
+        tgt_sizes = res["tgt_sizes"]
+
+        if not isinstance(pixel_values, (torch.Tensor, list)):
+            raise ValueError(
+                "Incorrect type of pixel values. " f"Got type: {type(pixel_values)}"
+            )
+
+        if not isinstance(tgt_sizes, (torch.Tensor, list)):
+            raise ValueError(
+                "Incorrect type of target sizes. " f"Got type: {type(tgt_sizes)}"
+            )
+
+        if len(pixel_values) != len(tgt_sizes):
+            raise ValueError(
+                "Inconsistent batch lengths, found: "
+                f"{len(pixel_values)} vs. {len(tgt_sizes)}"
+            )
+
+        # Track slices per image (like vLLM's num_slices)
+        slices_per_image: List[int] = []
+        pixel_values_flat: List[torch.Tensor] = []
+        tgt_sizes_flat: List[torch.Tensor] = []
+        for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
+            # per image
+            if len(pixel_b) != len(tgt_b):
+                raise ValueError(
+                    "Inconsistent N lengths, found: " f"{len(pixel_b)} vs {len(tgt_b)}"
+                )
+            slices_per_image.append(len(pixel_b))
+            for pixel_n, tgt_n in zip(pixel_b, tgt_b):
+                pixel_values_flat += [pixel_n]
+                tgt_sizes_flat += [tgt_n]
+
+        pixel_values = pixel_values_flat
+
+        items = []
+        input_ids = res["input_ids"].flatten()
+        image_offsets = self.get_mm_items_offset_by_pair(
+            input_ids=input_ids, mm_start_id=self.im_start_id, mm_end_id=self.im_end_id
+        )
+        slice_offsets = self.get_mm_items_offset_by_pair(
+            input_ids=input_ids,
+            mm_start_id=self.slice_start_id,
+            mm_end_id=self.slice_end_id,
+        )
+        image_offsets.extend(slice_offsets)
+        image_offsets = sorted(image_offsets)
+
+        # Create one item per image, each with its own slices and offsets
+        if len(pixel_values) != 0:
+            pv_idx = 0
+            offset_idx = 0
+            for num_slices in slices_per_image:
+                items.append(
+                    MultimodalDataItem(
+                        feature=pixel_values[pv_idx : pv_idx + num_slices],
+                        offsets=image_offsets[offset_idx : offset_idx + num_slices],
+                        model_specific_data={
+                            "tgt_size": tgt_sizes_flat[pv_idx : pv_idx + num_slices]
+                        },
+                        modality=Modality.IMAGE,
+                    )
+                )
+                pv_idx += num_slices
+                offset_idx += num_slices
+
+        if (
+            "audio_features" in res
+            and res["audio_features"] is not None
+            and len(res["audio_features"]) != 0
+        ):
+            if self.audio_start_id is not None and self.audio_end_id is not None:
+                audio_offsets = self.get_mm_items_offset_by_pair(
+                    input_ids=input_ids,
+                    mm_start_id=self.audio_start_id,
+                    mm_end_id=self.audio_end_id,
+                )
+            else:
+                audio_offsets = None
+            item = MultimodalDataItem(
+                feature=[res["audio_features"]],
+                model_specific_data={"audio_feature_lens": res["audio_feature_lens"]},
+                offsets=audio_offsets,
+                modality=Modality.AUDIO,
+            )
+            items += [item]
+        return MultimodalProcessorOutput(
+            mm_items=items,
+            input_ids=input_ids.tolist(),
+            audio_start_id=self.audio_start_id,
+            audio_end_id=self.audio_end_id,
+            im_token_id=self.im_token_id,
+            im_start_id=self.im_start_id,
+            im_end_id=self.im_end_id,
+            slice_start_id=self.slice_start_id,
+            slice_end_id=self.slice_end_id,
+        )

python/sglang/srt/multimodal/processors/minicpmv4_6.py

@@ -0,0 +1,548 @@
+# Copyright 2026 The SGLang team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+"""sglang multimodal processor for MiniCPM-V 4.6.
+
+Ports per-image preprocessing + chat-template expansion sglang-side because
+no working HF ``MiniCPMV4_6Processor`` is reachable yet: transformers main
+does not ship one until 5.7+, and the released 4.6 checkpoints ship only a
+tokenizer (no remote-code processor), so ``AutoProcessor.from_pretrained``
+falls through to a bare tokenizer. Once a real processor is loadable, this
+module collapses to a thin wrapper that delegates to it.
+"""
+
+from __future__ import annotations
+
+import math
+from itertools import chain
+from typing import Any, List, Optional, Sequence, Tuple, Union
+
+import torch
+import torchvision.transforms.functional as F
+from PIL import Image
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.minicpmv import MiniCPMV4_6ForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+IMAGENET_STANDARD_MEAN = (0.5, 0.5, 0.5)
+IMAGENET_STANDARD_STD = (0.5, 0.5, 0.5)
+
+# Inner per-feature pad sentinel: prevents the next per-image
+# ``replace(image_token, ...)`` from clobbering a previous expansion's inner
+# pads. Swapped back to the real pad token once per modality after splicing.
+_PAD_PLACEHOLDER = "<|placeholder|>"
+
+
+def _ensure_divide(length: int, divisor: int) -> int:
+    return max(round(length / divisor) * divisor, divisor)
+
+
+def _to_chw_tensor(image) -> torch.Tensor:
+    """PIL / torch / numpy -> ``(C, H, W)`` float32 in ``[0, 255]``.
+
+    Image inputs from ``load_mm_data`` are PIL; video frames from sglang's
+    video decoder come back as numpy arrays.
+    """
+    if isinstance(image, torch.Tensor):
+        if image.dim() == 4:
+            image = image.squeeze(0)
+        if image.dim() != 3:
+            raise ValueError(f"expected 3-D image tensor, got {image.shape}")
+        if image.shape[0] not in (1, 3, 4):
+            image = image.permute(2, 0, 1).contiguous()
+        if image.shape[0] == 4:
+            image = image[:3]
+        if image.shape[0] == 1:
+            image = image.repeat(3, 1, 1)
+        return image.float()
+
+    if isinstance(image, Image.Image):
+        if image.mode != "RGB":
+            image = image.convert("RGB")
+        return F.pil_to_tensor(image).float()
+
+    import numpy as np
+
+    if isinstance(image, np.ndarray):
+        t = torch.from_numpy(image)
+        if t.dim() == 3 and t.shape[-1] in (1, 3, 4):
+            t = t.permute(2, 0, 1).contiguous()
+        if t.shape[0] == 4:
+            t = t[:3]
+        if t.shape[0] == 1:
+            t = t.repeat(3, 1, 1)
+        return t.float()
+
+    raise TypeError(f"Unsupported image type: {type(image)!r}")
+
+
+def _resize(image: torch.Tensor, height: int, width: int) -> torch.Tensor:
+    return F.resize(
+        image,
+        size=[height, width],
+        interpolation=F.InterpolationMode.BICUBIC,
+        antialias=True,
+    )
+
+
+def _divide_to_patches(
+    image: torch.Tensor, patch_h: int, patch_w: int
+) -> List[torch.Tensor]:
+    _, H, W = image.shape
+    if H % patch_h != 0 or W % patch_w != 0:
+        raise ValueError(f"image ({H}, {W}) not divisible by ({patch_h}, {patch_w})")
+    rows = H // patch_h
+    cols = W // patch_w
+    patches: List[torch.Tensor] = []
+    for r in range(rows):
+        for c in range(cols):
+            patches.append(
+                image[
+                    :, r * patch_h : (r + 1) * patch_h, c * patch_w : (c + 1) * patch_w
+                ]
+            )
+    return patches
+
+
+def _reshape_by_patch(image: torch.Tensor, patch_size: int) -> torch.Tensor:
+    """``(C, H, W) -> (C, P, H*W/P)`` NaViT packing."""
+    C = image.shape[0]
+    patches = torch.nn.functional.unfold(
+        image.unsqueeze(0), (patch_size, patch_size), stride=(patch_size, patch_size)
+    )
+    patches = patches.reshape(C, patch_size, patch_size, -1)
+    patches = patches.permute(0, 1, 3, 2).reshape(C, patch_size, -1)
+    return patches
+
+
+def _flatten_patches(
+    per_item_pv: List[List[torch.Tensor]],
+    per_item_ts: List[List[List[int]]],
+) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
+    """Per-item per-patch -> flat per-patch (source first, slices row-major)."""
+    flat_pv = list(chain.from_iterable(per_item_pv))
+    flat_ts = [
+        torch.tensor(ts, dtype=torch.int32) for ts in chain.from_iterable(per_item_ts)
+    ]
+    return flat_pv, flat_ts
+
+
+class MiniCPMV4_6ImageProcessor:
+    """Per-image preprocessing.
+
+    Pipeline: pick a slice grid (rows x cols, up to ``max_slice_nums``); resize
+    source and (optionally) tiles to multiples of ``patch_size * 4`` (factor 4
+    = the two successive 2x2 spatial merges: mid-ViT merger + DownsampleMLP);
+    rescale, normalize, and NaViT-pack each tile into ``(C, P, H*W/P)``.
+    """
+
+    def __init__(
+        self,
+        max_slice_nums: int = 9,
+        scale_resolution: int = 448,
+        patch_size: int = 14,
+        slice_mode: bool = True,
+        downsample_mode: str = "16x",
+        use_image_id: bool = True,
+        image_mean: Sequence[float] = IMAGENET_STANDARD_MEAN,
+        image_std: Sequence[float] = IMAGENET_STANDARD_STD,
+        rescale_factor: float = 1.0 / 255.0,
+    ) -> None:
+        self.max_slice_nums = max_slice_nums
+        self.scale_resolution = scale_resolution
+        self.patch_size = patch_size
+        self.slice_mode = slice_mode
+        self.downsample_mode = downsample_mode
+        self.use_image_id = use_image_id
+        self.image_mean = torch.tensor(image_mean, dtype=torch.float32).view(3, 1, 1)
+        self.image_std = torch.tensor(image_std, dtype=torch.float32).view(3, 1, 1)
+        self.rescale_factor = rescale_factor
+
+    def _find_best_resize(
+        self,
+        image_size: Tuple[int, int],
+        allow_upscale: bool = False,
+    ) -> Tuple[int, int]:
+        height, width = image_size
+        scale = self.scale_resolution
+        # factor 4 = two successive 2x2 spatial merges (mid-ViT + DownsampleMLP)
+        divisor = self.patch_size * 4
+        if (height * width > scale * scale) or allow_upscale:
+            aspect_ratio = width / height
+            height = int(scale / math.sqrt(aspect_ratio))
+            width = int(height * aspect_ratio)
+        best_w = _ensure_divide(width, divisor)
+        best_h = _ensure_divide(height, divisor)
+        return best_h, best_w
+
+    def _get_refine_size(
+        self,
+        image_size: Tuple[int, int],
+        grid: Tuple[int, int],
+        allow_upscale: bool = False,
+    ) -> Tuple[int, int]:
+        height, width = image_size
+        grid_y, grid_x = grid
+        refine_w = _ensure_divide(width, grid_x)
+        refine_h = _ensure_divide(height, grid_y)
+        bh, bw = self._find_best_resize(
+            (refine_h // grid_y, refine_w // grid_x),
+            allow_upscale=allow_upscale,
+        )
+        return bh * grid_y, bw * grid_x
+
+    def _get_sliced_grid(
+        self, image_size: Tuple[int, int]
+    ) -> Optional[Tuple[int, int]]:
+        original_h, original_w = image_size
+        scale = self.scale_resolution
+        log_ratio = math.log(original_w / original_h)
+        ratio = original_w * original_h / (scale * scale)
+        multiple = min(math.ceil(ratio), self.max_slice_nums)
+        if multiple <= 1:
+            return None
+
+        best_grid = (1, 1)
+        min_error = float("inf")
+        for num_slices in (multiple - 1, multiple, multiple + 1):
+            if num_slices == 1 or num_slices > self.max_slice_nums:
+                continue
+            for num_rows in range(1, num_slices + 1):
+                if num_slices % num_rows != 0:
+                    continue
+                num_cols = num_slices // num_rows
+                error = abs(log_ratio - math.log(num_rows / num_cols))
+                if error < min_error:
+                    # Ref returns ``[cols, rows]``; preserve the convention so
+                    # downstream code matches HF.
+                    best_grid = (num_cols, num_rows)
+                    min_error = error
+        return best_grid
+
+    def _normalize(self, t: torch.Tensor) -> torch.Tensor:
+        t = t * self.rescale_factor
+        return (t - self.image_mean.to(t.dtype)) / self.image_std.to(t.dtype)
+
+    def __call__(self, images: List) -> dict:
+        return self.preprocess(images)
+
+    def preprocess(self, images: List) -> dict:
+        """Returns ``{pixel_values, tgt_sizes, grids, num_patches_per_image}``.
+
+        Per image, ``pixel_values[i]`` is a list whose first entry is the
+        source patch and remaining entries are slice tiles in row-major grid
+        order. ``grids[i]`` is ``[cols, rows]`` (zeros if no slicing).
+        """
+        per_image_pv: List[List[torch.Tensor]] = []
+        per_image_ts: List[List[List[int]]] = []
+        all_grids: List[List[int]] = []
+        num_patches_per_image: List[int] = []
+
+        for image in images:
+            chw = _to_chw_tensor(image)
+            H0, W0 = chw.shape[-2], chw.shape[-1]
+            best_grid = self._get_sliced_grid((H0, W0)) if self.slice_mode else None
+
+            allow_upscale_src = best_grid is None
+            src_h, src_w = self._find_best_resize(
+                (H0, W0), allow_upscale=allow_upscale_src
+            )
+            source = _resize(chw, src_h, src_w)
+
+            patches: List[torch.Tensor] = [source]
+            patch_h = patch_w = 0
+            if best_grid is not None:
+                refine_h, refine_w = self._get_refine_size(
+                    (H0, W0), best_grid, allow_upscale=True
+                )
+                refined = _resize(chw, refine_h, refine_w)
+                grid_y, grid_x = best_grid
+                patch_h = refine_h // grid_y
+                patch_w = refine_w // grid_x
+                patches.extend(_divide_to_patches(refined, patch_h, patch_w))
+
+            patches = [self._normalize(p) for p in patches]
+
+            pv = [_reshape_by_patch(patches[0], self.patch_size)]
+            ts = [[src_h // self.patch_size, src_w // self.patch_size]]
+            for p in patches[1:]:
+                pv.append(_reshape_by_patch(p, self.patch_size))
+                ts.append([patch_h // self.patch_size, patch_w // self.patch_size])
+
+            per_image_pv.append(pv)
+            per_image_ts.append(ts)
+            all_grids.append(list(best_grid) if best_grid is not None else [0, 0])
+            num_patches_per_image.append(len(pv))
+
+        return {
+            "pixel_values": per_image_pv,
+            "tgt_sizes": per_image_ts,
+            "grids": all_grids,
+            "num_patches_per_image": num_patches_per_image,
+        }
+
+
+class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
+    """4.6-only mm processor.
+
+    The legacy ``MiniCPMMultimodalProcessor`` stays for 2.6/4.0/4.5 because its
+    ``_processor.tokenizer`` shape and ``(<image>./</image>)`` placeholder
+    format don't fit 4.6.
+    """
+
+    models = [MiniCPMV4_6ForConditionalGeneration]
+    support_dynamic_frame_expansion = False
+    gpu_image_decode = False
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        # ``_processor`` is either the bare tokenizer (current state — no
+        # ``MiniCPMV4_6Processor`` shipped) or a real processor whose
+        # ``.tokenizer`` exposes the same.
+        self.tokenizer = getattr(_processor, "tokenizer", _processor)
+
+        vision_cfg = getattr(hf_config, "vision_config", None)
+        patch_size = (
+            getattr(vision_cfg, "patch_size", 14) if vision_cfg is not None else 14
+        )
+        downsample_mode = getattr(hf_config, "downsample_mode", "16x")
+        # Per-image preprocessor; reused for video frames (HF ref's
+        # video slicing geometry matches image slicing exactly).
+        self.image_processor = MiniCPMV4_6ImageProcessor(
+            max_slice_nums=9,
+            scale_resolution=448,
+            patch_size=patch_size,
+            slice_mode=True,
+            downsample_mode=downsample_mode,
+            use_image_id=True,
+        )
+
+        self.image_token = "<|image_pad|>"
+        self.video_token = "<|video_pad|>"
+        self.image_token_id = getattr(hf_config, "image_token_id", None)
+        if self.image_token_id is None:
+            self.image_token_id = self._token_id(self.image_token)
+        self.video_token_id = getattr(hf_config, "video_token_id", None)
+        if self.video_token_id is None:
+            self.video_token_id = self._token_id(self.video_token)
+
+        # ``<image>``/``<slice>`` wrap the expanded regions for both images and
+        # video frames; only the inner per-feature pad token differs.
+        self.image_start_token = "<image>"
+        self.image_end_token = "</image>"
+        self.slice_start_token = "<slice>"
+        self.slice_end_token = "</slice>"
+        self.image_id_start_token = "<image_id>"
+        self.image_id_end_token = "</image_id>"
+
+        self.image_start_id = self._token_id(self.image_start_token)
+        self.image_end_id = self._token_id(self.image_end_token)
+        self.slice_start_id = self._token_id(self.slice_start_token)
+        self.slice_end_id = self._token_id(self.slice_end_token)
+
+        self.pad_divisor = 16 if downsample_mode != "4x" else 4
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.image_token,
+            image_token_id=self.image_token_id,
+            video_token=self.video_token,
+            video_token_id=self.video_token_id,
+        ).build(_processor)
+
+    def _token_id(self, token: str):
+        try:
+            ids = self.tokenizer.convert_tokens_to_ids([token])
+            if ids and ids[0] is not None:
+                return int(ids[0])
+        except Exception:
+            pass
+        return None
+
+    def _expand_frame(
+        self,
+        tgt_sizes: List[List[int]],
+        grid: List[int],
+    ) -> str:
+        """``<image>...</image>`` (+ optional ``<slice>...</slice>`` rows) for
+        one image or video frame; inner pads are ``_PAD_PLACEHOLDER`` (caller
+        swaps back after splicing).
+        """
+        h0, w0 = tgt_sizes[0]
+        n_src = (h0 * w0) // self.pad_divisor
+        out = self.image_start_token + _PAD_PLACEHOLDER * n_src + self.image_end_token
+
+        if len(tgt_sizes) > 1 and grid and grid[0] > 0 and grid[1] > 0:
+            grid_y, grid_x = int(grid[0]), int(grid[1])
+            h_s, w_s = tgt_sizes[1]
+            n_slice = (h_s * w_s) // self.pad_divisor
+            slice_chunk = (
+                self.slice_start_token
+                + _PAD_PLACEHOLDER * n_slice
+                + self.slice_end_token
+            )
+            row_chunks = [slice_chunk * grid_x for _ in range(grid_y)]
+            out += "\n".join(row_chunks)
+        return out
+
+    def _expand_media(
+        self,
+        index: int,
+        frames: Sequence[Tuple[List[List[int]], List[int]]],
+    ) -> str:
+        """One image or one video. Image is a single-frame video."""
+        body = "".join(self._expand_frame(ts, grid) for ts, grid in frames)
+        return f"{self.image_id_start_token}{index}{self.image_id_end_token}" + body
+
+    async def process_mm_data_async(
+        self,
+        image_data: Sequence[Union[str, bytes]],
+        audio_data: Sequence[Union[str, bytes]],
+        input_text,
+        request_obj,
+        **kwargs: Any,
+    ):
+        # ``TokenizerManager`` does not pass ``video_data`` through the
+        # processor signature; read it off the request the way qwen_vl does.
+        video_data = getattr(request_obj, "video_data", None) or kwargs.get(
+            "video_data"
+        )
+        base = await self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            image_data=image_data,
+            video_data=video_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base is None:
+            return None
+
+        prompt: str = base.input_text or ""
+        images = base.images or []
+        videos = base.videos or []
+
+        # Image: one "frame" per image. Video: per-frame nesting kept so each
+        # frame becomes its own ``<image>...</image>`` block in the expansion.
+        img_per_pv, img_per_ts, img_grids = self._preprocess_images(images)
+        vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos(videos)
+
+        prompt = self._splice_expansions(
+            prompt,
+            (
+                self._expand_media(i, [(ts, gd)])
+                for i, (ts, gd) in enumerate(zip(img_per_ts, img_grids))
+            ),
+            (
+                self._expand_media(i, list(zip(fts, fgd)))
+                for i, (fts, fgd) in enumerate(zip(vid_per_ts, vid_grids))
+            ),
+        )
+
+        input_ids: List[int] = self.tokenizer.encode(prompt, add_special_tokens=False)
+        input_ids_tensor = torch.tensor(input_ids, dtype=torch.long)
+
+        # Each patch's pad tokens are guaranteed contiguous (the expansion
+        # functions wrap them in ``<image>...</image>`` / ``<slice>...</slice>``
+        # with nothing else in between), so a per-token-id contiguous-run scan
+        # — base's ``get_mm_items_offset`` — gives one (start, end) per patch.
+        mm_items: List[MultimodalDataItem] = []
+        mm_items.extend(
+            self._build_items(
+                input_ids_tensor,
+                self.image_token_id,
+                _flatten_patches(img_per_pv, img_per_ts),
+                Modality.IMAGE,
+            )
+        )
+        # Video: extra ``per-frame -> per-patch`` nesting; pre-flatten one
+        # level so ``_flatten_patches`` sees the same shape as image.
+        vid_pv_flat = [list(chain.from_iterable(v)) for v in vid_per_pv]
+        vid_ts_flat = [list(chain.from_iterable(v)) for v in vid_per_ts]
+        mm_items.extend(
+            self._build_items(
+                input_ids_tensor,
+                self.video_token_id,
+                _flatten_patches(vid_pv_flat, vid_ts_flat),
+                Modality.VIDEO,
+            )
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids,
+            im_token_id=self.image_token_id,
+            im_start_id=self.image_start_id,
+            im_end_id=self.image_end_id,
+            slice_start_id=self.slice_start_id,
+            slice_end_id=self.slice_end_id,
+        )
+
+    def _preprocess_images(self, images):
+        if not images:
+            return [], [], []
+        out = self.image_processor.preprocess(images)
+        return out["pixel_values"], out["tgt_sizes"], out["grids"]
+
+    def _preprocess_videos(self, videos):
+        per_video_pv: List[List[List[torch.Tensor]]] = []
+        per_video_ts: List[List[List[List[int]]]] = []
+        per_video_grids: List[List[List[int]]] = []
+        for frames in videos:
+            out = self.image_processor.preprocess(list(frames))
+            per_video_pv.append(out["pixel_values"])
+            per_video_ts.append(out["tgt_sizes"])
+            per_video_grids.append(out["grids"])
+        return per_video_pv, per_video_ts, per_video_grids
+
+    def _splice_expansions(self, prompt, image_expansions, video_expansions):
+        # The chat template emits exactly one marker per media item; a
+        # sequential ``replace(..., n=1)`` walk lines them up by left-to-right
+        # order. Expansions carry ``_PAD_PLACEHOLDER`` for inner pads so the
+        # next replace doesn't trip on a previous expansion's pads — we swap
+        # placeholders back to the real pad token in one pass per modality.
+        for token, expansions in (
+            (self.image_token, image_expansions),
+            (self.video_token, video_expansions),
+        ):
+            for expansion in expansions:
+                if token not in prompt:
+                    break
+                prompt = prompt.replace(token, expansion, 1)
+            prompt = prompt.replace(_PAD_PLACEHOLDER, token)
+        return prompt
+
+    def _build_items(
+        self,
+        input_ids: torch.Tensor,
+        pad_token_id: int,
+        flat: Tuple[List[torch.Tensor], List[torch.Tensor]],
+        modality: Modality,
+    ) -> List[MultimodalDataItem]:
+        flat_pv, flat_ts = flat
+        runs = self.get_mm_items_offset(input_ids, pad_token_id)
+        if len(runs) != len(flat_pv):
+            raise RuntimeError(
+                f"[minicpmv4_6] {modality} pad run / feature count mismatch: "
+                f"{len(runs)} runs vs {len(flat_pv)} patches"
+            )
+        return [
+            MultimodalDataItem(
+                feature=[pv],
+                offsets=[run],
+                model_specific_data={"tgt_size": [ts]},
+                modality=modality,
+            )
+            for run, pv, ts in zip(runs, flat_pv, flat_ts)
+        ]

python/sglang/srt/multimodal/processors/minimax_m3_vl.py

@@ -0,0 +1,283 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+"""HF processor classes live in sglang.srt.configs.minimax_vl_processor to avoid circular imports with model classes."""
+
+import math
+import re
+from typing import Dict, List, Optional, Tuple, Union
+
+import torch
+import torchvision
+from torchvision.transforms import InterpolationMode
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.minimax_m3_vl import MiniMaxM3SparseForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils import round_up
+
+
+def get_hw_multiple_of(
+    image_size: Tuple[int, int],
+    multiple: int,
+    max_size: Union[None, int, Tuple[int, int]] = None,
+) -> Tuple[int, int]:
+    w, h = image_size
+
+    if isinstance(max_size, int):
+        ratio = 1.0
+        max_dim = max(w, h)
+        if max_dim > max_size:
+            ratio = max_size / max_dim
+        new_w = round_up(round(w * ratio), multiple)
+        new_h = round_up(round(h * ratio), multiple)
+        return new_w, new_h
+
+    new_w = round_up(w, multiple)
+    new_h = round_up(h, multiple)
+
+    if max_size is not None:
+        assert isinstance(max_size, (list, tuple)) and len(max_size) == 2
+        max_w, max_h = max_size
+        assert max_w % multiple == 0 and max_h % multiple == 0
+
+        if new_w > max_w or new_h > max_h:
+            new_w_ = min((new_w * max_w) // new_w, (new_w * max_h) // new_h)
+            new_h_ = min((new_h * max_w) // new_w, (new_h * max_h) // new_h)
+            new_w = new_w_
+            new_h = new_h_
+
+            new_w = (
+                new_w
+                if new_w % multiple == 0
+                else new_w + (multiple - new_w % multiple)
+            )
+            new_h = (
+                new_h
+                if new_h % multiple == 0
+                else new_h + (multiple - new_h % multiple)
+            )
+
+        assert new_w % multiple == 0 and new_h % multiple == 0
+        assert new_w <= max_w and new_h <= max_h
+
+    return new_w, new_h
+
+
+def _compute_sampled_frame_indices(
+    total_frames: int,
+    video_fps: float,
+    fps: float,
+    max_frames: Optional[int] = None,
+) -> List[int]:
+    """Frame indices must match SFT extract_frame.py constant-mode sampling (>=1/fps apart, always keep last) or eval diverges from training."""
+    if total_frames <= 0 or video_fps <= 0 or fps <= 0:
+        return [0] if total_frames > 0 else []
+
+    read_time_interval = 1.0 / fps
+    eps = 1e-4
+
+    indices: List[int] = []
+    prev_kept_ts = -float("inf")
+    while True:
+        if not indices:
+            target_frame = 0
+        else:
+            target_ts = prev_kept_ts + read_time_interval - eps
+            target_frame = math.ceil(target_ts * video_fps)
+            target_frame = max(target_frame, indices[-1] + 1)
+        if target_frame >= total_frames:
+            break
+        indices.append(target_frame)
+        prev_kept_ts = target_frame / video_fps
+
+    last_frame_idx = total_frames - 1
+    last_ts = last_frame_idx / video_fps
+    if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
+        indices.append(last_frame_idx)
+
+    if not indices:
+        indices = [0]
+    if max_frames is not None and len(indices) > max_frames > 0:
+        last = indices[-1]
+        if max_frames == 1:
+            # max_frames == 1 would divide by (max_frames - 1) == 0 below; keep only the last frame.
+            indices = [last]
+        else:
+            step = len(indices) / (max_frames - 1)
+            indices = [indices[int(i * step)] for i in range(max_frames - 1)]
+            indices.append(last)
+    return indices
+
+
+async def get_video_tensor(
+    vr,
+    image_factor: int,
+    max_size: Tuple[int, int],
+    fps: Optional[float] = None,
+    frame_max_size: Optional[int] = None,
+    max_frames: Optional[int] = None,
+) -> Tuple[torch.Tensor, dict]:
+    if fps is None:
+        fps = 1.0
+    if frame_max_size is None:
+        frame_max_size = max_size[0]
+    if fps <= 0:
+        raise ValueError(f"video fps must be > 0, got {fps}")
+
+    if isinstance(vr, torch.Tensor):
+        video_tchw = vr
+        _, _, height, width = video_tchw.shape
+        resized_width, resized_height = get_hw_multiple_of(
+            (width, height), image_factor, max_size
+        )
+        resized = torchvision.transforms.functional.resize(
+            video_tchw,
+            [resized_height, resized_width],
+            interpolation=InterpolationMode.BICUBIC,
+        )
+        return resized, {
+            "total_num_frames": resized.shape[0],
+            "fps": None,
+            "frames_indices": None,
+        }
+
+    total_frames = len(vr)
+    video_fps = vr.avg_fps
+    if video_fps <= 0 or total_frames <= 0:
+        raise ValueError(
+            f"Invalid video metadata: fps={video_fps}, frames={total_frames}"
+        )
+    indices = _compute_sampled_frame_indices(total_frames, video_fps, fps, max_frames)
+    video_tchw = vr.get_frames_as_tensor(indices)
+    video_tchw = video_tchw.permute(0, 3, 1, 2).float()
+
+    _, _, height, width = video_tchw.shape
+    resized_width, resized_height = get_hw_multiple_of(
+        (width, height), image_factor, frame_max_size
+    )
+    resized = torchvision.transforms.functional.resize(
+        video_tchw,
+        [resized_height, resized_width],
+        interpolation=InterpolationMode.BICUBIC,
+    )
+    return resized, {
+        "total_num_frames": total_frames,
+        "fps": video_fps,
+        "frames_indices": indices,
+    }
+
+
+class MiniMaxM3VLProcessor(BaseMultimodalProcessor):
+    models = [
+        MiniMaxM3SparseForConditionalGeneration,
+    ]
+
+    gpu_image_decode = False
+
+    # M3's tokenizer has no pad_token.
+    tokenizer_padding = False
+
+    IMAGE_TOKEN = "]<]image[>["
+    VIDEO_TOKEN = "]<]video[>["
+    IMAGE_START_TOKEN = "]<]start of image[>["
+    IMAGE_END_TOKEN = "]<]end of image[>["
+
+    @staticmethod
+    def _token_id(tokenizer, token):
+        token_id = tokenizer.convert_tokens_to_ids(token)
+        assert token_id is not None, f"token id for {token!r} not found"
+        return token_id
+
+    @property
+    def spatial_merge_size(self):
+        return self._processor.image_processor.merge_size
+
+    def _video_resize_config(self):
+        video_processor = self._processor.video_processor
+        image_factor = video_processor.patch_size * video_processor.merge_size
+        # Newer M3 video processors expose max_pixels (area) instead of max_size; derive an equivalent (max_w, max_h) cap.
+        max_size = getattr(video_processor, "max_size", None)
+        if max_size is None:
+            max_pixels = getattr(video_processor, "max_pixels", None)
+            if max_pixels is not None:
+                side = int(math.isqrt(int(max_pixels)))
+                side -= side % image_factor
+                max_size = (side, side)
+            else:
+                max_size = video_processor._max_size_from_size(video_processor.size)
+        assert max_size is not None, "video processor max_size is required"
+        return image_factor, max_size
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        tokenizer = _processor.tokenizer
+        assert tokenizer is not None, "tokenizer is required"
+
+        self.IM_TOKEN_ID = self._token_id(tokenizer, self.IMAGE_TOKEN)
+        self.VIDEO_TOKEN_ID = self._token_id(tokenizer, self.VIDEO_TOKEN)
+        self.IM_START_TOKEN_ID = self._token_id(tokenizer, self.IMAGE_START_TOKEN)
+        self.IM_END_TOKEN_ID = self._token_id(tokenizer, self.IMAGE_END_TOKEN)
+        self.video_fps = self.video_config.pop("fps", None)
+        self.video_frame_max_size = self.video_config.pop("frame_max_size", None)
+        self.video_max_frames = self.video_config.pop("max_frames", None)
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+            image_token_regex=re.compile(
+                r"<image>|<\|image\|>|<\|image_pad\|>|\]\<\]image\[\>\["
+            ),
+            video_token=self.VIDEO_TOKEN,
+            video_token_id=self.VIDEO_TOKEN_ID,
+            video_token_regex=re.compile(r"<video>|<\|video\|>|\]\<\]video\[\>\["),
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: Optional[List],
+        audio_data: Optional[List],
+        input_text: str,
+        request_obj,
+        **kwargs,
+    ) -> Dict:
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        video_metadata = None
+        if base_output.videos:
+            image_factor, max_size = self._video_resize_config()
+            videos_processed = [
+                await get_video_tensor(
+                    video,
+                    image_factor=image_factor,
+                    max_size=max_size,
+                    fps=self.video_fps,
+                    frame_max_size=self.video_frame_max_size,
+                    max_frames=self.video_max_frames,
+                )
+                for video in base_output.videos
+            ]
+            base_output.videos, video_metadata = map(list, zip(*videos_processed))
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output=base_output,
+            mm_tokens=self.mm_tokens,
+            video_metadata=video_metadata,
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist() if hasattr(input_ids, "tolist") else input_ids,
+            mm_items=mm_items,
+            im_start_id=self.IM_START_TOKEN_ID,
+            im_end_id=self.IM_END_TOKEN_ID,
+            im_token_id=self.IM_TOKEN_ID,
+            video_token_id=self.VIDEO_TOKEN_ID,
+        )

python/sglang/srt/multimodal/processors/mlama.py

@@ -0,0 +1,38 @@
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.mllama import MllamaForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class MllamaImageProcessor(BaseMultimodalProcessor):
+    models = [MllamaForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self._processor.image_token,
+            image_token_id=self._processor.image_token_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
+    ):
+        base_out = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_out, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/mllama4.py

@@ -0,0 +1,50 @@
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.mllama4 import Llama4ForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class Mllama4ImageProcessor(BaseMultimodalProcessor):
+    models = [Llama4ForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.vision_config = hf_config.vision_config
+        self.text_config = hf_config.text_config
+        self.IM_START_TOKEN_ID = hf_config.boi_token_index
+        self.IM_END_TOKEN_ID = hf_config.eoi_token_index
+        self.IM_TOKEN_ID = hf_config.image_token_index
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=_processor.image_token,
+            image_token_id=self.IM_TOKEN_ID,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        # Process the prompt and images
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_start_id=self.IM_START_TOKEN_ID,
+            im_end_id=self.IM_END_TOKEN_ID,
+            im_token_id=self.IM_TOKEN_ID,
+        )

python/sglang/srt/multimodal/processors/moss_vl.py

@@ -0,0 +1,612 @@
+import asyncio
+import os
+import re
+import tempfile
+from typing import Dict, List, Optional, Tuple, Union
+from urllib.parse import unquote, urlparse
+
+import pybase64
+import requests
+import torch
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.moss_vl import MossVLForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    SGL_USE_CUDA_IPC,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils.cuda_ipc_transport_utils import CudaIpcTensorTransportProxy
+
+
+class MossVLImageProcessor(SGLangBaseProcessor):
+    models = [MossVLForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.image_only_mm_tokens = MultimodalSpecialTokens(
+            image_token="<|image|>",
+            image_token_regex=re.compile(re.escape("<|image|>")),
+        ).build(_processor)
+        self.image_token_id = getattr(hf_config, "image_token_id", None)
+        self.vision_seq_pad_multiple = 1
+
+    def _build_mm_items(
+        self, processor_output: Dict, input_ids: torch.Tensor
+    ) -> List[MultimodalDataItem]:
+        pixel_values = processor_output.get("pixel_values")
+        if pixel_values is None:
+            return []
+
+        item = MultimodalDataItem(
+            modality=Modality.IMAGE,
+            feature=pixel_values,
+            model_specific_data={},
+        )
+
+        grid_thw = processor_output.get("grid_thw")
+        if grid_thw is not None:
+            item.set("grid_thw", grid_thw)
+
+        return [item]
+
+    def _build_vision_token_info(
+        self,
+        grid_thw: Optional[torch.Tensor],
+        media_nums_per_sample: Optional[List[int]],
+    ) -> List[dict]:
+        if grid_thw is None:
+            return []
+
+        grid_thw = torch.as_tensor(grid_thw, dtype=torch.long)
+        if grid_thw.ndim == 1:
+            grid_thw = grid_thw.unsqueeze(0)
+        if grid_thw.numel() == 0:
+            return []
+
+        tokens_per_media = (grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]) // (
+            self.spatial_merge_size**2
+        )
+
+        if media_nums_per_sample is None:
+            media_nums_per_sample = [grid_thw.shape[0]]
+
+        batch_size = len(media_nums_per_sample)
+        if batch_size == 1:
+            total_len = 0
+            for i in range(grid_thw.shape[0]):
+                num_tokens = tokens_per_media[i].item()
+                num_frames = grid_thw[i, 0].item()
+                total_len += num_tokens + num_frames
+
+            if total_len % self.vision_seq_pad_multiple != 0:
+                max_seq_len = (
+                    (total_len + self.vision_seq_pad_multiple - 1)
+                    // self.vision_seq_pad_multiple
+                    * self.vision_seq_pad_multiple
+                )
+            else:
+                max_seq_len = total_len
+
+            sample_info = {
+                "medias": [],
+                "total_length": total_len,
+                "pad_start": total_len,
+                "pad_end": max_seq_len,
+            }
+
+            current_seq_len = 0
+            for media_idx in range(grid_thw.shape[0]):
+                num_tokens = tokens_per_media[media_idx].item()
+                t, h, w = grid_thw[media_idx].tolist()
+                num_frames = t
+                tokens_per_frame = num_tokens // num_frames
+                chunk_len = num_frames * (tokens_per_frame + 1)
+
+                sample_info["medias"].append(
+                    {
+                        "start": current_seq_len,
+                        "end": current_seq_len + chunk_len,
+                        "length": chunk_len,
+                        "num_frames": num_frames,
+                        "grid_h": h,
+                        "grid_w": w,
+                        "vision_tokens_per_frame": tokens_per_frame,
+                        "has_separator": True,
+                    }
+                )
+                current_seq_len += chunk_len
+
+            return [sample_info]
+
+        tokens_per_sample = []
+        media_idx = 0
+        for num_medias_in_sample in media_nums_per_sample:
+            sample_tokens = 0
+            for i in range(num_medias_in_sample):
+                num_tokens = tokens_per_media[media_idx + i].item()
+                num_frames = grid_thw[media_idx + i, 0].item()
+                sample_tokens += num_tokens + num_frames
+            tokens_per_sample.append(sample_tokens)
+            media_idx += num_medias_in_sample
+
+        max_seq_len = max(tokens_per_sample)
+        if max_seq_len % self.vision_seq_pad_multiple != 0:
+            max_seq_len = (
+                (max_seq_len + self.vision_seq_pad_multiple - 1)
+                // self.vision_seq_pad_multiple
+                * self.vision_seq_pad_multiple
+            )
+
+        vision_token_info = []
+        media_idx = 0
+        for sample_idx, num_medias_in_sample in enumerate(media_nums_per_sample):
+            sample_info = {
+                "medias": [],
+                "total_length": tokens_per_sample[sample_idx],
+                "pad_start": tokens_per_sample[sample_idx],
+                "pad_end": max_seq_len,
+            }
+
+            seq_offset = 0
+            for _ in range(num_medias_in_sample):
+                num_tokens = tokens_per_media[media_idx].item()
+                t, h, w = grid_thw[media_idx].tolist()
+                num_frames = t
+                tokens_per_frame = num_tokens // num_frames
+                media_length = num_tokens + num_frames
+
+                sample_info["medias"].append(
+                    {
+                        "start": seq_offset,
+                        "end": seq_offset + media_length,
+                        "length": media_length,
+                        "num_frames": num_frames,
+                        "grid_h": h,
+                        "grid_w": w,
+                        "vision_tokens_per_frame": tokens_per_frame,
+                        "has_separator": True,
+                    }
+                )
+
+                seq_offset += media_length
+                media_idx += 1
+
+            vision_token_info.append(sample_info)
+
+        return vision_token_info
+
+    def _compute_position_ids(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        is_image_token = input_ids == self.image_token_id
+        if attention_mask is not None:
+            is_padding = attention_mask == 0
+        else:
+            is_padding = torch.zeros_like(input_ids, dtype=torch.bool)
+
+        is_regular_token = ~(is_image_token | is_padding)
+        cumulative_regular = is_regular_token.long().cumsum(dim=1)
+        base_position_ids = cumulative_regular - is_regular_token.long()
+        base_position_ids = base_position_ids.masked_fill(is_padding, 0)
+        return base_position_ids.unsqueeze(0).expand(3, -1, -1).clone()
+
+    def _compute_vision_position_ids(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        vision_token_info: List[dict],
+        max_vision_seq_len: int,
+        attention_mask: Optional[torch.Tensor],
+    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        batch_size = input_ids.shape[0]
+        device = input_ids.device
+
+        image_token_indices = (input_ids == self.image_token_id).nonzero()
+
+        flat_eff_h = []
+        flat_eff_w = []
+        flat_vis_starts = []
+
+        for info in vision_token_info:
+            medias = info.get("medias", [])
+            for media in medias:
+                num_frames = media["num_frames"]
+                h, w = media["grid_h"], media["grid_w"]
+                eh, ew = h // self.spatial_merge_size, w // self.spatial_merge_size
+                start = media["start"]
+                tok_per_frame = media["vision_tokens_per_frame"]
+                stride = tok_per_frame + 1
+                for f in range(num_frames):
+                    flat_eff_h.append(eh)
+                    flat_eff_w.append(ew)
+                    flat_vis_starts.append(start + f * stride)
+
+        vision_pos_ids = torch.zeros(
+            (3, batch_size, max_vision_seq_len),
+            dtype=torch.long,
+            device=device,
+        )
+
+        if len(flat_eff_h) == 0 or len(image_token_indices) == 0:
+            rope_deltas = (
+                position_ids.max(dim=0).values.max(dim=-1).values
+                + 1
+                - input_ids.shape[1]
+            )
+            return vision_pos_ids, position_ids, rope_deltas
+
+        num_matches = min(len(flat_eff_h), len(image_token_indices))
+        flat_eff_h = torch.tensor(
+            flat_eff_h[:num_matches], device=device, dtype=torch.long
+        )
+        flat_eff_w = torch.tensor(
+            flat_eff_w[:num_matches], device=device, dtype=torch.long
+        )
+        flat_vis_starts = torch.tensor(
+            flat_vis_starts[:num_matches], device=device, dtype=torch.long
+        )
+
+        target_indices = image_token_indices[:num_matches]
+        batch_rows = target_indices[:, 0]
+        text_cols = target_indices[:, 1]
+
+        max_hw = torch.maximum(flat_eff_h, flat_eff_w)
+        shifts = max_hw + 1
+
+        shift_map = torch.zeros(
+            (batch_size, input_ids.shape[1]), dtype=torch.long, device=device
+        )
+        shift_map[batch_rows, text_cols] = shifts
+        cum_shifts = shift_map.cumsum(dim=1)
+
+        orig_pos = position_ids[0, batch_rows, text_cols]
+        shifts_before = cum_shifts[batch_rows, text_cols] - shifts
+        t_vals = orig_pos + shifts_before
+
+        new_pos_ids = position_ids + cum_shifts.unsqueeze(0)
+        img_token_mask = torch.zeros_like(input_ids, dtype=torch.bool)
+        img_token_mask[batch_rows, text_cols] = True
+        new_pos_ids[:, img_token_mask] -= 1
+
+        if attention_mask is not None:
+            padding_mask = (attention_mask == 0).unsqueeze(0)
+            new_pos_ids.masked_fill_(padding_mask, 0)
+
+        position_ids = new_pos_ids
+
+        unique_shapes = torch.unique(
+            torch.stack([flat_eff_h, flat_eff_w], dim=1), dim=0
+        )
+        for shape in unique_shapes:
+            eh, ew = shape[0].item(), shape[1].item()
+            mask = (flat_eff_h == eh) & (flat_eff_w == ew)
+
+            sub_t_vals = t_vals[mask]
+            sub_batch_rows = batch_rows[mask]
+            sub_vis_starts = flat_vis_starts[mask]
+            num_frames_sub = sub_t_vals.shape[0]
+            if num_frames_sub == 0:
+                continue
+
+            y_grid = (
+                torch.arange(eh, device=device)
+                .view(1, eh, 1)
+                .expand(num_frames_sub, -1, ew)
+            )
+            x_grid = (
+                torch.arange(ew, device=device)
+                .view(1, 1, ew)
+                .expand(num_frames_sub, eh, -1)
+            )
+            t_grid = sub_t_vals.view(-1, 1, 1).expand(-1, eh, ew)
+
+            h_grid = t_grid + y_grid
+            w_grid = t_grid + x_grid
+
+            flat_t = t_grid.reshape(-1)
+            flat_h = h_grid.reshape(-1)
+            flat_w = w_grid.reshape(-1)
+
+            tokens_per_frame = eh * ew
+            seq_offsets = torch.arange(tokens_per_frame, device=device).unsqueeze(0)
+            abs_seq_offsets = seq_offsets + sub_vis_starts.unsqueeze(1)
+
+            flat_seq_inds = abs_seq_offsets.reshape(-1)
+            flat_batch_inds = (
+                sub_batch_rows.unsqueeze(1).expand(-1, tokens_per_frame).reshape(-1)
+            )
+
+            valid_mask = flat_seq_inds < max_vision_seq_len
+            if valid_mask.any():
+                final_b = flat_batch_inds[valid_mask]
+                final_s = flat_seq_inds[valid_mask]
+                vision_pos_ids[0, final_b, final_s] = flat_t[valid_mask]
+                vision_pos_ids[1, final_b, final_s] = flat_h[valid_mask]
+                vision_pos_ids[2, final_b, final_s] = flat_w[valid_mask]
+
+        sep_vals = t_vals + max_hw
+        sep_indices = flat_vis_starts + (flat_eff_h * flat_eff_w)
+        valid_sep_mask = sep_indices < max_vision_seq_len
+        if valid_sep_mask.any():
+            final_b = batch_rows[valid_sep_mask]
+            final_s = sep_indices[valid_sep_mask]
+            vals = sep_vals[valid_sep_mask]
+            vision_pos_ids[0, final_b, final_s] = vals
+            vision_pos_ids[1, final_b, final_s] = vals
+            vision_pos_ids[2, final_b, final_s] = vals
+
+        max_pos = position_ids.max(dim=0).values.max(dim=-1).values
+        rope_deltas = max_pos + 1 - input_ids.shape[1]
+        return vision_pos_ids, position_ids, rope_deltas
+
+    def _compute_position_metadata(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: Optional[torch.Tensor],
+        grid_thw: Optional[torch.Tensor],
+        media_nums_per_sample: Optional[List[int]],
+    ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor], List[dict]]:
+        position_ids = self._compute_position_ids(input_ids, attention_mask)
+
+        if grid_thw is None:
+            max_pos = position_ids.max(dim=0).values.max(dim=-1).values
+            rope_deltas = (max_pos + 1 - input_ids.shape[1]).unsqueeze(1)
+            return position_ids, rope_deltas, None, []
+
+        vision_token_info = self._build_vision_token_info(
+            grid_thw, media_nums_per_sample
+        )
+        max_vision_seq_len = 0
+        if vision_token_info:
+            max_vision_seq_len = max(
+                info.get("pad_end", 0) for info in vision_token_info
+            )
+
+        if max_vision_seq_len == 0:
+            max_pos = position_ids.max(dim=0).values.max(dim=-1).values
+            rope_deltas = (max_pos + 1 - input_ids.shape[1]).unsqueeze(1)
+            return position_ids, rope_deltas, None, vision_token_info
+
+        vision_position_ids, position_ids, rope_deltas = (
+            self._compute_vision_position_ids(
+                input_ids=input_ids,
+                position_ids=position_ids,
+                vision_token_info=vision_token_info,
+                max_vision_seq_len=max_vision_seq_len,
+                attention_mask=attention_mask,
+            )
+        )
+        return (
+            position_ids,
+            rope_deltas.unsqueeze(1),
+            vision_position_ids,
+            vision_token_info,
+        )
+
+    def _compute_visible_frame_counts(
+        self, cross_attention_mask: Optional[Union[torch.Tensor, List]]
+    ) -> Optional[torch.Tensor]:
+        if cross_attention_mask is None:
+            return None
+
+        # HF Moss-VL processor outputs a bool mask with shape
+        # (batch_size, 1, text_len, num_frames), where True means masked.
+        cross_attention_mask = torch.as_tensor(cross_attention_mask, dtype=torch.bool)
+        visible_frame_counts = (~cross_attention_mask).sum(dim=-1, dtype=torch.int32)
+        return visible_frame_counts.reshape(-1)
+
+    def _resolve_file_url(self, value: str) -> str:
+        parsed = urlparse(value)
+        path = unquote(parsed.path or "")
+        if parsed.netloc and not path.startswith("/"):
+            path = f"/{path}"
+        return path
+
+    def _write_video_bytes_to_tempfile(
+        self, video_bytes: bytes, suffix: str = ".mp4"
+    ) -> str:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as f:
+            f.write(video_bytes)
+            return f.name
+
+    def _normalize_video_string(self, value: str) -> Tuple[str, Optional[str]]:
+        if value.startswith("file://"):
+            return self._resolve_file_url(value), None
+
+        if os.path.isfile(value):
+            return value, None
+
+        if value.startswith(("http://", "https://")):
+            timeout = int(os.getenv("REQUEST_TIMEOUT", "10"))
+            response = requests.get(value, stream=True, timeout=timeout)
+            response.raise_for_status()
+            suffix = os.path.splitext(urlparse(value).path)[1] or ".mp4"
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                return f.name, f.name
+
+        if value.startswith("data:"):
+            header, encoded = value.split(",", 1)
+            mime = header.split(";", 1)[0]
+            suffix = ".mp4"
+            if "/" in mime:
+                ext = mime.rsplit("/", 1)[-1]
+                if ext:
+                    suffix = f".{ext}"
+            temp_path = self._write_video_bytes_to_tempfile(
+                pybase64.b64decode(encoded, validate=True),
+                suffix=suffix,
+            )
+            return temp_path, temp_path
+
+        temp_path = self._write_video_bytes_to_tempfile(
+            pybase64.b64decode(value, validate=True)
+        )
+        return temp_path, temp_path
+
+    def _normalize_single_video_input(
+        self, video_input: Union[str, Dict]
+    ) -> Tuple[Union[str, Dict], List[str]]:
+        temp_paths: List[str] = []
+        if isinstance(video_input, dict):
+            normalized = dict(video_input)
+            video_path, temp_path = self._normalize_video_string(
+                normalized["video_path"]
+            )
+            normalized["video_path"] = video_path
+            if temp_path is not None:
+                temp_paths.append(temp_path)
+            return normalized, temp_paths
+
+        normalized_path, temp_path = self._normalize_video_string(video_input)
+        if temp_path is not None:
+            temp_paths.append(temp_path)
+        return normalized_path, temp_paths
+
+    async def _normalize_video_inputs_async(
+        self, video_data: Optional[List[Union[str, Dict]]]
+    ) -> Tuple[Optional[List[Union[str, Dict]]], List[str]]:
+        if not video_data:
+            return video_data, []
+
+        loop = asyncio.get_running_loop()
+        futures = [
+            loop.run_in_executor(
+                self.io_executor, self._normalize_single_video_input, v
+            )
+            for v in video_data
+        ]
+        results = await asyncio.gather(*futures)
+
+        normalized_inputs: List[Union[str, Dict]] = []
+        temp_paths: List[str] = []
+        for normalized_input, created_paths in results:
+            normalized_inputs.append(normalized_input)
+            temp_paths.extend(created_paths)
+        return normalized_inputs, temp_paths
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        normalized_video_data, temp_video_paths = (
+            await self._normalize_video_inputs_async(request_obj.video_data)
+        )
+
+        try:
+            base_output = await self.load_mm_data(
+                prompt=input_text,
+                image_data=image_data,
+                multimodal_tokens=self.image_only_mm_tokens,
+            )
+
+            processor_output = self.process_mm_data(
+                input_text=base_output.input_text,
+                images=base_output.images,
+                videos=normalized_video_data,
+            )
+            input_ids = torch.as_tensor(processor_output["input_ids"], dtype=torch.long)
+            attention_mask = processor_output.get("attention_mask")
+            if attention_mask is not None:
+                attention_mask = torch.as_tensor(attention_mask, dtype=torch.long)
+            grid_thw = processor_output.get("grid_thw")
+            if grid_thw is not None:
+                grid_thw = torch.as_tensor(grid_thw, dtype=torch.long)
+            media_nums_per_sample = processor_output.get("media_nums_per_sample")
+            visible_frame_counts = self._compute_visible_frame_counts(
+                processor_output.get("cross_attention_mask")
+            )
+
+            (
+                mrope_positions,
+                mrope_position_delta,
+                vision_position_ids,
+                vision_token_info,
+            ) = self._compute_position_metadata(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                grid_thw=grid_thw,
+                media_nums_per_sample=media_nums_per_sample,
+            )
+
+            input_ids = input_ids.flatten()
+            mm_items = self._build_mm_items(processor_output, input_ids)
+            if mm_items and vision_token_info:
+                mm_items[0].set("vision_token_info", vision_token_info[0])
+
+            if SGL_USE_CUDA_IPC:
+                for item in mm_items:
+                    if isinstance(item.feature, torch.Tensor) and item.feature.is_cuda:
+                        sync_flag, available_slice = (
+                            self.cudaipc_mmfeature_pool.return_a_slice_tensor_with_flag(
+                                item.feature
+                            )
+                        )
+                        if isinstance(available_slice, torch.Tensor):
+                            available_slice.copy_(
+                                item.feature.reshape(-1).view(torch.int8),
+                                non_blocking=True,
+                            )
+                            item.feature = CudaIpcTensorTransportProxy(
+                                data=available_slice,
+                                info_data=item.feature,
+                                sync_buffer_meta=sync_flag,
+                            )
+                    elif (
+                        isinstance(item.precomputed_embeddings, torch.Tensor)
+                        and item.precomputed_embeddings.is_cuda
+                    ):
+                        sync_flag, available_slice = (
+                            self.cudaipc_mmfeature_pool.return_a_slice_tensor_with_flag(
+                                item.precomputed_embeddings
+                            )
+                        )
+                        if isinstance(available_slice, torch.Tensor):
+                            flattened = item.precomputed_embeddings.reshape(-1)
+                            available_slice.copy_(
+                                flattened.view(torch.int8),
+                                non_blocking=True,
+                            )
+                            item.precomputed_embeddings = CudaIpcTensorTransportProxy(
+                                data=available_slice,
+                                info_data=item.precomputed_embeddings,
+                                sync_buffer_meta=sync_flag,
+                            )
+
+            return MultimodalProcessorOutput(
+                input_ids=input_ids.tolist(),
+                mm_items=mm_items,
+                im_token_id=self.image_token_id,
+                mrope_positions=mrope_positions.squeeze(1),
+                mrope_position_delta=mrope_position_delta,
+                media_nums_per_sample=media_nums_per_sample,
+                vision_position_ids=(
+                    vision_position_ids.squeeze(1)
+                    if vision_position_ids is not None
+                    else None
+                ),
+                visible_frame_counts=visible_frame_counts,
+            )
+        finally:
+            for temp_path in temp_video_paths:
+                try:
+                    os.unlink(temp_path)
+                except FileNotFoundError:
+                    pass

python/sglang/srt/multimodal/processors/nano_nemotron_vl.py

@@ -0,0 +1,509 @@
+# Copyright 2025 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import math
+from math import sqrt
+
+import numpy as np
+import torch
+from PIL import Image
+
+from sglang.srt.configs.nano_nemotron_vl import (
+    NemotronH_Nano_Omni_Reasoning_V3_Config,
+    NemotronH_Nano_VL_V2_Config,
+)
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.nano_nemotron_vl import (
+    NemotronH_Nano_Omni_Reasoning_V3,
+    NemotronH_Nano_VL_V2,
+)
+from sglang.srt.models.parakeet import ParakeetExtractor
+from sglang.srt.multimodal.audio_from_video import extract_audio_from_video_bytes
+from sglang.srt.multimodal.evs import EVSProcessor
+from sglang.srt.multimodal.internvl_utils import (
+    compute_budgeted_image_sizes,
+    get_video_target_size_and_feature_size,
+    image_to_pixel_values,
+    resize_image_to_pixels,
+    video_to_pixel_values,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils.common import sample_video_frames
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_NUM_TILES = 12
+NUM_VIDEO_TILES = 1
+DESIRED_FPS = 2  # TODO: allow desired fps/num frames to be configurable
+MAX_FRAMES = 128
+
+
+class NanoNemotronVLImageProcessor(BaseMultimodalProcessor):
+    models = [NemotronH_Nano_VL_V2, NemotronH_Nano_Omni_Reasoning_V3]
+    gpu_image_decode = (
+        False  # NanoNemotronVL processes loaded image as PIL image explicitly
+    )
+
+    def __init__(self, hf_config, server_args, _image_processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _image_processor, *args, **kwargs)
+        self.evs = EVSProcessor(
+            hf_config,
+            {
+                NemotronH_Nano_VL_V2_Config: NemotronH_Nano_VL_V2,
+                NemotronH_Nano_Omni_Reasoning_V3_Config: NemotronH_Nano_Omni_Reasoning_V3,
+            },
+        )
+        Image.MAX_IMAGE_PIXELS = None
+        self.image_size = hf_config.image_size
+        self.VIDEO_CONTEXT_TOKEN = hf_config.video_context_token
+        self.IMG_CONTEXT_TOKEN = hf_config.img_context_token
+        self.IMG_START_TOKEN = hf_config.img_start_token
+        self.IMG_END_TOKEN = hf_config.img_end_token
+        self.num_image_token = int(
+            (self.image_size // hf_config.patch_size) ** 2
+            * (hf_config.downsample_ratio**2)
+        )
+        if hasattr(self._processor, "tokenizer"):
+            tokenizer = self._processor.tokenizer
+        else:
+            tokenizer = self._processor
+        self.tokenizer = tokenizer
+
+        self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START_TOKEN)
+        self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END_TOKEN)
+
+        # Audio support: initialize Parakeet extractor if sound_config is present
+        self.audio_extractor: ParakeetExtractor | None = None
+        self.AUDIO_CONTEXT_TOKEN = getattr(
+            hf_config, "audio_context_token", "<so_embedding>"
+        )
+        self.AUDIO_START_TOKEN = getattr(hf_config, "audio_start_token", "<so_start>")
+        self.AUDIO_END_TOKEN = getattr(hf_config, "audio_end_token", "<so_end>")
+
+        audio_token_str = None
+        audio_token_id = None
+        if getattr(hf_config, "sound_config", None) is not None:
+            self.audio_extractor = ParakeetExtractor(hf_config.sound_config)
+            audio_token_str = self.AUDIO_CONTEXT_TOKEN
+            audio_token_id = tokenizer.convert_tokens_to_ids(self.AUDIO_CONTEXT_TOKEN)
+            self.audio_start_token_id = tokenizer.convert_tokens_to_ids(
+                self.AUDIO_START_TOKEN
+            )
+            self.audio_end_token_id = tokenizer.convert_tokens_to_ids(
+                self.AUDIO_END_TOKEN
+            )
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMG_CONTEXT_TOKEN,
+            image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT_TOKEN),
+            video_token=self.VIDEO_CONTEXT_TOKEN,
+            video_token_id=tokenizer.convert_tokens_to_ids(self.VIDEO_CONTEXT_TOKEN),
+            audio_token=audio_token_str,
+            audio_token_id=audio_token_id,
+        ).build(_image_processor)
+
+        # Normalization config (mean/std) and tiling behavior
+        self.norm_mean = hf_config.norm_mean
+        self.norm_std = hf_config.norm_std
+        self.use_thumbnail = hf_config.use_thumbnail
+
+        # Dynamic resolution config
+        self.dynamic_resolution = getattr(hf_config, "dynamic_resolution", False)
+        self.min_num_patches = getattr(hf_config, "min_num_patches", 0)
+        self.max_num_patches = getattr(hf_config, "max_num_patches", 0)
+        self.patch_size = hf_config.patch_size
+        self.downsample_ratio = hf_config.downsample_ratio
+
+        # Video temporal compression config
+        self.video_temporal_patch_size = getattr(
+            hf_config, "video_temporal_patch_size", 1
+        )
+        self.video_target_num_patches = getattr(
+            hf_config, "video_target_num_patches", 0
+        )
+        self.video_maintain_aspect_ratio = getattr(
+            hf_config, "video_maintain_aspect_ratio", True
+        )
+
+        self.max_model_len = getattr(server_args, "context_length", None) or 8192
+
+        self.PLACEHOLDER = self.tokenizer.unk_token
+        assert isinstance(self.PLACEHOLDER, str)
+        self.PLACEHOLDER_ID = tokenizer.convert_tokens_to_ids(self.PLACEHOLDER)
+        assert isinstance(self.PLACEHOLDER_ID, int)
+
+    def preprocess_image(
+        self, image: Image.Image, *, max_num_tiles: int = DEFAULT_NUM_TILES
+    ) -> torch.Tensor:
+        return image_to_pixel_values(
+            image,
+            input_size=self.image_size,
+            max_num_tiles=max_num_tiles,
+            use_thumbnail=self.use_thumbnail,
+            mean=self.norm_mean,
+            std=self.norm_std,
+        ).to(dtype=torch.bfloat16)
+
+    def render_image(self, *, num_tiles: int):
+        return f"{self.IMG_START_TOKEN}{self.IMG_CONTEXT_TOKEN * self.num_image_token * num_tiles}{self.IMG_END_TOKEN}"
+
+    def render_image_dynamic(self, *, num_tokens: int):
+        return f"{self.IMG_START_TOKEN}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}"
+
+    def render_tubelet(
+        self,
+        tubelet_index: int,
+        frame_indices: list[int],
+        timestamps: list[float],
+        num_tokens: int,
+    ):
+        """Render a tubelet (group of T frames) for temporal compression."""
+        if len(frame_indices) == 1:
+            return self.render_frame(
+                frame_indices[0], timestamp=timestamps[0], num_tokens=num_tokens
+            )
+        parts = " and ".join(
+            f"frame {fi + 1} sampled at {ts:.2f} seconds"
+            for fi, ts in zip(frame_indices, timestamps)
+        )
+        return f"{parts}: {self.PLACEHOLDER}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}"
+
+    def render_frame(self, frame_index: int, *, timestamp: float, num_tokens: int):
+        return f"Frame {frame_index + 1} sampled at {timestamp:.2f} seconds: {self.PLACEHOLDER}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}"
+
+    @staticmethod
+    def parse_video(video) -> tuple[np.ndarray, list[float]]:
+        frames = sample_video_frames(
+            video, desired_fps=DESIRED_FPS, max_frames=MAX_FRAMES
+        )
+        video_array = video.get_frames_at(frames)
+        avg_fps = video.avg_fps
+        if avg_fps > 0:
+            frame_duration_ms = int(1000 / avg_fps)
+        else:
+            frame_duration_ms = 0
+        timestamps = [i * frame_duration_ms / 1000.0 for i in frames]
+        return video_array, timestamps
+
+    def render_audio(self, *, num_tokens: int):
+        return (
+            f"{self.AUDIO_START_TOKEN}"
+            f"{self.AUDIO_CONTEXT_TOKEN * num_tokens}"
+            f"{self.AUDIO_END_TOKEN}"
+        )
+
+    async def process_mm_data_async(
+        self, image_data, audio_data, input_text, request_obj, **kwargs
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            audio_data=audio_data if self.audio_extractor else None,
+            multimodal_tokens=self.mm_tokens,
+            discard_alpha_channel=True,
+            audio_sample_rate=(
+                self.audio_extractor.sampling_rate if self.audio_extractor else None
+            ),
+        )
+
+        videos = [self.parse_video(video) for video in base_output.videos]
+
+        T = self.video_temporal_patch_size
+
+        if T > 1:
+            tubelets_per_video = [math.ceil(len(frames) / T) for frames, _ in videos]
+            if self.video_target_num_patches > 0 and videos:
+                frame_h, frame_w = videos[0][0][0].shape[:2]
+                target_w, target_h, tokens_per_tubelet = (
+                    get_video_target_size_and_feature_size(
+                        frame_w,
+                        frame_h,
+                        self.video_target_num_patches,
+                        self.video_maintain_aspect_ratio,
+                        self.patch_size,
+                        self.downsample_ratio,
+                    )
+                )
+                ds = int(1 / self.downsample_ratio)
+                rows = target_h // self.patch_size // ds
+                cols = target_w // self.patch_size // ds
+            else:
+                tokens_per_tubelet = self.num_image_token
+                rows = cols = int(sqrt(tokens_per_tubelet))
+            create_data_items, tokens_per_frame = self.evs.static_size_data_items(
+                frames_per_video=tubelets_per_video,
+                num_images=len(base_output.images),
+                rows=rows,
+                cols=cols,
+            )
+        else:
+            rows = cols = int(sqrt(self.num_image_token))
+            create_data_items, tokens_per_frame = self.evs.static_size_data_items(
+                frames_per_video=[len(frames) for frames, _ in videos],
+                num_images=len(base_output.images),
+                rows=rows,
+                cols=cols,
+            )
+
+        prompt = input_text
+        image_is_dynamic = False
+        num_tokens_per_image = []
+        image_feature = None
+        if base_output.images and self.dynamic_resolution:
+            image_is_dynamic = True
+            image_sizes = [(img.width, img.height) for img in base_output.images]
+            text_only = input_text.replace(self.IMG_CONTEXT_TOKEN, "")
+            text_tokens = len(
+                self.tokenizer(text_only, add_special_tokens=False)["input_ids"]
+            )
+            total_token_budget = self.max_model_len - text_tokens
+            budgeted_sizes = compute_budgeted_image_sizes(
+                image_sizes,
+                total_token_budget,
+                self.patch_size,
+                self.downsample_ratio,
+                self.min_num_patches,
+                self.max_num_patches,
+            )
+            preprocessed_images = []
+            for image, (target_w, target_h, n_tokens) in zip(
+                base_output.images, budgeted_sizes
+            ):
+                pv = resize_image_to_pixels(
+                    image,
+                    target_w,
+                    target_h,
+                    mean=self.norm_mean,
+                    std=self.norm_std,
+                )
+                preprocessed_images.append(pv.to(dtype=torch.bfloat16))
+                num_tokens_per_image.append(n_tokens)
+            rendered_images = [
+                self.render_image_dynamic(num_tokens=nt) for nt in num_tokens_per_image
+            ]
+            prompt = prompt.replace(self.IMG_CONTEXT_TOKEN, "".join(rendered_images), 1)
+            image_feature = preprocessed_images
+        elif base_output.images:
+            preprocessed_images = [
+                self.preprocess_image(image) for image in base_output.images
+            ]
+            rendered_images = [
+                self.render_image(num_tiles=image.shape[0])
+                for image in preprocessed_images
+            ]
+            prompt = prompt.replace(self.IMG_CONTEXT_TOKEN, "".join(rendered_images), 1)
+            image_feature = torch.cat(preprocessed_images, dim=0)
+
+        video_feature = None
+        T = self.video_temporal_patch_size
+        if base_output.videos:
+            preprocessed_videos = []
+            for (video_array, timestamps), tpf in zip(
+                videos, tokens_per_frame, strict=True
+            ):
+                if self.video_target_num_patches > 0:
+                    frames_tensors = []
+                    for frame in video_array:
+                        pv, _ = video_to_pixel_values(
+                            Image.fromarray(frame, mode="RGB"),
+                            patch_size=self.patch_size,
+                            downsample_ratio=self.downsample_ratio,
+                            target_num_patches=self.video_target_num_patches,
+                            maintain_aspect_ratio=self.video_maintain_aspect_ratio,
+                            mean=self.norm_mean,
+                            std=self.norm_std,
+                        )
+                        frames_tensors.append(pv.to(dtype=torch.bfloat16))
+                else:
+                    frames_tensors = [
+                        self.preprocess_image(
+                            Image.fromarray(frame, mode="RGB"),
+                            max_num_tiles=NUM_VIDEO_TILES,
+                        )
+                        for frame in video_array
+                    ]
+                preprocessed_video = torch.cat(frames_tensors, dim=0)
+                preprocessed_videos.append(preprocessed_video)
+
+                if T > 1:
+                    num_frames = len(video_array)
+                    num_tubelets = math.ceil(num_frames / T)
+                    rendered_parts = []
+                    for ti in range(num_tubelets):
+                        start_fi = ti * T
+                        end_fi = min(start_fi + T, num_frames)
+                        fi_list = list(range(start_fi, end_fi))
+                        ts_list = [timestamps[fi] for fi in fi_list]
+                        rendered_parts.append(
+                            self.render_tubelet(
+                                ti, fi_list, ts_list, num_tokens=tpf[ti]
+                            )
+                        )
+                    prompt = prompt.replace(
+                        self.VIDEO_CONTEXT_TOKEN, "\n".join(rendered_parts), 1
+                    )
+                else:
+                    rendered_frames = [
+                        self.render_frame(
+                            i,
+                            timestamp=timestamp,
+                            num_tokens=num_tokens,
+                        )
+                        for i, (timestamp, num_tokens) in enumerate(
+                            zip(timestamps, tpf, strict=True)
+                        )
+                    ]
+                    prompt = prompt.replace(
+                        self.VIDEO_CONTEXT_TOKEN, "".join(rendered_frames), 1
+                    )
+            video_feature = torch.cat(preprocessed_videos, dim=0)
+
+        # Extract audio from video if requested and no explicit audio provided
+        use_audio_in_video = getattr(request_obj, "use_audio_in_video", False)
+        extracted_audios: list[np.ndarray] = []
+        if (
+            use_audio_in_video
+            and base_output.videos
+            and not base_output.audios
+            and self.audio_extractor is not None
+        ):
+            for video_wrapper in base_output.videos:
+                video_bytes = video_wrapper.source_bytes
+                if video_bytes is not None:
+                    audio_array = extract_audio_from_video_bytes(
+                        video_bytes,
+                        target_sr=self.audio_extractor.sampling_rate,
+                    )
+                    if audio_array is not None:
+                        extracted_audios.append(audio_array)
+
+        all_audios: list[np.ndarray] = (
+            list(base_output.audios) if base_output.audios else []
+        )
+        all_audios.extend(extracted_audios)
+
+        # Process audio data through the Parakeet feature extractor
+        audio_items: list[MultimodalDataItem] = []
+        if all_audios and self.audio_extractor is not None:
+            extractor = self.audio_extractor
+            for audio in all_audios:
+                num_tokens = extractor.audio_token_count(len(audio))
+                rendered = self.render_audio(num_tokens=num_tokens)
+                if self.AUDIO_CONTEXT_TOKEN in prompt:
+                    prompt = prompt.replace(self.AUDIO_CONTEXT_TOKEN, rendered, 1)
+                else:
+                    prompt = prompt + rendered
+
+            extracted = extractor(
+                all_audios,
+                sampling_rate=extractor.sampling_rate,
+                return_tensors="pt",
+            )
+            input_features = extracted.input_features
+            attention_mask = extracted.attention_mask
+            clip_counts = extracted.audio_num_clips
+
+            clip_offset = 0
+            for audio_idx, num_clips in enumerate(clip_counts):
+                audio_features = input_features[clip_offset : clip_offset + num_clips]
+                audio_mask = attention_mask[clip_offset : clip_offset + num_clips]
+                clip_offset += num_clips
+                audio_items.append(
+                    MultimodalDataItem(
+                        modality=Modality.AUDIO,
+                        feature=audio_features,
+                        model_specific_data={
+                            "feature_attention_mask": audio_mask,
+                            "audio_num_clips": num_clips,
+                        },
+                    )
+                )
+
+        prompt_ids = self.tokenizer(
+            prompt, add_special_tokens=False, return_tensors="pt"
+        )["input_ids"].flatten()
+        offsets = self.get_mm_items_offset(prompt_ids, self.mm_tokens.image_token_id)
+        img_offsets = [
+            (start, end)
+            for start, end in offsets
+            if prompt_ids[start - 1] == self.img_start_token_id
+        ]
+        video_offsets = [
+            (start, end)
+            for start, end in offsets
+            if prompt_ids[start - 1] == self.PLACEHOLDER_ID
+        ]
+        # Cleanup:
+        prompt_ids[prompt_ids == self.PLACEHOLDER_ID] = self.img_start_token_id
+
+        # Compute audio offsets
+        if audio_items:
+            audio_token_id = self.mm_tokens.audio_token_id
+            audio_offsets_list = self.get_mm_items_offset(prompt_ids, audio_token_id)
+            for item, offset in zip(audio_items, audio_offsets_list):
+                item.offsets = [offset]
+
+        prompt_ids_list = prompt_ids.tolist()
+
+        if image_is_dynamic and image_feature is not None:
+            items = []
+            for i, (pv, offset) in enumerate(zip(image_feature, img_offsets)):
+                items.append(
+                    MultimodalDataItem(
+                        modality=Modality.IMAGE,
+                        feature=pv,
+                        offsets=[offset],
+                        model_specific_data={
+                            "num_tokens": num_tokens_per_image[i],
+                            "is_dynamic": True,
+                        },
+                    )
+                )
+            if video_feature is not None:
+                items.append(
+                    MultimodalDataItem(
+                        modality=Modality.VIDEO,
+                        feature=video_feature,
+                        offsets=video_offsets,
+                    )
+                )
+        else:
+            items = create_data_items(
+                image=image_feature,
+                image_offsets=img_offsets,
+                video=video_feature,
+                video_offsets=video_offsets,
+                input_ids_list=prompt_ids_list,
+            )
+        items.extend(audio_items)
+
+        return MultimodalProcessorOutput(
+            input_ids=prompt_ids_list,
+            mm_items=items,
+            im_start_id=self.img_start_token_id,
+            im_end_id=self.img_end_token_id,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.image_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id if audio_items else None,
+            audio_start_id=(self.audio_start_token_id if audio_items else None),
+            audio_end_id=(self.audio_end_token_id if audio_items else None),
+        )

python/sglang/srt/multimodal/processors/nvila.py

@@ -0,0 +1,80 @@
+from typing import Any
+
+import torch.nn as nn
+from transformers.configuration_utils import PretrainedConfig
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+from sglang.srt.managers.io_struct import GenerateReqInput
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.jet_vlm import JetVLMForConditionalGeneration
+from sglang.srt.models.nvila import NVILAForConditionalGeneration
+from sglang.srt.models.nvila_lite import NVILALiteForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.server_args import ServerArgs
+
+NUM_VIDEO_FRAMES = 8
+
+
+class NVILAMultimodalProcessor(BaseMultimodalProcessor):
+    models: list[type[nn.Module]] = [
+        NVILAForConditionalGeneration,
+        NVILALiteForConditionalGeneration,
+        JetVLMForConditionalGeneration,
+    ]
+
+    def __init__(
+        self,
+        hf_config: PretrainedConfig,
+        server_args: ServerArgs,
+        _processor: ProcessorMixin,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self._processor: ProcessorMixin
+
+        tokenizer: PreTrainedTokenizerBase = getattr(self._processor, "tokenizer")
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=tokenizer.image_token,
+            image_token_id=hf_config.image_token_id,
+            video_token=tokenizer.video_token,
+            video_token_id=hf_config.video_token_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data,
+        audio_data,
+        input_text,
+        request_obj: GenerateReqInput,
+        **kwargs,
+    ) -> dict[str, Any] | None:
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=self.mm_tokens,
+            image_data=request_obj.image_data,  # type: ignore
+            video_data=request_obj.video_data,  # type: ignore
+        )
+
+        for i, video in enumerate(base_output.videos):  # type: ignore
+            base_output.videos[i] = [x.asnumpy() for x in video]  # type: ignore
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output,
+            self.mm_tokens,
+            do_sample_frames=True,
+            num_frames=NUM_VIDEO_FRAMES,
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+        )

python/sglang/srt/multimodal/processors/paddleocr_vlm.py

@@ -0,0 +1,30 @@
+# Reference: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleocr-genai-vllm-server:latest
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from sglang.srt.models.paddleocr_vl import PaddleOCRVLForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
+from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor
+
+
+class PaddleOCRVLImageProcessor(QwenVLImageProcessor):
+    models = [PaddleOCRVLForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>",
+            image_token_id=hf_config.image_token_id,
+            video_token_id=hf_config.video_token_id,
+        ).build(_processor)

python/sglang/srt/multimodal/processors/phi4mm.py

@@ -0,0 +1,101 @@
+import logging
+from typing import List, Union
+
+from transformers.processing_utils import ProcessorMixin
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.phi4mm import Phi4MMForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# It is an adapter of hf phi4 mm processor to make it work for sglang
+# Ref: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/main/processing_phi4mm.py#L693
+class Phi4MMProcessorAdapter(ProcessorMixin):
+    def __init__(self, _processor) -> None:
+        self._processor = _processor
+
+    def __call__(self, **kwargs):
+        result = self._processor(**kwargs)
+
+        # Map HuggingFace output keys to sglang standard keys
+        key_mapping = {
+            "input_image_embeds": "pixel_values",
+            "input_audio_embeds": "audio_features",
+            "audio_embed_sizes": "audio_feature_lens",
+        }
+        for hf_key, sglang_key in key_mapping.items():
+            if hf_key in result:
+                result[sglang_key] = result[hf_key]
+                del result[hf_key]
+
+        # Filter out None or empty tensors from the result.
+        # This prevents the sglang function base_processor.collect_mm_items_from_processor_output()
+        # from misclassifying audio content as image content, and vice versa.
+        filtered_result = {
+            k: v
+            for k, v in result.items()
+            if v is not None and (not hasattr(v, "numel") or v.numel() > 0)
+        }
+        return filtered_result
+
+
+class Phi4MMMultimodalProcessor(BaseMultimodalProcessor):
+    models = [Phi4MMForCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        self.processor = Phi4MMProcessorAdapter(_processor)
+        super().__init__(hf_config, server_args, self.processor, *args, **kwargs)
+
+        # the following CONSTANTS come from hugging-face microsoft/Phi-4-multimodal-instruct's processing_phi4mm.py file
+        # ref: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/main/processing_phi4mm.py
+        self.IMAGE_TOKEN = "<|endoftext10|>"
+        self.AUDIO_TOKEN = "<|endoftext11|>"
+        self.IM_TOKEN_ID = 200010
+        self.AUDIO_TOKEN_ID = 200011
+        self.AUDIO_SAMPLE_RATE = 16000
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+            audio_token=self.AUDIO_TOKEN,
+            audio_token_id=self.AUDIO_TOKEN_ID,
+        ).build(self.processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        audio_data,
+        input_text,
+        request_obj,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+            audio_sample_rate=self.AUDIO_SAMPLE_RATE,
+        )
+
+        if base_output.audios is not None:
+            # hugging-face microsoft/Phi-4-multimodal-instruct's processing_phi4mm.py file requires the audio input to be tuple of (audio, sample_rate)
+            # ref: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/main/processing_phi4mm.py
+            base_output.audios = [
+                (audio, self.AUDIO_SAMPLE_RATE) for audio in base_output.audios
+            ]
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+        )

python/sglang/srt/multimodal/processors/pixtral.py

@@ -0,0 +1,134 @@
+import math
+from typing import List, Union
+
+from transformers import PreTrainedTokenizerBase
+from transformers.models.pixtral.image_processing_pixtral import (
+    _num_image_tokens as _get_pixtral_hf_num_image_tokens,
+)
+
+from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
+from sglang.srt.models.pixtral import (
+    PixtralForConditionalGeneration,
+    PixtralVisionModel,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class PixtralProcessor(BaseMultimodalProcessor):
+    models = [PixtralVisionModel, PixtralForConditionalGeneration]
+    gpu_image_decode = False  # Pixtral processes loaded image as PIL image explicitly
+
+    PAD_TOKEN = "<pad>"
+    DEFAULT_IMAGE_TOKEN = "[IMG]"
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.IM_TOKEN_ID = getattr(
+            hf_config, "image_token_index", PixtralVisionModel.DEFAULT_IMAGE_TOKEN_ID
+        )
+
+        self.vision_config = hf_config.vision_config
+        self.image_size = self.vision_config.image_size
+        self.patch_size = self.vision_config.patch_size
+
+        # spatial_merge_size may live on vision_config (Mistral native) or
+        # on the top-level config (HF native Mistral3Config).
+        self._spatial_merge_size = getattr(
+            self.vision_config,
+            "spatial_merge_size",
+            getattr(hf_config, "spatial_merge_size", 1),
+        )
+
+        self._processor.patch_size = self.patch_size
+        if self._spatial_merge_size > 1:
+            self._processor.spatial_merge_size = self._spatial_merge_size
+
+        tokenizer = (
+            _processor
+            if isinstance(_processor, PreTrainedTokenizerBase)
+            else _processor.tokenizer
+        )
+        self.image_token = getattr(_processor, "image_token", self.DEFAULT_IMAGE_TOKEN)
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.image_token,
+            image_token_id=self.IM_TOKEN_ID,
+        ).build(_processor)
+        tokenizer.add_special_tokens(
+            {
+                "pad_token": getattr(hf_config, "pad_token", self.PAD_TOKEN),
+            }
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        mm_data = await self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=self.mm_tokens,
+            image_data=image_data,
+            return_text=True,
+        )
+        if mm_data.images:
+            effective_patch = self.patch_size * self._spatial_merge_size
+            image_nrows = []
+            for img in mm_data.images:
+                w, h = img.size
+                ratio = max(w / self.image_size, h / self.image_size)
+                if ratio > 1:
+                    w = int(math.floor(w / ratio))
+                    h = int(math.floor(h / ratio))
+                nrows, _ = _get_pixtral_hf_num_image_tokens(
+                    (h, w), (effective_patch, effective_patch)
+                )
+                image_nrows.append(nrows)
+
+            mm_items, input_ids, _ = self.process_and_combine_mm_data(
+                mm_data, self.mm_tokens
+            )
+
+            # For multi-image: split single IMAGE mm_item into per-image items
+            if len(mm_data.images) > 1:
+                from sglang.srt.managers.schedule_batch import MultimodalDataItem
+
+                old_item = next(
+                    item for item in mm_items if item.modality == Modality.IMAGE
+                )
+                all_offsets = old_item.offsets
+                old_feature = old_item.feature
+                old_image_sizes = getattr(old_item, "image_sizes", None)
+
+                mm_items = [
+                    item for item in mm_items if item.modality != Modality.IMAGE
+                ]
+                offset_idx = 0
+                for i, img in enumerate(mm_data.images):
+                    nr = image_nrows[i]
+                    item_offsets = all_offsets[offset_idx : offset_idx + nr]
+                    offset_idx += nr
+                    new_item = MultimodalDataItem(modality=Modality.IMAGE)
+                    new_item.feature = old_feature[i : i + 1]
+                    new_item.offsets = item_offsets
+                    if old_image_sizes is not None:
+                        new_item.model_specific_data["image_sizes"] = old_image_sizes[
+                            i : i + 1
+                        ]
+                    mm_items.append(new_item)
+        else:
+            mm_items, input_ids, _ = self.process_and_combine_mm_data(
+                mm_data, self.mm_tokens
+            )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_token_id=self.IM_TOKEN_ID,
+        )

python/sglang/srt/multimodal/processors/points_v15_chat.py

@@ -0,0 +1,43 @@
+# Copy from qwen_vl.py, adapted for points-v15-chat
+
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.points_v15_chat import POINTSV15ChatModel
+from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor
+
+
+class POINTSV15ChatProcessor(QwenVLImageProcessor):
+    models = [POINTSV15ChatModel]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        # Compatible with POINTSV15Chat
+        hf_config.vision_start_token_id = None
+        hf_config.vision_end_token_id = None
+        hf_config.video_token_id = None
+
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/qwen3_asr.py

@@ -0,0 +1,98 @@
+import re
+from typing import Union
+
+import torch
+
+from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
+from sglang.srt.models.qwen3_asr import Qwen3ASRForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+AUDIO_PLACEHOLDER = "<|audio_start|><|audio_pad|><|audio_end|>"
+
+DEFAULT_ASR_PROMPT = (
+    f"<|im_start|>user\n"
+    f"{AUDIO_PLACEHOLDER}"
+    f"<|im_end|>\n"
+    f"<|im_start|>assistant\n"
+)
+
+
+class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor):
+    models = [Qwen3ASRForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.AUDIO_TOKEN = AUDIO_PLACEHOLDER
+        self.AUDIO_TOKEN_REGEX = re.compile(
+            r"<\|audio_start\|>(?:<\|audio_pad\|>)+<\|audio_end\|>"
+        )
+        tokenizer = self._processor.tokenizer
+        self.audio_start_id = tokenizer.convert_tokens_to_ids("<|audio_start|>")
+        self.audio_token_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
+        self.audio_end_id = tokenizer.convert_tokens_to_ids("<|audio_end|>")
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=self.AUDIO_TOKEN,
+            audio_token_regex=self.AUDIO_TOKEN_REGEX,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+        self.ATTR_NAME_TO_MODALITY.update({"feature_attention_mask": Modality.AUDIO})
+
+    def _build_transcription_prompt(self, input_text: Union[str, list]) -> str:
+        # TODO: support `force_language`
+        if isinstance(input_text, list):
+            input_text = self._tokenizer.decode(input_text)
+        if not input_text or not input_text.strip():
+            return DEFAULT_ASR_PROMPT
+        return input_text
+
+    def compute_mrope_positions(self, input_ids, mm_items):
+        if isinstance(input_ids, list):
+            seq_len = len(input_ids)
+        else:
+            seq_len = input_ids.shape[-1] if input_ids.dim() > 1 else input_ids.shape[0]
+        positions = torch.arange(seq_len, dtype=torch.long)
+        mrope_positions = positions.unsqueeze(0).expand(3, -1).clone()
+        return mrope_positions, torch.tensor([0], dtype=torch.long)
+
+    async def process_mm_data_async(
+        self,
+        audio_data=None,
+        input_text=None,
+        request_obj=None,
+        **kwargs,
+    ):
+        if not audio_data:
+            return None
+
+        prompt = self._build_transcription_prompt(input_text)
+
+        base_output = await self.load_mm_data(
+            prompt=prompt,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base_output is None:
+            return None
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        mrope_positions, mrope_position_delta = self.compute_mrope_positions(
+            input_ids, mm_items
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            audio_start_id=self.audio_start_id,
+            audio_token_id=self.audio_token_id,
+            audio_end_id=self.audio_end_id,
+            mrope_positions=mrope_positions,
+            mrope_position_delta=mrope_position_delta,
+        )

python/sglang/srt/multimodal/processors/qwen_audio.py

@@ -0,0 +1,117 @@
+import re
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.qwen2_audio import Qwen2AudioForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
+    models = [Qwen2AudioForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.AUDIO_TOKEN = "<|audio_bos|><|AUDIO|><|audio_eos|>"
+        self.AUDIO_TOKEN_REGEX = re.compile(
+            r"<\|audio_bos\|>(?:<\|AUDIO\|>)+<\|audio_eos\|>"
+        )
+        # Collect special token ids
+        tokenizer = self._processor.tokenizer
+        self.audio_start_id = tokenizer.convert_tokens_to_ids("<|audio_bos|>")
+        self.audio_token_id = tokenizer.convert_tokens_to_ids("<|AUDIO|>")
+        self.audio_end_id = tokenizer.convert_tokens_to_ids("<|audio_eos|>")
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=self.AUDIO_TOKEN,
+            audio_token_regex=self.AUDIO_TOKEN_REGEX,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+        self.ATTR_NAME_TO_MODALITY.update({"feature_attention_mask": Modality.AUDIO})
+
+    def get_mm_data(self, prompt, embeddings, **kwargs):
+        audio_feature_lens = kwargs.get("audio_feature_lens", None)
+
+        # Convert audio_feature_lens to token counts for build_input_ids
+        output_lengths = None
+        input_lengths = None
+        if audio_feature_lens is not None:
+            if audio_feature_lens.dim() > 1:
+                audio_feature_lens = audio_feature_lens.flatten()
+            input_lengths = (audio_feature_lens - 1) // 2 + 1
+            output_lengths = (input_lengths - 2) // 2 + 1
+
+        input_ids, offsets, modality_list = self.build_input_ids(
+            prompt,
+            audio_seq_lens=output_lengths,
+        )
+
+        mm_items = []
+        consumed_per_modality = {}
+
+        for modality, offset in zip(modality_list, offsets):
+            num_tokens = offset[1] - offset[0] + 1
+            embedding_start = consumed_per_modality.get(modality, 0)
+            embedding_slice = embeddings[modality][
+                embedding_start : embedding_start + num_tokens
+            ]
+            consumed_per_modality[modality] = embedding_start + num_tokens
+            mm_items.append(
+                MultimodalDataItem(
+                    modality=modality,
+                    offsets=[offset],
+                    precomputed_embeddings=embedding_slice,
+                )
+            )
+
+        if mm_items:
+            mm_items[0].audio_feature_lens = output_lengths
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids,
+            audio_start_id=self.audio_start_id,
+            audio_token_id=self.audio_token_id,
+            audio_end_id=self.audio_end_id,
+        )
+
+    async def process_mm_data_async(
+        self,
+        audio_data,
+        input_text,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base_output is None:
+            return None
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        assert (
+            "feature_attention_mask" in ret
+        ), "feature_attention_mask not found in processor output"
+        input_lengths = ret["feature_attention_mask"].sum(dim=-1)
+        input_lengths = (input_lengths - 1) // 2 + 1
+        output_lengths = (input_lengths - 2) // 2 + 1
+
+        mm_items[0].audio_feature_lens = output_lengths
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            audio_start_id=self.audio_start_id,
+            audio_token_id=self.audio_token_id,
+            audio_end_id=self.audio_end_id,
+        )

python/sglang/srt/multimodal/processors/qwen_vl.py

@@ -0,0 +1,835 @@
+import math
+import os
+import re
+import time
+from typing import List, Optional, Union
+
+import numpy as np
+import torch
+import torchvision
+from PIL import Image
+from torchvision.transforms import InterpolationMode
+
+from sglang.srt.environ import envs
+from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.interns2preview import InternS2PreviewForConditionalGeneration
+from sglang.srt.models.qwen2_5_vl import Qwen2_5_VLForConditionalGeneration
+from sglang.srt.models.qwen2_vl import Qwen2VLForConditionalGeneration
+from sglang.srt.models.qwen3_5 import (
+    Qwen3_5ForConditionalGeneration,
+    Qwen3_5MoeForConditionalGeneration,
+)
+from sglang.srt.models.qwen3_5_mtp import Qwen3_5ForCausalLMMTP
+from sglang.srt.models.qwen3_omni_moe import Qwen3OmniMoeForConditionalGeneration
+from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration
+from sglang.srt.models.qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils import cpu_has_amx_support, is_cpu
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
+from sglang.utils import logger
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = envs.SGLANG_IMAGE_MAX_PIXELS.get()
+MAX_RATIO = 200
+RESIZE_RESAMPLE = getattr(Image, envs.SGLANG_RESIZE_RESAMPLE.get(), None)
+if envs.SGLANG_RESIZE_RESAMPLE.is_set() and RESIZE_RESAMPLE is None:
+    logger.warning(
+        f"Invalid RESIZE_RESAMPLE value: '{envs.SGLANG_RESIZE_RESAMPLE.get()}'. "
+        f"Ignoring and using default."
+    )
+VIDEO_TOTAL_PIXELS = int(
+    float(os.environ.get("VIDEO_MAX_PIXELS", 128000 * 28 * 28 * 0.9))
+)
+
+VIDEO_MIN_PIXELS = 128 * 28 * 28
+VIDEO_MAX_PIXELS = 768 * 28 * 28
+FRAME_FACTOR = 2
+FPS = 2.0
+FPS_MIN_FRAMES = 4
+FPS_MAX_FRAMES = 768
+
+
+_is_cpu_amx_available = cpu_has_amx_support()
+_is_cpu = is_cpu()
+if _is_cpu and _is_cpu_amx_available:
+    try:
+        import transformers
+
+        from sglang.srt.layers.amx_utils import fast_preprocess_cpu
+
+        transformers.models.qwen2_vl.image_processing_qwen2_vl_fast.Qwen2VLImageProcessorFast._preprocess = (
+            fast_preprocess_cpu
+        )
+    except Exception as e:
+        logger.warning(
+            f"Failed to hack Qwen2VLImageProcessorFast with AMX optimization: {e}"
+        )
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = IMAGE_FACTOR,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_nframes(
+    ele: dict,
+    total_frames: int,
+    video_fps: int | float,
+) -> int:
+    """calculate the number of frames for video used for model inputs.
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+            support either `fps` or `nframes`:
+                - nframes: the number of frames to extract for model inputs.
+                - fps: the fps to extract frames for model inputs.
+                    - min_frames: the minimum number of frames of the video, only used when fps is provided.
+                    - max_frames: the maximum number of frames of the video, only used when fps is provided.
+        total_frames (int): the original total number of frames of the video.
+        video_fps (int | float): the original fps of the video.
+
+    Raises:
+        ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
+
+    Returns:
+        int: the number of frames for video used for model inputs.
+    """
+    assert not (
+        "fps" in ele and "nframes" in ele
+    ), "Only accept either `fps` or `nframes`"
+    if "nframes" in ele:
+        nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
+    else:
+        fps = ele.get("fps", FPS)
+        min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
+        max_frames = floor_by_factor(
+            ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR
+        )
+        nframes = total_frames / video_fps * fps
+        if nframes > total_frames:
+            logger.warning(
+                f"smart_nframes: nframes[{nframes}] > total_frames[{total_frames}]"
+            )
+        nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+        nframes = floor_by_factor(nframes, FRAME_FACTOR)
+    if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
+        raise ValueError(
+            f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}."
+        )
+    return nframes
+
+
+# process video, qwen-specific
+async def preprocess_video(
+    vr,
+    image_factor: int = IMAGE_FACTOR,
+    video_config: dict = {},
+) -> torch.Tensor:
+    # preprocessed video
+    is_video_obj = isinstance(vr, VideoDecoderWrapper)
+    if not is_video_obj:
+        return vr, None
+    entry_time = time.perf_counter()
+
+    total_frames, video_fps = len(vr), vr.avg_fps
+
+    nframes = smart_nframes(
+        video_config, total_frames=total_frames, video_fps=video_fps
+    )
+    idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
+    idx = np.unique(idx)
+
+    video = vr.get_frames_as_tensor(idx.tolist())
+
+    video = video.permute(0, 3, 1, 2)  # NHWC -> TCHW
+
+    nframes, _, height, width = video.shape
+    min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS)
+    total_pixels = video_config.get("total_pixels", VIDEO_TOTAL_PIXELS)
+    max_pixels = max(
+        min(
+            video_config.get("max_pixels", VIDEO_MAX_PIXELS),
+            total_pixels / nframes * FRAME_FACTOR,
+        ),
+        int(min_pixels * 1.05),
+    )
+
+    get_batch_time = time.perf_counter()
+
+    max_pixels_supposed = video_config.get("max_pixels", max_pixels)
+
+    if max_pixels_supposed > max_pixels:
+        logger.warning(
+            f"The given max_pixels[{max_pixels_supposed}] exceeds limit[{max_pixels}]."
+        )
+    max_pixels = min(max_pixels_supposed, max_pixels)
+    if "resized_height" in video_config and "resized_width" in video_config:
+        resized_height, resized_width = smart_resize(
+            video_config["resized_height"],
+            video_config["resized_width"],
+            factor=image_factor,
+        )
+    else:
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=image_factor,
+            min_pixels=min_pixels,
+            max_pixels=max_pixels,
+        )
+    smart_resize_time = time.perf_counter()
+    video = torchvision.transforms.functional.resize(
+        video,
+        [resized_height, resized_width],
+        interpolation=InterpolationMode.BILINEAR,
+    )
+    video = video.pin_memory()
+    video_metadata = {
+        "fps": video_fps,
+        "duration": total_frames / video_fps,
+        "total_num_frames": total_frames,
+        "frames_indices": idx,
+        "video_backend": "torchvision",
+    }
+    torchvision_resize_time = time.perf_counter()
+    logger.debug(
+        f"[preprocess_video Perf], "
+        f"get_batch_time: {(get_batch_time - entry_time) * 1000:.2f} ms, "
+        f"smart_resize_time: {(smart_resize_time - get_batch_time) * 1000:.2f} ms, "
+        f"torchvision_resize_time: {(torchvision_resize_time - smart_resize_time) * 1000:.2f} ms, "
+        f"total_time: {(torchvision_resize_time - entry_time) * 1000:.2f} ms"
+    )
+    return video, video_metadata
+
+
+# Compatible with Qwen-VL & Qwen-Omni Series
+class QwenVLImageProcessor(SGLangBaseProcessor):
+    supports_transformers_backend = True
+    models = [
+        Qwen2VLForConditionalGeneration,
+        Qwen2_5_VLForConditionalGeneration,
+        Qwen3VLForConditionalGeneration,
+        Qwen3VLMoeForConditionalGeneration,
+        Qwen3_5ForConditionalGeneration,
+        Qwen3_5MoeForConditionalGeneration,
+        Qwen3_5ForCausalLMMTP,
+        InternS2PreviewForConditionalGeneration,
+        Qwen3OmniMoeForConditionalGeneration,
+    ]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        self.model_type = hf_config.model_type
+        if hf_config.model_type == "qwen3_omni_moe":
+            hf_config = hf_config.thinker_config
+
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        self.IM_START_TOKEN_ID = hf_config.vision_start_token_id
+        self.IM_END_TOKEN_ID = hf_config.vision_end_token_id
+        self.IM_TOKEN_ID = hf_config.image_token_id
+        self.VIDEO_TOKEN_ID = hf_config.video_token_id
+
+        self.vision_start_token_id = hf_config.vision_start_token_id
+        self.vision_end_token_id = getattr(hf_config, "vision_end_token_id", None)
+
+        self.audio_start_token_id = getattr(hf_config, "audio_start_token_id", None)
+        self.audio_token_id = getattr(hf_config, "audio_token_id", None)
+
+        self._spatial_merge_size = self.hf_config.vision_config.spatial_merge_size
+        self._tokens_per_second = getattr(
+            self.hf_config.vision_config, "tokens_per_second", None
+        )
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|vision_start|><|image_pad|><|vision_end|>",
+            image_token_id=hf_config.image_token_id,
+            # The regex that matches expanded image tokens.
+            image_token_regex=re.compile(
+                r"<\|vision_start\|>(?:<\|image_pad\|>)+<\|vision_end\|>"
+            ),
+            video_token_id=self.VIDEO_TOKEN_ID,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+    @property
+    def spatial_merge_size(self):
+        return self._spatial_merge_size
+
+    def build_input_ids_with_timestamps(
+        self, prompt, embeddings, img_grid_thw, video_grid_thw, video_timestamps
+    ):
+        """
+        Build input_ids with timestamps for qwen3_vl models.
+        """
+        if not isinstance(prompt, list):
+            prompt = self._processor.tokenizer.encode(prompt)
+
+        img_token_id = getattr(self, "IM_TOKEN_ID", None)
+        video_token_id = getattr(self, "VIDEO_TOKEN_ID", None)
+        spatial_merge_size = self.spatial_merge_size
+        vision_start_token_id = getattr(self, "vision_start_token_id", None)
+        vision_end_token_id = getattr(self, "vision_end_token_id", None)
+
+        input_ids = []
+        offsets = []
+        modality_list = []
+        cur_idx = 0
+
+        vision_start_indices = []
+        for i in range(len(prompt) - 1):
+            if img_token_id is not None and prompt[i + 1] == img_token_id:
+                vision_start_indices.append((i, Modality.IMAGE))
+            elif video_token_id is not None and prompt[i + 1] == video_token_id:
+                vision_start_indices.append((i, Modality.VIDEO))
+
+        img_idx = 0
+        video_idx = 0
+        for mm_start_idx, modality in vision_start_indices:
+            modality_list.append(modality)
+            video_tokens = None
+            if modality == Modality.IMAGE:
+                mm_token_num = img_grid_thw[img_idx].prod() // (spatial_merge_size**2)
+                mm_token_id = img_token_id
+                img_idx += 1
+            elif modality == Modality.VIDEO:
+                curr_timestamps = video_timestamps[video_idx]
+                num_frames = video_grid_thw[video_idx][0]
+                frame_seqlen = video_grid_thw[video_idx][1:].prod().item() // (
+                    spatial_merge_size**2
+                )
+                video_tokens = []
+                _current_offset = len(input_ids) + mm_start_idx + 1 - cur_idx
+                # take single frame as one mm_item
+                for frame_idx in range(num_frames):
+                    if frame_idx > 0:
+                        modality_list.append(Modality.VIDEO)
+                    curr_time = curr_timestamps[frame_idx]
+                    timestamp_text = f"<{curr_time:.1f} seconds>"
+                    timestamp_tokens = self._processor.tokenizer.encode(
+                        timestamp_text, add_special_tokens=False
+                    )
+                    video_tokens.extend(timestamp_tokens)
+                    _current_offset += len(timestamp_tokens)
+                    if vision_start_token_id is not None:
+                        video_tokens.append(vision_start_token_id)
+                        _current_offset += 1
+                    video_tokens.extend([video_token_id] * frame_seqlen)
+                    if vision_end_token_id is not None:
+                        video_tokens.append(vision_end_token_id)
+                    offsets.append(
+                        (_current_offset, _current_offset + frame_seqlen - 1)
+                    )
+                    _current_offset += (
+                        frame_seqlen + 1
+                        if vision_end_token_id is not None
+                        else frame_seqlen
+                    )  # for vision_end_token_id
+                mm_token_num = len(video_tokens)
+                mm_token_id = None
+                video_idx += 1
+            else:
+                logger.warning(
+                    f"{modality} modality is not supported for qwen3_vl models with timestamps."
+                )
+                continue
+            assert cur_idx <= mm_start_idx
+            input_ids.extend(prompt[cur_idx : mm_start_idx + 1])
+            if modality == Modality.VIDEO:
+                input_ids.extend(video_tokens)
+            else:
+                mm_offset_start = len(input_ids)
+                input_ids.extend([mm_token_id] * mm_token_num)
+                offsets.append((mm_offset_start, len(input_ids) - 1))
+            cur_idx = mm_start_idx + 2  # jump to vision_end_id
+        else:
+            input_ids.extend(prompt[cur_idx:])
+
+        return input_ids, offsets, modality_list
+
+    def compute_mrope_positions(self, input_ids, mm_items):
+        image_grid_thw = self._concat_mm_item_grid(
+            mm_items, "image_grid_thw", Modality.IMAGE
+        )
+        video_grid_thw = self._concat_mm_item_grid(
+            mm_items, "video_grid_thw", Modality.VIDEO
+        )
+
+        input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
+            spatial_merge_size=self._spatial_merge_size,
+            image_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            vision_start_token_id=self.vision_start_token_id,
+            model_type=self.model_type,
+            tokens_per_second=self._tokens_per_second,
+            input_ids=input_ids_tensor,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+        )
+        return mrope_positions.squeeze(1), mrope_position_delta
+
+    @staticmethod
+    def _get_processor_output_value(ret, key):
+        if ret is None:
+            return None
+        return ret.get(key) if hasattr(ret, "get") else getattr(ret, key, None)
+
+    def _get_precomputed_mrope_from_output(self, ret):
+        mrope_positions = self._get_processor_output_value(ret, "mrope_positions")
+        mrope_position_delta = self._get_processor_output_value(
+            ret, "mrope_position_delta"
+        )
+        if mrope_positions is None or mrope_position_delta is None:
+            return None
+
+        mrope_positions = torch.as_tensor(mrope_positions)
+        if mrope_positions.ndim == 3:
+            if mrope_positions.shape[1] != 1:
+                return None
+            mrope_positions = mrope_positions.squeeze(1)
+        if mrope_positions.ndim != 2 or mrope_positions.shape[0] != 3:
+            return None
+
+        mrope_position_delta = torch.as_tensor(mrope_position_delta)
+        if mrope_position_delta.ndim <= 1:
+            mrope_position_delta = mrope_position_delta.reshape(-1, 1)
+        return mrope_positions, mrope_position_delta
+
+    @staticmethod
+    def _as_grid_batch(value):
+        if value is None:
+            return None
+        if isinstance(value, torch.Tensor):
+            return value.unsqueeze(0) if value.ndim == 1 else value
+        tensor = torch.as_tensor(value, dtype=torch.long)
+        return tensor.unsqueeze(0) if tensor.ndim == 1 else tensor
+
+    def _compute_image_only_mrope_positions_from_offsets(
+        self,
+        input_len: int,
+        mm_items: List[MultimodalDataItem],
+        dtype: torch.dtype,
+        device: torch.device,
+    ) -> Optional[tuple[torch.Tensor, torch.Tensor]]:
+        """instead of calling get_rope_index, build mrope position from mm_items.offsets and image_grid_thw of each image
+        basically a simplified version of get_rope_index for image-only reqs
+        """
+        if self.model_type not in (
+            "qwen3_vl",
+            "qwen3_vl_moe",
+            "qwen3_5",
+            "qwen3_5_moe",
+            "intern_s2_preview",
+        ):
+            return None
+
+        image_items = [item for item in mm_items if item.is_image()]
+        if not image_items or len(image_items) != len(mm_items):
+            return None
+
+        spatial_merge_size = self._spatial_merge_size
+        sorted_items = sorted(image_items, key=lambda item: item.offsets[0][0])
+        position_segments = []
+        st = 0
+        next_pos = 0
+
+        for item in sorted_items:
+            if item.offsets is None or len(item.offsets) != 1:
+                return None
+
+            start, end = item.offsets[0]
+            if start < st or end >= input_len:
+                return None
+
+            text_len = start - st
+            if text_len > 0:
+                position_segments.append(
+                    torch.arange(text_len, dtype=dtype, device=device)
+                    .view(1, -1)
+                    .expand(3, -1)
+                    + next_pos
+                )
+                next_pos += text_len
+
+            grid = self._as_grid_batch(item.model_specific_data.get("image_grid_thw"))
+            if grid is None or grid.shape[0] != 1:
+                return None
+            t, h, w = [int(x) for x in grid[0].tolist()]
+            llm_grid_t = t
+            llm_grid_h = h // spatial_merge_size
+            llm_grid_w = w // spatial_merge_size
+            num_image_tokens = llm_grid_t * llm_grid_h * llm_grid_w
+            if num_image_tokens != end - start + 1:
+                return None
+
+            t_index = (
+                torch.arange(llm_grid_t, dtype=dtype, device=device)
+                .view(-1, 1)
+                .expand(llm_grid_t, llm_grid_h * llm_grid_w)
+                .reshape(-1)
+            )
+            h_index = (
+                torch.arange(llm_grid_h, dtype=dtype, device=device)
+                .view(1, -1, 1)
+                .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+                .reshape(-1)
+            )
+            w_index = (
+                torch.arange(llm_grid_w, dtype=dtype, device=device)
+                .view(1, 1, -1)
+                .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+                .reshape(-1)
+            )
+            position_segments.append(
+                torch.stack([t_index, h_index, w_index]) + next_pos
+            )
+            next_pos += max(llm_grid_t, llm_grid_h, llm_grid_w)
+            st = end + 1
+
+        if st < input_len:
+            text_len = input_len - st
+            position_segments.append(
+                torch.arange(text_len, dtype=dtype, device=device)
+                .view(1, -1)
+                .expand(3, -1)
+                + next_pos
+            )
+
+        mrope_positions = torch.cat(position_segments, dim=1).unsqueeze(1)
+        mrope_position_delta = (mrope_positions.max() + 1 - input_len).reshape(1, 1)
+        return mrope_positions, mrope_position_delta
+
+    @classmethod
+    def _concat_mm_item_grid(cls, mm_items: list[MultimodalDataItem], key, modality):
+        grids = []
+        for item in mm_items:
+            if not item.is_modality(modality):
+                continue
+            grid = cls._as_grid_batch(item.model_specific_data.get(key))
+            if grid is not None:
+                grids.append(grid)
+        if not grids:
+            return None
+        if len(grids) == 1:
+            return grids[0]
+        return torch.cat(grids, dim=0)
+
+    @classmethod
+    def _get_grid_from_output_or_items(
+        cls, ret, mm_items, key, modality, input_data=None
+    ):
+        grid = cls._get_processor_output_value(ret, key)
+        if grid is None:
+            grid = cls._concat_mm_item_grid(mm_items, key, modality)
+        if grid is None and input_data and isinstance(input_data[0], dict):
+            grid = input_data[0].get(key)
+        return grid
+
+    def get_mm_data(self, prompt, embeddings, **kwargs):
+        img_grid_thw = kwargs.get("img_grid_thw", None)
+        video_grid_thw = kwargs.get("video_grid_thw", None)
+        audio_feature_lens = kwargs.get("audio_feature_lens", None)
+        video_timestamps = kwargs.get("video_timestamps", None)
+        second_per_grid_ts = kwargs.get("second_per_grid_ts", None)
+
+        audio_seq_lens = None
+        if audio_feature_lens is not None:
+            if self.model_type == "qwen3_omni_moe":
+                # apply _get_feat_extract_lengths to get seq_lens
+                input_lengths_leave = audio_feature_lens % 100
+                feat_lengths = (input_lengths_leave - 1) // 2 + 1
+                audio_seq_lens = (
+                    ((feat_lengths - 1) // 2 + 1 - 1) // 2
+                    + 1
+                    + (audio_feature_lens // 100) * 13
+                )
+            elif self.model_type == "qwen2_5_omni":
+                audio_seq_lens = (audio_feature_lens - 1) // 2 + 1
+                audio_seq_lens = (audio_seq_lens - 2) // 2 + 1
+
+        if (
+            self.model_type
+            in [
+                "qwen3_vl",
+                "qwen3_vl_moe",
+                "qwen3_5",
+                "qwen3_5_moe",
+                "intern_s2_preview",
+            ]
+            and video_timestamps is not None
+        ):
+            input_ids, offsets, modality_list = self.build_input_ids_with_timestamps(
+                prompt, embeddings, img_grid_thw, video_grid_thw, video_timestamps
+            )
+        else:
+            input_ids, offsets, modality_list = self.build_input_ids(
+                prompt, img_grid_thw, video_grid_thw, audio_seq_lens=audio_seq_lens
+            )
+        assert all(isinstance(modality, Modality) for modality in modality_list)
+
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
+            spatial_merge_size=self._spatial_merge_size,
+            image_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            vision_start_token_id=self.vision_start_token_id,
+            model_type=self.model_type,
+            input_ids=torch.tensor(input_ids, dtype=torch.long).unsqueeze(0),
+            image_grid_thw=img_grid_thw,
+            video_grid_thw=video_grid_thw,
+            second_per_grid_ts=second_per_grid_ts,
+            use_audio_in_video=False,
+            audio_seqlens=(
+                audio_feature_lens if self.model_type == "qwen3_omni_moe" else None
+            ),
+            audio_token_id=getattr(self.hf_config, "audio_token_id", None),
+            audio_start_token_id=self.audio_start_token_id,
+            position_id_per_seconds=getattr(
+                self.hf_config, "position_id_per_seconds", None
+            ),
+            tokens_per_second=self._tokens_per_second,
+        )
+        mrope_positions = mrope_positions.squeeze(1)
+
+        mm_items = []
+        consumed_per_modality = {}
+
+        for modality, offset in zip(modality_list, offsets):
+            num_tokens = offset[1] - offset[0] + 1
+            embedding_start = consumed_per_modality.get(modality, 0)
+            embedding_slice = embeddings[modality][
+                embedding_start : embedding_start + num_tokens
+            ]
+            consumed_per_modality[modality] = embedding_start + num_tokens
+            mm_items.append(
+                MultimodalDataItem(
+                    modality=modality,
+                    offsets=[offset],
+                    precomputed_embeddings=embedding_slice,
+                )
+            )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=mm_items,
+            im_start_id=self.IM_START_TOKEN_ID,
+            im_end_id=self.IM_END_TOKEN_ID,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+            mrope_positions=mrope_positions,
+            mrope_position_delta=mrope_position_delta,
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        entry_time = time.perf_counter()
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            audio_data=request_obj.audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        load_time = time.perf_counter()
+        rid = getattr(request_obj, "rid", "anonymous_rid")
+
+        video_metadata = None
+        if base_output.videos and not isinstance(base_output.videos[0], dict):
+            videos_processed = [
+                await preprocess_video(video, video_config=self.video_config)
+                for video in base_output.videos
+            ]
+            base_output.videos, video_metadata = map(list, zip(*videos_processed))
+
+        preprocess_time = time.perf_counter()
+
+        # NOTE: for qwen3-vl, video_meta need to be passed in, since do_sample_frames is already done in preprocess_video
+        if self.hf_config.model_type in (
+            "qwen3_vl",
+            "qwen3_vl_moe",
+            "qwen3_5",
+            "qwen3_5_moe",
+            "intern_s2_preview",
+        ):
+            mm_items, input_ids, ret = self.process_and_combine_mm_data(
+                base_output,
+                self.mm_tokens,
+                video_metadata=video_metadata,
+                do_sample_frames=False,
+            )
+        else:
+            mm_items, input_ids, ret = self.process_and_combine_mm_data(
+                base_output, self.mm_tokens
+            )
+
+        audio_feature_lengths = None
+
+        if self.model_type == "qwen3_omni_moe":
+            audio_item = next((mm for mm in mm_items if mm.is_audio()), None)
+            if audio_item:
+                audio_feature_lengths = torch.sum(
+                    audio_item.feature_attention_mask, dim=1
+                )
+
+        second_per_grid_ts = self._get_processor_output_value(ret, "second_per_grid_ts")
+        if second_per_grid_ts is None:
+            second_per_grid_ts = self._get_processor_output_value(
+                ret, "video_second_per_grid"
+            )
+
+        process_time = time.perf_counter()
+
+        input_ids = input_ids.flatten()
+        base_input_ids = getattr(base_output, "input_ids", None)
+        if (
+            isinstance(base_input_ids, list)
+            and len(base_input_ids) == input_ids.numel()
+        ):
+            # reuse preprocess input if it already carries list of input_ids
+            input_ids_list = base_input_ids
+        else:
+            input_ids_list = input_ids.tolist()
+
+        # look for if padded_input_ids already exists before computing
+        padded_input_ids = self._get_processor_output_value(ret, "padded_input_ids")
+        if padded_input_ids is None:
+            padded_input_ids = MultimodalProcessorOutput.build_padded_input_ids(
+                input_ids_list, mm_items
+            )
+        elif isinstance(padded_input_ids, torch.Tensor):
+            # reuse existing padded_input_ids
+            padded_input_ids = padded_input_ids.flatten().tolist()
+        else:
+            padded_input_ids = list(padded_input_ids)
+
+        image_grid_thw = self._get_grid_from_output_or_items(
+            ret, mm_items, "image_grid_thw", Modality.IMAGE, image_data
+        )
+        video_grid_thw = self._get_grid_from_output_or_items(
+            ret,
+            mm_items,
+            "video_grid_thw",
+            Modality.VIDEO,
+            request_obj.video_data,
+        )
+
+        mrope_result = self._get_precomputed_mrope_from_output(ret)
+        if mrope_result is None:
+            if (
+                video_grid_thw is None
+                and second_per_grid_ts is None
+                and audio_feature_lengths is None
+            ):
+                mrope_result = self._compute_image_only_mrope_positions_from_offsets(
+                    input_len=input_ids.numel(),
+                    mm_items=mm_items,
+                    dtype=input_ids.dtype,
+                    device=input_ids.device,
+                )
+        if mrope_result is None:
+            mrope_result = MRotaryEmbedding.get_rope_index(
+                spatial_merge_size=self._spatial_merge_size,
+                image_token_id=self.mm_tokens.image_token_id,
+                video_token_id=self.mm_tokens.video_token_id,
+                vision_start_token_id=self.vision_start_token_id,
+                model_type=self.model_type,
+                tokens_per_second=self._tokens_per_second,
+                # use the expanded token ids
+                input_ids=input_ids.unsqueeze(0),
+                image_grid_thw=image_grid_thw,
+                video_grid_thw=video_grid_thw,
+                second_per_grid_ts=second_per_grid_ts,
+                use_audio_in_video=False,
+                audio_seqlens=audio_feature_lengths,
+                audio_token_id=getattr(self.hf_config, "audio_token_id", None),
+                audio_start_token_id=self.audio_start_token_id,
+                position_id_per_seconds=getattr(
+                    self.hf_config, "position_id_per_seconds", None
+                ),
+            )
+
+        mrope_positions, mrope_position_delta = mrope_result
+        if mrope_positions.ndim == 3:
+            mrope_positions = mrope_positions.squeeze(1)
+        get_rope_index_time = time.perf_counter()
+        logger.debug(
+            f"[QwenVLProcessor Perf] {rid=}, "
+            f"load_time: {(load_time - entry_time) * 1000:.2f} ms, "
+            f"preprocess_time: {(preprocess_time - load_time) * 1000:.2f} ms, "
+            f"process_time: {(process_time - preprocess_time) * 1000:.2f} ms, "
+            f"get_rope_index_time: {(get_rope_index_time - process_time) * 1000:.2f} ms, "
+            f"total_time: {(get_rope_index_time - entry_time) * 1000:.2f} ms"
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids_list,
+            padded_input_ids=padded_input_ids,
+            mm_items=mm_items,
+            im_start_id=self.vision_start_token_id,
+            im_end_id=self.vision_end_token_id,
+            im_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
+            audio_token_id=self.mm_tokens.audio_token_id,
+            mrope_positions=mrope_positions,
+            mrope_position_delta=mrope_position_delta,
+        )

python/sglang/srt/multimodal/processors/sarashina2_vision.py

@@ -0,0 +1,82 @@
+from typing import List, Union
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.sarashina2_vision import Sarashina2VisionForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class Sarashina2VisionProcessor(BaseMultimodalProcessor):
+    models = [Sarashina2VisionForCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        # Sarashina2Vision specific tokens (default is <|file|>)
+        self.IMAGE_TOKEN = "<|file|>"
+        self.IM_TOKEN_ID = getattr(hf_config, "image_token_index", 14)
+        self.IM_START_ID = getattr(hf_config, "start_image_token_index", 102397)
+        self.IM_END_ID = getattr(hf_config, "end_image_token_index", 102398)
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+        ).build(_processor)
+
+        # Patch the processor's image processor to handle parameter compatibility
+        if hasattr(_processor, "image_processor") and hasattr(
+            _processor.image_processor, "_preprocess"
+        ):
+            original_preprocess = _processor.image_processor._preprocess
+
+            def patched_preprocess(*args, **kwargs):
+                # Filter kwargs to only include parameters that the custom _preprocess method accepts
+                # Based on Sarashina2VisionImageProcessor._preprocess signature
+                allowed_params = {
+                    "do_resize",
+                    "resample",
+                    "do_rescale",
+                    "rescale_factor",
+                    "do_normalize",
+                    "image_mean",
+                    "image_std",
+                    "do_convert_rgb",
+                    "data_format",
+                    "input_data_format",
+                }
+                filtered_kwargs = {
+                    k: v for k, v in kwargs.items() if k in allowed_params
+                }
+                return original_preprocess(*args, **filtered_kwargs)
+
+            _processor.image_processor._preprocess = patched_preprocess
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        """Process image data for Sarashina2Vision model using standard SGLang pattern."""
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output=base_output,
+            mm_tokens=self.mm_tokens,
+        )
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_token_id=self.mm_tokens.image_token_id,
+            im_start_id=self.IM_START_ID,
+            im_end_id=self.IM_END_ID,
+        )

python/sglang/srt/multimodal/processors/step3_vl.py

@@ -0,0 +1,574 @@
+import math
+import re
+from itertools import product
+from typing import List, Optional, Union
+
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import transforms
+from torchvision.transforms import InterpolationMode
+from torchvision.transforms import functional as F
+from transformers import BatchFeature, ProcessorMixin, TensorType
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.step3_vl import Step3VLForConditionalGeneration
+from sglang.srt.models.step3_vl_10b import StepVLForConditionalGeneration
+from sglang.srt.models.step3p7 import Step3p7ForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    MultimodalSpecialTokens,
+)
+
+Step3Image = Union[Image.Image, torch.Tensor]
+ImageWithPatches = tuple[Step3Image, list[Step3Image], list[int] | None]
+
+
+class GPUToTensor(torch.nn.Module):
+
+    def forward(
+        self, raw_image: Union[np.ndarray, Image.Image, torch.Tensor]
+    ) -> torch.Tensor:
+        if isinstance(raw_image, torch.Tensor):
+            image_tensor = raw_image
+            if image_tensor.ndim != 3:
+                raise TypeError(
+                    f"Expected CHW image tensor, got shape {tuple(image_tensor.shape)}"
+                )
+            if image_tensor.shape[0] == 1:
+                image_tensor = image_tensor.repeat(3, 1, 1)
+            elif image_tensor.shape[0] != 3:
+                raise TypeError(
+                    f"Expected CHW image tensor with 1 or 3 channels, got shape {tuple(image_tensor.shape)}"
+                )
+            if image_tensor.dtype == torch.uint8:
+                image_tensor = image_tensor.to(torch.float32).div(255)
+            elif not image_tensor.is_floating_point():
+                image_tensor = image_tensor.to(torch.float32)
+            return image_tensor.contiguous()
+        if isinstance(raw_image, Image.Image):
+            image_tensor = transforms.ToTensor()(raw_image)
+            if torch.cuda.is_available():
+                image_tensor = image_tensor.to(torch.device("cuda"))
+            return image_tensor
+        if raw_image.ndim == 2:
+            raw_image = raw_image[:, :, None].repeat(3, -1)
+        if torch.cuda.is_available():
+            device = torch.device("cuda")
+        else:
+            device = torch.device("cpu")
+        image_tensor = torch.from_numpy(raw_image).to(device)
+        image_tensor = torch.permute(image_tensor, (2, 0, 1)).contiguous()
+        if image_tensor.dtype == torch.uint8:
+            image_tensor = image_tensor.to(torch.float32).div(255)
+        return image_tensor
+
+
+class Step3VisionProcessor:
+    def __init__(self, size, interpolation_mode="bicubic", patch_size=None):
+        mean = [0.48145466, 0.4578275, 0.40821073]
+        std = [0.26862954, 0.26130258, 0.27577711]
+        patch_size = patch_size if patch_size is not None else size
+
+        self.transform = transforms.Compose(
+            [
+                GPUToTensor(),
+                transforms.Normalize(mean, std),
+                transforms.Resize(
+                    (size, size),
+                    interpolation=(
+                        InterpolationMode.BICUBIC
+                        if interpolation_mode == "bicubic"
+                        else InterpolationMode.BILINEAR
+                    ),
+                    antialias=True,
+                ),
+            ]
+        )
+
+        self.patch_transform = (
+            transforms.Compose(
+                [
+                    GPUToTensor(),
+                    transforms.Normalize(mean, std),
+                    transforms.Resize(
+                        (patch_size, patch_size),
+                        interpolation=(
+                            InterpolationMode.BICUBIC
+                            if interpolation_mode == "bicubic"
+                            else InterpolationMode.BILINEAR
+                        ),
+                        antialias=True,
+                    ),
+                ]
+            )
+            if patch_size is not None
+            else None
+        )
+
+    def __call__(self, image, is_patch=False):
+        if is_patch:
+            return {"pixel_values": self.patch_transform(image).unsqueeze(0)}
+        else:
+            return {"pixel_values": self.transform(image).unsqueeze(0)}
+
+
+class ImagePatcher:
+    def get_image_size(self, img: Step3Image) -> tuple[int, int]:
+        if isinstance(img, Image.Image):
+            return img.size
+        if isinstance(img, torch.Tensor):
+            if img.ndim != 3:
+                raise TypeError(
+                    f"Expected CHW image tensor, got shape {tuple(img.shape)}"
+                )
+            return int(img.shape[-1]), int(img.shape[-2])
+        raise TypeError(f"Unsupported image type: {type(img)}")
+
+    def determine_window_size(self, long: int, short: int) -> int:
+        if long <= 728:
+            return short if long / short > 1.5 else 0
+        return min(short, 504) if long / short > 4 else 504
+
+    def slide_window(
+        self,
+        width: int,
+        height: int,
+        sizes: list[tuple[int, int]],
+        steps: list[tuple[int, int]],
+        img_rate_thr: float = 0.6,
+    ) -> tuple[list[tuple[int, int, int, int]], tuple[int, int]]:
+        assert 1 >= img_rate_thr >= 0, "The `img_rate_thr` should lie in 0~1"
+        windows = []
+        # Sliding windows.
+        for size, step in zip(sizes, steps):
+            size_w, size_h = size
+            step_w, step_h = step
+
+            x_num = 1 if width <= size_w else math.ceil((width - size_w) / step_w + 1)
+            x_start = [step_w * i for i in range(x_num)]
+            if len(x_start) > 1 and x_start[-1] + size_w > width:
+                x_start[-1] = width - size_w
+
+            y_num = 1 if height <= size_h else math.ceil((height - size_h) / step_h + 1)
+            y_start = [step_h * i for i in range(y_num)]
+            if len(y_start) > 1 and y_start[-1] + size_h > height:
+                y_start[-1] = height - size_h
+
+            start = np.array(list(product(y_start, x_start)), dtype=int)
+            start[:, [0, 1]] = start[:, [1, 0]]
+            windows.append(np.concatenate([start, start + size], axis=1))
+        windows = np.concatenate(windows, axis=0)
+
+        return [
+            (int(box[0]), int(box[1]), int(box[2] - box[0]), int(box[3] - box[1]))
+            for box in windows
+        ], (x_num, y_num)
+
+    def square_pad(self, img: Step3Image) -> Step3Image:
+        w, h = self.get_image_size(img)
+        if w == h:
+            return img
+        size = max(w, h)
+        if isinstance(img, Image.Image):
+            padded = Image.new(img.mode, (size, size), 0)
+            padded.paste(img, (0, 0))
+            return padded
+        return torch.nn.functional.pad(img, (0, size - w, 0, size - h), value=0)
+
+    def get_image_size_for_padding(
+        self, img_width: int, img_height: int
+    ) -> tuple[int, int]:
+        ratio = img_width / img_height
+        if min(img_height, img_width) < 32 and (ratio > 4 or ratio < 1 / 4):
+            new_size = max(img_height, img_width)
+            return new_size, new_size
+        return img_width, img_height
+
+    def get_image_size_for_preprocess(
+        self, img_width: int, img_height: int
+    ) -> tuple[int, int]:
+
+        if max(img_height, img_width) > 3024:
+            scale_factor = 3024 / max(img_height, img_width)
+            img_width = int(img_width * scale_factor)
+            img_height = int(img_height * scale_factor)
+            return img_width, img_height
+        else:
+            return img_width, img_height
+
+    def get_image_size_for_crop(
+        self, img_width: int, img_height: int, window_size: int
+    ):
+        w_ratio = img_width / window_size
+        h_ratio = img_height / window_size
+
+        if w_ratio < 1:
+            width_new = img_width
+        else:
+            decimal_w = w_ratio - img_width // window_size
+            w_ratio = int(w_ratio) + 1 if decimal_w > 0.2 else int(w_ratio)
+            width_new = window_size * w_ratio
+        if h_ratio < 1:
+            height_new = img_height
+        else:
+            decimal_h = h_ratio - img_height // window_size
+            h_ratio = int(h_ratio) + 1 if decimal_h > 0.2 else int(h_ratio)
+            height_new = window_size * h_ratio
+        return int(width_new), int(height_new)
+
+    def resize(self, img: Step3Image, size: tuple[int, int]) -> Step3Image:
+        if isinstance(img, Image.Image):
+            return img.resize(size, Image.Resampling.BILINEAR)
+        return F.resize(
+            img,
+            [size[1], size[0]],
+            interpolation=InterpolationMode.BILINEAR,
+            antialias=True,
+        ).contiguous()
+
+    def patch_crop(
+        self, img: Step3Image, i: int, j: int, th: int, tw: int
+    ) -> Step3Image:
+        if isinstance(img, Image.Image):
+            return img.crop((j, i, j + tw, i + th))
+        return img[:, i : i + th, j : j + tw].contiguous()
+
+    def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]:
+        img_width, img_height = self.get_image_size_for_padding(img_width, img_height)
+        img_width, img_height = self.get_image_size_for_preprocess(
+            img_width, img_height
+        )
+        window_size = self.determine_window_size(
+            max(img_height, img_width), min(img_height, img_width)
+        )
+        if window_size == 0:
+            return 0, 0
+        else:
+            img_width, img_height = self.get_image_size_for_crop(
+                img_width, img_height, window_size
+            )
+            center_list, (x_num, y_num) = self.slide_window(
+                img_width,
+                img_height,
+                [(window_size, window_size)],
+                [(window_size, window_size)],
+            )
+            full_rows = (len(center_list) - 1) // x_num + 1
+            if len(center_list) > 0 and len(center_list) % x_num == 0:
+                full_rows -= 1
+            return len(center_list), full_rows
+
+    def __call__(
+        self, img: Step3Image
+    ) -> tuple[Step3Image, list[Step3Image], list[bool] | None]:
+        img_width, img_height = self.get_image_size(img)
+        new_img_width, new_img_height = self.get_image_size_for_padding(
+            img_width, img_height
+        )
+        if new_img_width != img_width or new_img_height != img_height:
+            img = self.square_pad(img)
+            img_width, img_height = self.get_image_size(img)
+
+        new_img_width, new_img_height = self.get_image_size_for_preprocess(
+            img_width, img_height
+        )
+        img = self.resize(img, (new_img_width, new_img_height))
+        window_size = self.determine_window_size(
+            max(new_img_height, new_img_width), min(new_img_height, new_img_width)
+        )
+        if window_size == 0:
+            return img, [], None
+        else:
+            new_img_width, new_img_height = self.get_image_size_for_crop(
+                new_img_width, new_img_height, window_size
+            )
+            if (new_img_width, new_img_height) != (img_width, img_height):
+                img_for_crop = self.resize(img, (new_img_width, new_img_height))
+            else:
+                img_for_crop = img
+
+            patches = []
+            newlines = []
+            center_list, (x_num, y_num) = self.slide_window(
+                new_img_width,
+                new_img_height,
+                [(window_size, window_size)],
+                [(window_size, window_size)],
+            )
+            for patch_id, center_lf_point in enumerate(center_list):
+                x, y, patch_w, patch_h = center_lf_point
+                big_patch = self.patch_crop(img_for_crop, y, x, patch_h, patch_w)
+                patches.append(big_patch)
+                if (patch_id + 1) % x_num == 0:
+                    newlines.append(patch_id)
+
+            if newlines and newlines[-1] == len(patches) - 1:
+                newlines.pop()
+
+            return (
+                img,
+                patches,
+                (
+                    [i in newlines for i in range(len(patches))]
+                    if len(patches) > 0
+                    else None
+                ),
+            )
+
+
+class Step3VLProcessor:
+    def __init__(
+        self,
+        config,
+        tokenizer,
+    ) -> None:
+        super().__init__()
+
+        self.config = config
+        if isinstance(tokenizer, ProcessorMixin):
+            tokenizer = tokenizer.tokenizer
+        self.tokenizer = tokenizer
+
+        self.image_size = 728
+        self.patch_size = 504
+        self.image_preprocessor = Step3VisionProcessor(
+            self.image_size, "bilinear", self.patch_size
+        )
+
+        self.num_image_feature_size = 169
+        self.num_patch_feature_size = 81
+        self.image_token = "<im_patch>"
+        self.image_feature_placeholder = self.image_token * self.num_image_feature_size
+        self.patch_feature_placeholder = self.image_token * self.num_patch_feature_size
+
+        self.patcher = ImagePatcher()
+
+    @property
+    def image_token_id(self) -> int:
+        return self.tokenizer.get_vocab()[self.image_token]
+
+    def get_num_image_tokens(self, img_width: int, img_height: int) -> int:
+        num_patches, num_newlines = self.patcher.get_num_patches(img_width, img_height)
+
+        return (
+            num_patches * (self.num_patch_feature_size + 2)
+            + self.num_image_feature_size
+            + 2
+            + num_newlines
+        )
+
+    def _split_images(self, images: list[Image.Image]) -> list[ImageWithPatches]:
+        result = []
+        for img in images:
+            result.append(self.patcher(img))
+        return result
+
+    def _convert_images_to_pixel_values(
+        self,
+        images: list[Step3Image],
+        is_patch: bool = False,
+    ) -> list[torch.Tensor]:
+        return [
+            self.image_preprocessor(img, is_patch=is_patch)["pixel_values"]
+            for img in images
+        ]
+
+    def _get_patch_repl(
+        self,
+        num_patches: int,
+        patch_newline_mask: list[bool] | None,
+    ) -> tuple[str, list[int]]:
+        text = ""
+        token_ids = []
+        for i in range(num_patches):
+            assert len(patch_newline_mask) == num_patches
+            text += f"<patch_start>{self.patch_feature_placeholder}<patch_end>"
+            token_ids.extend(
+                [self.tokenizer.convert_tokens_to_ids("<patch_start>")]
+                + [self.image_token_id] * self.num_patch_feature_size
+                + [self.tokenizer.convert_tokens_to_ids("<patch_end>")]
+            )
+            if patch_newline_mask and patch_newline_mask[i]:
+                text += "<patch_newline>"
+                token_ids.append(
+                    self.tokenizer.convert_tokens_to_ids("<patch_newline>")
+                )
+        return text, token_ids
+
+    def _get_image_repl(
+        self,
+        num_images: int,
+    ) -> tuple[str, list[int]]:
+        text = f"<im_start>{self.image_feature_placeholder}<im_end>"
+        token_ids = (
+            [self.tokenizer.convert_tokens_to_ids("<im_start>")]
+            + [self.image_token_id] * self.num_image_feature_size
+            + [self.tokenizer.convert_tokens_to_ids("<im_end>")]
+        )
+        return text * num_images, token_ids * num_images
+
+    def _get_image_repl_features(
+        self,
+        num_images: int,
+        num_patches: int,
+        patch_new_line_idx: Optional[list[bool]],
+    ) -> tuple[str, list[int]]:
+        if num_patches > 0:
+            patch_repl, patch_repl_ids = self._get_patch_repl(
+                num_patches, patch_new_line_idx
+            )
+        else:
+            patch_repl = ""
+            patch_repl_ids = []
+        image_repl, image_repl_ids = self._get_image_repl(num_images)
+        return patch_repl + image_repl, patch_repl_ids + image_repl_ids
+
+    def replace_placeholder(self, text: str, placeholder: str, repls: list[str]) -> str:
+        parts = text.split(placeholder)
+
+        if len(parts) - 1 != len(repls):
+            raise ValueError(
+                "The number of placeholders does not match the number of replacements."  # noqa: E501
+            )
+
+        result = [parts[0]]
+        for i, repl in enumerate(repls):
+            result.append(repl)
+            result.append(parts[i + 1])
+
+        return "".join(result)
+
+    def __call__(
+        self,
+        text: Optional[Union[str, list[str]]] = None,
+        images: Optional[Union[Image.Image, list[Image.Image]]] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        *args,
+        **kwargs,
+    ) -> BatchFeature:
+        if text is None:
+            text = []
+        if not isinstance(text, list):
+            text = [text]
+        if images is None:
+            images = []
+        if not isinstance(images, list):
+            images = [images]
+
+        if len(images) == 0:
+            image_inputs = {}
+            text_inputs = self.tokenizer(text)
+        else:
+            splitted_images_data = self._split_images(images)
+            pixel_values_lst = []
+            patch_pixel_values_lst = []
+            patch_newline_mask_lst = []
+            image_repl_str_lst = []
+            image_repl_ids_lst = []
+            num_patches = []
+            for (
+                raw_img,
+                img_patches,
+                patch_newline_mask,
+            ) in splitted_images_data:  # noqa: E501
+                pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
+
+                if len(img_patches) > 0:
+                    patch_pixel_values_lst.extend(
+                        self._convert_images_to_pixel_values(img_patches, is_patch=True)
+                    )
+                num_patches.append(len(img_patches))
+
+                image_repl_str, image_repl_ids = self._get_image_repl_features(
+                    1, len(img_patches), patch_newline_mask
+                )
+                image_repl_str_lst.append(image_repl_str)
+                image_repl_ids_lst.extend(image_repl_ids)
+
+                if patch_newline_mask is not None:
+                    patch_newline_mask_lst.extend(patch_newline_mask)
+
+            image_inputs = {
+                "pixel_values": torch.cat(pixel_values_lst),
+                "num_patches": num_patches,
+            }
+            if patch_pixel_values_lst:
+                image_inputs["patch_pixel_values"] = torch.cat(patch_pixel_values_lst)
+            if patch_newline_mask_lst:
+                image_inputs["patch_newline_mask"] = torch.tensor(
+                    patch_newline_mask_lst, dtype=torch.bool
+                )
+
+            text = [
+                self.replace_placeholder(t, self.image_token, image_repl_str_lst)
+                for t in text
+            ]
+            text_inputs = self.tokenizer(text)
+
+        return BatchFeature(
+            {
+                **text_inputs,
+                **image_inputs,
+            },
+            tensor_type=return_tensors,
+        )
+
+
+################################################
+
+
+class Step3VLImageProcessor(SGLangBaseProcessor):
+    models = [
+        Step3VLForConditionalGeneration,
+        StepVLForConditionalGeneration,
+        Step3p7ForConditionalGeneration,
+    ]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        # TODO, check _processor is tokenizer or processor.
+        processor = Step3VLProcessor(hf_config, _processor)
+        super().__init__(hf_config, server_args, processor, *args, **kwargs)
+        self.IM_TOKEN = "<im_patch>"
+        self.IM_TOKEN_ID = self._processor.tokenizer.get_vocab()[self.IM_TOKEN]
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IM_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+            image_token_regex=re.compile(r"(?:<im_patch>)"),
+        ).build(_processor)
+
+        mean = [0.48145466, 0.4578275, 0.40821073]
+        std = [0.26862954, 0.26130258, 0.27577711]
+
+    def preprocess(self, image):
+        return {"pixel_values": self.transform(image).unsqueeze(0)}
+
+    def __call__(self, image):
+        return self.preprocess(image)
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text: str | List[int],
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/transformers_auto.py

@@ -0,0 +1,218 @@
+from typing import Optional
+
+import torch
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.utils import load_image
+
+
+def _first_attr(obj, names: tuple[str, ...], default=None):
+    for name in names:
+        value = getattr(obj, name, None)
+        if value is not None:
+            return value
+    return default
+
+
+def _uses_mrope(hf_config) -> bool:
+    text_config = getattr(hf_config, "text_config", hf_config)
+    rope_scaling = getattr(text_config, "rope_scaling", None) or {}
+    if isinstance(rope_scaling, dict) and "mrope_section" in rope_scaling:
+        return True
+    rope_type = str(getattr(text_config, "rope_type", "")).lower()
+    return "mrope" in rope_type
+
+
+class TransformersAutoMultimodalProcessor(BaseMultimodalProcessor):
+    """Generic multimodal processor for the Transformers backend.
+
+    Unlike model-specific processors that rely on regex-based token matching
+    in the raw prompt, this processor applies the HF processor directly to
+    the prompt text + raw media.  This handles models like Gemma3 where the
+    chat template uses a marker (``<start_of_image>``) that the HF processor
+    internally expands into placeholder tokens.
+    """
+
+    models = []
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=getattr(_processor, "image_token", None),
+            video_token=getattr(_processor, "video_token", None),
+            audio_token=getattr(_processor, "audio_token", None),
+            image_token_id=_first_attr(
+                hf_config,
+                ("image_token_id", "image_token_index", "im_token_id"),
+            ),
+            video_token_id=_first_attr(
+                hf_config,
+                ("video_token_id",),
+            ),
+            audio_token_id=_first_attr(
+                hf_config,
+                ("audio_token_id",),
+            ),
+        ).build(_processor)
+
+        self._is_mrope = _uses_mrope(hf_config)
+        if self._is_mrope:
+            vision_config = getattr(hf_config, "vision_config", None)
+            self._spatial_merge_size = getattr(vision_config, "spatial_merge_size", 2)
+            self._tokens_per_second = getattr(vision_config, "tokens_per_second", None)
+            self._vision_start_token_id = _first_attr(
+                hf_config, ("vision_start_token_id",)
+            )
+            self._model_type = getattr(hf_config, "model_type", "")
+
+    def _compute_mrope_positions(
+        self,
+        input_ids: list[int],
+        image_grid_thw: Optional[torch.Tensor] = None,
+        video_grid_thw: Optional[torch.Tensor] = None,
+    ):
+        from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
+
+        input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
+            spatial_merge_size=self._spatial_merge_size,
+            image_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id or -1,
+            vision_start_token_id=self._vision_start_token_id,
+            model_type=self._model_type,
+            input_ids=input_ids_tensor,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            tokens_per_second=self._tokens_per_second,
+        )
+        return mrope_positions.squeeze(1), mrope_position_delta
+
+    def _load_images(self, image_data) -> list:
+        """Download / decode images from URLs, file paths, or base64."""
+        if not image_data:
+            return []
+        images = []
+        for data in image_data:
+            img, _ = load_image(data)
+            if img.mode != "RGB":
+                img = img.convert("RGB")
+            images.append(img)
+        return images
+
+    def _apply_hf_processor(self, text: str, images=None, videos=None):
+        """Run the HF processor on text + media and return the full output.
+
+        This is the key method that makes the generic processor work for
+        models with non-trivial token expansion (Gemma3, PaliGemma, etc.).
+        The HF processor handles chat-template expansion, image token
+        insertion, and tokenization in one shot.
+        """
+        kwargs = {}
+        if images:
+            kwargs["images"] = images
+        if videos:
+            kwargs["videos"] = videos
+        return self._processor(text=text, return_tensors="pt", **kwargs)
+
+    def _build_mm_items(
+        self, processor_output: dict, input_ids: torch.Tensor
+    ) -> list[MultimodalDataItem]:
+        """Extract MultimodalDataItem objects from the HF processor output."""
+        items = self.collect_mm_items_from_processor_output(processor_output)
+
+        modality_to_token_id = {
+            Modality.IMAGE: self.mm_tokens.image_token_id,
+            Modality.VIDEO: self.mm_tokens.video_token_id,
+            Modality.AUDIO: self.mm_tokens.audio_token_id,
+        }
+
+        for item in items:
+            token_id = modality_to_token_id.get(item.modality)
+            if token_id is not None:
+                item.offsets = self.get_mm_items_offset(input_ids, token_id)
+
+        return items
+
+    async def process_mm_data_async(
+        self,
+        image_data,
+        audio_data,
+        input_text,
+        request_obj,
+        **kwargs,
+    ):
+        video_data = getattr(request_obj, "video_data", None)
+        if video_data is not None and not isinstance(video_data, list):
+            video_data = [video_data]
+
+        # Load raw media
+        images = self._load_images(image_data)
+        # TODO: video / audio loading when needed
+
+        # Apply HF processor — handles token expansion internally
+        processor_output = self._apply_hf_processor(
+            text=input_text,
+            images=images or None,
+            videos=video_data or None,
+        )
+
+        input_ids = processor_output["input_ids"].flatten()
+
+        # Build mm_items from processor output
+        mm_items = self._build_mm_items(processor_output, input_ids)
+
+        ret = MultimodalProcessorOutput(
+            input_ids=input_ids.tolist(),
+            mm_items=mm_items,
+        )
+
+        # Propagate token_type_ids for models that need it (Gemma3, PaliGemma)
+        token_type_key = (
+            "mm_token_type_ids"
+            if "mm_token_type_ids" in processor_output
+            else "token_type_ids"
+        )
+        if token_type_key in processor_output:
+            ret.token_type_ids = processor_output[token_type_key].flatten().tolist()
+
+        if self.mm_tokens.image_token_id is not None:
+            ret.im_token_id = self.mm_tokens.image_token_id
+        if self.mm_tokens.video_token_id is not None:
+            ret.video_token_id = self.mm_tokens.video_token_id
+        if self.mm_tokens.audio_token_id is not None:
+            ret.audio_token_id = self.mm_tokens.audio_token_id
+
+        image_start_id = _first_attr(
+            self.hf_config,
+            ("image_start_token_id", "vision_start_token_id", "im_start_id"),
+        )
+        image_end_id = _first_attr(
+            self.hf_config,
+            ("image_end_token_id", "vision_end_token_id", "im_end_id"),
+        )
+        if image_start_id is not None:
+            ret.im_start_id = image_start_id
+        if image_end_id is not None:
+            ret.im_end_id = image_end_id
+
+        # M-RoPE positions (Qwen2.5-VL, Qwen3-VL)
+        if self._is_mrope:
+            image_grid_thw = processor_output.get("image_grid_thw")
+            video_grid_thw = processor_output.get("video_grid_thw")
+            mrope_positions, mrope_position_delta = self._compute_mrope_positions(
+                ret.input_ids,
+                image_grid_thw=image_grid_thw,
+                video_grid_thw=video_grid_thw,
+            )
+            ret.mrope_positions = mrope_positions
+            ret.mrope_position_delta = mrope_position_delta
+
+        return ret

python/sglang/srt/multimodal/processors/unlimited_ocr.py

@@ -0,0 +1,119 @@
+"""Standalone UNLIMITED-OCR processor."""
+
+import hashlib
+import logging
+from typing import List, Union
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.models.unlimited_ocr import UnlimitedOCRForCausalLM
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+_IMAGE_MODE_PRESETS = {
+    "tiny": (512, 512, False),
+    "small": (640, 640, False),
+    "base": (1024, 1024, False),
+    "large": (1280, 1280, False),
+    "gundam": (1024, 640, True),
+}
+_DEFAULT_MODE = "gundam"
+
+
+def _resolve_mode(images_config, num_images: int = 1) -> dict:
+    """Return processor kwargs from images_config (or default)."""
+    mode = _DEFAULT_MODE
+    if images_config:
+        mode = images_config.get("image_mode", _DEFAULT_MODE)
+    key = mode.strip().lower()
+    preset = _IMAGE_MODE_PRESETS.get(key)
+    if preset is None:
+        logger.error(
+            f"Unknown image_mode '{mode}'. Supported: {', '.join(_IMAGE_MODE_PRESETS)}"
+        )
+        raise ValueError(
+            f"Unknown image_mode '{mode}'. "
+            f"Supported: {', '.join(_IMAGE_MODE_PRESETS)}"
+        )
+    _MULTI_IMAGE_ALLOWED = ("tiny", "small", "base")
+    base_size, image_size, crop_mode = preset
+    if num_images > 1 and key not in _MULTI_IMAGE_ALLOWED:
+        raise ValueError(
+            f"image_mode='{mode}' is not supported with multiple images "
+            f"(got {num_images} images). "
+            f"Please use one of: {list(_MULTI_IMAGE_ALLOWED)}"
+        )
+    return dict(zip(("base_size", "image_size", "crop_mode"), preset))
+
+
+class UnlimitedOCRProcessor(BaseMultimodalProcessor):
+    """Multimodal processor for UNLIMITED-OCR model."""
+
+    models = [UnlimitedOCRForCausalLM]
+    gpu_image_decode = False
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        """Initialize UnlimitedOCRProcessor."""
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<image>", image_token_id=self._processor.image_token_id
+        ).build(_processor)
+
+    @staticmethod
+    def _mix_config_into_hash(mm_items, processor_kwargs):
+        """Mix images_config into mm_item hashes so that different configs
+        produce different pad_values, avoiding radix/embedding cache collisions."""
+        from sglang.srt.managers.mm_utils import hash_feature
+
+        config_bytes = str(sorted(processor_kwargs.items())).encode()
+        for item in mm_items:
+            if item.feature is not None:
+                base_hash = hash_feature(item.feature)
+            elif item.precomputed_embeddings is not None:
+                base_hash = hash_feature(item.precomputed_embeddings)
+            else:
+                continue
+            combined = hashlib.sha256(
+                base_hash.to_bytes(8, byteorder="big") + config_bytes
+            ).digest()[:8]
+            item.hash = int.from_bytes(combined, byteorder="big", signed=False)
+
+    async def process_mm_data_async(
+        self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
+    ):
+        """Process multimodal data asynchronously."""
+        request_obj = kwargs.get("request_obj")
+        images_config = (
+            getattr(request_obj, "images_config", None) if request_obj else None
+        )
+        processor_kwargs = _resolve_mode(images_config, num_images=len(image_data))
+
+        prefix = images_config.get("prefix", "") if images_config else ""
+
+        base_output = await self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=self.mm_tokens,
+            image_data=image_data,
+        )
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens, **processor_kwargs
+        )
+
+        if prefix:
+            prefix_ids = self._tokenizer.encode(prefix, add_special_tokens=False)
+            input_ids = torch.cat(
+                [input_ids, torch.tensor(prefix_ids, dtype=input_ids.dtype)]
+            )
+
+        self._mix_config_into_hash(mm_items, processor_kwargs)
+
+        return MultimodalProcessorOutput(
+            mm_items=mm_items,
+            input_ids=input_ids.tolist(),
+            im_token_id=self.mm_tokens.image_token_id,
+        )

python/sglang/srt/multimodal/processors/voxtral.py

@@ -0,0 +1,217 @@
+"""Multimodal processor for Voxtral (speech-to-text) models."""
+
+import math
+import re
+from typing import Dict, List, Optional
+
+import torch
+
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.voxtral import VoxtralForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+# Special token IDs for Voxtral audio (from tekken.json vocabulary)
+AUDIO_TOKEN_ID = 24  # [AUDIO]
+BEGIN_AUDIO_TOKEN_ID = 25  # [BEGIN_AUDIO]
+INST_TOKEN_ID = 3  # [INST]
+
+# Placeholder for load_mm_data regex matching.
+# encode("[AUDIO]") does NOT produce token 24; actual token insertion
+# is handled in _build_input_ids_with_audio.
+AUDIO_PLACEHOLDER = "[AUDIO]"
+AUDIO_PLACEHOLDER_REGEX = re.compile(r"\[AUDIO\]")
+
+
+class VoxtralMultimodalProcessor(BaseMultimodalProcessor):
+    models = [VoxtralForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        audio_config = getattr(hf_config, "audio_config", None)
+        self.audio_token_id = getattr(hf_config, "audio_token_id", AUDIO_TOKEN_ID)
+        self.sampling_rate = getattr(audio_config, "sampling_rate", 16000)
+        self.hop_length = getattr(audio_config, "hop_length", 160)
+        self.max_source_positions = getattr(audio_config, "max_source_positions", 1500)
+        self.conv_downsample = 2  # conv1 stride=1 * conv2 stride=2
+        self.downsample_factor = getattr(
+            audio_config,
+            "downsample_factor",
+            getattr(audio_config, "intermediate_size", 5120)
+            // getattr(audio_config, "hidden_size", 1280),
+        )
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=AUDIO_PLACEHOLDER,
+            audio_token_regex=AUDIO_PLACEHOLDER_REGEX,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+    def _compute_audio_token_count(self, n_samples: int) -> int:
+        """Compute the number of [AUDIO] tokens for a given audio length."""
+        mel_frames = n_samples / self.hop_length
+        chunk_size = self.max_source_positions * self.conv_downsample
+        n_chunks = math.ceil(mel_frames / chunk_size) if mel_frames > 0 else 1
+        tokens_per_chunk = self.max_source_positions // self.downsample_factor
+        return n_chunks * tokens_per_chunk
+
+    async def process_mm_data_async(
+        self,
+        image_data,
+        audio_data,
+        input_text,
+        request_obj,
+        **kwargs,
+    ) -> Optional[MultimodalProcessorOutput]:
+        if not audio_data:
+            return None
+
+        # Insert [AUDIO] placeholders into prompt for load_mm_data's regex
+        prompt_with_placeholders = self._insert_audio_placeholders(
+            input_text, len(audio_data)
+        )
+
+        # load_mm_data handles async loading, format detection, resampling.
+        # process_and_combine_mm_data cannot be used: HF VoxtralProcessor.__call__
+        # does not support audio (only apply_chat_template does).
+        base_output = await self.load_mm_data(
+            prompt=prompt_with_placeholders,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+            audio_sample_rate=self.sampling_rate,
+        )
+        if base_output is None:
+            return None
+
+        # Convert loaded audio to tensors
+        waveforms: List[torch.Tensor] = []
+        for audio in base_output.audios:
+            wav = torch.as_tensor(audio, dtype=torch.float32)
+            if wav.dim() > 1:
+                wav = wav.mean(dim=0)
+            waveforms.append(wav)
+
+        # Compute audio token counts and build input_ids with audio tokens
+        audio_token_counts = [
+            self._compute_audio_token_count(wav.shape[-1]) for wav in waveforms
+        ]
+        tokenizer = getattr(self._processor, "tokenizer", self._processor)
+        input_ids = self._build_input_ids_with_audio(
+            tokenizer, input_text, audio_token_counts
+        )
+
+        # Find offsets of [AUDIO] token runs and build mm_items
+        audio_offsets = self._find_audio_offsets(input_ids, self.audio_token_id)
+        mm_items = []
+        for i, wav in enumerate(waveforms):
+            item = MultimodalDataItem(feature=wav, modality=Modality.AUDIO)
+            if i < len(audio_offsets):
+                item.offsets = [audio_offsets[i]]
+            mm_items.append(item)
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=mm_items,
+            audio_token_id=self.audio_token_id,
+        )
+
+    @staticmethod
+    def _insert_audio_placeholders(prompt: str, n_audio: int) -> str:
+        """Insert [AUDIO] placeholder texts into the prompt for load_mm_data."""
+        placeholders = AUDIO_PLACEHOLDER * n_audio
+        # Insert after the last [INST] marker if present
+        last_inst = prompt.rfind("[INST]")
+        if last_inst >= 0:
+            insert_pos = last_inst + len("[INST]")
+            return prompt[:insert_pos] + placeholders + prompt[insert_pos:]
+        return placeholders + prompt
+
+    @staticmethod
+    def _find_audio_offsets(input_ids: List[int], audio_token_id: int) -> List[tuple]:
+        """Find consecutive runs of audio_token_id in input_ids."""
+        offsets = []
+        start = None
+        for i, tok_id in enumerate(input_ids):
+            if tok_id == audio_token_id:
+                if start is None:
+                    start = i
+            elif start is not None:
+                offsets.append((start, i - 1))
+                start = None
+        if start is not None:
+            offsets.append((start, len(input_ids) - 1))
+        return offsets
+
+    def _build_input_ids_with_audio(
+        self,
+        tokenizer,
+        input_text: str,
+        audio_token_counts: List[int],
+    ) -> List[int]:
+        """Build input_ids by tokenizing text and inserting audio tokens.
+
+        The input_text is a decoded Mistral prompt (from text-only
+        apply_chat_template).  We re-tokenize to get proper special tokens
+        (BOS, [INST], [/INST]), then insert [BEGIN_AUDIO] + [AUDIO]*N after
+        the last [INST].
+        """
+        messages = self._parse_mistral_prompt(input_text)
+        try:
+            input_ids = tokenizer.apply_chat_template(messages, tokenize=True)
+        except (ValueError, KeyError):
+            # Fallback if prompt parsing produces malformed messages
+            input_ids = tokenizer.encode(input_text)
+
+        # Insert audio tokens after the last [INST]
+        inst_positions = [i for i, t in enumerate(input_ids) if t == INST_TOKEN_ID]
+        insert_pos = (inst_positions[-1] + 1) if inst_positions else 1
+
+        audio_tokens = []
+        for count in audio_token_counts:
+            audio_tokens.append(BEGIN_AUDIO_TOKEN_ID)
+            audio_tokens.extend([AUDIO_TOKEN_ID] * count)
+
+        return input_ids[:insert_pos] + audio_tokens + input_ids[insert_pos:]
+
+    @staticmethod
+    def _parse_mistral_prompt(prompt: str) -> List[Dict[str, str]]:
+        """Parse a Mistral-formatted prompt into a list of messages."""
+        messages = []
+        text = prompt.strip()
+
+        for marker in ["<s>", "</s>"]:
+            text = text.replace(marker, "")
+        text = text.strip()
+
+        # Extract system prompt
+        system_match = re.search(
+            r"\[SYSTEM_PROMPT\]\s*(.*?)\s*\[/SYSTEM_PROMPT\]", text, re.DOTALL
+        )
+        if system_match:
+            messages.append(
+                {"role": "system", "content": system_match.group(1).strip()}
+            )
+            text = text[: system_match.start()] + text[system_match.end() :]
+            text = text.strip()
+
+        # Split by [INST] / [/INST]
+        parts = re.split(r"\[/?INST\]", text)
+        for i, part in enumerate(parts):
+            part = part.strip()
+            if not part:
+                continue
+            if i % 2 == 1:
+                messages.append({"role": "user", "content": part})
+            elif i > 0:
+                messages.append({"role": "assistant", "content": part})
+
+        if not messages:
+            messages.append({"role": "user", "content": text})
+
+        return messages

python/sglang/srt/multimodal/processors/whisper.py

@@ -0,0 +1,255 @@
+import logging
+from typing import Any, Dict, Optional
+
+from sglang.srt.entrypoints.openai.transcription_adapters.whisper import (
+    FUSED_AUTODETECT_FLAG,
+)
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalProcessorOutput,
+)
+from sglang.srt.models.whisper import WhisperForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor
+from sglang.srt.utils import load_audio
+
+logger = logging.getLogger(__name__)
+
+# ISO 639-1 supported languages for Whisper
+# From https://platform.openai.com/docs/guides/speech-to-text/supported-languages
+# Maps ISO 639-1 code -> Full language name
+ISO639_1_SUPPORTED_LANGS = {
+    "af": "Afrikaans",
+    "ar": "Arabic",
+    "hy": "Armenian",
+    "az": "Azerbaijani",
+    "be": "Belarusian",
+    "bs": "Bosnian",
+    "bg": "Bulgarian",
+    "ca": "Catalan",
+    "zh": "Chinese",
+    "hr": "Croatian",
+    "cs": "Czech",
+    "da": "Danish",
+    "nl": "Dutch",
+    "en": "English",
+    "et": "Estonian",
+    "fi": "Finnish",
+    "fr": "French",
+    "gl": "Galician",
+    "de": "German",
+    "el": "Greek",
+    "he": "Hebrew",
+    "hi": "Hindi",
+    "hu": "Hungarian",
+    "is": "Icelandic",
+    "id": "Indonesian",
+    "it": "Italian",
+    "ja": "Japanese",
+    "kn": "Kannada",
+    "kk": "Kazakh",
+    "ko": "Korean",
+    "lv": "Latvian",
+    "lt": "Lithuanian",
+    "mk": "Macedonian",
+    "ms": "Malay",
+    "mr": "Marathi",
+    "mi": "Maori",
+    "ne": "Nepali",
+    "no": "Norwegian",
+    "fa": "Persian",
+    "pl": "Polish",
+    "pt": "Portuguese",
+    "ro": "Romanian",
+    "ru": "Russian",
+    "sr": "Serbian",
+    "sk": "Slovak",
+    "sl": "Slovenian",
+    "es": "Spanish",
+    "sw": "Swahili",
+    "sv": "Swedish",
+    "tl": "Tagalog",
+    "ta": "Tamil",
+    "th": "Thai",
+    "tr": "Turkish",
+    "uk": "Ukrainian",
+    "ur": "Urdu",
+    "vi": "Vietnamese",
+    "cy": "Welsh",
+}
+
+# Reverse mapping: Full language name (lowercase) -> ISO 639-1 code
+LANG_NAME_TO_CODE = {
+    name.lower(): code for code, name in ISO639_1_SUPPORTED_LANGS.items()
+}
+
+
+def normalize_language_to_code(language: Optional[str]) -> Optional[str]:
+    """Convert a language input (full name or code) to ISO 639-1 code.
+
+    Args:
+        language: Language as full name (e.g., 'English', 'Spanish') or
+                  ISO 639-1 code (e.g., 'en', 'es'). Three-letter Whisper
+                  codes the model supports but that aren't in
+                  ISO639_1_SUPPORTED_LANGS (e.g., 'yue', 'haw', 'jw') are
+                  also accepted so that a code returned by fused autodetect
+                  round-trips cleanly when reused as ``language=`` later.
+
+    Returns:
+        Whisper language code or None if input is None
+    """
+    if language is None:
+        return None
+
+    language_lower = language.lower().strip()
+
+    # Check if it's already a valid ISO code
+    if language_lower in ISO639_1_SUPPORTED_LANGS:
+        return language_lower
+
+    # Check if it's a full language name
+    if language_lower in LANG_NAME_TO_CODE:
+        return LANG_NAME_TO_CODE[language_lower]
+
+    # Fused autodetect's FSM regex covers the full Whisper language-token
+    # vocab (see WHISPER_LANG_TOKEN_CODES), which is wider than the
+    # English-name-keyed ISO639_1_SUPPORTED_LANGS dict. Accept any code in
+    # that wider set too so that detection -> reuse-as-input round-trips.
+    # Lazy import to avoid top-level cycle with the openai entrypoint.
+    from sglang.srt.entrypoints.openai.transcription_adapters.whisper import (
+        WHISPER_LANG_TOKEN_CODES,
+    )
+
+    if language_lower in WHISPER_LANG_TOKEN_CODES:
+        return language_lower
+
+    # Not recognized
+    raise ValueError(
+        f"Language '{language}' not recognized. "
+        f"Use full name (e.g., 'English') or ISO 639-1 code (e.g., 'en')."
+    )
+
+
+class WhisperProcessor(BaseMultimodalProcessor):
+    models = [WhisperForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+        # Cache tokenizer for language token lookup
+        self._tokenizer = getattr(self._processor, "tokenizer", None)
+
+    def _pop_sampling_param(self, request_obj, key: str):
+        sampling_params = getattr(request_obj, "sampling_params", None) or {}
+        return sampling_params.pop(key, None)
+
+    def _get_language_token_id(self, language: Optional[str]) -> int:
+        # Default to English if not specified
+        if language is None:
+            language = "en"  # Default to English
+        language_token = f"<|{language}|>"
+        token_id = self._tokenizer.convert_tokens_to_ids(language_token)
+        # normalize_language_to_code accepts the full Whisper language-token
+        # vocab (including yue/haw/jw) so fused autodetect output round-trips.
+        # Older checkpoints (v1/v2) don't have every newer token in their
+        # vocab, in which case convert_tokens_to_ids returns the unk id.
+        # Raise a clean error here instead of silently feeding unk into the
+        # decoder and producing garbage.
+        unk_id = getattr(self._tokenizer, "unk_token_id", None)
+        if token_id is None or (unk_id is not None and token_id == unk_id):
+            raise ValueError(
+                f"Language '{language}' is not in this Whisper model's vocabulary. "
+                f"The '{language_token}' token may have been added in a later "
+                f"Whisper version than the loaded checkpoint."
+            )
+        return token_id
+
+    async def process_mm_data_async(
+        self,
+        image_data,
+        audio_data,
+        input_text,
+        request_obj,
+        **kwargs,
+    ) -> Optional[Dict[str, Any]]:
+        if not audio_data:
+            return None
+
+        if len(audio_data) != 1:
+            raise ValueError(
+                f"Whisper expects exactly 1 audio input, got {len(audio_data)}"
+            )
+
+        # Check if this is a fused auto-detect request (decoder prompt = [SOT] only,
+        # structured generation handles the rest via regex constraint).
+        detect_language = self._pop_sampling_param(request_obj, FUSED_AUTODETECT_FLAG)
+        # timestamp_granularities is a transcription-level field; it must be
+        # popped in both branches or it leaks into SamplingParams(**kwargs)
+        # downstream and TypeErrors. In the fused branch the FSM regex was
+        # already picked in build_fused_autodetect_params based on this value,
+        # so we only need to keep it here to pick the timestamp_token_id for
+        # the explicit-language branch.
+        timestamp_granularities = self._pop_sampling_param(
+            request_obj, "timestamp_granularities"
+        )
+
+        audios = [load_audio(audio) for audio in audio_data]
+
+        # Whisper expects input features padded to max_length (3000 frames = 30 seconds)
+        # This is the standard context length for Whisper
+        input_features = self._processor.feature_extractor(
+            audios[0],
+            sampling_rate=16000,
+            padding="max_length",  # Pad to 3000 frames
+            return_tensors="pt",
+        )["input_features"][0]
+
+        # Whisper is a pure speech-to-text model; text prompts are ignored.
+        # The full decoder sequence is:
+        #   <|startoftranscript|> <|lang|> <|transcribe|> [<|notimestamps|> | <|0.00|>]
+        #
+        # When language is known, we build this prefix explicitly below.
+        # When auto-detecting (_detect_language=True), we feed only <|startoftranscript|>
+        # and let SGLang's structured generation (regex) constrain the model to produce
+        # <|lang|><|transcribe|><|notimestamps|> as the first 3 decode tokens — this is
+        # equivalent to HuggingFace's forced_decoder_ids but uses SGLang's native API.
+
+        decoder_start_token_id = getattr(
+            self.hf_config, "decoder_start_token_id", 50258
+        )
+
+        if detect_language:
+            input_ids = [decoder_start_token_id]
+        else:
+            language = normalize_language_to_code(
+                self._pop_sampling_param(request_obj, "language")
+            )
+            language_token_id = self._get_language_token_id(language)
+
+            transcribe_token_id = self._tokenizer.convert_tokens_to_ids(
+                "<|transcribe|>"
+            )
+
+            # Use <|0.00|> to enable timestamp generation, or <|notimestamps|> to disable
+            if timestamp_granularities:
+                timestamp_token_id = self._tokenizer.convert_tokens_to_ids("<|0.00|>")
+            else:
+                timestamp_token_id = self._tokenizer.convert_tokens_to_ids(
+                    "<|notimestamps|>"
+                )
+
+            input_ids = [
+                decoder_start_token_id,
+                language_token_id,
+                transcribe_token_id,
+                timestamp_token_id,
+            ]
+
+        return MultimodalProcessorOutput(
+            input_ids=input_ids,
+            mm_items=[
+                MultimodalDataItem(
+                    feature=input_features,
+                    modality=Modality.AUDIO,
+                )
+            ],
+        )