chore: import upstream snapshot with attribution

swift/template/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from . import templates
+from .base import MaxLengthError, Template
+from .constant import TemplateType
+from .grounding import draw_bbox
+from .register import TEMPLATE_MAPPING, get_template, get_template_meta, register_template
+from .template_inputs import StdTemplateInputs, TemplateInputs
+from .template_meta import TemplateMeta
+from .utils import (ContextType, History, Messages, Prompt, Tool, Word, get_last_user_round, history_to_messages,
+                    messages_to_history, split_str_parts_by, update_generation_config_eos_token)
+from .vision_utils import load_file, load_image

swift/template/constant.py

@@ -0,0 +1,287 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+from typing import List
+
+
+class LLMTemplateType:
+    chatml = 'chatml'
+    default = 'default'
+    dummy = 'dummy'
+
+    qwen = 'qwen'
+    qwen2_5 = 'qwen2_5'
+    qwen2_5_math = 'qwen2_5_math'
+    qwen2_5_math_prm = 'qwen2_5_math_prm'
+    qwen3 = 'qwen3'
+    qwen3_guard = 'qwen3_guard'
+    qwen3_thinking = 'qwen3_thinking'
+    qwen3_nothinking = 'qwen3_nothinking'
+    qwen3_coder = 'qwen3_coder'
+    qwen3_emb = 'qwen3_emb'
+    qwen3_reranker = 'qwen3_reranker'
+    qwq_preview = 'qwq_preview'
+    qwq = 'qwq'
+    yufeng_xguard = 'yufeng_xguard'
+    marco_o1 = 'marco_o1'
+    modelscope_agent = 'modelscope_agent'
+
+    llama = 'llama'  # llama2
+    llama3 = 'llama3'
+    llama3_2 = 'llama3_2'
+    reflection = 'reflection'
+    megrez = 'megrez'
+    yi_coder = 'yi_coder'
+    sus = 'sus'
+    gpt_oss = 'gpt_oss'
+    seed_oss = 'seed_oss'
+
+    minimax = 'minimax'
+    minimax_m1 = 'minimax_m1'
+    minimax_m2 = 'minimax_m2'
+    minimax_m2_1 = 'minimax_m2_1'
+    minimax_m2_5 = 'minimax_m2_5'
+    minimax_m2_7 = 'minimax_m2_7'
+    minimax_vl = 'minimax_vl'
+
+    numina = 'numina'
+    ziya = 'ziya'
+    atom = 'atom'
+    mengzi = 'mengzi'
+    bge_reranker = 'bge_reranker'
+
+    chatglm2 = 'chatglm2'
+    chatglm4 = 'chatglm4'
+    glm4 = 'glm4'
+    glm4_z1_rumination = 'glm4_z1_rumination'
+    glm4_5 = 'glm4_5'
+    glm4_7 = 'glm4_7'
+    glm5_1 = 'glm5_1'
+    glm5_2 = 'glm5_2'
+    codegeex4 = 'codegeex4'
+    longwriter_llama = 'longwriter_llama'
+
+    internlm = 'internlm'
+    internlm2 = 'internlm2'
+    internlm3 = 'internlm3'
+
+    deepseek = 'deepseek'
+    deepseek_coder = 'deepseek_coder'
+    deepseek_v2_5 = 'deepseek_v2_5'
+    deepseek_r1 = 'deepseek_r1'
+    deepseek_v3_1 = 'deepseek_v3_1'
+    deepseek_v4 = 'deepseek_v4'
+
+    openbuddy = 'openbuddy'
+    openbuddy2 = 'openbuddy2'
+    baichuan = 'baichuan'
+    baichuan_m1 = 'baichuan_m1'
+    minicpm = 'minicpm'
+    minicpm5 = 'minicpm5'
+    telechat = 'telechat'
+    telechat2 = 'telechat2'
+
+    codefuse = 'codefuse'
+    codefuse_codellama = 'codefuse_codellama'
+
+    skywork = 'skywork'
+    skywork_o1 = 'skywork_o1'
+
+    mistral_nemo = 'mistral_nemo'
+    mistral_2501 = 'mistral_2501'
+    devstral = 'devstral'
+    zephyr = 'zephyr'
+    wizardlm2 = 'wizardlm2'
+    wizardlm2_moe = 'wizardlm2_moe'
+    gemma = 'gemma'
+    gemma3_text = 'gemma3_text'
+    phi3 = 'phi3'
+    phi4 = 'phi4'
+
+    ling = 'ling'
+    ling2 = 'ling2'
+    ring2 = 'ring2'
+    ring2_5 = 'ring2_5'
+    yuan = 'yuan'
+    xverse = 'xverse'
+    bluelm = 'bluelm'
+    orion = 'orion'
+    moonlight = 'moonlight'
+    kimi_k2 = 'kimi_k2'
+    mimo_rl = 'mimo_rl'
+    dots1 = 'dots1'
+    hunyuan_moe = 'hunyuan_moe'
+    hunyuan = 'hunyuan'
+    hy_v3_preview = 'hy_v3_preview'
+    hy_v3 = 'hy_v3'
+    ernie = 'ernie'
+    ernie_thinking = 'ernie_thinking'
+    longchat = 'longchat'
+
+    aya = 'aya'
+    c4ai = 'c4ai'
+    dbrx = 'dbrx'
+
+    bert = 'bert'
+    dummy = 'dummy'
+    minimind = 'minimind'
+    iquestcoder = 'iquestcoder'
+    youtu_llm = 'youtu_llm'
+    olmoe = 'olmoe'
+    olmoe_0924 = 'olmoe_0924'
+
+
+class RMTemplateType:
+    internlm2_reward = 'internlm2_reward'
+
+
+class MLLMTemplateType:
+    qwen_vl = 'qwen_vl'
+    qwen_audio = 'qwen_audio'
+    qwen2_vl = 'qwen2_vl'
+    qwen2_5_vl = 'qwen2_5_vl'
+    qwen2_5_omni = 'qwen2_5_omni'
+    qwen3_omni = 'qwen3_omni'
+    qwen2_audio = 'qwen2_audio'
+    qwen3_asr = 'qwen3_asr'
+    qwen3_tts = 'qwen3_tts'
+    qwen3_vl = 'qwen3_vl'
+    qwen3_vl_emb = 'qwen3_vl_emb'
+    qwen3_vl_reranker = 'qwen3_vl_reranker'
+    qwen3_5 = 'qwen3_5'
+
+    qwen2_gme = 'qwen2_gme'
+    qvq = 'qvq'
+    ovis1_6 = 'ovis1_6'
+    ovis1_6_llama3 = 'ovis1_6_llama3'
+    ovis2 = 'ovis2'
+    ovis2_5 = 'ovis2_5'
+    mimo_vl = 'mimo_vl'
+    midashenglm = 'midashenglm'
+
+    llama3_1_omni = 'llama3_1_omni'
+    llama3_2_vision = 'llama3_2_vision'
+    llama4 = 'llama4'
+
+    llava1_5_hf = 'llava1_5_hf'
+    llava1_6_mistral_hf = 'llava1_6_mistral_hf'
+    llava1_6_vicuna_hf = 'llava1_6_vicuna_hf'
+    llava1_6_yi_hf = 'llava1_6_yi_hf'
+    llama3_llava_next_hf = 'llama3_llava_next_hf'
+    llava_next_qwen_hf = 'llava_next_qwen_hf'
+    llava_onevision_hf = 'llava_onevision_hf'
+    llava_next_video_hf = 'llava_next_video_hf'
+
+    llava_llama3_1_hf = 'llava_llama3_1_hf'  # DaozeZhang
+    llava_llama3_hf = 'llava_llama3_hf'  # xtuner
+    # lmms-lab
+    llava1_6_mistral = 'llava1_6_mistral'
+    llava1_6_yi = 'llava1_6_yi'
+    llava_next_qwen = 'llava_next_qwen'
+    llama3_llava_next = 'llama3_llava_next'
+    llava_onevision1_5 = 'llava_onevision1_5'
+
+    yi_vl = 'yi_vl'
+    ernie_vl = 'ernie_vl'
+    ernie_vl_thinking = 'ernie_vl_thinking'
+
+    internvl = 'internvl'
+    internvl_phi3 = 'internvl_phi3'
+    internvl2 = 'internvl2'
+    internvl2_phi3 = 'internvl2_phi3'
+    internvl2_5 = 'internvl2_5'
+    internvl3_5 = 'internvl3_5'
+    internvl3_5_gpt = 'internvl3_5_gpt'
+    interns1 = 'interns1'
+    internvl_hf = 'internvl_hf'
+
+    jina_reranker_m0 = 'jina_reranker_m0'
+
+    xcomposer2 = 'ixcomposer2'
+    xcomposer2_4khd = 'xcomposer2_4khd'
+    xcomposer2_5 = 'xcomposer2_5'
+
+    cogagent_chat = 'cogagent_chat'
+    cogagent_vqa = 'cogagent_vqa'
+    cogvlm = 'cogvlm'
+    cogvlm2 = 'cogvlm2'
+    cogvlm2_video = 'cogvlm2_video'
+    chatglm4v = 'chatglm4v'
+    glm_edge_v = 'glm_edge_v'
+    glm4v = 'glm4v'
+    glm4_5v = 'glm4_5v'
+    glm_ocr = 'glm_ocr'
+
+    minicpmv = 'minicpmv'
+    minicpmv2_5 = 'minicpmv2_5'
+    minicpmv2_6 = 'minicpmv2_6'
+    minicpmv4 = 'minicpmv4'
+    minicpmv4_5 = 'minicpmv4_5'
+    minicpmv4_6 = 'minicpmv4_6'
+    minicpmo = 'minicpmo'
+    minicpmo4_5 = 'minicpmo4_5'
+
+    deepseek_vl = 'deepseek_vl'
+    deepseek_vl2 = 'deepseek_vl2'
+    deepseek_janus = 'deepseek_janus'
+    deepseek_janus_pro = 'deepseek_janus_pro'
+    deepseek_ocr = 'deepseek_ocr'
+    deepseek_ocr2 = 'deepseek_ocr2'
+    unlimited_ocr = 'unlimited_ocr'
+
+    mplug_owl2 = 'mplug_owl2'
+    mplug_owl3 = 'mplug_owl3'
+    mplug_owl3_241101 = 'mplug_owl3_241101'
+    doc_owl2 = 'doc_owl2'
+
+    emu3_chat = 'emu3_chat'
+    emu3_gen = 'emu3_gen'
+
+    got_ocr2 = 'got_ocr2'
+    got_ocr2_hf = 'got_ocr2_hf'
+    step_audio = 'step_audio'
+    step_audio2_mini = 'step_audio2_mini'
+    kimi_vl = 'kimi_vl'
+    kimi_k25 = 'kimi_k25'
+    keye_vl = 'keye_vl'
+    keye_vl_1_5 = 'keye_vl_1_5'
+    dots_ocr = 'dots_ocr'
+    sail_vl2 = 'sail_vl2'
+
+    idefics3 = 'idefics3'
+    pixtral = 'pixtral'
+    paligemma = 'paligemma'
+    phi3_vision = 'phi3_vision'
+    phi4_multimodal = 'phi4_multimodal'
+    florence = 'florence'
+    molmo = 'molmo'
+    molmo2 = 'molmo2'
+    megrez_omni = 'megrez_omni'
+    valley = 'valley'
+    gemma3_vision = 'gemma3_vision'
+    gemma3n = 'gemma3n'
+    gemma4 = 'gemma4'
+    gemma4_nothinking = 'gemma4_nothinking'
+    diffusion_gemma = 'diffusion_gemma'
+    mistral_2503 = 'mistral_2503'
+    mistral_2506 = 'mistral_2506'
+    mistral_2512 = 'mistral_2512'
+    mistral_2512_thinking = 'mistral_2512_thinking'
+    paddle_ocr = 'paddle_ocr'
+    paddle_ocr_1_5 = 'paddle_ocr_1_5'
+    hunyuan_ocr = 'hunyuan_ocr'
+    step3_vl = 'step3_vl'
+    minimax_m3_vl = 'minimax_m3_vl'
+
+
+class TemplateType(LLMTemplateType, MLLMTemplateType, RMTemplateType):
+
+    @classmethod
+    def get_template_name_list(cls) -> List[str]:
+        res = []
+        for k in cls.__dict__.keys():
+            if k.startswith('__'):
+                continue
+            value = cls.__dict__[k]
+            if isinstance(value, str):
+                res.append(value)
+        return res

swift/template/grounding.py

@@ -0,0 +1,75 @@
+import colorsys
+import itertools
+from copy import deepcopy
+from modelscope.hub.file_download import model_file_download
+from PIL import Image, ImageDraw, ImageFont
+from typing import Any, List, Literal
+
+
+def _shuffle_colors(nums: List[Any]) -> List[Any]:
+    if len(nums) == 1:
+        return nums
+
+    mid = len(nums) // 2
+
+    left = nums[:mid]
+    right = nums[mid:]
+    left = _shuffle_colors(left)
+    right = _shuffle_colors(right)
+    new_nums = []
+    for x, y in zip(left, right):
+        new_nums += [x, y]
+    new_nums += left[len(right):] or right[len(left):]
+    return new_nums
+
+
+def generate_colors():
+    vs_combinations = [(v, s) for v, s in itertools.product([0.7, 0.3, 1], [0.7, 0.3, 1])]
+    colors = [colorsys.hsv_to_rgb(i / 16, s, v) for v, s in vs_combinations for i in _shuffle_colors(list(range(16)))]
+    colors = [(int(r * 255), int(g * 255), int(b * 255)) for r, g, b in colors]
+    return _shuffle_colors(colors)
+
+
+colors = generate_colors()
+color_mapping = {}
+
+
+def _calculate_brightness(image, region: List[int]):
+    cropped_image = image.crop(region)
+    grayscale_image = cropped_image.convert('L')
+    pixels = list(grayscale_image.getdata())
+    average_brightness = sum(pixels) / len(pixels)
+    return average_brightness
+
+
+def draw_bbox(image: Image.Image,
+              ref: List[str],
+              bbox: List[List[int]],
+              norm_bbox: Literal['norm1000', 'none'] = 'norm1000'):
+    bbox = deepcopy(bbox)
+    # norm bbox
+    for i, box in enumerate(bbox):
+        for i in range(len(box)):
+            box[i] = int(box[i])
+        if norm_bbox == 'norm1000':
+            box[0] = box[0] / 1000 * image.width
+            box[2] = box[2] / 1000 * image.width
+            box[1] = box[1] / 1000 * image.height
+            box[3] = box[3] / 1000 * image.height
+
+    draw = ImageDraw.Draw(image)
+    # draw bbox
+    assert len(ref) == len(bbox), f'len(refs): {len(ref)}, len(bboxes): {len(bbox)}'
+    for (left, top, right, bottom), box_ref in zip(bbox, ref):
+        if box_ref not in color_mapping:
+            color_mapping[box_ref] = colors[len(color_mapping) % len(colors)]
+        color = color_mapping[box_ref]
+        draw.rectangle([(left, top), (right, bottom)], outline=color, width=3)
+    # draw text
+    file_path = model_file_download('Qwen/Qwen-VL-Chat', 'SimSun.ttf')
+    font = ImageFont.truetype(file_path, 20)
+    for (left, top, _, _), box_ref in zip(bbox, ref):
+        brightness = _calculate_brightness(
+            image, [left, top, min(left + 100, image.width),
+                    min(top + 20, image.height)])
+        draw.text((left, top), box_ref, fill='white' if brightness < 128 else 'black', font=font)

swift/template/register.py

@@ -0,0 +1,215 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+import os
+from typing import TYPE_CHECKING, Dict, Literal, Optional
+
+from swift.utils import Processor
+from .base import Template
+from .template_meta import TemplateMeta
+
+if TYPE_CHECKING:
+    from swift.model import ModelInfo, ModelMeta
+
+TEMPLATE_MAPPING: Dict[str, TemplateMeta] = {}
+
+
+def register_template(template_meta: TemplateMeta, *, exist_ok: bool = False) -> None:
+    template_type = template_meta.template_type
+    if not exist_ok and template_type in TEMPLATE_MAPPING:
+        raise ValueError(f'The `{template_type}` has already been registered in the TEMPLATE_MAPPING.')
+    TEMPLATE_MAPPING[template_type] = template_meta
+
+
+def _read_args_json_template_type(model_dir):
+    if not os.path.exists(os.path.join(model_dir, 'args.json')):
+        return
+    from swift.arguments import BaseArguments
+    args = BaseArguments.from_pretrained(model_dir)
+    return args.template
+
+
+def get_template_meta(model_info: 'ModelInfo',
+                      model_meta: 'ModelMeta',
+                      template_type: Optional[str] = None) -> TemplateMeta:
+    if template_type is None and model_info is not None:
+        template_type = _read_args_json_template_type(model_info.model_dir)
+    template_type = template_type or model_meta.template
+    if template_type is None:
+        candidates = model_meta.candidate_templates
+        if len(candidates) > 1 or len(candidates) == 0:
+            candidates_str = ''
+            if len(candidates) > 1:
+                candidates_str = f'Multiple possible types found: {candidates}. '
+            raise ValueError(
+                f'Failed to automatically match `template_type` for `{model_info.model_dir}`. {candidates_str}'
+                'Please specify `template_type` manually via `--template`. See documentation: '
+                'https://swift.readthedocs.io/en/latest/Instruction/Supported-models-and-datasets.html')
+        elif len(candidates) == 1:
+            template_type = candidates[0]
+    elif template_type not in TEMPLATE_MAPPING:
+        raise ValueError(f"template_type: '{template_type}' not in {list(TEMPLATE_MAPPING.keys())}")
+    template_meta = TEMPLATE_MAPPING[template_type]
+    return template_meta
+
+
+def get_template(
+    processor: Processor,
+    default_system: Optional[str] = None,
+    max_length: Optional[int] = None,
+    *,
+    template_type: Optional[str] = None,
+    truncation_strategy: Literal['raise', 'left', 'right', 'split'] = 'raise',
+    max_pixels: Optional[int] = None,  # h * w
+    agent_template: Optional[str] = None,
+    norm_bbox: Literal['norm1000', 'none', None] = None,
+    use_chat_template: bool = True,
+    remove_unused_columns: bool = True,
+    padding_side: Literal['left', 'right'] = 'right',
+    # train
+    padding_free: bool = False,
+    loss_scale: str = 'default',
+    is_binary_loss_scale: Optional[bool] = None,
+    sequence_parallel_size: int = 1,
+    # infer/deploy
+    template_backend: Literal['swift', 'jinja'] = 'swift',
+    # thinking
+    response_prefix: Optional[str] = None,
+    enable_thinking: Optional[bool] = None,
+    preserve_thinking: Optional[bool] = None,
+    add_non_thinking_prefix: bool = True,
+) -> 'Template':
+    """Get or create a template instance for model input/output formatting.
+
+    This function retrieves the appropriate template class based on the model type and initializes
+    it with the specified configuration. It handles automatic template type detection from model
+    metadata, validates configuration, and supports various modes including training, inference,
+    RLHF, and agent-based interactions.
+
+    The template system provides a unified interface for:
+    - Converting conversations to token sequences and back
+    - Handling multimodal inputs (images, videos, audio, bounding boxes)
+    - Managing different chat formats and special tokens
+    - Supporting various training strategies (standard, RLHF, KTO, embedding, etc.)
+    - Integrating with multiple inference engines (Transformers, vLLM, LMDeploy, SGLang)
+
+    Args:
+        processor (Processor): Processor object containing model information, metadata,
+            tokenizer, and preprocessing capabilities. Required for template initialization.
+        default_system (Optional[str], optional): Default system prompt to prepend to conversations.
+            If None, uses the template's default system prompt. Can be used to override the
+            model's built-in system message. Defaults to None.
+        max_length (Optional[int], optional): Maximum sequence length for tokenized inputs.
+            Sequences exceeding this length are handled according to truncation_strategy.
+            If None, set to the maximum length supported by the model. Defaults to None.
+        template_type (Optional[str], optional): Explicit template type identifier
+            (e.g., 'chatml', 'qwen', 'llama3'). If None, automatically detected from model
+            metadata or args.json in the model directory. Defaults to None.
+            Template auto-detection priority: explicit template_type > args.json > model metadata
+        truncation_strategy (Literal['raise', 'left', 'right', 'split'], optional):
+            Strategy for handling sequences that exceed max_length:
+            - 'raise': Raise MaxLengthError
+            - 'left': Truncate from the left, preserving recent context
+            - 'right': Truncate from the right, preserving initial context
+            - 'split': Split into multiple sequences of max_length
+            Defaults to 'raise'.
+        max_pixels (Optional[int], optional): Maximum number of pixels (height × width) for
+            image inputs in vision-language models. Images exceeding this limit are rescaled
+            proportionally. None means no limit. Defaults to None.
+        agent_template (Optional[str], optional): Template type for agent-based interactions
+            such as ReAct, function calling, or tool use. Examples: 'react', 'hermes'.
+            If None, uses the model's default agent template if available. Defaults to None.
+        norm_bbox (Literal['norm1000', 'none', None], optional): Bounding box normalization
+            strategy for grounding and detection tasks:
+            - 'norm1000': Normalize coordinates to [0, 1000] range
+            - 'none': Keep original pixel coordinates
+            - None: Use the default normalization of the corresponding model's template
+            Defaults to None.
+        use_chat_template (bool, optional): Whether to use the model's native chat template
+            format. If False, uses a simpler generation-only template without chat structure.
+            Defaults to True.
+        remove_unused_columns (bool, optional): Whether to remove dataset columns not used
+            by the model during data processing. Helps reduce memory usage. Defaults to True.
+        padding_side (Literal['left', 'right'], optional): Side to add padding tokens:
+            - 'left': Pad on the left (useful for batched inference)
+            - 'right': Pad on the right (standard for training)
+            Defaults to 'right'.
+        padding_free (bool, optional): Enable padding-free (packing) training where multiple
+            sequences are concatenated without padding tokens. Improves training efficiency.
+            Defaults to False.
+        loss_scale (str, optional): Loss scaling strategy identifier for different parts
+            of sequences. Controls the contribution value of tokens to the loss.
+            Defaults to 'default'.
+        is_binary_loss_scale (bool, optional): When `loss_scale` can only take values of `0` or `1`,
+            its semantics can be represented by `labels` instead — by setting the `labels` of
+            positions where `loss_scale` is `0` to `-100`, thereby ensuring compatibility with
+            `liger_kernel` and reducing memory usage. Defaults to `None` for automatic configuration.
+        sequence_parallel_size (int, optional): Number of devices for sequence parallelism
+            in distributed training. Splits long sequences across devices.
+            Defaults to 1 (no parallelism).
+        template_backend (Literal['swift', 'jinja'], optional): Template rendering engine:
+            - 'swift': Swift's native template engine with advanced features
+            - 'jinja': Jinja2 template engine
+            Defaults to 'swift'.
+        response_prefix (Optional[str], optional): Prefix string to add before model responses.
+            Useful for structured output, thinking tokens, or format indicators. If None,
+            uses template's default prefix based on thinking mode. Defaults to None.
+        enable_thinking (Optional[bool], optional): Controls whether thinking mode is enabled
+            during inference.
+        preserve_thinking (Optional[bool]): Whether to preserve historical thinking content
+            during inference and training.
+        add_non_thinking_prefix (bool, optional): This parameter only takes effect during
+            training and indicates whether to add a non-thinking prefix to data samples
+            whose assistant part does not start with the thinking tag '<think>'
+            (typically used in hybrid thinking models that contain non-thinking prefixes).
+
+    Returns:
+        Template: Initialized template instance configured with the specified parameters.
+            The template is ready to encode conversations, handle multimodal inputs, and
+            integrate with training or inference pipelines.
+
+    Raises:
+        ValueError: If template_type cannot be automatically determined and multiple or no
+            candidate templates are found. The error message will list candidates if multiple
+            are available and provide a link to supported models documentation.
+        KeyError: If the specified or detected template_type is not found in TEMPLATE_MAPPING.
+
+    Examples:
+        >>> from swift import get_processor, get_template
+        >>>
+        >>> # Basic usage with auto-detection
+        >>> processor = get_processor('Qwen/Qwen2.5-VL-7B-Instruct')
+        >>> template = get_template(processor)
+        >>>
+        >>> # Specify template type explicitly
+        >>> tokenizer = get_processor('Qwen/Qwen2.5-7B-Instruct-123')
+        >>> template = get_template(tokenizer, template_type='qwen2_5')
+    """
+    model_info = processor.model_info
+    model_meta = processor.model_meta
+    template_meta = get_template_meta(model_info, model_meta, template_type=template_type)
+    template_cls = template_meta.template_cls
+    return template_cls(
+        processor,
+        template_meta,
+        default_system,
+        max_length,
+        truncation_strategy=truncation_strategy,
+        max_pixels=max_pixels,
+        agent_template=agent_template,
+        norm_bbox=norm_bbox,
+        use_chat_template=use_chat_template,
+        remove_unused_columns=remove_unused_columns,
+        padding_side=padding_side,
+        # train
+        padding_free=padding_free,
+        loss_scale=loss_scale,
+        is_binary_loss_scale=is_binary_loss_scale,
+        sequence_parallel_size=sequence_parallel_size,
+        # infer/deploy
+        template_backend=template_backend,
+        # thinking
+        response_prefix=response_prefix,
+        enable_thinking=enable_thinking,
+        preserve_thinking=preserve_thinking,
+        add_non_thinking_prefix=add_non_thinking_prefix,
+    )

swift/template/template_inputs.py

@@ -0,0 +1,219 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import json
+from copy import deepcopy
+from dataclasses import dataclass, field, fields
+from PIL import Image
+from typing import Any, Dict, List, Optional, Union
+
+from swift.utils import get_logger
+from .utils import Messages, Tool, get_last_user_round, messages_to_history
+
+logger = get_logger()
+
+
+@dataclass
+class StdTemplateInputs:
+    # only user/tool/assistant
+    messages: List[Dict[str, str]]
+    # None: use default system; '': not use system
+    system: Optional[str] = None
+    tools: Optional[List[Tool]] = None
+
+    label: Optional[int] = None
+    channel: Optional[str] = None
+
+    images: List[Union[str, Image.Image]] = field(default_factory=list)
+    videos: List[str] = field(default_factory=list)
+    audios: List[str] = field(default_factory=list)
+    objects: Dict[str, Any] = field(default_factory=dict)
+
+    margin: Optional[float] = None  # for reward modeling
+    chat_template_kwargs: Dict[str, Any] = field(default_factory=dict)  # from dataset
+    extra_kwargs: Dict[str, Any] = field(default_factory=dict)
+
+    mm_processor_kwargs: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        self.image_idx = 0
+        self.audio_idx = 0
+        self.video_idx = 0
+        self.ref_idx = 0
+        self.bbox_idx = 0
+        if self.images and not isinstance(self.images, (list, tuple)):
+            self.images = [self.images]
+        if self.videos and not isinstance(self.videos, (list, tuple)):
+            self.videos = [self.videos]
+        if self.audios and not isinstance(self.audios, (list, tuple)):
+            self.audios = [self.audios]
+
+    def to_history(self):
+        if not self.messages:
+            return None
+        return messages_to_history(self.messages)
+
+    @property
+    def is_multimodal(self):
+        return bool(self.images or self.audios or self.videos or self.objects)
+
+    @classmethod
+    def from_dict(cls, inputs: Dict[str, Any]) -> 'StdTemplateInputs':
+        inputs = deepcopy(inputs)
+        kwargs = {}
+        for key in ['label', 'channel', 'margin', 'rejected_response']:
+            if key in inputs:
+                kwargs[key] = inputs[key]
+        messages = inputs['messages']
+        tools = inputs.get('tools')
+        objects = inputs.get('objects') or {}
+        chat_template_kwargs = inputs.get('chat_template_kwargs') or {}
+
+        if messages and messages[0]['role'] == 'system':
+            message = messages.pop(0)
+            system = message['content']
+        else:
+            system = None
+
+        for message in messages:
+            if message['role'] == 'tool_response':
+                message['role'] = 'tool'
+            if message['role'] in {'tool_call', 'tool'} and not isinstance(message['content'], str):
+                message['content'] = json.dumps(message['content'], ensure_ascii=False)
+
+        media_kwargs = StdTemplateInputs.remove_messages_media(messages)
+        for k in list(media_kwargs.keys()):
+            mm_data = media_kwargs[k]
+
+            inputs_mm_data = inputs.get(k)
+            if isinstance(inputs_mm_data, str):
+                inputs_mm_data = [inputs_mm_data]
+            inputs_mm_data = (inputs_mm_data or []).copy()
+            if mm_data:
+                assert not inputs_mm_data, f'self.{k}: {inputs_mm_data}'
+            else:
+                media_kwargs[k] = inputs_mm_data
+
+        all_keys = set(f.name for f in fields(StdTemplateInputs))
+        extra_kwargs = {k: v for k, v in inputs.items() if k not in all_keys}
+        return cls(
+            messages=messages,
+            system=system,
+            tools=tools,
+            objects=objects,
+            chat_template_kwargs=chat_template_kwargs,
+            extra_kwargs=extra_kwargs,
+            **kwargs,
+            **media_kwargs)
+
+    @staticmethod
+    def remove_messages_media(messages: Messages) -> Dict[str, Any]:
+        res = {'images': [], 'audios': [], 'videos': []}
+        for message in messages:
+            content = message['content']
+            if isinstance(content, str):
+                continue
+            elif (isinstance(content, list) and content
+                  and isinstance(content[0], int)) or (isinstance(content, dict) and 'token_ids' in content):
+                continue
+            # List[Dict[str, Any]]
+            new_content = ''
+            for item in content:
+                key: str = item['type']
+                value = item.get(key)
+                if key == 'text':
+                    new_content += value
+                    continue
+                # image/audio/video
+                # image_url/audio_url/video_url
+                if key.endswith('_url'):
+                    key = key[:-len('_url')]
+                new_content += f'<{key}>'
+                if isinstance(value, dict):
+                    value = value['url']
+                if value:
+                    res[f'{key}s'].append(value)
+            message['content'] = new_content
+        return res
+
+
+@dataclass
+class TemplateInputs:
+    chosen: StdTemplateInputs  # or Dict[str, Any]
+    rejected: Optional[StdTemplateInputs] = None
+    positive: List[StdTemplateInputs] = field(default_factory=list)  # or Dict[str, Any]
+    negative: List[StdTemplateInputs] = field(default_factory=list)
+
+    def __post_init__(self):
+        all_keys = set(f.name for f in fields(StdTemplateInputs))
+        for key in ['chosen', 'rejected', 'positive', 'negative']:
+            value_dict = getattr(self, key, None)
+            if not isinstance(value_dict, dict):
+                continue
+            if key in {'chosen', 'rejected'}:
+                setattr(self, key, StdTemplateInputs.from_dict(value_dict))
+            else:
+                res = []
+                for i in range(len(value_dict['messages'])):
+                    kwargs = {}
+                    for k in all_keys:
+                        val = value_dict.get(k)
+                        if val is None:
+                            continue
+                        kwargs[k] = val[i]
+                    res.append(StdTemplateInputs.from_dict(kwargs))
+                setattr(self, key, res)
+
+    @staticmethod
+    def _compat_rejected_response(inputs: Dict[str, Any]):
+        if 'rejected_response' not in inputs:
+            return
+        messages = inputs['messages']
+        assert len(messages) > 0, f'messages: {messages}'
+        idx = get_last_user_round(messages) + 1
+
+        rejected_response = inputs.pop('rejected_response')
+        if isinstance(rejected_response, str):
+            rejected_responses = [{'role': 'assistant', 'content': rejected_response}]
+        elif isinstance(rejected_response, list):
+            rejected_responses = rejected_response
+            for message in rejected_responses:
+                if message['role'] == 'user':
+                    raise ValueError(
+                        f"The 'user' role is not allowed in 'rejected_response' messages. Found: {message}")
+        else:
+            raise ValueError(f'rejected_response must be a str or list. rejected_response: {rejected_response}')
+        # Check that the response is different from the rejected_response.
+        if len(messages[idx:]) == 1 and len(rejected_responses) == 1:
+            response = messages[idx]['content']
+            rejected_response = rejected_responses[0]['content']
+            assert rejected_response != response, f'rejected_response: {rejected_response}, response: {response}'
+        inputs['rejected_messages'] = deepcopy(messages[:idx]) + rejected_responses
+
+    @classmethod
+    def from_dict(cls, inputs: Dict[str, Any]) -> 'TemplateInputs':
+        inputs = deepcopy(inputs)
+
+        has_rejected_messages = inputs.get('rejected_messages') is not None
+        cls._compat_rejected_response(inputs)
+        kwargs = {}
+        non_chosen_keys = ['rejected', 'positive', 'negative']
+        for prefix in ['chosen'] + non_chosen_keys:
+            if prefix == 'chosen':
+                std_inputs = {
+                    k: v
+                    for k, v in inputs.items() if not any(k.startswith(f'{p}_') for p in non_chosen_keys)
+                }
+            else:
+                std_inputs = {k[len(f'{prefix}_'):]: v for k, v in inputs.items() if k.startswith(f'{prefix}_')}
+            if std_inputs:
+                kwargs[prefix] = std_inputs
+
+        if not has_rejected_messages and kwargs.get('rejected') is not None:
+            chosen = kwargs['chosen']
+            rejected = kwargs['rejected']
+            # Supplement additional key-value pairs
+            for k, chosen_v in chosen.items():
+                rejected_v = rejected.get(k)
+                if chosen_v is not None and rejected_v is None:
+                    rejected[k] = chosen_v
+
+        return cls(**kwargs)

swift/template/template_meta.py

@@ -0,0 +1,145 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from transformers import PreTrainedTokenizerBase
+from typing import List, Optional, Type, Union
+
+from .base import Template
+from .utils import Prompt, Word
+
+
+@dataclass
+class TemplateMeta:
+    """
+    Examples:
+        chatml (with bos):
+            prefix: <s>
+            prompt: <|im_start|>user\n{{QUERY}}<|im_end|>\n<|im_start|>assistant\n
+            chat_sep: <|im_end|>\n
+            suffix: <|im_end|>
+            system_prefix: <s><|im_start|>system\n{{SYSTEM}}<|im_end|>\n
+
+        <s><|im_start|>system  # prefix or system_prefix
+        {{SYSTEM}}<|im_end|>
+        <|im_start|>user  # prompt
+        {{QUERY}}<|im_end|>
+        <|im_start|>assistant
+        {{RESPONSE}}<|im_end|>  # chat_sep
+        <|im_start|>user  # prompt
+        {{QUERY}}<|im_end|>
+        <|im_start|>assistant
+        {{RESPONSE}}<|im_end|>  # suffix
+    """
+    template_type: str
+    prefix: Prompt
+    prompt: Prompt
+    chat_sep: Optional[Prompt]
+    suffix: Prompt = field(default_factory=lambda: [['eos_token_id']])
+    template_cls: Type[Template] = Template
+    system_prefix: Optional[Prompt] = None
+    default_system: Optional[str] = None
+
+    auto_add_bos: bool = False
+    stop_words: List[Word] = field(default_factory=list)
+    agent_template: Optional[str] = None
+    # thinking
+    is_thinking: bool = False  # Automatically remove think content
+    thinking_prefix: str = ''
+    non_thinking_prefix: str = ''  # Automatically add non_thinking_prefix for hybrid thinking models
+    # During encoding, historical thinking content will be removed.
+    # This parameter represents the prefix for the historical part.
+    history_thinking_prefix: str = ''
+
+    def to_generate_template_meta(self) -> 'TemplateMeta':
+        self = deepcopy(self)
+        return TemplateMeta(
+            self.template_type,
+            prefix=[],
+            prompt=['{{QUERY}}'],
+            chat_sep=None,
+            template_cls=self.template_cls,
+            auto_add_bos=True,
+            stop_words=self.stop_words,
+        )
+
+    @staticmethod
+    def _has_system(prefix_or_prompt: Prompt) -> bool:
+        return any(['{{SYSTEM}}' in p for p in prefix_or_prompt])
+
+    @staticmethod
+    def _replace_system(prefix: Prompt) -> Prompt:
+        return [p.replace('{{SYSTEM}}', '') if isinstance(p, str) else p for p in prefix]
+
+    def _check_template_meta(self):
+        # check
+        for x in [self.prefix, self.prompt, self.suffix]:
+            assert isinstance(x, list)
+        for x in [self.chat_sep, self.system_prefix]:
+            assert x is None or isinstance(x, list)
+
+    def __post_init__(self):
+        # system
+        if self._has_system(self.prefix):
+            assert self.system_prefix is None, 'The prefix already contains {{SYSTEM}}.'
+            self.system_prefix = self.prefix
+            self.prefix = self._replace_system(self.prefix)
+
+        self.is_post_system = self._has_system(self.prompt)  # mistral_nemo
+        if self.is_post_system:
+            self.system_prompt = self.prompt
+            self.prompt = [context for context in self.prompt if '{{SYSTEM}}' not in context]
+
+        if self.system_prefix is None and not self.is_post_system:
+            self.support_system = False
+        else:
+            self.support_system = True
+        self.check_system(self.default_system)
+
+        self.support_multi_round = self.chat_sep is not None
+
+    @staticmethod
+    def _token_attr_to_id(tokenizer: PreTrainedTokenizerBase, value: Optional[Prompt]) -> Optional[Prompt]:
+        """Turn `eos_token_id` to token id
+
+        e.g. [['eos_token_id']] -> [[2]]
+        """
+        if value is None:
+            return None
+        res_value = []
+        for v in value:
+            if isinstance(v, list):
+                v = [getattr(tokenizer, sub_v) if isinstance(sub_v, str) else sub_v for sub_v in v]
+            res_value.append(v)
+        return res_value
+
+    def init(self, tokenizer: PreTrainedTokenizerBase) -> None:
+        for key in ['prefix', 'prompt', 'chat_sep', 'suffix', 'system_prefix']:
+            value = getattr(self, key)
+            value = self._token_attr_to_id(tokenizer, value)
+            setattr(self, key, value)
+
+        suffix_stop = self.suffix[-1] if self.suffix else None
+        if isinstance(suffix_stop, str):
+            suffix_stop = suffix_stop.strip()
+        self.suffix_stop = suffix_stop
+        if suffix_stop and suffix_stop not in self.stop_words:
+            self.stop_words.append(suffix_stop)
+        if tokenizer.eos_token not in self.stop_words:
+            self.stop_words.append(tokenizer.eos_token)
+
+        self.stop_token_id = tokenizer.eos_token_id
+        if suffix_stop:
+            if isinstance(suffix_stop, str):
+                stop_token_id = tokenizer.convert_tokens_to_ids(suffix_stop)
+            elif isinstance(suffix_stop, list) and len(suffix_stop) == 1:
+                stop_token_id = suffix_stop[0]
+            else:
+                stop_token_id = None
+            if stop_token_id is not None:
+                self.stop_token_id = stop_token_id
+
+    def check_system(self, system: Optional[str]) -> None:
+        if system is not None:
+            assert self.support_system, (
+                f'The template does not support `system`, template_type: {self.template_type}, system: {system}')

swift/template/templates/__init__.py

@@ -0,0 +1,3 @@
+from . import (baai, baidu, bert, deepseek, dots, gemma, glm, idefics3, internlm, internvl, kwai, llama, llava, llm,
+               megrez, microsoft, midashenglm, minicpm, minimax, minimind, mistral, molmo, moonshot, mplug, openbuddy,
+               pixtral, qwen, seed, stepfun, tencent, valley, yi)

swift/template/templates/baai.py

@@ -0,0 +1,202 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+import random
+import torch
+from PIL import Image
+from typing import Any, Dict, List
+
+from swift.utils import get_device
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..template_meta import TemplateMeta
+from ..utils import findall
+from .utils import DEFAULT_SYSTEM, EmptyTemplateMeta
+
+
+class Emu3GenTemplate(Template):
+
+    NULL_PROMPT_PROB = 0.1
+    COOKBOOK_SIZE = 32768
+    CFG_SCALE = os.environ.get('CFG_SCALE', 3.0)
+    GENERATION_RATIO = os.environ.get('GENERATION_RATIO', '1:1')
+    NEGATIVE_PROMPT = os.environ.get(
+        'NEGATIVE_PROMPT',
+        'lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, '
+        'worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry.')
+
+    def init_processor(self, processor) -> None:
+        if processor is None:
+            return
+        super().init_processor(processor)
+        self.bov = self.processor.tokenizer.encode(self.processor.visual_template[0].format(token_id=0))[0]
+        self.eov = self.processor.tokenizer.encode(self.processor.visual_template[0].format(token_id=self.COOKBOOK_SIZE
+                                                                                            - 1))[0]
+        self.h, self.w = self.processor.calculate_generate_size(self.GENERATION_RATIO, self.processor.image_area,
+                                                                self.processor.vision_tokenizer.spatial_scale_factor)
+        self.skip_prompt = False
+        self.apply_loss_on_only_vision = True
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        if self.is_training:
+            p_prob = random.random()
+            if p_prob < self.NULL_PROMPT_PROB:
+                prompt = ''
+            else:
+                prompt = inputs.to_history()['response']
+            image = self.smart_resize(inputs.images[0].convert('RGB'))
+            with torch.no_grad():
+                image = self.processor.image_processor(
+                    image, return_tensors='pt')['pixel_values'].to(device=self.processor.vision_tokenizer.device)
+                image_token_ids = self.processor.vision_tokenizer.encode(image).squeeze(0)
+            encoded = self._process_prompt_train(prompt, image_token_ids)
+        else:
+            prompt = inputs.to_history()['query']
+            encoded = self._process_prompt_test(prompt)
+            encoded = {key: encoded[key][0] for key in encoded.keys()}  # [1, L] -> [L]
+
+        return encoded
+
+    def _process_prompt_train(self, raw_prompt, image_token_ids):
+        image_prompt = self.format_image_prompt(image_token_ids)
+        prompt = self.tokenizer.bos_token + raw_prompt + image_prompt
+        sample = self.tokenizer(prompt, padding='max_length', return_token_type_ids=False)
+        labels = torch.tensor(sample['input_ids'])
+        if self.apply_loss_on_only_vision:
+            labels = torch.where(torch.logical_and(labels >= self.bov, labels <= self.eov), labels, -100)
+        sample['labels'] = labels.tolist()
+        return sample
+
+    def _process_prompt_test(self, raw_prompt):
+        # for supporting multi inputs, use list instead of single string
+        if isinstance(raw_prompt, str):
+            raw_prompt = [raw_prompt]
+        prompt_list = []
+        size_list = []
+        for text_prompt in raw_prompt:
+            prompt = self.processor.tokenizer.bos_token
+            image_prompt = (
+                self.processor.tokenizer.boi_token + self.processor.prefix_template.format(H=self.h, W=self.w)
+                + self.processor.tokenizer.img_token)
+            prompt += (text_prompt + image_prompt)
+            prompt_list.append(prompt)
+            size_list.append([self.h, self.w])
+        prompt_list = self.tokenizer(prompt_list, padding='longest', return_token_type_ids=False)
+        return prompt_list
+
+    def prepare_for_output(self, output: str) -> str:
+        return output
+
+    def prepare_generate_kwargs(self, generate_kwargs: Dict[str, Any], *, model=None) -> Dict[str, Any]:
+        from transformers import (LogitsProcessorList, PrefixConstrainedLogitsProcessor,
+                                  UnbatchedClassifierFreeGuidanceLogitsProcessor)
+
+        negative_prompt = self.NEGATIVE_PROMPT
+        neg_inputs = self._process_prompt_test(negative_prompt)
+        neg_inputs = {key: torch.tensor(val) for key, val in neg_inputs.items()}
+        batch_size = generate_kwargs['input_ids'].shape[0]
+        h = torch.tensor([self.h] * batch_size)
+        w = torch.tensor([self.w] * batch_size)
+
+        constrained_fn = self.processor.build_prefix_constrained_fn(h, w)
+        logits_processor = LogitsProcessorList([
+            UnbatchedClassifierFreeGuidanceLogitsProcessor(
+                self.CFG_SCALE,
+                model,
+                unconditional_ids=neg_inputs['input_ids'].to(get_device()),
+            ),
+            PrefixConstrainedLogitsProcessor(
+                constrained_fn,
+                num_beams=1,
+            ),
+        ])
+        res = super().prepare_generate_kwargs(generate_kwargs, model=model)
+        res['logits_processor'] = logits_processor
+        return res
+
+    def decode_generate_ids(self, generate_ids: List[int], **kwargs) -> Any:
+        mm_list = self.processor.decode(generate_ids)
+        for im in mm_list:
+            if not isinstance(im, Image.Image):
+                continue
+            return [{'type': 'image', 'image': im}]
+
+    def to_imgstr(self, image_tokens):
+        image_token_str = [[self.processor.visual_template[0].format(token_id=token_id) for token_id in token_row]
+                           for token_row in image_tokens]
+        image_row_str = [''.join(token_row) for token_row in image_token_str]
+        imgstr = self.tokenizer.eol_token.join(image_row_str)
+        return imgstr
+
+    def format_image_prompt(self, image_tokens):
+        h, w = image_tokens.shape
+        imgstr = self.to_imgstr(image_tokens)
+        image_prompt = (
+            self.tokenizer.boi_token + f'{h}*{w}' + self.tokenizer.img_token + imgstr + self.tokenizer.eol_token
+            + self.tokenizer.eof_token + self.tokenizer.eoi_token)
+        return image_prompt
+
+    def smart_resize(self, image):
+        w, h = image.size
+        current_area = h * w
+        target_ratio = (self.processor.image_area / current_area)**0.5
+        th = int(round(h * target_ratio))
+        tw = int(round(w * target_ratio))
+        image = image.resize((tw, th))
+        return image
+
+
+register_template(EmptyTemplateMeta(
+    MLLMTemplateType.emu3_gen,
+    template_cls=Emu3GenTemplate,
+))
+
+
+class Emu3ChatTemplate(Template):
+    system = 'You are a helpful assistant.'
+    image_placeholder = ['<|image token|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        # image
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        image_tokens = self.processor.tokenize_image(images)
+        image_prompts = []
+        idx_list = findall(input_ids, self.tokenizer.encode(self.image_placeholder))
+        # Create image prompts
+        for i in range(len(images)):
+            h, w = image_tokens[i].shape
+            imgstr = self.processor.to_imgstr(image_tokens[i])
+            image_prompt = (
+                self.tokenizer.boi_token + self.processor.prefix_template.format(H=h, W=w) + self.tokenizer.img_token
+                + imgstr + self.tokenizer.eol_token + self.tokenizer.eof_token + self.tokenizer.eoi_token)
+            image_prompts.append(self.tokenizer.encode(image_prompt))
+
+        # Insert image tokens into input_ids
+        input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                            lambda i: image_prompts[i])
+        return {'input_ids': input_ids, 'labels': labels, 'loss_scale': loss_scale}
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.emu3_chat,
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=[' User: {{QUERY}}. Assistant:'],
+        chat_sep=[['eos_token_id']],
+        suffix=[['eos_token_id']],
+        default_system=DEFAULT_SYSTEM,
+        template_cls=Emu3ChatTemplate))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.bge_reranker,
+        prefix=['<s> '],
+        chat_sep=[],
+        prompt=['{{QUERY}}</s></s> '],
+        suffix=['</s>'],
+    ))

swift/template/templates/baidu.py

@@ -0,0 +1,295 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import numpy as np
+import torch
+import torch.nn as nn
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class ERNIETemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|begin_of_sentence|>'])
+    prompt: Prompt = field(default_factory=lambda: ['User: {{QUERY}}\nAssistant: '])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|end_of_sentence|>'])
+    suffix: Prompt = field(default_factory=lambda: ['</s>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|begin_of_sentence|>{{SYSTEM}}\n'])
+
+
+register_template(ERNIETemplateMeta(LLMTemplateType.ernie))
+
+
+class ErnieThinkingTemplate(Template):
+
+    def _swift_prepare_inputs(self, inputs) -> None:
+        super()._swift_prepare_inputs(inputs)
+        for message in inputs.messages:
+            if message['role'] == 'assistant':
+                if '<response>' not in message['content']:
+                    if '</think>' in message['content']:
+                        message['content'] = message['content'].replace('</think>', '</think>\n\n<response>\n')
+                        message['content'] = message['content'] + '\n</response>'
+                        if '<think>\n' not in message['content']:
+                            message['content'] = message['content'].replace('<think>', '<think>\n')
+                    else:
+                        message['content'] = '<response>\n' + message['content'] + '\n</response>\n'
+
+
+@dataclass
+class ERNIEThinkingTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(
+        default_factory=lambda:
+        ['<|im_start|>system\n'
+         '<global_setting>\n'
+         'think_mode=True\n'
+         '</global_setting><|im_end|>\n\n'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['<|im_start|>user\n'
+                                 '{{QUERY}}<|im_end|>\n\n'
+                                 '<|im_start|>assistant\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>\n\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: [
+        '<|im_start|>system\n'
+        '<system_setting>\n'
+        '{{SYSTEM}}\n'
+        '</system_setting>\n\n'
+        '<global_setting>\n'
+        'think_mode=True\n'
+        '</global_setting><|im_end|>\n\n'
+    ])
+
+
+register_template(
+    ERNIEThinkingTemplateMeta(
+        LLMTemplateType.ernie_thinking,
+        template_cls=ErnieThinkingTemplate,
+        is_thinking=True,
+        thinking_prefix='<think>\n'))
+
+
+class PaddleOCRTemplate(Template):
+    image_token = '<|IMAGE_PLACEHOLDER|>'
+    image_token_id = 100295
+    skip_prompt = False
+    version = 'v1'
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, self.image_token_id)
+        processor = self.processor
+        images = inputs.images
+        if images:
+            processor_kwargs = {}
+            if self.version == 'v1_5' and inputs.chat_template_kwargs:
+                for key in ['shortest_edge', 'longest_edge']:
+                    value = inputs.chat_template_kwargs.get(key, None)
+                    if value:
+                        processor_kwargs[key] = value
+                if processor_kwargs:
+                    processor_kwargs = {'size': processor_kwargs}
+            image_inputs = processor.image_processor(images=images, return_tensors='pt', **processor_kwargs)
+            image_inputs['pixel_values'] = image_inputs['pixel_values']
+            image_grid_thw = image_inputs['image_grid_thw']
+            merge_size = processor.image_processor.merge_size**2
+
+            def _get_new_tokens(i):
+                img_tokens: List[int] = [self.image_token_id] * (image_grid_thw[i].prod() // merge_size)
+                return img_tokens
+
+            encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['image_grid_thw'] = image_grid_thw
+
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        embedding = model.get_input_embeddings()
+        device = embedding.weight.device
+        input_ids = inputs['input_ids']
+        inputs_embeds = embedding(input_ids).to(device=device)
+        pixel_values = inputs.get('pixel_values')
+        image_grid_thw = inputs.get('image_grid_thw')
+        if pixel_values is not None:
+            siglip_position_ids = list()
+            image_grid_hws = list()
+            sample_indices = list()
+            cu_seqlens = [0]
+            pixel_values = pixel_values.unsqueeze(0).to(device=device)
+            for idx, thw in enumerate(image_grid_thw):
+                thw_tuple = tuple(thw.detach().cpu().numpy().tolist())
+                numel = np.prod(thw_tuple)
+                image_grid_hws.append(thw_tuple)
+                image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
+                siglip_position_ids.append(image_position_ids)
+                sample_indices.append(torch.full((numel, ), idx, dtype=torch.int64))
+                cu_seqlens.append(cu_seqlens[-1] + numel)
+
+            siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(pixel_values.device)
+            cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values.device)
+            sample_indices = torch.concat(sample_indices, dim=0).to(pixel_values.device)
+
+            vision_outputs = model.visual(
+                pixel_values=pixel_values,
+                image_grid_thw=image_grid_hws,
+                position_ids=siglip_position_ids,
+                vision_return_embed_list=True,
+                interpolate_pos_encoding=True,
+                sample_indices=sample_indices,
+                cu_seqlens=cu_seqlens,
+                return_pooler_output=False,
+                use_rope=True,
+                window_size=-1,
+            )
+            image_embeds = vision_outputs.last_hidden_state
+            image_embeds = model.mlp_AR(image_embeds, image_grid_thw)
+            n_image_tokens = (input_ids == self.image_token_id).sum().item()
+            image_embeds = torch.cat(image_embeds, dim=0)
+            n_image_features = image_embeds.shape[0]
+            if n_image_tokens != n_image_features:
+                raise ValueError('Image features and image tokens do not match: tokens: '
+                                 f'{n_image_tokens}, features {n_image_features}')
+
+            mask = input_ids == self.image_token_id
+            mask_unsqueezed = mask.unsqueeze(-1)
+            mask_expanded = mask_unsqueezed.expand_as(inputs_embeds)
+            image_mask = mask_expanded.to(inputs_embeds.device)
+
+            image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+
+            inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(ERNIETemplateMeta(MLLMTemplateType.paddle_ocr, template_cls=PaddleOCRTemplate))
+
+
+class ERNIE_VLTemplate(Template):
+    placeholder_tokens = ['<|IMAGE_PLACEHOLDER|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [f'Picture {index + 1}:<|IMAGE_PLACEHOLDER|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded['loss_scale']
+        image_token = self._tokenize('<|IMAGE_PLACEHOLDER|>')[0]
+        idx_list = findall(input_ids, image_token)
+        if idx_list:
+            split_token = self._tokenize('\n')[0]
+            new_inputs = self.processor(
+                text=['\n'.join(['<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>'] * len(idx_list))],
+                images=inputs.images,
+                videos=inputs.videos,
+                padding=True,
+                return_tensors='pt',
+            )
+            splited_tokens = self._split_list(new_inputs['input_ids'][0].tolist(), split_token)
+            # Insert image tokens into input_ids
+            input_ids_len = len(input_ids)
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                lambda i: splited_tokens[i])
+            idx_list.append(input_ids_len)
+            splited_tokens.append([])
+            token_type_ids = []
+            position_ids = []
+            text_i, image_i, n_text_token = 0, 0, 0
+            for i, idx in enumerate(idx_list):
+                image_idx = image_i + len(splited_tokens[i])
+                text_len = idx - text_i
+                token_type_ids.append(torch.tensor([0] * (text_len))[None])
+                token_type_ids.append(new_inputs['token_type_ids'][:, image_i:image_idx])
+                text_position_ids = torch.arange(0, text_len)[None, :, None]
+                start_idx = 0
+                if position_ids:
+                    start_idx = position_ids[-1][0, -1].max() + 1
+                position_ids.append(torch.concat([text_position_ids + start_idx for _ in range(3)], dim=2))
+                n_text_token += text_len
+                position_ids.append(new_inputs['position_ids'][:, image_i:image_idx] + n_text_token)
+                text_i = idx + 1
+                n_text_token -= 1  # '\n'
+                image_i = image_idx + 1
+            token_type_ids = torch.cat(token_type_ids, dim=1)
+            position_ids = torch.cat(position_ids, dim=1)
+            encoded.update(new_inputs)
+            encoded['token_type_ids'] = token_type_ids
+            encoded['position_ids'] = position_ids
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = {}
+        for key in ['images', 'grid_thw', 'image_type_ids']:
+            res[key] = self.concat_tensor(batch, key, 0)
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+    def generate(self, model, *args, **kwargs):
+        kwargs['use_cache'] = False
+        return super().generate(model, *args, **kwargs)
+
+
+register_template(
+    ERNIETemplateMeta(
+        MLLMTemplateType.ernie_vl, template_cls=ERNIE_VLTemplate, is_thinking=True, thinking_prefix='<think>'))
+
+ERNIE_VL_SYSTEM = ('You are a multimodal AI assistant called ERNIE developed by Baidu based on the PaddlePaddle '
+                   'framework.')
+
+register_template(
+    ERNIETemplateMeta(
+        MLLMTemplateType.ernie_vl_thinking,
+        template_cls=ERNIE_VLTemplate,
+        is_thinking=True,
+        thinking_prefix='\n<think>\n',
+        default_system=ERNIE_VL_SYSTEM))
+
+
+class PaddleOCR1_5Template(PaddleOCRTemplate):
+    version = 'v1_5'
+    skip_prompt = True
+    support_padding_free = True
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+        base_model = self.get_base_model(model)
+        input_ids = inputs['input_ids']
+        pixel_values = inputs.pop('pixel_values')
+        image_grid_thw = inputs.get('image_grid_thw')
+        inputs_embeds = base_model.model.language_model.embed_tokens(input_ids)
+        if pixel_values is not None:
+            image_embeds = base_model.model.get_image_features(
+                pixel_values, image_grid_thw, return_dict=True).pooler_output
+            image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+            image_mask = base_model.model.get_placeholder_mask(
+                input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds)
+            inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    ERNIETemplateMeta(
+        MLLMTemplateType.paddle_ocr_1_5, prompt=['User: {{QUERY}}\nAssistant:\n'], template_cls=PaddleOCR1_5Template))

swift/template/templates/bert.py

@@ -0,0 +1,5 @@
+from ..constant import LLMTemplateType
+from ..register import TemplateMeta, register_template
+
+register_template(
+    TemplateMeta(LLMTemplateType.bert, prefix=[], prompt=['{{QUERY}}[SEP]'], chat_sep=['[SEP]'], auto_add_bos=True))

swift/template/templates/deepseek.py

@@ -0,0 +1,628 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from dataclasses import dataclass, field
+from PIL import Image, ImageOps
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from typing import Any, Dict, List, Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Prompt, findall
+
+
+@dataclass
+class DeepseekTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: [['bos_token_id']])
+    prompt: Prompt = field(default_factory=lambda: ['User: {{QUERY}}\n\nAssistant:'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: [['eos_token_id']])
+    suffix: Prompt = field(default_factory=lambda: [['eos_token_id']])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: [['bos_token_id'], '{{SYSTEM}}\n\n'])
+
+
+register_template(DeepseekTemplateMeta(LLMTemplateType.deepseek, ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.deepseek_coder,
+        prefix=['{{SYSTEM}}'],
+        prompt=['### Instruction:\n{{QUERY}}\n### Response:\n'],
+        chat_sep=['\n<|EOT|>\n'],
+        suffix=['\n<|EOT|>'],
+        stop_words=['<|EOT|>'],
+        default_system=('You are an AI programming assistant, utilizing the Deepseek Coder model, '
+                        'developed by Deepseek Company, and you only answer questions related to computer science. '
+                        'For politically sensitive questions, security and privacy issues, '
+                        'and other non-computer science questions, you will refuse to answer\n')))
+
+
+class DeepseekVLTemplate(Template):
+    image_placeholder = ['<image_placeholder>']
+    skip_prompt = False
+    use_model = True
+    placeholder_tokens = ['<image_placeholder>']
+
+    image_token_num_per_image: int = 576
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        is_janus = getattr(self, 'is_janus', False)
+
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        processor = self.processor
+        input_ids, labels = encoded['input_ids'], encoded['labels']
+
+        if not inputs.generate_mode:  # understanding task
+            idx_list = findall(input_ids, processor.image_id)  # '<image_placeholder>'
+            new_input_ids, new_labels = [], []
+            lo = 0
+            for hi in idx_list:
+                new_input_ids += input_ids[lo:hi]
+                if labels is not None:
+                    new_labels += labels[lo:hi]
+                image_tokens = [processor.image_id] * processor.num_image_tokens
+                if is_janus:
+                    image_tokens = [processor.image_start_id] + image_tokens + [processor.image_end_id]
+                new_input_ids += image_tokens
+                new_labels += [-100] * len(image_tokens)
+                lo = hi + 1
+            new_input_ids += input_ids[lo:]
+            if labels is not None:
+                new_labels += labels[lo:]
+            else:
+                new_labels = None
+            if is_janus:
+                from janus.models.processing_vlm import VLChatProcessorOutput
+            else:
+                from deepseek_vl.models.processing_vlm import VLChatProcessorOutput
+
+            images_outputs = processor.image_processor(images, return_tensors='pt')
+            output = VLChatProcessorOutput(
+                sft_format=None,
+                input_ids=torch.tensor(new_input_ids),
+                pixel_values=images_outputs.pixel_values,
+                num_image_tokens=torch.tensor([processor.num_image_tokens] * len(idx_list)))
+            encoded = {'output': output, 'input_ids': new_input_ids, 'labels': new_labels}
+            return encoded
+
+        else:  # image generation task
+            if self.is_training:
+                raise NotImplementedError('Only support the inference of generation of Janus series models.')
+            sft_format = self.tokenizer.decode(input_ids)
+            prompt = sft_format + processor.image_start_tag
+            input_ids = processor.tokenizer.encode(prompt)
+            input_ids = torch.LongTensor(input_ids)
+
+            encoded = {'input_ids': input_ids, 'labels': labels, 'generate_mode': inputs.generate_mode}
+            return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not inputs.get('generate_mode'):
+            inputs['pixel_values'] = inputs['pixel_values'].to(dtype=self.model_info.torch_dtype)
+            inputs_embeds = model.prepare_inputs_embeds(**inputs)
+            return {'inputs_embeds': inputs_embeds}
+        else:
+            return inputs
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        gene_img_list = [b.get('generate_mode') for b in batch]
+        if all(gene_img_list):
+            generate_mode = True
+        elif not any(gene_img_list):
+            generate_mode = False
+        else:
+            raise NotImplementedError('Do not support understanding and image generation tasks in one batch.')
+
+        if not generate_mode:
+            output = self.fetch_inputs(batch, ['output'])['output']
+            batched_output = dict(self.processor.batchify(output))
+            res = super()._data_collator(batch, padding_to=padding_to)
+            return {**batched_output, **res}
+        else:
+            res = super()._data_collator(batch, padding_to=padding_to)
+            res['generate_mode'] = generate_mode
+            return res
+
+    def generate(self, model, *args, **kwargs):
+        if not kwargs.get('generate_mode'):
+            return super().generate(model, *args, **kwargs)
+
+        else:
+            # generate how many number of images for each prompt, it is named parallel_size in the author's code
+            parallel_size = kwargs['generation_config'].num_return_sequences
+            temperature = kwargs['generation_config'].temperature
+            cfg_weight = get_env_args('cfg_weight', float, 5.0)
+
+            input_ids = kwargs['input_ids']  # [bsz, max_input_token_num]
+            bsz, max_input_token_num = input_ids.shape
+            tokens = torch.zeros((bsz, parallel_size * 2, max_input_token_num),
+                                 dtype=torch.int).cuda()  # [bsz, parallel_size*2, max_input_token_num]
+            for i in range(parallel_size * 2):
+                tokens[:, i, :] = input_ids
+                if i % 2 != 0:
+                    tokens[:, i, 1:-1] = self.processor.pad_id
+
+            inputs_embeds = model.language_model.get_input_embeddings()(
+                tokens)  # [bsz, parallel_size*2, max_input_token_num, 2048]
+
+            generated_tokens = torch.zeros(
+                (bsz, parallel_size, self.image_token_num_per_image),
+                dtype=torch.int).cuda()  # [bsz, 16, image_token_num_per_image] placeholder for the generated tokens
+
+            # set the first two dimensions into one dimension for batch size
+            inputs_embeds = inputs_embeds.reshape(bsz * parallel_size * 2, max_input_token_num, -1)
+            generated_tokens = generated_tokens.reshape(bsz * parallel_size, self.image_token_num_per_image)
+
+            for i in range(self.image_token_num_per_image):  # generate the tokens of image in a auto-regression way
+                outputs = model.language_model.model(
+                    inputs_embeds=inputs_embeds,
+                    use_cache=True,
+                    past_key_values=outputs.past_key_values if i != 0 else None)
+                hidden_states = outputs.last_hidden_state
+
+                logits = self.model.gen_head(hidden_states[:, -1, :])
+                logit_cond = logits[0::2, :]
+                logit_uncond = logits[1::2, :]
+
+                logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
+                probs = torch.softmax(logits / temperature, dim=-1)
+
+                next_token = torch.multinomial(probs, num_samples=1)
+                generated_tokens[:, i] = next_token.squeeze(dim=-1)  # [parallel_size, self.image_token_num_per_image]
+
+                next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1)
+                img_embeds = model.prepare_gen_img_embeds(next_token)  # [parallel_size * 2, 2048]
+                inputs_embeds = img_embeds.unsqueeze(dim=1)  # [parallel_size * 2, 1, 2048]
+
+            # no need to reset the original first two dimensions, waiting for the update of the upper layer
+            # inputs_embeds = inputs_embeds.reshape(bsz, parallel_size*2, -1)
+            # generated_tokens = generated_tokens.reshape(bsz, parallel_size, self.image_token_num_per_image)
+
+            return {'sequences': generated_tokens}
+
+    def decode_generate_ids(self, generate_ids: List[int], **kwargs) -> Any:
+        if 'template_inputs' not in kwargs or not kwargs['template_inputs'].generate_mode:
+            return super().decode_generate_ids(generate_ids, **kwargs)
+        else:
+            img_size = get_env_args('img_size', int, 384)
+            patch_size = 16
+
+            num_to_decode = 1  # for now, generate_ids is a 1D list
+
+            generate_ids = torch.tensor(generate_ids).unsqueeze(0)  # [num_to_decode=1, self.image_token_num_per_image]
+
+            dec = self.model.gen_vision_model.decode_code(
+                generate_ids.to(dtype=torch.int),
+                shape=[num_to_decode, 8, img_size // patch_size, img_size // patch_size])
+            dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)  # [num_to_decode, H, W, ch=3]
+
+            dec = np.clip((dec + 1) / 2 * 255, 0, 255)
+
+            visual_img = np.zeros((num_to_decode, img_size, img_size, 3), dtype=np.uint8)
+            visual_img[:, :, :] = dec
+
+            img_list = []
+            for i in range(num_to_decode):
+                cur_img = Image.fromarray(visual_img[i])
+                img_list.append({'type': 'image', 'image': cur_img})
+            return img_list
+
+
+@dataclass
+class DeepseekVLTemplateMeta(DeepseekTemplateMeta):
+    default_system: Optional[str] = ('You are a helpful language and vision assistant. '
+                                     'You are able to understand the visual content that the user provides, '
+                                     'and assist the user with a variety of tasks using natural language.')
+
+
+register_template(DeepseekVLTemplateMeta(
+    MLLMTemplateType.deepseek_vl,
+    template_cls=DeepseekVLTemplate,
+))
+
+
+class DeepseekJanus(DeepseekVLTemplate):
+    is_janus = True
+    image_placeholder = ['<image_placeholder>\n']
+
+
+register_template(DeepseekVLTemplateMeta(MLLMTemplateType.deepseek_janus, template_cls=DeepseekJanus))
+
+
+class DeepseekOCR(Template):
+    version = 'v1'
+    image_placeholder = ['<image>\n']
+
+    def init_env_args(self):
+        # Delay loading dynamic modules that require specific transformers versions
+        # These will be loaded lazily in _preprocess_image when actually needed
+        # This avoids triggering transformers version compatibility issues for vllm backend
+        super().init_env_args()
+        self._BasicImageTransform = None
+        self._dynamic_preprocess = None
+        self.crop_mode = get_env_args('crop_mode', bool, True)
+        self.base_size = get_env_args('base_size', int, 1024)
+        # image_size will be set after detecting version (v1: 640, v2: 768)
+        self._image_size_override = get_env_args('image_size', int, None)
+
+    @property
+    def image_size(self):
+        if self._image_size_override is not None:
+            return self._image_size_override
+        return 768 if self.version == 'v2' else 640
+
+    @property
+    def crop_threshold(self):
+        # v1: 640, v2: 768
+        return 768 if self.version == 'v2' else 640
+
+    def _load_dynamic_modules(self):
+        """Lazily load dynamic modules from model repository."""
+        if self._BasicImageTransform is None:
+            model_dir = self.model_info.model_dir
+            model_type_name = 'deepseekocr2' if self.version == 'v2' else 'deepseekocr'
+            self._BasicImageTransform = get_class_from_dynamic_module(f'modeling_{model_type_name}.BasicImageTransform',
+                                                                      model_dir)
+            self._dynamic_preprocess = get_class_from_dynamic_module(f'modeling_{model_type_name}.dynamic_preprocess',
+                                                                     model_dir)
+
+    @property
+    def BasicImageTransform(self):
+        self._load_dynamic_modules()
+        return self._BasicImageTransform
+
+    @property
+    def dynamic_preprocess(self):
+        self._load_dynamic_modules()
+        return self._dynamic_preprocess
+
+    def _preprocess_image(self, images, image_token_id):
+        # Code borrowed from
+        # https://modelscope.cn/models/deepseek-ai/DeepSeek-OCR/file/view/master/modeling_deepseekocr.py?status=1
+        # https://modelscope.cn/models/deepseek-ai/DeepSeek-OCR-2/file/view/master/modeling_deepseekocr2.py?status=1
+        crop_mode = self.crop_mode
+        patch_size = 16
+        downsample_ratio = 4
+        valid_img_tokens = 0
+        w, h = images[0].size
+        ratio = 1 - ((max(w, h) - min(w, h)) / (max(w, h)))
+        crop_threshold = self.crop_threshold
+        image_size = self.image_size
+
+        image_transform = self.BasicImageTransform(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True)
+        images_list, images_crop_list = [], []
+        tokenized_str = []
+        images_spatial_crop = []
+        for image in images:
+            if crop_mode:
+                if image.size[0] <= crop_threshold and image.size[1] <= crop_threshold:
+                    crop_ratio = [1, 1]
+                else:
+                    if crop_mode:
+                        images_crop_raw, crop_ratio = self.dynamic_preprocess(image)
+                    else:
+                        crop_ratio = [1, 1]
+                """process the global view"""
+                global_view = ImageOps.pad(
+                    image, (self.base_size, self.base_size), color=tuple(int(x * 255) for x in image_transform.mean))
+
+                if self.base_size == 1024:
+                    valid_img_tokens += int(256 * ratio)
+                elif self.base_size == 1280:
+                    valid_img_tokens += int(400 * ratio)
+
+                images_list.append(image_transform(global_view).to(torch.bfloat16))
+                width_crop_num, height_crop_num = crop_ratio
+
+                images_spatial_crop.append([width_crop_num, height_crop_num])
+
+                if width_crop_num > 1 or height_crop_num > 1:
+                    """process the local views"""
+                    for i in range(len(images_crop_raw)):
+                        images_crop_list.append(image_transform(images_crop_raw[i]).to(torch.bfloat16))
+
+                if image_size == 640:
+                    valid_img_tokens += len(images_crop_list) * 100
+                elif image_size == 768:
+                    valid_img_tokens += len(images_crop_list) * 144
+
+                num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
+                num_queries_base = math.ceil((self.base_size // patch_size) / downsample_ratio)
+                """add image tokens"""
+                # v1: adds newline token after each row, v2: no newline tokens in rows
+                if self.version == 'v2':
+                    tokenized_image = ([image_token_id] * num_queries_base) * num_queries_base
+                    tokenized_image += [image_token_id]
+                    if width_crop_num > 1 or height_crop_num > 1:
+                        tokenized_image += ([image_token_id] * (num_queries * width_crop_num)) * (
+                            num_queries * height_crop_num)
+                else:
+                    tokenized_image = ([image_token_id] * num_queries_base + [image_token_id]) * num_queries_base
+                    tokenized_image += [image_token_id]
+                    if width_crop_num > 1 or height_crop_num > 1:
+                        tokenized_image += ([image_token_id] * (num_queries * width_crop_num) + [image_token_id]) * (
+                            num_queries * height_crop_num)
+                tokenized_str.append(tokenized_image)
+            else:
+                """process the global view"""
+                if image_size <= crop_threshold:
+                    image = image.resize((image_size, image_size))
+                global_view = ImageOps.pad(
+                    image, (image_size, image_size), color=tuple(int(x * 255) for x in image_transform.mean))
+                images_list.append(image_transform(global_view).to(torch.bfloat16))
+
+                if self.base_size == 1024:
+                    valid_img_tokens += int(256 * ratio)
+                elif self.base_size == 1280:
+                    valid_img_tokens += int(400 * ratio)
+                elif self.base_size == 640:
+                    valid_img_tokens += int(100 * 1)
+                elif self.base_size == 512:
+                    valid_img_tokens += int(64 * 1)
+                elif self.base_size == 768:
+                    valid_img_tokens += int(144 * 1)
+
+                width_crop_num, height_crop_num = 1, 1
+
+                images_spatial_crop.append([width_crop_num, height_crop_num])
+                """add image tokens"""
+                num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
+
+                # v1: adds newline token after each row, v2: no newline tokens in rows
+                if self.version == 'v2':
+                    tokenized_image = ([image_token_id] * num_queries) * num_queries
+                    tokenized_image += [image_token_id]
+                else:
+                    tokenized_image = ([image_token_id] * num_queries + [image_token_id]) * num_queries
+                    tokenized_image += [image_token_id]
+                tokenized_str.append(tokenized_image)
+        if len(images_list) == 0:
+            images_ori = torch.zeros((1, 3, self.image_size, self.image_size))
+            images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
+            images_crop = torch.zeros((1, 3, self.base_size, self.base_size))
+
+        else:
+            images_ori = torch.stack(images_list, dim=0)
+            images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
+            if images_crop_list:
+                images_crop = torch.stack(images_crop_list, dim=0)
+            else:
+                images_crop = torch.zeros((1, 3, self.base_size, self.base_size))
+        return tokenized_str, images_ori, images_crop, images_spatial_crop
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        image_token = self._tokenize('<image>')
+        idx_list = findall(input_ids, image_token)
+        if idx_list:
+            tokenized_str, images_ori, images_crop, images_spatial_crop = self._preprocess_image(
+                inputs.images, image_token[0])
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                lambda i: tokenized_str[i])
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded['loss_scale'] = loss_scale
+            encoded['images'] = [(images_crop, images_ori)]
+            encoded['images_seq_mask'] = (torch.tensor(input_ids) == image_token[0])[None]
+            encoded['images_spatial_crop'] = images_spatial_crop
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        images = self.gather_list(batch, 'images')
+        if images:
+            res['images'] = images
+        images_seq_mask = [x['images_seq_mask'] for x in batch if x.get('images_seq_mask') is not None]
+        images_spatial_crop = self.concat_tensor(batch, 'images_spatial_crop', 0)
+        padding_side = self.padding_side if self.is_training else 'left'
+        if images_seq_mask:
+            max_len = max([x.shape[1] for x in images_seq_mask])
+            res['images_seq_mask'] = torch.concat([
+                F.pad(x, (0, max_len - x.shape[1]) if padding_side == 'right' else (max_len - x.shape[1], 0))
+                for x in images_seq_mask
+            ])
+        if images_spatial_crop is not None:
+            res['images_spatial_crop'] = images_spatial_crop
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.deepseek_ocr,
+        prefix=['<｜begin▁of▁sentence｜>'],
+        prompt=['{{QUERY}}'],
+        chat_sep=None,
+        template_cls=DeepseekOCR))
+
+
+class DeepseekOCR2(DeepseekOCR):
+    version = 'v2'
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.deepseek_ocr2,
+        prefix=['<｜begin▁of▁sentence｜>'],
+        prompt=['{{QUERY}}'],
+        chat_sep=None,
+        template_cls=DeepseekOCR2))
+
+
+class UnlimitedOCR(DeepseekOCR):
+    image_placeholder = ['<image>']  # Remove trailing newline; override the parent class default
+
+    def init_env_args(self):
+        super().init_env_args()
+        self._device_fixed = False  # Instance variable; avoid sharing state across multiple instances.
+
+    def _fix_device(self):
+        if not self._device_fixed and self.model is not None:
+            try:
+                vision_device = next(self.model.model.vision_model.parameters()).device
+                self.model.model.image_newline.data = self.model.model.image_newline.data.to(vision_device)
+                self.model.model.view_seperator.data = self.model.model.view_seperator.data.to(vision_device)
+                self._device_fixed = True
+            except Exception:
+                pass
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        self._fix_device()
+        return super()._encode(inputs)
+
+    def _load_dynamic_modules(self):
+        if self._BasicImageTransform is None:
+            model_dir = self.model_info.model_dir
+            self._BasicImageTransform = get_class_from_dynamic_module('modeling_unlimitedocr.BasicImageTransform',
+                                                                      model_dir)
+            self._dynamic_preprocess = get_class_from_dynamic_module('modeling_unlimitedocr.dynamic_preprocess',
+                                                                     model_dir)
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.unlimited_ocr,
+        prefix=[['bos_token_id']],
+        prompt=['{{QUERY}}'],
+        chat_sep=None,
+        template_cls=UnlimitedOCR,
+    ))
+
+
+@dataclass
+class DeepseekV2_5TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<｜begin▁of▁sentence｜>{{SYSTEM}}'])
+    prompt: Prompt = field(default_factory=lambda: ['<｜User｜>{{QUERY}}<｜Assistant｜>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<｜end▁of▁sentence｜>'])
+    suffix: Prompt = field(default_factory=lambda: ['<｜end▁of▁sentence｜>'])
+
+
+register_template(DeepseekV2_5TemplateMeta(LLMTemplateType.deepseek_v2_5))
+
+register_template(DeepseekV2_5TemplateMeta(LLMTemplateType.deepseek_r1, is_thinking=True, thinking_prefix='<think>\n'))
+
+
+class DeepseekV3_1Template(Template):
+    jinja_enable_thinking_key = 'thinking'
+    non_thinking_prefix_only_after_user = True
+
+
+register_template(
+    DeepseekV2_5TemplateMeta(
+        LLMTemplateType.deepseek_v3_1,
+        agent_template='deepseek_v3_1',
+        is_thinking=True,
+        template_cls=DeepseekV3_1Template,
+        thinking_prefix='<think>',
+        non_thinking_prefix='</think>',
+        history_thinking_prefix='</think>'))
+
+REASONING_EFFORT_MAX = (
+    'Reasoning Effort: Absolute maximum with no shortcuts permitted.\n'
+    'You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve '
+    'the root cause, rigorously stress-testing your logic against all potential paths, edge cases, '
+    'and adversarial scenarios.\n'
+    'Explicitly write out your entire deliberation process, documenting every intermediate step, '
+    'considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n')
+
+
+class DeepseekV4Template(DeepseekV3_1Template):
+
+    def init_env_args(self):
+        super().init_env_args()
+        # reasoning_effort: "max", "high", or None
+        self.reasoning_effort = get_env_args('reasoning_effort', str, None)
+        if self.reasoning_effort is None:
+            self.reasoning_effort = 'high' if self.enable_thinking else None
+        self.enable_thinking = self.reasoning_effort in ('max', 'high')
+        self.chat_template_kwargs['reasoning_effort'] = self.reasoning_effort
+
+    def _get_enable_thinking(self, inputs=None):
+        reasoning_effort = None if inputs is None else inputs.chat_template_kwargs.get('reasoning_effort')
+        if reasoning_effort is not None:
+            return reasoning_effort in ('max', 'high')
+        return super()._get_enable_thinking(inputs)
+
+    def _get_system(self, inputs):
+        system = super()._get_system(inputs)
+        reasoning_effort = inputs.chat_template_kwargs.get('reasoning_effort')
+        if reasoning_effort is None:
+            reasoning_effort = self.reasoning_effort
+        if reasoning_effort == 'max' and self._get_enable_thinking(inputs):
+            system = REASONING_EFFORT_MAX + (system or '')
+        return system
+
+
+register_template(
+    DeepseekV2_5TemplateMeta(
+        LLMTemplateType.deepseek_v4,
+        agent_template='deepseek_v4',
+        is_thinking=True,
+        template_cls=DeepseekV4Template,
+        thinking_prefix='<think>',
+        non_thinking_prefix='</think>',
+        history_thinking_prefix='</think>'))
+
+
+class DeepseekVL2Template(DeepseekVLTemplate):
+    image_placeholder = ['<image>\n']
+    placeholder_tokens = ['<image>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from deepseek_vl2.models.processing_deepseek_vl_v2 import VLChatProcessorOutput
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        processor = self.processor
+        input_ids, labels = encoded['input_ids'], encoded['labels']
+        images_seq_mask = [False] * len(input_ids)
+        idx_list = findall(input_ids, processor.image_token_id)  # '<image>'
+        _, images_list, _, images_spatial_crop, num_image_tokens = processor.tokenize_with_images(
+            '<image>' * len(images), images, cropping=len(images) <= 2)
+        new_num_tokens = 0
+        for idx, n_image_tokens in zip(idx_list, num_image_tokens):
+            image_tokens = [processor.image_token_id] * n_image_tokens
+            input_ids = input_ids[:idx] + image_tokens + input_ids[idx + 1:]
+            if labels is not None:
+                labels = labels[:idx] + [-100] * n_image_tokens + labels[idx + 1:]
+            images_seq_mask = images_seq_mask[:idx] + [True] * n_image_tokens + images_seq_mask[idx + 1:]
+            new_num_tokens += n_image_tokens - 1
+
+        output = VLChatProcessorOutput(
+            sft_format=None,
+            input_ids=torch.tensor(input_ids),
+            target_ids=torch.tensor(input_ids),
+            images=torch.stack(images_list) if images_list else torch.zeros((0, 3, 384, 384)),
+            images_seq_mask=torch.tensor(images_seq_mask),
+            images_spatial_crop=torch.tensor(images_spatial_crop),
+            num_image_tokens=num_image_tokens)
+        output.images = output.images.to(dtype=self.model_info.torch_dtype)
+        encoded = {'output': output, 'input_ids': input_ids, 'labels': labels}
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs['images_seq_mask'] = inputs['images_seq_mask'].to(torch.bool)
+        inputs['images_spatial_crop'] = inputs['images_spatial_crop'].to(torch.long)
+        inputs_embeds = model.prepare_inputs_embeds(**inputs)
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    DeepseekV2_5TemplateMeta(
+        MLLMTemplateType.deepseek_vl2,
+        prompt=['<|User|>: {{QUERY}}\n\n<|Assistant|>:'],
+        template_cls=DeepseekVL2Template,
+    ))
+
+register_template(
+    DeepseekVLTemplateMeta(
+        MLLMTemplateType.deepseek_janus_pro,
+        prompt=['<|User|>: {{QUERY}}\n\n<|Assistant|>:'],
+        template_cls=DeepseekJanus))

swift/template/templates/dots.py

@@ -0,0 +1,62 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from typing import Any, Dict, List, Literal
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from .utils import TemplateMeta
+
+
+class DotsOCRTemplate(Template):
+    image_token_id = 151665
+    placeholder_tokens = ['<|imgpad|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        from qwen_vl_utils import fetch_image
+        assert media_type == 'image'
+        inputs.images[index] = fetch_image({'image': inputs.images[index]})
+        if self.mode == 'lmdeploy':
+            return ['<|img|>', [-100], '<|endofimg|>']
+        else:
+            return ['<|img|><|imgpad|><|endofimg|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+
+        images = inputs.images
+        media_token = self.image_token_id
+        media_inputs = processor.image_processor(images=images, videos=None, return_tensors='pt', do_resize=False)
+        media_grid_thw = media_inputs['image_grid_thw']
+        idx_list = findall(input_ids, media_token)
+        merge_length = processor.image_processor.merge_size**2
+
+        def _get_new_tokens(i):
+            token_len = (media_grid_thw[i].prod() // merge_length)
+            return [media_token] * token_len
+
+        input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+        encoded.update(media_inputs)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.dots_ocr,
+        prefix=[''],
+        prompt=['<|user|>{{QUERY}}<|endofuser|><|assistant|>'],
+        chat_sep=['<|endofassistant|>'],
+        suffix=['<|endofassistant|>'],
+        system_prefix=['<|system|>{{SYSTEM}}<|endofsystem|>\n'],
+        template_cls=DotsOCRTemplate,
+    ))

swift/template/templates/gemma.py

@@ -0,0 +1,444 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import numpy as np
+import torch
+import torch.nn.functional as F
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_logger, upper_bound
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word, findall
+from ..vision_utils import load_audio, load_vllm_video
+
+logger = get_logger()
+
+
+@dataclass
+class GemmaTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<bos>'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['<start_of_turn>user\n{{QUERY}}<end_of_turn>\n<start_of_turn>model\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<end_of_turn>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<end_of_turn>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<bos><start_of_turn>system\n{{SYSTEM}}<end_of_turn>\n'])
+
+
+register_template(GemmaTemplateMeta(LLMTemplateType.gemma))
+
+
+class PaliGemmaTemplate(Template):
+    placeholder_tokens = ['<image>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        if self.mode == 'vllm':
+            self.prompt = ['{{QUERY}}']
+            return []
+        else:
+            self.prompt = ['{{QUERY}}\n']
+            return ['<image>' * self.processor.image_seq_length + '<bos>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        raw_image = inputs.images
+        processor = self.processor
+        if encoded['labels'] is not None:
+            n = upper_bound(0, len(encoded['labels']), lambda idx: encoded['labels'][idx] == -100)
+            n2 = len(encoded['labels']) - n
+            encoded['token_type_ids'] = [0] * n + [1] * n2
+        else:
+            encoded['token_type_ids'] = [0] * len(encoded['input_ids'])
+        if raw_image:
+            model_inputs = processor(text='<image>' * len(raw_image), images=raw_image, return_tensors='pt')
+            encoded['pixel_values'] = model_inputs['pixel_values'].to(self.model_info.torch_dtype)
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.paligemma,
+        prefix=[],
+        prompt=['{{QUERY}}\n'],
+        chat_sep=None,
+        suffix=['<eos>'],
+        template_cls=PaliGemmaTemplate,
+    ))
+
+
+@dataclass
+class Gemma3TextTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<bos>'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['<start_of_turn>user\n{{QUERY}}<end_of_turn>\n<start_of_turn>model\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<end_of_turn>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<end_of_turn>'])
+
+
+class Gemma3Template(Template):
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        if inputs.system is not None:
+            system = inputs.system
+            inputs.system = None
+            inputs.messages[0]['content'] = system + '\n\n' + inputs.messages[0]['content']
+        for message in inputs.messages:
+            if message['role'] == 'assistant' and isinstance(message['content'], str):
+                message['content'] = message['content'].strip('\n')
+        return super()._swift_encode(inputs)
+
+
+register_template(Gemma3TextTemplateMeta(LLMTemplateType.gemma3_text, template_cls=Gemma3Template))
+
+
+class Gemma3VisionTemplate(Gemma3Template):
+    boi_token_id = 255999
+    placeholder_tokens = ['<start_of_image>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<start_of_image>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
+
+        encoded = super()._encode(inputs)
+        if inputs.images:
+            input_ids = encoded['input_ids']
+            labels = encoded['labels']
+            loss_scale = encoded.get('loss_scale', None)
+            idx_list = findall(input_ids, self.boi_token_id)
+            img_tokens = self._tokenize(self.processor.full_image_sequence)
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                lambda _: img_tokens)
+
+            # TODO: customize
+            processor_kwargs = Gemma3ProcessorKwargs._defaults['images_kwargs']
+            image_inputs = self.processor.image_processor(inputs.images, **processor_kwargs)
+            image_inputs['pixel_values'] = torch.as_tensor(np.array(image_inputs['pixel_values']))
+            image_inputs.pop('num_crops')
+
+            array_ids = np.array(input_ids)
+            mm_token_type_ids = np.zeros_like(input_ids)
+            mm_token_type_ids[array_ids == self.processor.image_token_id] = 1
+            encoded['token_type_ids'] = mm_token_type_ids.tolist()
+            encoded['input_ids'] = input_ids
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['labels'] = labels
+            encoded['loss_scale'] = loss_scale
+        return encoded
+
+
+register_template(GemmaTemplateMeta(MLLMTemplateType.gemma3_vision, template_cls=Gemma3VisionTemplate))
+
+
+class Gemma3nTemplate(Gemma3Template):
+    boi_token_id = 255999
+    boa_token_id = 256000
+    placeholder_tokens = ['<start_of_image>', '<start_of_audio>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            if self.mode == 'vllm':
+                return ['<image_soft_token>']
+            else:
+                return ['\n\n<start_of_image>']
+        elif media_type == 'audio':
+            if self.mode == 'vllm':
+                raise ValueError('Audio is not supported in vLLM')
+            inputs.audios[index] = load_audio(inputs.audios[index], self.processor.feature_extractor.sampling_rate)
+            return ['<start_of_audio>']
+        else:
+            raise ValueError(f'Unsupported media type: {media_type}. Supported types are: image, audio')
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from transformers.models.gemma3n.processing_gemma3n import Gemma3nProcessorKwargs
+
+        # Input validation
+        if not inputs.images and not inputs.audios and not inputs.messages:
+            raise ValueError('Provide at least one of `images`, `audios`, or `messages`.')
+
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+
+        # Initialize token_type_ids and other outputs
+        array_ids = np.array(input_ids)
+        mm_token_type_ids = np.zeros_like(input_ids)
+
+        # Handle images
+        if inputs.images:
+            idx_list = findall(input_ids, self.boi_token_id)
+            img_tokens = self._tokenize(processor.full_image_sequence[2:])
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                lambda _: img_tokens)
+
+            # Process images
+            processor_kwargs = Gemma3nProcessorKwargs._defaults.get('images_kwargs', {})
+            image_inputs = processor.image_processor(inputs.images, **processor_kwargs)
+            image_inputs['pixel_values'] = torch.as_tensor(
+                np.array(image_inputs['pixel_values']), dtype=self.model_info.torch_dtype)
+            if 'num_crops' in image_inputs:
+                image_inputs.pop('num_crops')
+            encoded.update(image_inputs)
+
+        # Handle audios
+        if inputs.audios:
+            audio_idx_list = findall(input_ids, self.boa_token_id)
+            if audio_idx_list:
+                # Get audio token sequence from processor
+                audio_tokens = self._tokenize(processor.full_audio_sequence)
+                input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, audio_idx_list,
+                                                                    lambda _: audio_tokens)
+
+                # Process audios
+                processor_kwargs = Gemma3nProcessorKwargs._defaults.get('audio_kwargs', {})
+                audio_inputs = processor.feature_extractor(inputs.audios, **processor_kwargs)
+
+                if 'input_features' in audio_inputs:
+                    audio_inputs['input_features'] = torch.tensor(audio_inputs['input_features']).to(
+                        self.model_info.torch_dtype)
+                if 'input_features_mask' in audio_inputs:
+                    audio_inputs['input_features_mask'] = torch.tensor(audio_inputs['input_features_mask'])
+                encoded.update(audio_inputs)
+
+        # Update array_ids after token extension
+        array_ids = np.array(input_ids)
+        mm_token_type_ids = np.zeros_like(input_ids)
+
+        if hasattr(processor, 'image_token_id') and processor.image_token_id is not None:
+            mm_token_type_ids[array_ids == processor.image_token_id] = 1
+
+        if hasattr(processor, 'audio_token_id') and processor.audio_token_id is not None:
+            mm_token_type_ids[array_ids == processor.audio_token_id] = 3
+
+        encoded['token_type_ids'] = mm_token_type_ids.tolist()
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Handle multimodal data collation for Gemma3n, including audio features"""
+        res = super()._data_collator_mm_data(batch)
+
+        # Handle audio features like other templates do
+        input_features = [b['input_features'] for b in batch if b.get('input_features') is not None]
+        input_features_mask = [b['input_features_mask'] for b in batch if b.get('input_features_mask') is not None]
+
+        if input_features:
+            res['input_features'] = torch.concat(input_features)
+        if input_features_mask:
+            res['input_features_mask'] = torch.concat(input_features_mask)
+
+        return res
+
+
+register_template(GemmaTemplateMeta(MLLMTemplateType.gemma3n, template_cls=Gemma3nTemplate))
+
+
+class Gemma4Template(Template):
+    placeholder_tokens = ['<|image|>', '<|audio|>', '<|video|>']
+    non_thinking_prefix_only_after_user = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|image|>']
+        elif media_type == 'audio':
+            if self.mode != 'vllm':
+                inputs.audios[index] = load_audio(inputs.audios[index], self.processor.feature_extractor.sampling_rate)
+            return ['<|audio|>']
+        elif media_type == 'video':
+            if self.mode == 'vllm':
+                num_frames = self.processor.video_processor.num_frames
+                video_data, video_metadatas = load_vllm_video(inputs.videos[index], num_frames)
+                inputs.videos[index] = [(video_data, video_metadatas)]
+            return ['<|video|>']
+
+    def _get_system(self, inputs: StdTemplateInputs) -> Optional[str]:
+        system = super()._get_system(inputs)
+        if self._get_enable_thinking(inputs):
+            system = '<|think|>\n' + (system or '')
+        return system
+
+    def _add_non_thinking_prefix(self, inputs: StdTemplateInputs, thinking_prefix: str = '<|channel>thought'):
+        return super()._add_non_thinking_prefix(inputs, thinking_prefix=thinking_prefix)
+
+    def _remove_thinking_content(self, content: str, thinking_suffix: str = '<channel|>') -> str:
+        return super()._remove_thinking_content(content, thinking_suffix=thinking_suffix)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        split_token = self._tokenize('\n')
+        media_inputs = self.processor(
+            text='\n'.join(['<|image|>'] * len(inputs.images) + ['<|video|>'] * len(inputs.videos)
+                           + ['<|audio|>'] * len(inputs.audios)),
+            audio=inputs.audios or None,
+            images=inputs.images or None,
+            videos=inputs.videos or None,
+            return_tensors='pt',
+            add_special_tokens=False,
+        )
+        splited_tokens = self._split_list(media_inputs['input_ids'][0].tolist(), split_token)
+        media_inputs.pop('input_ids')
+        media_inputs.pop('attention_mask')
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        mm_mask = [False] * len(input_ids)
+
+        idx_list = []
+        for key in ['image', 'video', 'audio']:
+            token_id = getattr(self.config, f'{key}_token_id', None)
+            if token_id is None:
+                continue
+            idx_list += findall(input_ids, token_id)
+        sorted_order = sorted(range(len(idx_list)), key=lambda i: idx_list[i])
+        idx_list = [idx_list[i] for i in sorted_order]
+        splited_tokens = [splited_tokens[i] for i in sorted_order]
+
+        def _get_new_tokens(i):
+            return splited_tokens[i]
+
+        if idx_list:
+            input_ids, labels, loss_scale, mm_mask = self._extend_tokens(
+                input_ids, labels, loss_scale, idx_list, _get_new_tokens, mm_mask=mm_mask)
+        for key in [
+                'pixel_values', 'image_position_ids', 'pixel_values_videos', 'video_position_ids', 'input_features',
+                'input_features_mask'
+        ]:
+            if key in media_inputs:
+                encoded[key] = media_inputs[key]
+        # unpad input_features
+        # https://github.com/vllm-project/vllm/blob/v0.23.0/vllm/model_executor/models/gemma4_mm.py#L747-L758
+        if 'input_features' in encoded and 'input_features_mask' in encoded:
+            masks = encoded['input_features_mask']
+            features = encoded['input_features']
+            if isinstance(masks, torch.Tensor) and masks.ndim >= 2:
+                bool_masks = masks.bool()
+                encoded['input_features'] = torch.stack([f[m] for f, m in zip(features, bool_masks)])
+                encoded['input_features_mask'] = torch.stack([m[m] for m in bool_masks])
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        encoded['mm_token_type_ids'] = self.create_mm_token_type_ids(input_ids, mm_mask)
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        for key in ['image_position_ids', 'video_position_ids']:
+            value = [b[key] for b in batch if b.get(key) is not None]
+            if value:
+                res[key] = torch.concat(value)
+        input_features = [b['input_features'] for b in batch if b.get('input_features') is not None]
+        if input_features:
+            input_features_mask = [b['input_features_mask'] for b in batch if b.get('input_features_mask') is not None]
+            max_len = max([x.shape[1] for x in input_features_mask])
+            res['input_features'] = torch.concat([F.pad(x, (0, 0, 0, max_len - x.shape[1])) for x in input_features])
+            res['input_features_mask'] = torch.concat(
+                [F.pad(x, (0, max_len - x.shape[1])) for x in input_features_mask])
+        return res
+
+
+@dataclass
+class Gemma4TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<bos>'])
+    prompt: Prompt = field(default_factory=lambda: ['<|turn>user\n{{QUERY}}<turn|>\n<|turn>model\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<turn|>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<turn|>\n'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<bos><|turn>system\n{{SYSTEM}}<turn|>\n'])
+    stop_words: List[Word] = field(default_factory=lambda: ['<eos>', '<turn|>', '<|tool_response>'])
+
+
+register_template(
+    Gemma4TemplateMeta(MLLMTemplateType.gemma4_nothinking, template_cls=Gemma4Template, agent_template='gemma4'))
+
+register_template(
+    Gemma4TemplateMeta(
+        MLLMTemplateType.gemma4,
+        template_cls=Gemma4Template,
+        agent_template='gemma4',
+        is_thinking=True,
+        non_thinking_prefix='<|channel>thought\n<channel|>'))
+
+
+class DiffusionGemmaTemplate(Gemma4Template):
+    is_encoder_decoder = True
+    skip_prompt = True
+
+    @property
+    def loss_scale(self):
+        loss_scale = super().loss_scale
+        if self.is_training and loss_scale.base_strategy != 'last_round':
+            logger.warning_once('DiffusionGemmaTemplate only supports the `last_round` base strategy for loss scaling. '
+                                'Setting loss_scale.base_strategy to `last_round`.')
+        loss_scale.base_strategy = 'last_round'
+        return loss_scale
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        inputs = super()._data_collator(batch, padding_to=padding_to)
+        if self.is_training:
+            inputs = self._update_inputs(inputs)
+        return inputs
+
+    # Code reference: https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/DiffusionGemma_(26B-A4B)-Sudoku.ipynb  # noqa
+    def _update_inputs(self, inputs):
+        canvas_length = self.config.canvas_length
+        if inputs['labels'].shape[0] > 1:
+            raise ValueError('per_device_train_batch_size must be 1 for diffusion gemma')
+        first_idx = (inputs['labels'] != -100).int().argmax().item()
+        prompt_ids = inputs['input_ids'][:, :first_idx]
+        # reserve one slot at the end of the canvas for the explicit eos token expected by
+        # the diffusion sampler as the termination signal.
+        response_length = inputs['input_ids'].shape[1] - first_idx
+        if response_length > canvas_length - 1:
+            raise ValueError(f'response length ({response_length}) exceeds canvas_length-1 ({canvas_length - 1}); '
+                             'please use a shorter response or increase canvas_length.')
+        canvas_content = inputs['input_ids'][:, first_idx:first_idx + canvas_length - 1]
+        # x0: clean canvas padded to canvas_length; loss is only computed on response + eos.
+        device = prompt_ids.device
+        eos_token_id = self.tokenizer.eos_token_id
+        pad_token_id = self.tokenizer.pad_token_id
+        x0 = torch.full((prompt_ids.shape[0], canvas_length), pad_token_id, dtype=torch.long, device=device)
+        n = canvas_content.shape[1]
+        x0[:, :n] = canvas_content
+        # explicitly append eos as the canvas-end signal expected by the diffusion sampler.
+        # without it, sampler keeps denoising the trailing positions during inference and emits garbage.
+        x0[:, n] = eos_token_id
+        labels = x0.clone()
+        labels[:, n + 1:] = -100
+
+        # forward diffusion: per-sample noise level t ∈ [min, max], replace tokens with random vocab ids
+        t = torch.empty((), device=device).uniform_(0.1, 1.)
+        noise_mask = torch.rand(canvas_length, device=device) < t
+        random_tokens = torch.randint(0, self.config.text_config.vocab_size, (canvas_length, ), device=device)
+        decoder_input_ids = torch.where(noise_mask, random_tokens, x0)
+        return {'input_ids': prompt_ids, 'decoder_input_ids': decoder_input_ids, 'labels': labels}
+
+    def compute_sft_loss(self, model, inputs: Dict[str, Any], num_items_in_batch: Optional[int] = None, trainer=None):
+        if trainer.args.gradient_checkpointing:
+            raise ValueError('Gradient checkpointing is not supported for diffusion gemma')
+        outputs = model(**inputs)
+        logits = outputs.logits.view(-1, outputs.logits.shape[-1])
+        labels = inputs['labels'].view(-1)
+        outputs.loss = F.cross_entropy(logits, labels, reduction='sum')
+        outputs.loss = outputs.loss / num_items_in_batch
+        return outputs
+
+
+register_template(
+    Gemma4TemplateMeta(
+        MLLMTemplateType.diffusion_gemma,
+        template_cls=DiffusionGemmaTemplate,
+        agent_template='gemma4',
+        is_thinking=True,
+        non_thinking_prefix='<|channel>thought\n<channel|>'))

swift/template/templates/glm.py

@@ -0,0 +1,680 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import inspect
+import torch
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args, get_packed_seq_params
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word, findall
+from ..vision_utils import load_batch, load_video_cogvlm2, load_video_hf
+
+
+@dataclass
+class GLMTemplateMeta(TemplateMeta):
+    auto_add_bos: bool = True
+
+
+class GLM4Template(Template):
+    strip_newline = True
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        res_context_list, loss_scale_list, answer_len = super()._swift_encode(inputs)
+        if self.strip_newline:
+            for i, res_context in enumerate(res_context_list):
+                # The last round or is tool_call.
+                if isinstance(res_context, str) and (res_context.endswith('<|assistant|>\n')
+                                                     or res_context.endswith('<think></think>\n')) and (
+                                                         i + 1 >= len(res_context_list)
+                                                         or '<|observation|>' in res_context_list[i + 1]):
+                    res_context_list[i] = res_context_list[i][:-len('\n')]
+        return res_context_list, loss_scale_list, answer_len
+
+    def decode_generate_ids(self, *args, **kwargs):
+        response = super().decode_generate_ids(*args, **kwargs)
+        return response.lstrip('\n') if self.strip_newline else response
+
+
+register_template(
+    GLMTemplateMeta(
+        LLMTemplateType.chatglm2,
+        prefix=['{{SYSTEM}}'],
+        prompt=['[Round {{ROUND1}}]\n\n问：{{QUERY}}\n\n答：'],
+        chat_sep=['\n\n']))
+
+
+@dataclass
+class ChatGLM4TemplateMeta(GLMTemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['<|user|>\n{{QUERY}}<|assistant|>\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=list)
+    suffix: Prompt = field(default_factory=lambda: ['<|user|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|system|>\n{{SYSTEM}}'])
+
+    agent_template: str = 'chatglm4'
+    stop_words: List[Word] = field(default_factory=lambda: ['<|endoftext|>', '<|user|>', '<|observation|>'])
+
+
+@dataclass
+class GLM4TemplateMeta(ChatGLM4TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['[gMASK]<sop>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['[gMASK]<sop><|system|>\n{{SYSTEM}}'])
+    agent_template: str = 'glm4'
+
+
+@dataclass
+class GLM4_5TemplateMeta(GLM4TemplateMeta):
+    agent_template: str = 'glm4_5'
+    is_thinking: bool = True
+    non_thinking_prefix: str = '<think></think>\n'
+    history_thinking_prefix: str = '<think></think>\n'
+
+
+class ChatGLM4VTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        if self.mode == 'vllm':
+            return ['<|begin_of_image|><|endoftext|><|end_of_image|>']
+        return [[-100]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, -100)
+        if idx_list:
+            idx = idx_list[0]
+            image = inputs.images[0]
+            placeholder = '<|begin_of_image|><|endoftext|><|end_of_image|>'
+            placeholder_id = self.processor.encode(placeholder, add_special_tokens=False)
+            input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
+            if labels is not None:
+                labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
+            messages = inputs.messages
+            messages[0]['image'] = image
+            inputs2: Dict[str, Any] = self.processor.apply_chat_template(messages, return_dict=True)
+            encoded['images'] = inputs2['images']
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['position_ids'] = list(range(len(input_ids)))
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = torch.concat(images)
+        return res
+
+
+class GLM4vPackingTemplateMixin:
+    support_padding_free = True  # https://github.com/huggingface/transformers/issues/39685
+    use_model = True
+
+    def packing_row(self, row: List[Dict[str, Any]]) -> Dict[str, Any]:
+        for r in row:
+            r_copy = r.copy()
+            r_copy['input_ids'] = torch.tensor(r_copy['input_ids'])[None]
+            r.update(self._get_position_ids(r_copy))
+        packed = super().packing_row(row)
+        return packed
+
+    def _get_position_ids(self, inputs: Dict[str, Any]):
+        base_model = self.get_base_model(self._get_model())
+        attention_mask = inputs.get('attention_mask_2d')
+        if attention_mask is None:
+            attention_mask = inputs.get('attention_mask')
+        kwargs = {}
+        input_ids = inputs['input_ids']
+        get_rope_index = base_model.model.get_rope_index
+        if 'mm_token_type_ids' in inspect.signature(get_rope_index).parameters:
+            kwargs['mm_token_type_ids'] = self.create_mm_token_type_ids(input_ids)
+        elif not self.is_training:
+            return {}
+        position_ids, _ = get_rope_index(
+            input_ids,
+            image_grid_thw=inputs.get('image_grid_thw'),
+            video_grid_thw=inputs.get('video_grid_thw'),
+            attention_mask=attention_mask,
+            **kwargs)
+        return {'position_ids': self._concat_text_position_ids(position_ids)}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if not self.padding_free:
+            res.update(self._get_position_ids(res))
+        if 'position_ids' in res and self.is_training:
+            position_ids = res['position_ids']
+            res['position_ids'] = position_ids[1:]
+            res['text_position_ids'] = text_position_ids = position_ids[0]
+            # https://github.com/huggingface/transformers/pull/40194
+            if text_position_ids.shape[0] == 1:
+                res.update(get_packed_seq_params(text_position_ids))
+        return res
+
+    def _patch_create_causal_mask(self, modeling_module):
+        create_causal_mask = modeling_module.create_causal_mask
+
+        def new_create_causal_mask(*args, **kwargs):
+            position_ids = kwargs.get('position_ids')
+            if position_ids is not None and position_ids.dim() == 3:
+                kwargs['position_ids'] = None
+            return create_causal_mask(*args, **kwargs)
+
+        modeling_module.create_causal_mask = new_create_causal_mask
+
+
+register_template(
+    ChatGLM4TemplateMeta(MLLMTemplateType.chatglm4v, template_cls=ChatGLM4VTemplate, suffix=['<|endoftext|>']))
+
+register_template(ChatGLM4TemplateMeta(LLMTemplateType.chatglm4, template_cls=GLM4Template))
+
+
+class GLM4VTemplate(GLM4vPackingTemplateMixin, Template):
+    begin_of_image_token = 151339
+    end_of_image_token = 151340
+    begin_of_video_token = 151341
+    end_of_video_token = 151342
+    placeholder_tokens = ['<|image|>', '<|video|>']
+
+    def init_processor(self, processor) -> None:
+        if processor is None:
+            return
+        super().init_processor(processor)
+        if not getattr(GLM4VTemplate, '_patched', False) and self.padding_free:
+            GLM4VTemplate._patched = True
+            from transformers.models.glm4v import modeling_glm4v
+            self._patch_create_causal_mask(modeling_glm4v)
+        self.image_token = self._tokenize('<|image|>')[0]
+        self.video_token = self._tokenize('<|video|>')[0]
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        # TODO: model video infer bug
+        if self.mode == 'vllm':
+            if media_type == 'image':
+                return ['<|begin_of_image|><|image|><|end_of_image|>']
+            elif media_type == 'video':
+                return ['<|begin_of_video|><|video|><|end_of_video|>']
+        assert media_type in ['image']
+        if media_type == 'image':
+            return [[-100]]
+        elif media_type == 'video':
+            return [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        image_idx_list = findall(input_ids, -100)
+        video_idx_list = findall(input_ids, -200)
+        if image_idx_list:
+            images = inputs.images
+            image_inputs = processor.image_processor(images=images, return_tensors='pt')
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['image_grid_thw'] = image_grid_thw = image_inputs['image_grid_thw']
+            merge_length = processor.image_processor.merge_size**2
+            added_tokens_len = 0
+            for i, idx in enumerate(image_idx_list):
+                num_image_tokens = image_grid_thw[i].prod() // merge_length
+                image_tokens = [self.begin_of_image_token
+                                ] + [self.image_token] * num_image_tokens + [self.end_of_image_token]
+
+                input_ids = input_ids[:added_tokens_len + idx] + image_tokens + input_ids[added_tokens_len + idx + 1:]
+                if labels is not None:
+                    labels = labels[:added_tokens_len + idx] + [-100] * len(image_tokens) + labels[added_tokens_len
+                                                                                                   + idx + 1:]
+                added_tokens_len += len(image_tokens) - 1
+
+        if video_idx_list:
+            # TODO: model video infer bug
+            assert len(
+                video_idx_list) <= 1, f'GLM4.1V model only support 1 video, but detected {len(video_idx_list)} <video> '
+            assert not image_idx_list, "GLM4.1V model doesn't support inputs containing both video and images"
+
+            video_fnames = inputs.videos
+            import numpy as np
+            from transformers.image_utils import load_image
+            from transformers.video_utils import load_video
+            video_metadata = []
+            videos = []
+            for fname in video_fnames:
+                if isinstance(fname, (list, tuple)) and isinstance(fname[0], str):
+                    video = [np.array(load_image(image_fname)) for image_fname in fname]
+                    # create a 4D video because `load_video` always returns a 4D array
+                    video = np.stack(video)
+                    metadata = None
+                else:
+                    video, metadata = load_video(fname)
+                videos.append(video)
+                video_metadata.append(metadata)
+            videos = [videos]
+            video_metadata = [video_metadata]
+
+            videos_inputs = processor.video_processor(videos=videos, video_metadata=video_metadata, return_tensors='pt')
+            encoded['pixel_values_videos'] = videos_inputs['pixel_values_videos']
+            encoded['video_grid_thw'] = video_grid_thw = videos_inputs['video_grid_thw']
+            timestamps = videos_inputs.pop('timestamps')
+            num_frames = len(video_grid_thw)
+            video_structure = [self.begin_of_video_token]
+            if hasattr(timestamps, 'tolist'):
+                timestamps_list = timestamps.tolist()[0]
+            else:
+                timestamps_list = timestamps[0] if isinstance(timestamps[0], list) else timestamps
+            unique_timestamps = []
+            for idx in range(0, len(timestamps_list)):
+                unique_timestamps.append(timestamps_list[idx])
+            selected_timestamps = unique_timestamps[:num_frames]
+            while len(selected_timestamps) < num_frames:
+                selected_timestamps.append(selected_timestamps[-1] if selected_timestamps else 0)
+            merge_length = processor.video_processor.merge_size**2
+            added_tokens_len = 0
+            for frame_idx in range(num_frames):
+                timestamp_sec = selected_timestamps[frame_idx]
+                num_image_tokens = video_grid_thw[frame_idx].prod() // merge_length
+                timestamp_sec_token = processor.tokenizer(str(timestamp_sec))['input_ids']
+                frame_structure = [self.begin_of_image_token] + [self.image_token] * num_image_tokens + \
+                    [self.end_of_image_token] + timestamp_sec_token
+                video_structure += frame_structure
+            video_structure += [self.end_of_video_token]
+
+            for i, idx in enumerate(video_idx_list):
+                # BUG in GLM4.1V?: All video placeholder take same tokens
+                # https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/models/glm4v/processing_glm4v.py#L165-L194
+                input_ids = input_ids[:added_tokens_len + idx] + video_structure + \
+                    input_ids[added_tokens_len + idx + 1:]
+                if labels is not None:
+                    labels = labels[:added_tokens_len + idx] + [-100] * len(video_structure) + \
+                        labels[added_tokens_len + idx + 1:]
+                added_tokens_len += len(video_structure) - 1
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        # TODO: check video
+        if not self.is_training:
+            return inputs
+        input_ids = inputs['input_ids']
+        inputs_embeds = model.get_input_embeddings()(input_ids)
+        inputs_embeds = self._get_inputs_embeds_hf(inputs_embeds, inputs, model.visual, self.processor, model.config)
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(GLM4TemplateMeta(LLMTemplateType.glm4, template_cls=GLM4Template, thinking_prefix='<think>'))
+register_template(GLM4TemplateMeta(MLLMTemplateType.glm4v, template_cls=GLM4VTemplate))
+
+
+class GLM4_5Template(GLM4Template):
+
+    def _jinja_encode(self, inputs: StdTemplateInputs):
+        for message in inputs.messages:
+            if message['role'] == 'assistant' and isinstance(message['content'],
+                                                             str) and message['content'].endswith('<|observation|>'):
+                message['content'] = message['content'][:-len('<|observation|>')]
+        return super()._jinja_encode(inputs)
+
+
+register_template(GLM4_5TemplateMeta(LLMTemplateType.glm4_5, template_cls=GLM4_5Template))
+
+
+@dataclass
+class GLM4_7TemplateMeta(GLM4_5TemplateMeta):
+    prompt: Prompt = field(default_factory=lambda: ['<|user|>{{QUERY}}<|assistant|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['[gMASK]<sop><|system|>{{SYSTEM}}'])
+
+    thinking_prefix: str = '<think>'
+    non_thinking_prefix: str = '</think>'
+    history_thinking_prefix: str = '</think>'
+
+
+register_template(GLM4_7TemplateMeta(
+    LLMTemplateType.glm4_7,
+    template_cls=GLM4_5Template,
+    agent_template='glm4_7',
+))
+
+register_template(GLM4_7TemplateMeta(
+    LLMTemplateType.glm5_1,
+    template_cls=GLM4_5Template,
+    agent_template='glm5_1',
+))
+
+
+class GLM5_2Template(GLM4_5Template):
+
+    def init_env_args(self):
+        super().init_env_args()
+        # reasoning_effort: "max" or "high"
+        self.reasoning_effort = get_env_args('reasoning_effort', str, 'max')
+        self.chat_template_kwargs['reasoning_effort'] = self.reasoning_effort
+
+    def _get_system(self, inputs):
+        system = super()._get_system(inputs)
+        reasoning_effort = inputs.chat_template_kwargs.get('reasoning_effort')
+        if reasoning_effort is None:
+            reasoning_effort = self.reasoning_effort
+        if self._get_enable_thinking(inputs):
+            effort_str = f'Reasoning Effort: {reasoning_effort.capitalize()}'
+            if system:
+                system = f'{effort_str}<|system|>{system}'
+            else:
+                system = effort_str
+        return system
+
+
+register_template(
+    GLM4_7TemplateMeta(
+        LLMTemplateType.glm5_2,
+        template_cls=GLM5_2Template,
+        agent_template='glm5_1',
+        non_thinking_prefix='<think></think>',
+        history_thinking_prefix='<think></think>',
+    ))
+
+
+class GLM4_5VTemplate(GLM4vPackingTemplateMixin, GLM4_5Template):
+    placeholder_tokens = ['<|image|>', '<|video|>']
+    strip_newline = False
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|begin_of_image|><|image|><|end_of_image|>']
+        elif media_type == 'video':
+            return ['<|begin_of_video|><|video|><|end_of_video|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        for mm_type in ['image', 'video']:
+            mm_token = f'<|{mm_type}|>'
+            mm_token_id = self._tokenize(mm_token)[0]
+
+            idx_list = findall(input_ids, mm_token_id)
+            if idx_list:
+                split_token = self._tokenize('\n')[0]
+                mm_data = getattr(inputs, f'{mm_type}s')
+                if mm_type == 'image':
+                    kwargs = {'images': mm_data}
+                else:
+                    videos, video_metadata = load_video_hf(mm_data)
+                    kwargs = {'videos': [videos], 'video_metadata': [video_metadata]}
+                mm_inputs = self.processor(text='\n'.join([mm_token] * len(mm_data)), return_tensors='pt', **kwargs)
+                splited_tokens = self._split_list(mm_inputs['input_ids'][0].tolist(), split_token)
+                for key in ['input_ids', 'token_type_ids', 'attention_mask']:
+                    mm_inputs.pop(key, None)
+                input_ids, encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                    input_ids, encoded['labels'], encoded['loss_scale'], idx_list, lambda i: splited_tokens[i])
+                encoded.update(mm_inputs)
+        encoded['input_ids'] = input_ids
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+        input_ids = inputs['input_ids']
+        base_model = self.get_base_model(model)
+        inputs_embeds = base_model.model.language_model.embed_tokens(input_ids)
+        inputs_embeds = self._get_inputs_embeds_hf(inputs_embeds, inputs, model.visual, self.processor, model.config)
+        return {'inputs_embeds': inputs_embeds}
+
+    def init_processor(self, processor) -> None:
+        super().init_processor(processor)
+        if not getattr(GLM4_5VTemplate, '_patched', False) and self.padding_free:
+            GLM4_5VTemplate._patched = True
+            from transformers.models.glm4v_moe import modeling_glm4v_moe
+            self._patch_create_causal_mask(modeling_glm4v_moe)
+
+
+register_template(GLM4_5TemplateMeta(MLLMTemplateType.glm4_5v, template_cls=GLM4_5VTemplate))
+
+glm4z1rumination_system = (
+    '你是一个专业的深度研究助手，通过提供的工具与模拟浏览器交互，来帮助用户完成深度信息调研和报告撰写任务。'
+    '今年是 2025 年。\n\n'
+    '<核心要求>\n'
+    '- 首先分解用户请求，得到包含多个子要求的列表\n'
+    '- 制定初始研究计划\n'
+    '- 进行多轮迭代搜索和页面浏览（at least 10 function calls）：\n'
+    '    * 根据已获得的信息调整研究计划和关键词\n'
+    '    * 打开页面阅读，从发现的内容中识别新的关键概念/名词\n'
+    '    * 从搜索结果中提取新的关键词继续搜索\n'
+    '    * 访问并仔细阅读相关页面，识别新的关键概念/名词\n\n'
+    '<重要配置>\n'
+    '- 采用语言\n'
+    '    * 搜索关键词：英文\n'
+    '    * 思考：英文\n\n'
+    '<可调用的工具列表>\n'
+    '[{"name": "search", "description": "Execute a search query and return search results. '
+    'Use this function when you need to find information about a specific topic.", '
+    '"parameters": {"type": "object", "properties": {"query": {"type": "string", '
+    '"description": "Search query string, use English words unless it is a proper name in Chinese"}}, '
+    '"required": ["query"], "additionalProperties": false}}, '
+    '{"name": "click", "description": "Click a link in the search results and navigate to the corresponding page. '
+    'Use this function when you need to view detailed content of a specific search result.", '
+    '"parameters": {"type": "object", "properties": {"link_id": {"type": "integer", '
+    '"description": "The link ID to click (from the sequence number in search results)"}}, '
+    '"required": ["link_id"], "additionalProperties": false}}, '
+    '{"name": "open", "description": "Open a specific website. Get content from any website with its URL.", '
+    '"parameters": {"type": "object", "properties": {"url": {"type": "string", '
+    '"description": "The target website URL or domain"}}, "required": ["url"], "additionalProperties": false}}, '
+    '{"name": "finish", "description": "Finish the task. '
+    'Use this function when you have found the information you need.", '
+    '"parameters": {"type": "object", "properties": {}, "additionalProperties": false}}]')
+
+register_template(
+    GLM4TemplateMeta(
+        LLMTemplateType.glm4_z1_rumination,
+        template_cls=GLM4Template,
+        default_system=glm4z1rumination_system,
+        is_thinking=True))
+
+codegeex4_system = '你是一位智能编程助手，你叫CodeGeeX。你会为用户回答关于编程、代码、计算机方面的任何问题，并提供格式规范、可以执行、准确安全的代码，并在必要时提供详细的解释。'
+
+register_template(ChatGLM4TemplateMeta(LLMTemplateType.codegeex4, default_system=codegeex4_system))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.longwriter_llama, ['[INST]'], ['{{QUERY}}[/INST]'], ['[INST]'], ['<|end_of_text|>'],
+        system_prefix=['<<SYS>>\n{{SYSTEM}}\n<</SYS>>\n\n']))
+
+
+class CogTemplate(Template):
+    placeholder_tokens = ['<|reserved_special_token_0|>']
+
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        return []
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        model = self.model
+        image = inputs.images or []
+        history_inputs = inputs.to_history()
+        inputs2 = model.build_conversation_input_ids(
+            self.processor, query=history_inputs['query'], history=history_inputs['history'], images=image)
+        image_token_len = inputs2['token_type_ids'].sum().item()
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        encoded['token_type_ids'] = [0] + [1] * image_token_len + [0] * len(input_ids[1:])
+        encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * image_token_len + input_ids[1:]
+        if labels is not None:
+            encoded['labels'] = labels[:1] + [-100] * image_token_len + labels[1:]
+        if len(image) > 0:
+            encoded['images'] = [[img.to(dtype=self.model_info.torch_dtype)] for img in inputs2['images']]
+            if 'cross_images' in inputs2:
+                # is cogagent
+                encoded['cross_images'] = [[cross_img.to(dtype=self.model_info.torch_dtype)]
+                                           for cross_img in inputs2['cross_images']]
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        keys = ['images', 'cross_images']
+        for key in keys:
+            if key in batch[0]:
+                res[key] = [b[key][0] for b in batch]
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.cogagent_chat,
+        prefix=['<s>'],
+        prompt=[' [INST] {{QUERY}} [/INST] '],
+        chat_sep=[],
+        suffix=['</s>'],
+        template_cls=CogTemplate,
+    ))
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.cogagent_vqa,
+        prefix=['<s>'],
+        prompt=['<EOI>Question: {{QUERY}} Answer:'],
+        chat_sep=None,
+        suffix=['</s>'],
+        template_cls=CogTemplate))
+
+
+@dataclass
+class CogVLMTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: [['bos_token_id']])
+    prompt: Prompt = field(default_factory=lambda: ['Question: {{QUERY}} Answer:'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['\n'])
+
+
+register_template(CogVLMTemplateMeta(MLLMTemplateType.cogvlm, template_cls=CogTemplate))
+
+register_template(CogVLMTemplateMeta(MLLMTemplateType.cogvlm2, template_cls=CogTemplate))
+
+
+class Cog2VideoTemplate(CogTemplate):
+    use_model = True
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        model = self.model
+        encoded = super(CogTemplate, self)._encode(inputs)
+        videos_path = inputs.videos or []
+        video = load_batch(videos_path, load_video_cogvlm2)
+        history_inputs = inputs.to_history()
+        inputs2 = model.build_conversation_input_ids(
+            self.processor,
+            query=history_inputs['query'],
+            history=history_inputs['history'],
+            images=video,
+            template_version='chat')
+        video_token_len = inputs2['token_type_ids'].sum().item()
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        encoded['token_type_ids'] = [0] + [1] * video_token_len + [0] * len(input_ids[1:])
+        encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * video_token_len + input_ids[1:]
+        if labels is not None:
+            encoded['labels'] = labels[:1] + [-100] * video_token_len + labels[1:]
+        if len(video) > 0:
+            dtype = model.dtype
+            encoded['images'] = [[img.to(dtype=dtype)] for img in inputs2['images']]
+        return encoded
+
+
+register_template(CogVLMTemplateMeta(
+    MLLMTemplateType.cogvlm2_video,
+    template_cls=Cog2VideoTemplate,
+))
+
+
+class GLMEdgeVTemplate(Template):
+    placeholder_tokens = ['<|begin_of_image|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<|begin_of_image|>' * 578]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            encoded['pixel_values'] = torch.tensor(self.processor(images).pixel_values)
+        return encoded
+
+
+register_template(
+    ChatGLM4TemplateMeta(
+        MLLMTemplateType.glm_edge_v,
+        prompt=['<|user|>\\n{{QUERY}}\\n<|assistant|>\\n'],
+        chat_sep=['\\n'],
+        system_prefix=['<|system|>\\n{{SYSTEM}}\\n'],
+        suffix=['<|endoftext|>'],
+        template_cls=GLMEdgeVTemplate,
+    ))
+
+
+class GLMOCRTemplate(Template):
+    begin_of_image_token = 59256
+    end_of_image_token = 59257
+    placeholder_tokens = ['<|image|>']
+
+    def init_processor(self, processor) -> None:
+        if processor is None:
+            return
+        super().init_processor(processor)
+        self.image_token = self._tokenize('<|image|>')[0]
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type in ['image']
+        if self.mode == 'vllm':
+            return ['<|begin_of_image|><|image|><|end_of_image|>']
+        return [[-100]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        image_idx_list = findall(input_ids, -100)
+        if image_idx_list:
+            images = inputs.images
+            image_inputs = processor.image_processor(images=images, return_tensors='pt')
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['image_grid_thw'] = image_grid_thw = image_inputs['image_grid_thw']
+            merge_length = processor.image_processor.merge_size**2
+            added_tokens_len = 0
+            for i, idx in enumerate(image_idx_list):
+                num_image_tokens = image_grid_thw[i].prod() // merge_length
+                image_tokens = [self.begin_of_image_token
+                                ] + [self.image_token] * num_image_tokens + [self.end_of_image_token]
+
+                input_ids = input_ids[:added_tokens_len + idx] + image_tokens + input_ids[added_tokens_len + idx + 1:]
+                if labels is not None:
+                    labels = labels[:added_tokens_len + idx] + [-100] * len(image_tokens) + labels[added_tokens_len
+                                                                                                   + idx + 1:]
+                added_tokens_len += len(image_tokens) - 1
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+        input_ids = inputs['input_ids']
+        inputs_embeds = model.get_input_embeddings()(input_ids)
+        inputs_embeds = self._get_inputs_embeds_hf(inputs_embeds, inputs, model.visual, self.processor, model.config)
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(GLM4TemplateMeta(
+    MLLMTemplateType.glm_ocr,
+    template_cls=GLMOCRTemplate,
+))

swift/template/templates/idefics3.py

@@ -0,0 +1,37 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from typing import Any, Dict
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import align_image_inputs
+
+
+class Idefics3Template(Template):
+    placeholder_tokens = ['<image>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        processor = self.processor
+        prompt = self.processor.decode(encoded['input_ids'])
+        if images:
+            image_inputs = processor(text=prompt, images=images, return_tensors='pt', add_special_tokens=False)
+            image_token = 128257  # <image>
+            encoded['input_ids'], encoded['labels'] = align_image_inputs(encoded['input_ids'], encoded['labels'],
+                                                                         image_inputs['input_ids'][0], image_token)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.idefics3,
+        prefix=['<|begin_of_text|>'],
+        prompt=['User:{{QUERY}}<end_of_utterance>\nAssistant:'],
+        chat_sep=['<end_of_utterance>\n'],
+        suffix=['<end_of_utterance>'],
+        system_prefix=['System:{{SYSTEM}}<end_of_utterance>\n'],
+        template_cls=Idefics3Template,
+    ))

swift/template/templates/internlm.py

@@ -0,0 +1,194 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from dataclasses import dataclass, field
+from PIL import Image
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType, RMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word
+from ..vision_utils import load_file
+from .utils import ChatmlTemplateMeta
+
+INTERNLM_SYSTEM = (
+    'You are an AI assistant whose name is InternLM (书生·浦语).\n'
+    '- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). '
+    'It is designed to be helpful, honest, and harmless.\n'
+    '- InternLM (书生·浦语) can understand and communicate fluently in the language chosen '
+    'by the user such as English and 中文.')
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.internlm,
+        prefix=['<s>'],
+        prompt=['<|User|>:{{QUERY}}\n<|Bot|>:'],
+        chat_sep=['<eoa>\n'],
+        suffix=['<eoa>'],
+        default_system=INTERNLM_SYSTEM,
+        system_prefix=['<s><|System|>:{{SYSTEM}}\n']))
+
+register_template(ChatmlTemplateMeta(LLMTemplateType.internlm2, default_system=INTERNLM_SYSTEM))
+
+register_template(ChatmlTemplateMeta(RMTemplateType.internlm2_reward, suffix=['<|im_end|>\n<|reward|>']))
+
+
+class InternLMXComposer2Template(Template):
+    image_placeholder = ['</s>']
+    version = 'v2'
+    skip_prompt = False
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'video':
+            inputs.images.insert(inputs.image_idx, inputs.videos[index])
+            inputs.image_idx += 1
+        return self.image_placeholder
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        model = self.model
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+
+        if self.version == 'v2.5':
+            hd_num = 24
+            if len(images) > 1:
+                hd_num = 6
+            hd_num = get_env_args('hd_num', int, hd_num)
+            images_origin = images
+            images = []
+            for image in images_origin:
+                if isinstance(image, Image.Image):
+                    Image_transform = get_class_from_dynamic_module('ixc_utils.Image_transform', model.model_dir)
+                    images.append(Image_transform(image, hd_num=hd_num))
+                else:
+                    load_video = get_class_from_dynamic_module('ixc_utils.load_video', model.model_dir)
+                    frame2img = get_class_from_dynamic_module('ixc_utils.frame2img', model.model_dir)
+                    Video_transform = get_class_from_dynamic_module('ixc_utils.Video_transform', model.model_dir)
+                    image = load_video(load_file(image))
+                    image = frame2img(image, model.font)
+                    images.append(Video_transform(image, hd_num=hd_num))
+        elif self.version == 'v2-4khd':
+            hd_num = 55
+            hd_num = get_env_args('hd_num', int, hd_num)
+            HD_transform = get_class_from_dynamic_module('ixc_utils.HD_transform', model.model_dir)
+            images = [HD_transform(image, hd_num=hd_num) for image in images]
+        images = [model.vis_processor(image).to(model.dtype) for image in images]
+        encoded['images'] = images
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        batch_size = len(inputs['input_ids'])
+        res = []
+        im_mask = []
+        length = inputs['length']
+        for i in range(batch_size):
+            input_ids = inputs['input_ids'][i].tolist()[:length[i]]
+            input_ids.append(2)  # add dummy </s>
+            labels = inputs.get('labels')
+            if labels is not None:
+                labels = labels[i].tolist()[:length[i]]
+                labels.append(2)
+            else:
+                labels = []
+            images = inputs['images'][i]
+            res_inputs_embeds = []
+            res_labels = []
+            wrap_im_mask = []
+            pre_i, i, idx = 0, 0, 0
+            device = model.device
+            internlm2_model = model.model
+            if not hasattr(internlm2_model, 'tok_embeddings'):
+                internlm2_model = internlm2_model.model
+            tok_embeddings = internlm2_model.tok_embeddings
+            if len(images) > 0:
+                images = torch.concat([model.img2emb(image[None])[0] for image in images], dim=0)
+            add_bos = False
+            while i < len(input_ids):
+                if input_ids[i] == 2:  # replace_token
+                    res_input_ids = torch.tensor(([1] if add_bos else []) + input_ids[pre_i:i], device=device)
+                    if not add_bos and self.version != 'v2.5':
+                        add_bos = True
+                    res_inputs_embeds.append(tok_embeddings(res_input_ids[None])[0])
+                    wrap_im_mask += [0] * len(res_input_ids)
+                    res_labels += ([-100] if add_bos else []) + labels[pre_i:i]
+                    if len(images) > 0 and idx < images.shape[0]:
+                        res_inputs_embeds.append(images[idx].to(device))
+                        wrap_im_mask += [1] * images.shape[1]
+                        res_labels += [-100] * images.shape[1]
+                    idx += 1
+                    i += 1
+                    pre_i = i
+                    continue
+                i += 1
+            if len(labels) == 0:
+                res_labels = None
+            im_mask.append(torch.tensor(wrap_im_mask, dtype=torch.bool, device=device))
+            res.append({'inputs_embeds': torch.concat(res_inputs_embeds, dim=0), 'labels': res_labels})
+        res = Template._data_collator(self, res)
+        res['im_mask'] = self._pad_sequence(im_mask, 0)
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        res['length'] = [len(b['input_ids']) for b in batch]
+        res.update(self.fetch_inputs(batch, ['images']))
+        return res
+
+
+@dataclass
+class Xcomposer2TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['[UNUSED_TOKEN_146]user\n{{QUERY}}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['[UNUSED_TOKEN_145]\n'])
+    suffix: Prompt = field(default_factory=lambda: ['[UNUSED_TOKEN_145]'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<s>[UNUSED_TOKEN_146]system\n{{SYSTEM}}[UNUSED_TOKEN_145]\n'])
+    stop_words: List[Word] = field(default_factory=lambda: ['<|im_end|>'])
+
+
+register_template(
+    Xcomposer2TemplateMeta(
+        MLLMTemplateType.xcomposer2,
+        template_cls=InternLMXComposer2Template,
+        default_system=('You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).\n'
+                        '- InternLM-XComposer (浦语·灵笔) is a conversational language model that is developed by '
+                        'Shanghai AI Laboratory (上海人工智能实验室). '
+                        'It is designed to be helpful, honest, and harmless.\n'
+                        '- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen '
+                        'by the user such as English and 中文.'),
+    ))
+
+
+class InternLMXComposer2_5Template(InternLMXComposer2Template):
+    system = ('You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).\n'
+              '- InternLM-XComposer (浦语·灵笔) is a multi-modality conversational language model '
+              'that is developed by Shanghai AI Laboratory (上海人工智能实验室). '
+              'It is designed to be helpful, honest, and harmless.\n'
+              '- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen '
+              'by the user such as English and 中文.\n'
+              '- InternLM-XComposer (浦语·灵笔) is capable of comprehending and articulating responses effectively '
+              'based on the provided image.')
+    version = 'v2.5'
+
+
+class InternLMXComposer2_4khdTemplate(InternLMXComposer2Template):
+    version = 'v2-4khd'
+
+
+register_template(
+    Xcomposer2TemplateMeta(
+        MLLMTemplateType.xcomposer2_5,
+        template_cls=InternLMXComposer2_5Template,
+        default_system=InternLMXComposer2_5Template.system))
+
+register_template(
+    Xcomposer2TemplateMeta(
+        MLLMTemplateType.xcomposer2_4khd,
+        template_cls=InternLMXComposer2_4khdTemplate,
+        default_system=InternLMXComposer2_5Template.system))

swift/template/templates/internvl.py

@@ -0,0 +1,365 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from functools import partial
+from torch import nn
+from typing import Any, Dict, List, Literal
+
+from swift.utils import get_env_args, is_deepspeed_enabled
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from ..vision_utils import load_video_internvl, transform_image
+from .llm import GptOssTemplateMeta, GptTemplate
+from .microsoft import Phi3TemplateMeta
+from .utils import ChatmlTemplateMeta
+
+
+class InternvlTemplate(Template):
+    skip_prompt = False
+    num_image_token = None
+    placeholder_tokens = ['<IMG_CONTEXT>']
+    support_padding_free = True
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.input_size = get_env_args('input_size', int, 448)
+        self.max_num = get_env_args('max_num', int, 12)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            image_context = ['<image>\n']
+        else:
+            image_context = ['<img>', [-100], '</img>\n']
+        return image_context
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        idx_list = findall(input_ids, -100)
+        pixel_values = None
+        images = inputs.images
+        if images:
+            labels = encoded.get('labels')
+            if self.num_image_token is None:
+                self.num_image_token = int((self.input_size // 14)**2 * (0.5**2))
+            pixel_values_images = [transform_image(image, self.input_size, self.max_num) for image in images]
+            pixel_values = torch.cat(pixel_values_images, dim=0).to(self.model_info.torch_dtype)
+            image_bs = pixel_values.shape[0]
+
+            idx, idx2 = idx_list[0], idx_list[-1]  # remove [-100, -100]
+            img_tokens: List[int] = self.processor.encode(
+                '<IMG_CONTEXT>', add_special_tokens=False) * self.num_image_token * image_bs
+            input_ids = input_ids[:idx] + img_tokens + input_ids[idx2 + 1:]
+            if labels is not None:
+                labels = labels[:idx] + [-100] * len(img_tokens) + labels[idx2 + 1:]
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+        encoded['pixel_values'] = pixel_values
+        return encoded
+
+    def forward_context(self, model, inputs):
+        model_name = model.language_model.__class__.__name__.lower()
+        if self.padding_free and 'internlm2' in model_name:
+            position_ids = inputs['position_ids']
+            modeling_module = model.language_model.model.layers[0].attention.__class__
+            return self._patch_flash_attention_forward(modeling_module, position_ids, use_new_func=True)
+        else:
+            return super().forward_context(model, inputs)
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        embedding = model.get_input_embeddings()
+        device = embedding.weight.device
+        input_ids = inputs['input_ids']
+        inputs_embeds = embedding(input_ids).to(device=device)
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            pixel_values = pixel_values.to(device=device)
+            vit_embeds = model.extract_feature(pixel_values).to(device=device)
+            selected = (input_ids == self.processor.encode('<IMG_CONTEXT>', add_special_tokens=False)[0])
+            inputs_embeds[selected] = vit_embeds.reshape(-1, vit_embeds.shape[-1]).to(dtype=inputs_embeds.dtype)
+        elif is_deepspeed_enabled():
+            dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype)
+            vit_embeds = model.extract_feature(dummy_pixel_values).to(device=device)
+            inputs_embeds += vit_embeds.mean() * 0.
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.internvl,
+        default_system='You are an AI assistant whose name is InternLM (书生·浦语).',
+        template_cls=InternvlTemplate,
+        auto_add_bos=True))
+register_template(
+    Phi3TemplateMeta(
+        MLLMTemplateType.internvl_phi3,
+        default_system='You are an AI assistant whose name is Phi-3.',
+        template_cls=InternvlTemplate,
+        auto_add_bos=True))
+
+
+class Internvl2Template(InternvlTemplate):
+    VIDEO_SEGMENTS = 8
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.video_max_num = get_env_args('video_max_num', int, 1)
+        self.video_segments = get_env_args('video_segments', int, self.VIDEO_SEGMENTS)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        image_context = super().replace_tag('image', index, inputs)
+        if media_type == 'image':
+            return image_context
+        elif media_type == 'video':
+            load_video = partial(load_video_internvl, num_segments=self.video_segments)
+            return self.replace_video2image(load_video, inputs, lambda i: [f'Frame{i + 1}: '] + image_context)
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<ref>{ref}</ref>']
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<box>[{bbox}]</box>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super(InternvlTemplate, self)._encode(inputs)
+        input_ids = encoded['input_ids']
+        idx_list = findall(input_ids, -100)
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        images = inputs.images
+        if images:
+            has_video = bool(inputs.videos)
+            if self.num_image_token is None:
+                self.num_image_token = int((self.input_size // 14)**2 * (0.5**2))
+            max_num = self.max_num
+            if has_video:
+                max_num = self.video_max_num
+            pixel_values = [transform_image(image, self.input_size, max_num) for image in images]
+            num_patches = [pv.shape[0] for pv in pixel_values]
+            pixel_values = torch.cat(pixel_values).to(self.model_info.torch_dtype)
+        else:
+            pixel_values = None
+            num_patches = []
+        assert len(num_patches) == len(
+            idx_list), f'len(num_patches): {len(num_patches)}, len(idx_list): {len(idx_list)}'
+
+        def _get_new_tokens(i):
+            img_tokens: List[int] = self.processor.encode(
+                '<IMG_CONTEXT>', add_special_tokens=False) * self.num_image_token * num_patches[i]
+            return img_tokens
+
+        encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+            input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+        encoded['pixel_values'] = pixel_values
+        return encoded
+
+
+_internvl2_system = '你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。'
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.internvl2,
+        default_system=_internvl2_system,
+        template_cls=Internvl2Template,
+    ))
+
+register_template(
+    Phi3TemplateMeta(
+        MLLMTemplateType.internvl2_phi3,
+        default_system=_internvl2_system,
+        template_cls=Internvl2Template,
+    ))
+
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.internvl2_5,
+        template_cls=Internvl2Template,
+        default_system='你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。'))
+
+register_template(ChatmlTemplateMeta(MLLMTemplateType.internvl3_5, template_cls=Internvl2Template))
+
+
+class Internvl3_5GPTTemplate(Internvl2Template, GptTemplate):
+    pass
+
+
+register_template(GptOssTemplateMeta(MLLMTemplateType.internvl3_5_gpt, template_cls=Internvl3_5GPTTemplate))
+
+
+class InternvlhfTemplate(Internvl2Template):
+
+    def init_env_args(self):
+        Template.init_env_args(self)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type in ['image', 'video']
+        if media_type == 'video':
+            if self.mode == 'vllm':
+                return Template.replace_tag(self, 'video', index, inputs)
+            else:
+                return [[-200]]
+        else:
+            if self.mode == 'vllm':
+                return ['<IMG_CONTEXT>']
+            else:
+                return ['<img>', [-100], '</img>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        import numpy as np
+        from transformers.image_utils import concatenate_list, make_flat_list_of_images
+        from transformers.video_utils import make_batched_videos
+
+        from swift.template.vision_utils import load_video_hf
+        encoded = super(InternvlTemplate, self)._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        images = inputs.images
+        videos = inputs.videos
+        image_num_patches_indices = np.array([0])
+        video_num_patches_indices = np.array([0])
+        video_patch_indices = np.array([0])
+        image_num_patches = []
+        video_num_patches = []
+        image_video_patches = []
+        image_idx_list = []
+        video_idx_list = []
+        image_pixel_values = None
+        video_pixel_values = None
+
+        if images:
+            # InternS1Processor
+            image_idx_list = findall(input_ids, -100)
+            images = make_flat_list_of_images(images)
+            image_inputs = self.processor.image_processor(images=images, crop_to_patches=True, return_tensors='pt')
+            image_num_patches = image_inputs.pop('num_patches')
+            image_pixel_values = image_inputs.pop('pixel_values').to(self.model_info.torch_dtype)
+            image_num_patches_indices = np.cumsum(image_num_patches)
+        if videos:
+            video_idx_list = findall(input_ids, -200)
+            videos, _ = load_video_hf(videos)
+            videos = make_batched_videos(videos)
+            video_inputs = self.processor.video_processor(videos=videos, return_tensors='pt')
+            video_pixel_values = video_inputs.pop('pixel_values_videos').to(self.model_info.torch_dtype)
+            num_frames_per_video = [len(video) for video in video_pixel_values]
+            video_num_patches = [1 for frames in num_frames_per_video for _ in range(frames)]
+            video_patch_indices = np.cumsum(num_frames_per_video)
+            video_num_patches_indices = np.cumsum(video_num_patches)
+            video_pixel_values = video_pixel_values.flatten(0, 1)
+
+        def merge_and_sort(image_idx_list: List[int], video_idx_list: List[int]) -> tuple:
+            """Merge and sort image and video index lists while preserving their relative order."""
+            merged = []
+            is_image_list = []
+            i, j = 0, 0
+
+            while i < len(image_idx_list) and j < len(video_idx_list):
+                if image_idx_list[i] < video_idx_list[j]:
+                    merged.append(image_idx_list[i])
+                    i += 1
+                    is_image_list.append(True)
+                else:
+                    merged.append(video_idx_list[j])
+                    j += 1
+                    is_image_list.append(False)
+            # Add remaining elements
+            merged.extend(image_idx_list[i:])
+            is_image_list.extend([True] * (len(image_idx_list) - i))
+            merged.extend(video_idx_list[j:])
+            is_image_list.extend([False] * (len(video_idx_list) - j))
+            return merged, is_image_list
+
+        # Merge and sort the index lists
+        idx_list, is_image_list = merge_and_sort(image_idx_list, video_idx_list)
+
+        # Validate the lengths
+        if images and len(image_idx_list) > 0:
+            assert len(image_num_patches_indices) == len(image_idx_list)
+        if videos and len(video_idx_list) > 0:
+            assert len(video_patch_indices) == len(video_idx_list)
+
+        def _get_new_tokens(i):
+            if is_image_list[i]:
+                # Find the corresponding image index
+                image_idx = sum(is_image_list[:i])
+                start = image_num_patches_indices[image_idx - 1] if image_idx > 0 else 0
+                end = image_num_patches_indices[image_idx]
+                image_seq_length = self.processor.image_seq_length
+                image_video_patches.append(image_pixel_values[start:end])
+                img_tokens: List[int] = self.processor.encode(
+                    '<IMG_CONTEXT>', add_special_tokens=False) * image_seq_length * image_num_patches[image_idx]
+            else:
+                # Find the corresponding video index
+                video_idx = i - sum(is_image_list[:i])
+                current_patch = video_patch_indices[video_idx - 1] if video_idx > 0 else 0
+                end_patch = video_patch_indices[video_idx]
+
+                start = video_num_patches_indices[current_patch] if video_idx > 0 else 0
+                end = video_num_patches_indices[end_patch - 1]
+                image_video_patches.append(video_pixel_values[start:end])
+                image_seq_length = self.processor.image_seq_length
+                num_patches = list(video_num_patches[current_patch:end_patch])
+                video_prompt = ''.join(
+                    f"Frame{i + 1}: <img>{'<IMG_CONTEXT>' * image_seq_length * num_patches[i]}</img>\n"
+                    for i in range(len(num_patches)))
+                img_tokens = self.processor.encode(video_prompt, add_special_tokens=False)
+            return img_tokens
+
+        encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+            input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+        if images or videos:
+            encoded['pixel_values'] = concatenate_list(image_video_patches)
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        embedding = model.get_input_embeddings()
+        device = embedding.weight.device
+        input_ids = inputs['input_ids']
+        inputs_embeds = embedding(input_ids).to(device=device)
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            pixel_values = pixel_values.to(device=device)
+            image_features = model.model.get_image_features(
+                pixel_values,
+                vision_feature_layer=self.config.vision_feature_layer,
+                vision_feature_select_strategy=self.config.vision_feature_select_strategy,
+            )
+            if hasattr(image_features, 'pooler_output'):
+                image_features = image_features.pooler_output
+            special_image_mask = input_ids == self.config.image_token_id
+            special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+            image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
+            inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)
+        elif is_deepspeed_enabled():
+            dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype)
+            image_features = model.model.get_image_features(
+                dummy_pixel_values,
+                vision_feature_layer=self.config.vision_feature_layer,
+                vision_feature_select_strategy=self.config.vision_feature_select_strategy,
+            )
+            if hasattr(image_features, 'pooler_output'):
+                image_features = image_features.pooler_output
+            inputs_embeds = inputs_embeds + image_features.mean() * 0.
+        return {'inputs_embeds': inputs_embeds}
+
+
+INTERNS1_DEFAULT_SYSTEM = ('You are an expert reasoner with extensive experience in all areas. '
+                           'You approach problems through systematic thinking and rigorous reasoning. '
+                           'Your response should reflect deep understanding and precise logical thinking, '
+                           'making your solution path and reasoning clear to others. '
+                           'Please put your thinking process within <think>...</think> tags.')
+
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.interns1,
+        template_cls=InternvlhfTemplate,
+        default_system=INTERNS1_DEFAULT_SYSTEM,
+        is_thinking=True,
+        thinking_prefix='<think>',
+    ))
+
+register_template(ChatmlTemplateMeta(MLLMTemplateType.internvl_hf, template_cls=InternvlhfTemplate))

swift/template/templates/kwai.py

@@ -0,0 +1,300 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import numpy as np
+import os
+import torch
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal
+
+from swift.utils import is_deepspeed_enabled, to_device
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Word, findall
+from .utils import ChatmlTemplateMeta
+
+
+@dataclass
+class KeyeTemplateMeta(ChatmlTemplateMeta):
+    auto_add_bos: bool = False
+    stop_words: List[Word] = field(default_factory=lambda: ['<|endoftext|>'])
+
+
+class KeyeVLTemplate(Template):
+    image_token_id = 151655
+    video_token_id = 151656
+    placeholder_tokens = ['<|image_pad|>', '<|video_pad|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        from keye_vl_utils import fetch_image, fetch_video
+        assert media_type in {'image', 'video'}
+        if media_type == 'image':
+            inputs.images[index] = fetch_image({'image': inputs.images[index]})
+            if getattr(self, 'mode', None) == 'lmdeploy':
+                return ['<|vision_start|>', [-100], '<|vision_end|>']
+            else:
+                return ['<|vision_start|><|image_pad|><|vision_end|>']
+        else:
+            video = inputs.videos[index]
+            video, video_kwargs = fetch_video({'video': video})
+            if isinstance(video, torch.Tensor):
+                video = video.to(torch.uint8)
+            inputs.videos[index] = video
+            for k, v in video_kwargs.items():
+                inputs.mm_processor_kwargs.setdefault(k, []).append(v)
+            return ['<|vision_start|><|video_pad|><|vision_end|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        for media_type in ['images', 'videos']:
+            mm_data = getattr(inputs, media_type)
+            if mm_data:
+                if media_type == 'images':
+                    media_token = self.image_token_id
+                    media_inputs = processor.image_processor(images=mm_data, return_tensors='pt', do_resize=False)
+                    media_grid_thw = media_inputs['image_grid_thw']
+                else:
+                    split_token = self._tokenize('\n')[0]
+                    media_inputs = processor(
+                        text=['\n'.join(['<|video_pad|>'] * len(mm_data))],
+                        videos=mm_data,
+                        return_tensors='pt',
+                        **inputs.mm_processor_kwargs)
+                    splited_tokens = self._split_list(media_inputs['input_ids'][0].tolist(), split_token)
+                    media_grid_thw = media_inputs['video_grid_thw']
+                    media_token = self.video_token_id
+                idx_list = findall(input_ids, media_token)
+                merge_length = processor.image_processor.merge_size**2
+
+                def _get_new_tokens(i):
+                    if media_type == 'images':
+                        token_len = (media_grid_thw[i].prod() // merge_length)
+                        return [media_token] * token_len
+                    else:
+                        return splited_tokens[i]
+
+                input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                    _get_new_tokens)
+                encoded.update(media_inputs)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+        input_ids = inputs['input_ids']
+        pixel_values = inputs.get('pixel_values')
+        pixel_values_videos = inputs.get('pixel_values_videos')
+        image_grid_thw = inputs.get('image_grid_thw')
+        video_grid_thw = inputs.get('video_grid_thw')
+
+        base_model = self.get_base_model(model)
+        if hasattr(base_model.model, 'embed_tokens'):
+            inputs_embeds = base_model.model.embed_tokens(input_ids)
+        else:
+            inputs_embeds = base_model.model.language_model.embed_tokens(input_ids)
+
+        # Get dtype from visual model, adapting for KeyeVL model structure
+        if hasattr(model.visual, 'get_dtype'):
+            dtype = model.visual.get_dtype()
+        else:
+            dtype = model.visual.dtype
+
+        if pixel_values is None and pixel_values_videos is None:  # plain-text
+            if is_deepspeed_enabled():
+                from PIL import Image
+                images = [Image.new('RGB', (32, 32), (0, 0, 0))]
+                media_inputs = self.processor.image_processor(images=images, return_tensors='pt')
+                device = input_ids.device
+                media_inputs = to_device(media_inputs, device)
+                pixel_values = media_inputs['pixel_values'].type(dtype)
+                # Convert to 5D format for KeyeVL: [num_patches, 3, 14, 14] -> [1, num_patches, 3, 14, 14]
+                pixel_values = pixel_values.unsqueeze(0)
+
+                # KeyeVL requires position_ids when pixel_values is 5D
+                num_patches = pixel_values.shape[1]
+                position_ids = torch.arange(num_patches, device=device)
+
+                # Create dummy grid that works with mlp_AR
+                # Assuming merge_size is 2, we need h and w divisible by merge_size
+                merge_size = getattr(self.processor.image_processor, 'merge_size', 2)
+                grid_size = int(np.sqrt(num_patches))
+
+                # Adjust grid_size to be divisible by merge_size
+                if grid_size % merge_size != 0:
+                    grid_size = ((grid_size + merge_size - 1) // merge_size) * merge_size
+
+                # For dummy case, use square layout that's compatible with mlp_AR
+                dummy_grid_hw = [(1, grid_size, grid_size)]
+                sample_indices = torch.zeros(num_patches, dtype=torch.int64, device=device)
+                cu_seqlens = torch.tensor([0, num_patches], dtype=torch.int32, device=device)
+
+                vision_outputs = model.visual(
+                    pixel_values=pixel_values,
+                    image_grid_thw=dummy_grid_hw,
+                    position_ids=position_ids,
+                    vision_return_embed_list=True,
+                    interpolate_pos_encoding=True,
+                    sample_indices=sample_indices,
+                    cu_seqlens=cu_seqlens,
+                    return_pooler_output=False,
+                    use_rope=True,
+                    window_size=-1,
+                )
+                image_embeds = vision_outputs.last_hidden_state
+                # Process through projector like in normal cases
+                image_embeds = model.mlp_AR(image_embeds, dummy_grid_hw)
+                # Concatenate all embeddings
+                image_embeds = torch.cat(image_embeds, dim=0)
+                inputs_embeds += image_embeds.mean() * 0.
+        else:
+            if pixel_values is not None:
+                pixel_values = pixel_values.type(dtype)
+                # KeyeVL expects 5D input: (batch_size, sequence_len, channel, height, width)
+                # where sequence_len is the total number of patches from all images
+                pixel_values = pixel_values.unsqueeze(0)  # [num_patches, 3, 14, 14] -> [1, num_patches, 3, 14, 14]
+
+                if image_grid_thw is not None:
+                    image_grid_hws = []
+                    for thw in image_grid_thw:
+                        if isinstance(thw, torch.Tensor):
+                            thw_tuple = tuple(thw.detach().cpu().numpy().tolist())
+                        else:
+                            thw_tuple = tuple(thw)
+                        image_grid_hws.append(thw_tuple)
+
+                    # Prepare position_ids and other parameters for KeyeVL
+                    siglip_position_ids = []
+                    sample_indices = []
+                    cu_seqlens = [0]
+
+                    for idx, thw_tuple in enumerate(image_grid_hws):
+                        numel = np.prod(thw_tuple)
+                        image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
+                        siglip_position_ids.append(image_position_ids)
+                        sample_indices.append(torch.full((numel, ), idx, dtype=torch.int64))
+                        cu_seqlens.append(cu_seqlens[-1] + numel)
+
+                    siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(pixel_values.device)
+                    cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values.device)
+                    sample_indices = torch.concat(sample_indices, dim=0).to(pixel_values.device)
+
+                    # Call KeyeVL visual model
+                    vision_outputs = model.visual(
+                        pixel_values=pixel_values,
+                        image_grid_thw=image_grid_hws,
+                        position_ids=siglip_position_ids,
+                        vision_return_embed_list=True,
+                        interpolate_pos_encoding=True,
+                        sample_indices=sample_indices,
+                        cu_seqlens=cu_seqlens,
+                        return_pooler_output=False,
+                        use_rope=True,
+                        window_size=-1,
+                    )
+                    image_embeds = vision_outputs.last_hidden_state
+
+                    # Process through projector
+                    image_embeds = model.mlp_AR(image_embeds, image_grid_thw)
+                    # Concatenate all image embeddings
+                    image_embeds = torch.cat(image_embeds, dim=0)
+                else:
+                    # Fallback for case without grid info
+                    num_patches = pixel_values.shape[1]
+                    position_ids = torch.arange(num_patches, device=pixel_values.device)
+                    vision_outputs = model.visual(pixel_values=pixel_values, position_ids=position_ids)
+                    image_embeds = vision_outputs.last_hidden_state.reshape(-1,
+                                                                            vision_outputs.last_hidden_state.shape[-1])
+
+                image_mask = (input_ids == model.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+                image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+            if pixel_values_videos is not None:
+                pixel_values_videos = pixel_values_videos.type(dtype)
+                # Same processing for videos: convert to 5D format
+                pixel_values_videos = pixel_values_videos.unsqueeze(
+                    0)  # [num_patches, 3, 14, 14] -> [1, num_patches, 3, 14, 14]
+
+                if video_grid_thw is not None:
+                    video_grid_hws = []
+                    for thw in video_grid_thw:
+                        if isinstance(thw, torch.Tensor):
+                            thw_tuple = tuple(thw.detach().cpu().numpy().tolist())
+                        else:
+                            thw_tuple = tuple(thw)
+                        video_grid_hws.append(thw_tuple)
+
+                    siglip_position_ids = []
+                    sample_indices = []
+                    cu_seqlens = [0]
+
+                    for idx, thw_tuple in enumerate(video_grid_hws):
+                        numel = np.prod(thw_tuple)
+                        video_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
+                        siglip_position_ids.append(video_position_ids)
+                        sample_indices.append(torch.full((numel, ), idx, dtype=torch.int64))
+                        cu_seqlens.append(cu_seqlens[-1] + numel)
+
+                    siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(pixel_values_videos.device)
+                    cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values_videos.device)
+                    sample_indices = torch.concat(sample_indices, dim=0).to(pixel_values_videos.device)
+
+                    vision_outputs = model.visual(
+                        pixel_values=pixel_values_videos,
+                        image_grid_thw=video_grid_hws,
+                        position_ids=siglip_position_ids,
+                        vision_return_embed_list=True,
+                        interpolate_pos_encoding=True,
+                        sample_indices=sample_indices,
+                        cu_seqlens=cu_seqlens,
+                        return_pooler_output=False,
+                        use_rope=True,
+                        window_size=-1,
+                    )
+                    video_embeds = vision_outputs.last_hidden_state
+                    video_embeds = model.mlp_AR(video_embeds, video_grid_thw)
+                    video_embeds = torch.cat(video_embeds, dim=0)
+                else:
+                    # Fallback for case without grid info
+                    num_patches = pixel_values_videos.shape[1]
+                    position_ids = torch.arange(num_patches, device=pixel_values_videos.device)
+                    vision_outputs = model.visual(pixel_values=pixel_values_videos, position_ids=position_ids)
+                    video_embeds = vision_outputs.last_hidden_state.reshape(-1,
+                                                                            vision_outputs.last_hidden_state.shape[-1])
+
+                video_mask = (input_ids == model.config.video_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+                video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
+
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        second_per_grid_ts = self.gather_list(batch, 'second_per_grid_ts')
+        if second_per_grid_ts:
+            res['second_per_grid_ts'] = second_per_grid_ts
+        return res
+
+
+# Register the Keye VL template
+register_template(KeyeTemplateMeta(MLLMTemplateType.keye_vl, template_cls=KeyeVLTemplate))
+
+
+class KeyeVL1_5Template(KeyeVLTemplate):
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return super(KeyeVLTemplate, self)._post_encode(model, inputs)
+
+
+register_template(
+    KeyeTemplateMeta(
+        MLLMTemplateType.keye_vl_1_5, template_cls=KeyeVL1_5Template, default_system='You are a helpful assistant.'))

swift/template/templates/llama.py

@@ -0,0 +1,216 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+import datetime as dt
+import torch
+import torch.nn as nn
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word, findall
+from ..vision_utils import load_batch
+
+# ref: https://github.com/facebookresearch/llama/blob/main/llama/generation.py
+LLAMA_DEFAULT_SYSTEM = (
+    'You are a helpful, respectful and honest assistant. '
+    'Always answer as helpfully as possible, while being safe. '
+    'Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. '
+    'Please ensure that your responses are socially unbiased and positive in nature.\n\n'
+    'If a question does not make any sense, or is not factually coherent, '
+    'explain why instead of answering something not correct. '
+    "If you don't know the answer to a question, please don't share false information.")
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.llama, ['<s>[INST] '], ['{{QUERY}} [/INST]'], ['</s><s>[INST] '], ['</s>'],
+        default_system=LLAMA_DEFAULT_SYSTEM,
+        system_prefix=['<s>[INST] <<SYS>>\n{{SYSTEM}}\n<</SYS>>\n\n']))
+
+
+@dataclass
+class Llama3TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|begin_of_text|>'])
+    prompt: Prompt = field(default_factory=lambda: [
+        '<|start_header_id|>user<|end_header_id|>\n\n{{QUERY}}<|eot_id|>'
+        '<|start_header_id|>assistant<|end_header_id|>\n\n'
+    ])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|eot_id|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|eot_id|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{{SYSTEM}}<|eot_id|>'])
+    agent_template: str = 'llama3'
+
+
+register_template(Llama3TemplateMeta(LLMTemplateType.llama3))
+
+
+def _get_llama3_2_prefix() -> Prompt:
+    now = dt.datetime.now()
+    date_string = now.strftime('%d %b %Y')
+    date_prompt = f'Cutting Knowledge Date: December 2023\nToday Date: {date_string}'
+    return [f'<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{date_prompt}\n\n'
+            '{{SYSTEM}}<|eot_id|>']
+
+
+@dataclass
+class Llama3_2TemplateMeta(Llama3TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: _get_llama3_2_prefix())
+    system_prefix: Optional[Prompt] = None
+
+
+register_template(Llama3_2TemplateMeta(LLMTemplateType.llama3_2))
+
+
+class Llama3_2VisionTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<|image|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from transformers.models.mllama.processing_mllama import (convert_sparse_cross_attention_mask_to_dense,
+                                                                  get_cross_attention_token_mask)
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            input_ids = encoded['input_ids']
+            processor = self.processor
+            image_features = processor.image_processor(images, return_tensors='pt')
+            num_tiles = image_features.pop('num_tiles')
+            encoded.update(image_features)
+
+            cross_attention_token_mask = [get_cross_attention_token_mask(input_ids, processor.image_token_id)]
+            cross_attention_mask = convert_sparse_cross_attention_mask_to_dense(
+                cross_attention_token_mask,
+                num_tiles=num_tiles,
+                max_num_tiles=processor.image_processor.max_image_tiles,
+                length=len(input_ids),
+            )
+            encoded['cross_attention_mask'] = torch.tensor(cross_attention_mask)
+
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        for key in ['aspect_ratio_ids', 'aspect_ratio_mask']:
+            value = [b[key] for b in batch if b.get(key) is not None]
+            if value:
+                res[key] = torch.concat(value)
+
+        cross_attention_mask = [
+            b['cross_attention_mask'][0] for b in batch if b.get('cross_attention_mask') is not None
+        ]
+        if cross_attention_mask:
+            res['cross_attention_mask'] = self._pad_sequence(cross_attention_mask, 0)
+        return res
+
+
+register_template(Llama3_2TemplateMeta(MLLMTemplateType.llama3_2_vision, template_cls=Llama3_2VisionTemplate))
+
+
+class Llama4Template(Template):
+    placeholder_tokens = ['<|patch|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        if self.mode == 'vllm':
+            return ['<|image|>']
+        return [[-100]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            split_token = self._tokenize('\n')
+            input_ids, labels = encoded['input_ids'], encoded['labels']
+            loss_scale = encoded['loss_scale']
+            idx_list = findall(input_ids, -100)
+            media_inputs = self.processor(
+                text='\n'.join(['<|image|>'] * len(idx_list)),
+                images=images,
+                add_special_tokens=False,
+                return_tensors='pt')
+            splited_tokens = self._split_list(media_inputs['input_ids'][0].tolist(), split_token)
+
+            encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, labels, loss_scale, idx_list, lambda i: splited_tokens[i])
+            encoded['pixel_values'] = media_inputs['pixel_values']
+        return encoded
+
+
+@dataclass
+class Llama4TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|begin_of_text|>'])
+    prompt: Prompt = field(
+        default_factory=lambda:
+        ['<|header_start|>user<|header_end|>\n\n{{QUERY}}<|eot|>'
+         '<|header_start|>assistant<|header_end|>\n\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|eot|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|eot|>'])
+    stop_words: List[Word] = field(default_factory=lambda: ['<|end_of_text|>', '<|eom|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|begin_of_text|><|header_start|>system<|header_end|>\n\n{{SYSTEM}}<|eot|>'])
+    agent_template: str = 'llama4'
+
+
+register_template(Llama4TemplateMeta(MLLMTemplateType.llama4, template_cls=Llama4Template))
+
+register_template(
+    Llama3TemplateMeta(
+        LLMTemplateType.reflection,
+        default_system=('You are a world-class AI system, capable of complex reasoning and reflection. '
+                        'Reason through the query inside <thinking> tags, and then provide your final '
+                        'response inside <output> tags. If you detect that you made a mistake in your reasoning '
+                        'at any point, correct yourself inside <reflection> tags.')))
+
+
+class Llama3_1OmniTemplate(Template):
+    skip_prompt = False
+    audio_placeholder = [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        import whisper
+        encoded = super()._encode(inputs)
+        audios = inputs.audios
+        if audios:
+            audios = load_batch(audios, whisper.load_audio)
+            n_mels = get_env_args('n_mels', int, 128)
+            for i, audio in enumerate(audios):
+                audio = whisper.pad_or_trim(audio)
+                audios[i] = whisper.log_mel_spectrogram(audio, n_mels=n_mels).permute(1, 0)
+            audios = torch.stack(audios)
+            encoded.update({'speech': audios, 'speech_lengths': torch.tensor([[audios.shape[1]]])})
+
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        speech = inputs.get('speech')
+        input_ids = inputs['input_ids']
+        labels = inputs.get('labels')
+        if speech is not None:
+            speech_lengths = inputs['speech_lengths']
+            speech = speech.to(model.dtype)
+            inputs_embeds, labels = model.prepare_inputs_labels_for_speech_and_text(input_ids, None, None, None, labels,
+                                                                                    speech, speech_lengths)[4:]
+        else:
+            inputs_embeds = model.get_model().embed_tokens(input_ids)
+        res = {'inputs_embeds': inputs_embeds}
+        if labels is not None:
+            res['labels'] = labels[0]
+        return res
+
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llama3_1_omni,
+        default_system=('You are a helpful language and speech assistant. '
+                        'You are able to understand the speech content that the user provides, '
+                        'and assist the user with a variety of tasks using natural language.'),
+        template_cls=Llama3_1OmniTemplate,
+    ))

swift/template/templates/llava.py

@@ -0,0 +1,416 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+import transformers
+from dataclasses import dataclass, field
+from packaging import version
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_video_llava
+from .llama import Llama3TemplateMeta
+from .qwen import QwenTemplateMeta
+from .utils import ChatmlTemplateMeta
+
+
+class LlavaHfTemplate(Template):
+    placeholder_tokens = ['<image>']
+
+    @property
+    def image_token_index(self):
+        if not hasattr(self, '_image_token_index'):
+            self._image_token_index = self.tokenizer.convert_tokens_to_ids(self.processor.image_token)
+        return self._image_token_index
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<image>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            if 'image_sizes' in image_inputs:
+                encoded['image_sizes'] = image_inputs['image_sizes']
+            if version.parse(transformers.__version__) >= version.parse('4.47'):
+                input_ids = encoded['input_ids']
+                labels = encoded['labels']
+                idx_list = findall(input_ids, self.image_token_index)  # <image>
+                height, width = image_inputs['pixel_values'][0].shape[-2:]
+                added_tokens_len = 0
+                for i, idx in enumerate(idx_list):
+                    if 'image_sizes' in image_inputs:
+                        orig_height, orig_width = image_inputs['image_sizes'][i].tolist()
+                        num_image_tokens = self.processor._get_number_of_features(orig_height, orig_width, height,
+                                                                                  width)
+                    else:
+                        num_image_tokens = (height // self.processor.patch_size) * (
+                            width // self.processor.patch_size) + self.processor.num_additional_image_tokens
+                    if self.processor.vision_feature_select_strategy == 'default':
+                        num_image_tokens -= 1
+                    input_ids = input_ids[:added_tokens_len + idx] + [self.image_token_index] * num_image_tokens \
+                        + input_ids[added_tokens_len + idx + 1:]
+                    if labels is not None:
+                        labels = labels[:added_tokens_len + idx] + [-100] * num_image_tokens \
+                            + labels[added_tokens_len + idx + 1:]
+                    added_tokens_len += num_image_tokens - 1
+                encoded['input_ids'] = input_ids
+                encoded['labels'] = labels
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.llava1_5_hf,
+        prefix=['<s>'],
+        prompt=['USER: {{QUERY}}\nASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        system_prefix=['<s>{{SYSTEM}}\n'],
+        template_cls=LlavaHfTemplate,
+    ))
+
+
+class LlavaVideoHfTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<image>\n']
+        assert media_type == 'video'
+        media_file = inputs.videos[index]
+        if media_file.rsplit('.', 1)[-1] in {'jpg', 'png'}:
+            return ['<image>\n']
+        else:
+            inputs.videos[index] = load_video_llava(inputs.videos[index])
+            return ['<video>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        videos = inputs.videos or []
+        if len(videos) > 0:
+            video_processor = self.processor.video_processor
+            video_inputs = video_processor(videos, return_tensors='pt').to(self.model_info.torch_dtype)
+            encoded['pixel_values_videos'] = video_inputs['pixel_values_videos']
+        if len(images) > 0:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['image_sizes'] = image_inputs['image_sizes']
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.llava_next_video_hf,
+        prefix=['{{SYSTEM}} '],
+        prompt=['USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=[' '],
+        suffix=[['eos_token_id']],
+        template_cls=LlavaVideoHfTemplate,
+        auto_add_bos=True,
+    ))
+
+
+class Llava1_6HfTemplate(LlavaHfTemplate):
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        for b in batch:
+            pixel_values = b.get('pixel_values')
+            if pixel_values is not None:
+                b['pixel_values'] = pixel_values.squeeze(0)  # 5d -> 4d
+        res = super()._data_collator(batch, padding_to=padding_to)
+        return res
+
+
+@dataclass
+class LlavaMistralTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>[INST] '])
+    prompt: Prompt = field(default_factory=lambda: ['{{QUERY}} [/INST]'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['</s>[INST] '])
+    suffix: Prompt = field(default_factory=lambda: ['</s>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<<SYS>>\n{{system}}\n<</SYS>>\n\n'])
+
+
+register_template(LlavaMistralTemplateMeta(MLLMTemplateType.llava1_6_mistral_hf, template_cls=Llava1_6HfTemplate))
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.llava1_6_vicuna_hf,
+        prefix=['<s>'],
+        prompt=['USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        default_system=('A chat between a curious human and an artificial intelligence assistant. '
+                        "The assistant gives helpful, detailed, and polite answers to the human's questions."),
+        system_prefix=['<s>{{SYSTEM}} '],
+        template_cls=Llava1_6HfTemplate))
+
+
+class LLava1_6YiHfTemplate(Llava1_6HfTemplate):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            return [[64000], '\n']
+        else:
+            return super().replace_tag(media_type, index, inputs)
+
+
+register_template(ChatmlTemplateMeta(
+    MLLMTemplateType.llava1_6_yi_hf,
+    template_cls=LLava1_6YiHfTemplate,
+))
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llama3_llava_next_hf,
+        template_cls=Llava1_6HfTemplate,
+        agent_template=None,
+    ))
+
+register_template(
+    QwenTemplateMeta(MLLMTemplateType.llava_next_qwen_hf, template_cls=Llava1_6HfTemplate, agent_template=None))
+
+
+class LlavaOneVisionHfTemplate(Llava1_6HfTemplate):
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, 151646)  # <image>
+        processor = self.processor
+        if images:
+            image_processor = processor.image_processor
+            image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+            height, width = image_inputs['pixel_values'][0].shape[-2:]
+            added_tokens_len = 0
+            for idx, pixel_v, image_size in zip(idx_list, image_inputs['pixel_values'], image_inputs['image_sizes']):
+                if isinstance(image_size, torch.Tensor):
+                    image_size = image_size.tolist()
+                orig_height, orig_width = image_size
+                num_image_tokens = processor._get_number_of_features(orig_height, orig_width, height, width)
+                input_ids = input_ids[:added_tokens_len
+                                      + idx] + [151646] * num_image_tokens + input_ids[added_tokens_len + idx + 1:]
+                if labels is not None:
+                    labels = labels[:added_tokens_len + idx] + [-100] * num_image_tokens + labels[added_tokens_len + idx
+                                                                                                  + 1:]
+                added_tokens_len += num_image_tokens - 1
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            if 'image_sizes' in image_inputs:
+                encoded['image_sizes'] = image_inputs['image_sizes']
+        return encoded
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.llava_onevision_hf,
+        default_system=None,
+        template_cls=LlavaOneVisionHfTemplate,
+        agent_template=None,
+    ))
+
+
+class LlavaLlama3_1HfTemplate(LlavaHfTemplate):
+    # DaozeZhang
+    system = ('You are a helpful language and vision assistant. '
+              'You are able to understand the visual content that the user provides, '
+              'and assist the user with a variety of tasks using natural language.')
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        if len(encoded['pixel_values'].shape) == 5:  # (1, num_patch, 3, H/W, W/H)
+            encoded['pixel_values'] = torch.squeeze(encoded['pixel_values'], dim=0)  # (num_patch, 3, H/W, W/H)
+        return encoded
+
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llava_llama3_1_hf,
+        default_system=LlavaLlama3_1HfTemplate.system,
+        template_cls=LlavaLlama3_1HfTemplate,
+        agent_template=None,
+    ))
+
+
+class LLavaLlama3HfTemplate(Template):
+    # xtuner
+    image_placeholder = ['<image>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        raw_image = inputs.images
+        if raw_image:
+            pixel_values = self.processor.image_processor(raw_image, return_tensors='pt')['pixel_values']
+            encoded['pixel_values'] = pixel_values.to(self.model_info.torch_dtype)
+        return encoded
+
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llava_llama3_hf,
+        template_cls=LLavaLlama3HfTemplate,
+        agent_template=None,
+    ))
+
+
+class LLavaTemplate(Template):
+    skip_prompt = False
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-200], '\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        image_sizes = [x.size for x in images]
+        from llava.mm_utils import process_images
+        model = self.model.model
+        if not hasattr(model, 'vision_tower'):
+            model = model.model
+        image_processor = model.vision_tower.image_processor
+        if images:
+            images_tensor = process_images(images, image_processor, model.config)
+            encoded['images'] = images_tensor.to(model.dtype).squeeze(0)
+            encoded['image_sizes'] = image_sizes
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = images
+            res['image_sizes'] = sum([b['image_sizes'] for b in batch if 'image_sizes' in b], start=[])
+        return res
+
+
+register_template(LlavaMistralTemplateMeta(MLLMTemplateType.llava1_6_mistral, template_cls=LLavaTemplate))
+
+register_template(ChatmlTemplateMeta(MLLMTemplateType.llava1_6_yi, template_cls=LLavaTemplate))
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llama3_llava_next,
+        template_cls=LLavaTemplate,
+        default_system=('You are a helpful language and vision assistant. '
+                        'You are able to understand the visual content that the user provides, '
+                        'and assist the user with a variety of tasks using natural language.'),
+        agent_template=None,
+    ))
+
+register_template(QwenTemplateMeta(MLLMTemplateType.llava_next_qwen, template_cls=LLavaTemplate, agent_template=None))
+
+
+class LLavaOneVision1_5Template(Template):
+    image_token_id = 151655
+    video_token_id = 151656
+    placeholder_tokens = ['<|image_pad|>', '<|video_pad|>']
+    use_model = True
+    support_padding_free = True
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.bbox_format = get_env_args('QWENVL_BBOX_FORMAT', str, 'legacy')
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        from qwen_vl_utils import fetch_image, fetch_video
+        assert media_type in {'image', 'video'}
+        if media_type == 'image':
+            inputs.images[index] = fetch_image({'image': inputs.images[index]})
+            if self.mode == 'lmdeploy':
+                return ['<|vision_start|>', [-100], '<|vision_end|>']
+            else:
+                return ['<|vision_start|><|image_pad|><|vision_end|>']
+        else:
+            video = inputs.videos[index]
+            video, video_kwargs = fetch_video({'video': video}, return_video_sample_fps=True)
+            inputs.mm_processor_kwargs.setdefault('fps', []).append(video_kwargs)
+            tokens = ['<|vision_start|><|video_pad|><|vision_end|>']
+            if isinstance(video, torch.Tensor):
+                video = video.to(torch.uint8)
+            inputs.videos[index] = video
+            return tokens
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        if self.bbox_format == 'legacy':
+            return [f'<|object_ref_start|>{ref}<|object_ref_end|>']
+        else:
+            return [ref]
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        if self.bbox_format == 'legacy':
+            return [f'<|box_start|>{self._get_bbox_str(bbox)}<|box_end|>']
+        else:
+            return [str(bbox)]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        for media_type in ['images', 'videos']:
+            mm_data = getattr(inputs, media_type)
+            if mm_data:
+                if media_type == 'images':
+                    media_token = self.image_token_id
+                    media_inputs = processor.image_processor(images=mm_data, return_tensors='pt', do_resize=False)
+                    media_grid_thw = media_inputs['image_grid_thw']
+                else:
+                    kwargs = {}
+                    if hasattr(processor, 'video_processor'):
+                        processor_func = processor.video_processor
+                    else:
+                        processor_func = processor.image_processor
+                        kwargs['images'] = None
+                    media_inputs = processor_func(videos=mm_data, return_tensors='pt', do_resize=False, **kwargs)
+                    media_grid_thw = media_inputs['video_grid_thw']
+                    media_token = self.video_token_id
+                idx_list = findall(input_ids, media_token)
+                merge_length = processor.image_processor.merge_size**2
+
+                def _get_new_tokens(i):
+                    token_len = (media_grid_thw[i].prod() // merge_length)
+                    return [media_token] * token_len
+
+                input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                    _get_new_tokens)
+                encoded.update(media_inputs)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+        input_ids = inputs['input_ids']
+        base_model = self.get_base_model(model)
+        if hasattr(base_model.model, 'embed_tokens'):
+            inputs_embeds = base_model.model.embed_tokens(input_ids)
+        else:
+            inputs_embeds = base_model.model.language_model.embed_tokens(input_ids)
+        inputs_embeds = self._get_inputs_embeds_hf(inputs_embeds, inputs, model.visual, self.processor, model.config)
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    QwenTemplateMeta(MLLMTemplateType.llava_onevision1_5, template_cls=LLavaOneVision1_5Template, agent_template=None))

swift/template/templates/llm.py

@@ -0,0 +1,568 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Prompt
+from .llama import Llama3_2TemplateMeta
+from .qwen import Qwen2VLTemplate, QwenTemplateMeta
+from .utils import DEFAULT_SYSTEM, ChatmlTemplateMeta
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.default,
+        prefix=[],
+        prompt=['### Human:\n{{QUERY}}\n\n### Assistant:\n'],
+        chat_sep=['\n\n'],
+        default_system=DEFAULT_SYSTEM,
+        system_prefix=['{{SYSTEM}}\n\n'],
+        auto_add_bos=True))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.modelscope_agent,
+        prefix=[],
+        prompt=[' \n\n<|user|>:{{QUERY}} \n\n<|assistant|>:'],
+        chat_sep=[],
+        suffix=[' \n\n</s>'],
+        system_prefix=[' \n\n<|system|>:{{SYSTEM}}'],
+        default_system=DEFAULT_SYSTEM,
+    ))
+
+
+class GMETemplate(Qwen2VLTemplate):
+
+    def _preprocess_inputs(self, inputs: StdTemplateInputs) -> None:
+        super()._preprocess_inputs(inputs)
+        if inputs.messages[-1]['role'] != 'assistant':
+            inputs.messages.append({'role': 'assistant', 'content': ''})
+        return inputs
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen2_gme, template_cls=GMETemplate, suffix=['<|endoftext|>']))
+
+
+class JinaRerankerM0Template(Qwen2VLTemplate):
+
+    def _preprocess_inputs(self, inputs: StdTemplateInputs) -> None:
+        super()._preprocess_inputs(inputs)
+        instruction = ''
+        if inputs.system is not None:
+            instruction = inputs.system
+            inputs.system = None
+        query = inputs.messages[0]['content']
+        document = inputs.messages[1]['content']
+        user_message = instruction + '\n' + '**Query**:\n' + query + '\n' + '**Document**:\n' + document
+        inputs.messages = [{'role': 'user', 'content': user_message}]
+        return inputs
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.jina_reranker_m0,
+        template_cls=JinaRerankerM0Template,
+        prefix=[],
+        chat_sep=[],
+        prompt=['{{QUERY}}']))
+
+register_template(
+    TemplateMeta(LLMTemplateType.baichuan, prefix=['{{SYSTEM}}'], prompt=[[195], '{{QUERY}}', [196]], chat_sep=[]))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.baichuan_m1,
+        prefix=[],
+        prompt=['<C_Q>{{QUERY}}<C_A>'],
+        chat_sep=[],
+        suffix=['<C_A>'],
+        system_prefix=['<B_SYS>{{SYSTEM}}'],
+        default_system=DEFAULT_SYSTEM,
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.numina,
+        prefix=[['bos_token_id']],
+        prompt=['### Problem: {{QUERY}}\n### Solution: '],
+        chat_sep=['\n'],
+        system_prefix=[['bos_token_id'], '{{SYSTEM}}']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.mistral_nemo,
+        prefix=['<s>[INST] '],
+        prompt=['{{SYSTEM}}\n\n', '{{QUERY}}[/INST]'],
+        chat_sep=['</s>[INST] '],
+        suffix=['</s>']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.xverse,
+        prefix=['{{SYSTEM}}'],
+        prompt=['Human: {{QUERY}}\n\nAssistant: '],
+        chat_sep=[['eos_token_id']]))
+
+register_template(TemplateMeta(LLMTemplateType.yuan, prefix=[], prompt=['{{QUERY}}<sep>'], chat_sep=None))
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ziya,
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=['<human>:{{QUERY}}\n<bot>:'],
+        chat_sep=['\n']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.skywork,
+        prefix=['<s>{{SYSTEM}}'],
+        prompt=['</s><s>[USER]{{QUERY}}[SEP][BOT]'],
+        chat_sep=None,
+        suffix=['[SEP]</s>']))
+
+register_template(
+    Llama3_2TemplateMeta(
+        LLMTemplateType.skywork_o1,
+        default_system=(
+            'You are Skywork-o1, a thinking model developed by Skywork AI, specializing in solving complex problems '
+            "involving mathematics, coding, and logical reasoning through deep thought. When faced with a user's "
+            'request, you first engage in a lengthy and in-depth thinking process to explore possible solutions to '
+            'the problem. After completing your thoughts, you then provide a detailed explanation of the solution '
+            'process in your response.'),
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.bluelm,
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=['[|Human|]:{{QUERY}}[|AI|]:'],
+        chat_sep=[]))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.codefuse_codellama,
+        prefix=['{{SYSTEM}}'],
+        prompt=['<|role_start|>human<|role_end|>{{QUERY}}<|role_start|>bot<|role_end|>'],
+        chat_sep=[]))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.codefuse,
+        prefix=[],
+        prompt=['<s>human\n{{QUERY}}\n<s>bot\n'],
+        chat_sep=[['eos_token_id'], '\n'],
+        system_prefix=['<s>system\n{{SYSTEM}}\n']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.zephyr,
+        prefix=[],
+        prompt=['<|user|>\n{{QUERY}}</s>\n<|assistant|>\n'],
+        chat_sep=['</s>\n'],
+        suffix=['</s>'],
+        system_prefix=['<|system|>\n{{SYSTEM}}</s>\n']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.sus,
+        prefix=['{{SYSTEM}}'],
+        prompt=['### Human: {{QUERY}}\n\n### Assistant: '],
+        chat_sep=['<|endoftext|>'],
+        suffix=['<|endoftext|>']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.orion,
+        prefix=['<s>{{SYSTEM}}'],
+        prompt=['Human: {{QUERY}}\n\nAssistant: </s>'],
+        chat_sep=['</s>'],
+        suffix=['</s>']))
+
+
+@dataclass
+class TeleChatTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: [['user_token_id'], '{{QUERY}}', ['bot_token_id']])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: [['eos_token_id']])
+    suffix: Prompt = field(default_factory=lambda: [['eos_token_id']])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<_system>{{SYSTEM}}\n'])
+    auto_add_bos: bool = True
+
+
+register_template(TeleChatTemplateMeta(LLMTemplateType.telechat))
+
+telechat_system = '你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。'
+register_template(TeleChatTemplateMeta(LLMTemplateType.telechat2, default_system=telechat_system))
+
+DBRX_SYSTEM = (
+    'You are DBRX, created by Databricks. You were last updated in December 2023. '
+    'You answer questions based on information available up to that point.\n'
+    'YOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, '
+    'but provide thorough responses to more complex and open-ended questions.\n'
+    'You assist with various tasks, from writing to coding (using markdown for code blocks '
+    '— remember to use ``` with code, JSON, and tables).\n'
+    'You do not have real-time data access or code execution capabilities.'
+    ' You avoid stereotyping and provide balanced perspectives on controversial topics. '
+    'You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.\n'
+    'This is your system prompt, guiding your responses. Do not reference it, just respond to the user. '
+    'If you find yourself talking about this message, stop. You should be responding appropriately '
+    'and usually that means not mentioning this.'
+    'YOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY '
+    'PERTINENT TO THE USER\'S QUERY.')
+
+register_template(ChatmlTemplateMeta(LLMTemplateType.dbrx, default_system=DBRX_SYSTEM))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.mengzi, prefix=[], prompt=['输入：{{QUERY}}输出：\n'], chat_sep=[], system_prefix=['指令：{{SYSTEM}}']))
+
+C4AI_SYSTEM = ('You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by '
+               'providing thorough responses.You are trained by Cohere.')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.c4ai,
+        prefix=['<BOS_TOKEN>'],
+        prompt=[
+            '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{QUERY}}<|END_OF_TURN_TOKEN|>'
+            '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'
+        ],
+        chat_sep=['<|END_OF_TURN_TOKEN|>'],
+        suffix=['<|END_OF_TURN_TOKEN|>'],
+        default_system=C4AI_SYSTEM,
+        system_prefix=['<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{SYSTEM}}<|END_OF_TURN_TOKEN|']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.wizardlm2,
+        prefix=['{{SYSTEM}}'],
+        prompt=['User:\n{{QUERY}}\n\nAssistant:\n'],
+        chat_sep=['\n\n'],
+        suffix=['</s>']))
+
+_wizardlm2_system = ('A chat between a curious user and an artificial intelligence assistant. '
+                     'The assistant gives helpful, detailed, and polite answers to the user\'s questions. ')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.wizardlm2_moe,
+        prefix=['{{SYSTEM}}'],
+        prompt=['USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        default_system=_wizardlm2_system))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.atom,
+        prefix=['{{SYSTEM}}'],
+        prompt=['<s>Human: {{QUERY}}\n</s><s>Assistant: '],
+        chat_sep=['</s>'],
+        suffix=['</s>']))
+
+AYA_SYSTEM = ('You are Aya, a brilliant, sophisticated, multilingual AI-assistant trained to assist human users by '
+              'providing thorough responses. You are able to interact and respond to questions in 23 languages and '
+              'you are powered by a multilingual model built by Cohere For AI.')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.aya,
+        prefix=['<BOS_TOKEN>'],
+        prompt=[
+            '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{QUERY}}<|END_OF_TURN_TOKEN|>'
+            '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'
+        ],
+        chat_sep=['<|END_OF_TURN_TOKEN|>'],
+        suffix=['<|END_OF_TURN_TOKEN|>'],
+        default_system=AYA_SYSTEM,
+        system_prefix=['<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{SYSTEM}}<|END_OF_TURN_TOKEN|']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ling,
+        prefix=[],
+        system_prefix=['<role>SYSTEM</role>{{SYSTEM}}'],
+        prompt=['<role>HUMAN</role>{{QUERY}}<role>ASSISTANT</role>'],
+        chat_sep=[],
+        suffix=['<|endoftext|>'],
+    ))
+
+register_template(
+    QwenTemplateMeta(
+        LLMTemplateType.mimo_rl,
+        default_system='You are MiMo, an AI assistant developed by Xiaomi.',
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.dots1,
+        prefix=['<|system|>{{SYSTEM}}<|endofsystem|>'],
+        prompt=['<|userprompt|>{{QUERY}}<|endofuserprompt|><|response|>'],
+        chat_sep=['<|endofresponse|>'],
+        suffix=['<|endofresponse|>'],
+        default_system='You are a helpful assistant.',
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.hunyuan_moe,
+        prefix=['<|startoftext|>'],
+        system_prefix=['<|startoftext|>{{SYSTEM}}<|extra_4|>'],
+        prompt=['{{QUERY}}<|extra_0|>'],
+        chat_sep=['<|eos|><|startoftext|>'],
+        suffix=['<|eos|>'],
+    ))
+
+
+class HunyuanTemplate(Template):
+
+    def _remove_thinking_content(self, content: str) -> str:
+        content = content.split('<answer>')[-1].rstrip()
+        if content.endswith('</answer>'):
+            content = content[:-len('</answer>')]
+        return self.template_meta.history_thinking_prefix + content.strip()
+
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.hunyuan,
+        prefix=['<｜hy_begin▁of▁sentence｜>'],
+        system_prefix=['<｜hy_begin▁of▁sentence｜>{{SYSTEM}}<｜hy_place▁holder▁no▁3｜>'],
+        prompt=['<｜hy_User｜>{{QUERY}}<｜hy_Assistant｜>'],
+        chat_sep=['<｜hy_place▁holder▁no▁2｜>'],
+        suffix=['<｜hy_place▁holder▁no▁2｜>'],
+        template_cls=HunyuanTemplate,
+        is_thinking=True,
+        non_thinking_prefix='<think>\n\n</think>\n',
+        agent_template='hunyuan_hermes'))
+
+
+class HyV3PreviewTemplate(Template):
+    HYTK = ''
+
+    def init_env_args(self):
+        super().init_env_args()
+        # reasoning_effort: "no_think", "low", "high" (deep chain-of-thought)
+        # TODO: sample level
+        self.reasoning_effort = get_env_args('reasoning_effort', str, None)
+        if self.reasoning_effort is None:
+            self.reasoning_effort = 'high' if self.enable_thinking else 'no_think'
+        self.enable_thinking = self.reasoning_effort != 'no_think'
+        self.chat_template_kwargs['reasoning_effort'] = self.reasoning_effort
+
+    def _get_enable_thinking(self, inputs=None):
+        reasoning_effort = None if inputs is None else inputs.chat_template_kwargs.get('reasoning_effort')
+        if reasoning_effort is not None:
+            return reasoning_effort != 'no_think'
+        return super()._get_enable_thinking(inputs)
+
+    def _get_system(self, inputs):
+        system = super()._get_system(inputs)
+        reasoning_effort = inputs.chat_template_kwargs.get('reasoning_effort')
+        if reasoning_effort is None:
+            reasoning_effort = self.reasoning_effort
+        if inputs.tools:
+            # For tool calls, append reasoning_mode after </tool_calls> in the tool instruction
+            system = system.replace(
+                f'you should print </tool_calls{self.HYTK}>',
+                f'you should print </tool_calls{self.HYTK}><｜reasoning_mode{self.HYTK}｜>'
+                f'reasoning_effort:{reasoning_effort}')
+        else:
+            # For non-tool calls, append reasoning_mode to the system/prefix area
+            mode_str = f'<｜reasoning_mode{self.HYTK}｜>reasoning_effort:{reasoning_effort}'
+            system = (system or '') + mode_str
+        return system
+
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.hy_v3_preview,
+        prefix=['<｜hy_begin▁of▁sentence｜>'],
+        system_prefix=['<｜hy_begin▁of▁sentence｜>{{SYSTEM}}'],
+        prompt=['<｜hy_User｜>{{QUERY}}<｜hy_Assistant｜>'],
+        chat_sep=['<｜hy_eos｜>'],
+        suffix=['<｜hy_eos｜>'],
+        template_cls=HyV3PreviewTemplate,
+        is_thinking=True,
+        thinking_prefix='<think>',
+        non_thinking_prefix='<think></think>',
+        history_thinking_prefix='<think></think>',
+        agent_template='hy_v3_preview'))
+
+
+class HyV3Template(HyV3PreviewTemplate):
+    HYTK = ':opensource'
+
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.hy_v3,
+        prefix=['<｜hy_begin_of_sentence:opensource｜>'],
+        system_prefix=['<｜hy_begin_of_sentence:opensource｜>{{SYSTEM}}'],
+        prompt=['<｜hy_User:opensource｜>{{QUERY}}<｜hy_Assistant:opensource｜>'],
+        chat_sep=['<｜hy_eos:opensource｜>'],
+        suffix=['<｜hy_eos:opensource｜>'],
+        template_cls=HyV3Template,
+        is_thinking=True,
+        thinking_prefix='<think:opensource>',
+        non_thinking_prefix='<think:opensource></think:opensource>',
+        history_thinking_prefix='<think:opensource></think:opensource>',
+        agent_template='hy_v3'))
+
+
+class GptTemplate(Template):
+    support_padding_free = False
+
+    def _get_gpt_oss_prefix(self):
+        today = datetime.now().strftime('%Y-%m-%d')
+        return ('<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\n'
+                f'Knowledge cutoff: 2024-06\nCurrent date: {today}\n\nReasoning: medium\n\n'
+                '# Valid channels: analysis, commentary, final. '
+                'Channel must be included for every message.<|end|>')
+
+    def _swift_prepare_inputs(self, inputs: StdTemplateInputs):
+        super()._swift_prepare_inputs(inputs)
+        messages = inputs.messages
+        if self.use_chat_template:
+            if inputs.system is None:
+                inputs.system = self._get_gpt_oss_prefix()
+            elif not inputs.system.startswith('<|start|>'):
+                inputs.system = self._get_gpt_oss_prefix() + (
+                    f'<|start|>developer<|message|># Instructions\n\n{inputs.system}<|end|>')
+            for i, message in enumerate(messages):
+                if message['role'] == 'assistant' and isinstance(message['content'], str):
+                    if not message['content'].startswith('<|channel|>'):
+                        message['content'] = '<|channel|>final<|message|>' + message['content']
+
+
+@dataclass
+class GptOssTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['{{SYSTEM}}'])
+    prompt: Prompt = field(default_factory=lambda: ['<|start|>user<|message|>{{QUERY}}<|end|><|start|>assistant'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|return|>'])
+
+
+register_template(GptOssTemplateMeta(LLMTemplateType.gpt_oss, template_cls=GptTemplate))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.longchat,
+        prefix=[],
+        system_prefix=['SYSTEM:{{SYSTEM}}'],
+        prompt=[' [Round {{ROUND0}}] USER:{{QUERY}} ASSISTANT:'],
+        chat_sep=['</longcat_s>'],
+        suffix=['</longcat_s>'],
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ling2,
+        prefix=['<role>SYSTEM</role>detailed thinking off<|role_end|>'],
+        system_prefix=['<role>SYSTEM</role>{{SYSTEM}}\ndetailed thinking off<|role_end|>'],
+        prompt=['<role>HUMAN</role>{{QUERY}}<|role_end|><role>ASSISTANT</role>'],
+        chat_sep=['<|role_end|>'],
+        suffix=['<|role_end|>'],
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ring2,
+        prefix=[],
+        system_prefix=['<role>SYSTEM</role>{{SYSTEM}}'],
+        prompt=['<role>HUMAN</role>{{QUERY}}<role>ASSISTANT</role>'],
+        chat_sep=[],
+        suffix=['<|endoftext|>'],
+        is_thinking=True,
+        thinking_prefix='<think>\n',
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ring2_5,
+        prefix=[],
+        system_prefix=['<role>SYSTEM</role>\n{{SYSTEM}}\n\n'],
+        prompt=['<role>HUMAN</role>\n{{QUERY}}<|role_end|>\n\n<role>ASSISTANT</role>\n'],
+        chat_sep=['<|role_end|>\n\n'],
+        suffix=['<|role_end|>\n\n'],
+        is_thinking=True,
+    ))
+
+register_template(
+    QwenTemplateMeta(
+        LLMTemplateType.iquestcoder,
+        default_system='You are LoopCoder, a helpful assistant developed by IQuest.',
+    ))
+
+
+class YoutuLLMTemplate(Template):
+
+    def _remove_thinking_content(self, content: str) -> str:
+        if '</think>' in content:
+            content = content.rsplit('</think>', 1)[-1].lstrip('\n')
+        return self.template_meta.history_thinking_prefix + content.strip()
+
+    def _add_non_thinking_prefix(self, inputs) -> None:
+        messages = inputs.messages
+        non_thinking_prefix = self.template_meta.non_thinking_prefix
+        if non_thinking_prefix and messages:
+            # Find the last assistant message
+            for i in range(len(messages) - 1, -1, -1):
+                message = messages[i]
+                if message['role'] == 'assistant' and isinstance(message['content'], str):
+                    if '<think>' not in message['content'] and '</think>' not in message['content']:
+                        message['content'] = non_thinking_prefix + message['content']
+                    break
+
+    def _remove_history_thinking(self, inputs) -> None:
+        messages = inputs.messages
+        first_tool_index = len(messages)
+        for i, message in enumerate(messages):
+            if message['role'] == 'tool' or (message['role'] == 'user' and isinstance(message.get('content'), str)
+                                             and message['content'].startswith('<tool_response>')
+                                             and message['content'].endswith('</tool_response>')):
+                first_tool_index = i
+                break
+        # Only remove thinking content for assistant messages before first_tool_index - 1
+        for i, message in enumerate(messages):
+            if message['role'] == 'assistant' and isinstance(message['content'], str):
+                is_last = (i == len(messages) - 1)
+                if not is_last and i < first_tool_index - 1:
+                    message['content'] = self._remove_thinking_content(message['content'])
+
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.youtu_llm,
+        template_cls=YoutuLLMTemplate,
+        prefix=[['bos_token_id']],
+        system_prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=['<|User|>{{QUERY}}<|Assistant|>'],
+        chat_sep=['<|end_of_text|>'],
+        suffix=['<|end_of_text|>'],
+        is_thinking=True,
+        non_thinking_prefix='<think>\n\n</think>\n\n',
+        agent_template='youtu',
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.olmoe,
+        prefix=['|||IP_ADDRESS|||'],
+        system_prefix=['|||IP_ADDRESS|||<|system|>\n{{SYSTEM}}\n'],
+        prompt=['<|user|>\n{{QUERY}}\n<|assistant|>\n'],
+        chat_sep=['|||IP_ADDRESS|||\n'],
+        suffix=['|||IP_ADDRESS|||'],
+        stop_words=['<|endoftext|>'],
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.olmoe_0924,
+        prefix=['<|endoftext|>'],
+        system_prefix=['<|endoftext|><|system|>\n{{SYSTEM}}\n'],
+        prompt=['<|user|>\n{{QUERY}}\n<|assistant|>\n'],
+        chat_sep=['<|endoftext|>\n'],
+        suffix=['<|endoftext|>'],
+        stop_words=['<|endoftext|>'],
+    ))

swift/template/templates/megrez.py

@@ -0,0 +1,96 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+import torch.nn as nn
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class MegrezTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|role_start|>system<|role_end|>{{SYSTEM}}<|turn_end|>'])
+    prompt: Prompt = field(default_factory=lambda:
+                           ['<|role_start|>user<|role_end|>{{QUERY}}<|turn_end|><|role_start|>assistant<|role_end|>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|turn_end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|turn_end|>'])
+    default_system: str = '你是Megrez-3B-Instruct，将针对用户的问题给出详细的、积极的回答。'
+
+
+register_template(MegrezTemplateMeta(LLMTemplateType.megrez))
+
+
+class MegrezOmniTemplate(Template):
+    skip_prompt = False
+    placeholder_tokens = ['<|unk|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [[-1], '\n']
+        elif media_type == 'audio':
+            return [f'Audio {index + 1}: ', [-2], '\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+
+        for mm_key in ['images', 'audios']:
+            mm_data = getattr(inputs, mm_key)
+            if not mm_data:
+                continue
+            if mm_key == 'images':
+                idx_list = findall(input_ids, -1)
+                encoding = self.processor.process_image(
+                    mm_data,
+                    return_tensors='pt',
+                )
+                text = self.processor.insert_image_feature_placeholders(
+                    '<s>'.join(['(<image>./</image>)'] * len(mm_data)), encoding)
+                encoded['image_encoding'] = encoding
+            else:
+                idx_list = findall(input_ids, -2)
+                encoding = self.processor.process_audio(
+                    mm_data,
+                    return_tensors='pt',
+                )
+                text = self.processor.insert_audio_feature_placeholders(
+                    '<s>'.join(['(<audio>./</audio>)'] * len(mm_data)), encoding)
+                encoded['audio_encoding'] = encoding
+
+            padding = text.split('<s>')
+
+            def _get_new_tokens(i):
+                return self._tokenize(padding[i])
+
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_tokens)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        _, inputs_embeds, _ = model.compose_embeddings(inputs)
+        inputs.pop('position_ids', None)
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        new_batch = []
+        for b in batch:
+            text_encodings = {'input_ids': torch.tensor(b['input_ids'])}
+            multimodal_inputs = {'image_encoding': b.get('image_encoding'), 'audio_encoding': b.get('audio_encoding')}
+            new_batch.append(self.processor.merge_encodings(text_encodings, multimodal_inputs))
+        res.update(self.processor.data_collator(new_batch))
+        return res
+
+
+register_template(MegrezTemplateMeta(MLLMTemplateType.megrez_omni, template_cls=MegrezOmniTemplate))

swift/template/templates/microsoft.py

@@ -0,0 +1,209 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import json
+import torch
+from dataclasses import dataclass, field
+from torch import nn
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_file
+
+
+class FlorenceTemplate(Template):
+    # If it's an encoder-decoder architecture, the default settings are
+    # loss_scale: 'last_round' and skip_prompt: False.
+    is_encoder_decoder = True
+    skip_prompt = False
+
+    @staticmethod
+    def _add_default_tags(inputs: StdTemplateInputs) -> None:
+        return
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        return []
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [''.join(f'<loc_{box}>' for box in bbox)]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        processor = self.processor
+        inputs.query = inputs.to_history()['query']
+        new_query = processor._construct_prompts([inputs.query])[0]
+        for i in reversed(range(len(inputs.messages))):
+            if inputs.messages[i]['role'] == 'user':
+                inputs.messages[i]['content'] = new_query
+                break
+        encoded = super()._encode(inputs)
+        input_ids = encoded['prompt_input_ids']
+        images = inputs.images or []
+        labels = encoded['answer_labels']
+        if labels is not None:
+            labels = [0] + labels
+        if images:
+            pixel_values = processor.image_processor(
+                images, return_tensors='pt')['pixel_values'].to(self.model_info.torch_dtype)
+            encoded['pixel_values'] = pixel_values
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs_embeds = model.get_input_embeddings()(inputs['input_ids'])
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            image_features = model._encode_image(pixel_values)
+            inputs_embeds, inputs['attention_mask'] = model._merge_input_ids_with_image_features(
+                image_features, inputs_embeds)
+        return {'inputs_embeds': inputs_embeds}
+
+    def decode_generate_ids(self, generate_ids: List[int], **kwargs) -> Any:
+        response = super().decode_generate_ids(generate_ids, **kwargs)
+        template_inputs = kwargs.get('template_inputs')
+        images = template_inputs.images
+        image_size = None
+        if images:
+            image_size = (images[0].width, images[0].height)
+        query_before, query_sep, query_after = template_inputs.query.partition('>')
+        task = query_before + query_sep if query_sep else ''
+        return json.dumps(self.processor.post_process_generation(response, task=task, image_size=image_size))
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.florence,
+        prefix=['<s>'],
+        prompt=['{{QUERY}}</s>'],
+        chat_sep=None,
+        suffix=['</s>'],
+        template_cls=FlorenceTemplate,
+    ))
+
+
+@dataclass
+class Phi3TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['<|user|>\n{{QUERY}}<|end|>\n<|assistant|>\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|end|>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<|end|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|system|>\n{{SYSTEM}}<|end|>\n'])
+    auto_add_bos: bool = True
+
+
+register_template(Phi3TemplateMeta(LLMTemplateType.phi3))
+
+
+@dataclass
+class Phi4TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(
+        default_factory=lambda: ['<|im_start|>user<|im_sep|>{{QUERY}}<|im_end|><|im_start|>assistant<|im_sep|>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|im_start|>system<|im_sep|>{{SYSTEM}}<|im_end|>'])
+    auto_add_bos: bool = True
+
+
+register_template(Phi4TemplateMeta(LLMTemplateType.phi4))
+
+
+class Phi3VisionTemplate(Template):
+    image_placeholder = ['<|image|><s>\n']  # <|image|>\n
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            return [f'<|image_{index + 1}|>\n']  # <|image_1|>\n
+        else:
+            return super().replace_tag(media_type, index, inputs)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        images = inputs.images or []
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, 32044)  # '<|image|>'
+
+        if len(images) > 0:
+            processor = self.processor
+            encoded.update(processor.image_processor(images, return_tensors='pt'))
+            assert len(idx_list) == len(images), f'len(idx_list): {len(idx_list)}, len(images): {len(images)}'
+            res_input_ids = []
+            res_labels = []
+            num_img_tokens = encoded.pop('num_img_tokens').tolist()
+            idx_list.insert(0, -1)
+            for i in range(len(idx_list) - 1):
+                image_token_id = -i - 1
+                res_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]] + [image_token_id] * num_img_tokens[i]
+                if labels is not None:
+                    res_labels += labels[idx_list[i] + 1:idx_list[i + 1]] + [-100] * num_img_tokens[i]
+            res_input_ids += input_ids[idx_list[-1] + 1:]
+            input_ids = res_input_ids
+            if labels is not None:
+                res_labels += labels[idx_list[-1] + 1:]
+                labels = res_labels
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+
+class Phi4MMTemplate(Template):
+    placeholder_tokens = ['<|endoftext10|>', '<|endoftext11|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            if self.mode == 'vllm':
+                return [f'<|image_{index + 1}|>']  # <|image_1|>
+            return [[-100]]
+        elif media_type == 'audio':
+            import soundfile as sf
+            inputs.audios[index] = sf.read(load_file(inputs.audios[index]))
+            return [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        images_idx = findall(input_ids, -100)
+        audios_idx = findall(input_ids, -200)
+        text = '\n'.join(['<|image_1|>'] * len(inputs.images) + ['<|audio_1|>'] * len(inputs.audios))
+        new_encoded = self.processor(
+            text=text, images=inputs.images or None, audios=inputs.audios or None, return_tensors='pt')
+        placeholders = self._split_list(new_encoded.pop('input_ids')[0].tolist(), 198)
+
+        def _get_new_tokens(i):
+            return placeholders[i]
+
+        encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+            input_ids, labels, loss_scale, images_idx + audios_idx, _get_new_tokens)
+        new_encoded.pop('attention_mask')
+        encoded.update(new_encoded)
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        keys = [
+            'input_image_embeds', 'image_sizes', 'image_attention_mask', 'input_audio_embeds', 'audio_embed_sizes',
+            'input_mode'
+        ]
+        inputs = self.fetch_inputs(batch, keys)
+        for k, v in inputs.items():
+            inputs[k] = torch.concat(v)
+        res.update(inputs)
+        return res
+
+
+register_template(Phi3TemplateMeta(MLLMTemplateType.phi3_vision, template_cls=Phi3VisionTemplate))
+
+register_template(Phi3TemplateMeta(
+    MLLMTemplateType.phi4_multimodal,
+    template_cls=Phi4MMTemplate,
+))

swift/template/templates/midashenglm.py

@@ -0,0 +1,65 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+import torch.nn.functional as F
+from functools import partial
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from ..vision_utils import load_batch
+from .qwen import QwenTemplateMeta
+
+
+class MiDashengLMTemplate(Template):
+    placeholder_tokens = ['<|AUDIO|>']
+    skip_prompt = False
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.sampling_rate = get_env_args('sampling_rate', int, 16000)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'audio'
+        return ['<|AUDIO|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from transformers.audio_utils import load_audio
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        inputs.audios = load_batch(inputs.audios, partial(load_audio, sampling_rate=self.sampling_rate))
+        audio_token = self._tokenize('<|AUDIO|>')[0]
+        idx_list = findall(input_ids, audio_token)
+        if idx_list:
+            split_token = self._tokenize('\n')[0]
+            audio_inputs = self.processor(text='\n'.join(['<|AUDIO|>'] * len(inputs.audios)), audio=inputs.audios)
+            splited_tokens = self._split_list(audio_inputs['input_ids'][0].tolist(), split_token)
+
+            encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, encoded['labels'], encoded['loss_scale'], idx_list, lambda i: splited_tokens[i])
+            encoded['input_values'] = audio_inputs['input_values']
+            encoded['audio_length'] = audio_inputs['audio_length']
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+
+        input_values = [b['input_values'] for b in batch if b.get('input_values') is not None]
+        audio_lengths = [b['audio_length'] for b in batch if b.get('audio_length') is not None]
+
+        if input_values:
+            res['audio_length'] = torch.concat(audio_lengths)
+            for i in range(len(input_values)):
+                # TODO: check padding_side
+                pad_len = (res['audio_length'].max() - input_values[i].shape[1]).item()
+                input_values[i] = F.pad(input_values[i], (0, pad_len), 'constant', 0)
+            res['input_values'] = torch.concat(input_values)
+
+        return res
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.midashenglm, template_cls=MiDashengLMTemplate))

swift/template/templates/minicpm.py

@@ -0,0 +1,723 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import math
+import numpy as np
+import torch
+import transformers
+from dataclasses import dataclass, field
+from functools import partial
+from packaging import version
+from torch import nn
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args, get_packed_seq_params
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_audio, load_video_minicpmv_mplug_owl3
+from .llama import Llama3TemplateMeta
+from .qwen import Qwen2_5TemplateMeta, Qwen3MixedTemplateMeta, QwenTemplateMeta
+from .utils import ChatmlTemplateMeta
+
+
+@dataclass
+class MinicpmTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>{{SYSTEM}}'])
+    prompt: Prompt = field(default_factory=lambda: ['<用户>{{QUERY}}<AI>'])
+    chat_sep: Optional[Prompt] = field(default_factory=list)
+    suffix: Prompt = field(default_factory=lambda: ['</s>'])
+
+
+register_template(MinicpmTemplateMeta(LLMTemplateType.minicpm))
+
+
+def _remove_idx(arr: List[int], idx_list: List[int]) -> List[int]:
+    res = []
+    idx_set = set(idx_list)
+    for i, x in enumerate(arr):
+        if i not in idx_set:
+            res.append(x)
+    return res
+
+
+class MiniCPMVTemplate(Template):
+    is_v2_5 = False
+    use_model = True
+    skip_prompt = False
+    placeholder_tokens = ['<unk>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            return ['(<image>./</image>)\n']
+        else:
+            return [[-100]]
+
+    async def prepare_lmdeploy_turbomind_inputs(self, inputs: Dict[str, Any]) -> None:
+        images = inputs.pop('images', None) or []
+        if len(images) == 0:
+            return
+        input_ids = inputs['input_ids']
+        idx_list = findall(input_ids, -100)
+        idx_list.insert(0, -1)
+        new_input_ids = []
+        features = []
+        for i in range(len(idx_list) - 1):
+            new_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]]
+            context_list = ['<image>', [-100], '</image>']
+            feat = [x.squeeze() for x in images[i]['embeddings'].split(1)]
+            grid = images[i].get('grid')
+            if len(feat) > 1 and grid is not None:
+                context_list.append('<slice>')
+                for j in range(grid[1]):
+                    if j > 0:
+                        context_list.append('\n')
+                    for _ in range(grid[0]):
+                        context_list += ['<image>', [-100], '</image>']
+                context_list.append('</slice>\n')
+            new_input_ids += self._encode_context_list(context_list)[0]
+            features += feat
+        new_input_ids += input_ids[idx_list[-1] + 1:]
+        inputs['input_ids'] = new_input_ids
+        inputs['images'] = features
+        await super().prepare_lmdeploy_turbomind_inputs(inputs)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, -100)
+        idx = idx_list[0]
+        tgt_sizes = None
+        slice_mode = getattr(self.config, 'slice_mode', False)
+        if slice_mode:
+            if self.is_v2_5:
+                image_processor = self.processor.image_processor
+                image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+                placeholder = image_processor.get_slice_image_placeholder(image_inputs.image_sizes[0][0])
+                pixel_values = image_inputs['pixel_values']
+                tgt_sizes = image_inputs['tgt_sizes']
+            else:
+                images, placeholder = self.model.get_slice_image_placeholder(images[0], self.processor)
+                pixel_values = [[self.model.transform(img) for img in images]]
+            placeholder += '\n'
+            placeholder_id = self.processor.encode(placeholder, add_special_tokens=False)
+            input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
+            if labels is not None:
+                labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
+            input_tensor_ids = torch.tensor(input_ids)
+            image_start_idx = torch.where(input_tensor_ids == self.processor.im_start_id)[0]
+            image_start_idx += 1
+            image_end_idx = torch.where(input_tensor_ids == self.processor.im_end_id)[0]
+            valid_image_nums = max(len(image_start_idx), len(image_end_idx))
+            image_bound = [
+                torch.hstack(
+                    [image_start_idx[:valid_image_nums].unsqueeze(-1), image_end_idx[:valid_image_nums].unsqueeze(-1)])
+            ]
+        else:
+            placeholder = '<image>' + '<unk>' * self.config.query_num + '</image>\n'
+            placeholder_id = self.processor.encode(placeholder, add_special_tokens=False)
+            input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
+            if labels is not None:
+                labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
+            image_bound = [torch.tensor([[idx, idx + self.config.query_num]])]
+            pixel_values = [[self.model.transform(images[0])]]
+        encoded = {
+            'input_ids': input_ids,
+            'labels': labels,
+            'image_bound': image_bound,
+            'pixel_values': pixel_values,
+            'tgt_sizes': tgt_sizes
+        }
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs_embeds, _ = model.get_vllm_embedding(inputs)
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = {}
+        for k in ['pixel_values', 'image_bound', 'tgt_sizes']:
+            res[k] = self.gather_list(batch, k)
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+
+register_template(MinicpmTemplateMeta(MLLMTemplateType.minicpmv, template_cls=MiniCPMVTemplate))
+
+
+class MiniCPMV2_5Template(MiniCPMVTemplate):
+    is_v2_5 = True
+
+
+register_template(Llama3TemplateMeta(
+    MLLMTemplateType.minicpmv2_5,
+    template_cls=MiniCPMV2_5Template,
+))
+
+
+class MiniCPMV2_6Template(MiniCPMVTemplate):
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.max_num_frames = get_env_args('max_num_frames', int, 64)
+        self.max_slice_nums = get_env_args('max_slice_nums', int, None)
+        self.video_max_slice_nums = get_env_args('video_max_slice_nums', int, 1)  # or 2
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type in {'image', 'video'}
+        load_video = partial(load_video_minicpmv_mplug_owl3, max_num_frames=self.max_num_frames)
+        image_context = super().replace_tag('image', index, inputs)
+        if media_type == 'image':
+            return image_context
+        elif media_type == 'video':
+            return self.replace_video2image(load_video, inputs, lambda i: image_context)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        use_video = bool(inputs.videos)
+        use_image_id = True
+        max_slice_nums = self.max_slice_nums
+        if use_video:
+            max_slice_nums = self.video_max_slice_nums
+            use_image_id = False
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, -100)
+
+        image_processor = self.processor.image_processor
+        image_inputs = image_processor([images], return_tensors='pt',
+                                       max_slice_nums=max_slice_nums).to(self.model_info.torch_dtype)
+
+        def _get_new_tokens(i):
+            placeholder = image_processor.get_slice_image_placeholder(
+                image_inputs.image_sizes[0][i], image_idx=i, max_slice_nums=max_slice_nums, use_image_id=use_image_id)
+            placeholder += '\n'
+            return self.processor.encode(placeholder, add_special_tokens=False)
+
+        input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+
+        if inputs.images:
+            input_tensor_ids = torch.tensor(input_ids)
+            unk_token = self.processor.encode('<unk>', add_special_tokens=False)[0]
+            indices = (input_tensor_ids == unk_token).nonzero(as_tuple=True)[0].tolist()
+
+            ranges = []
+            start = indices[0]
+            for i in range(1, len(indices)):
+                if indices[i] != indices[i - 1] + 1:
+                    ranges.append([start, indices[i - 1] + 1])
+                    start = indices[i]
+            ranges.append([start, indices[-1] + 1])
+            image_bound = [torch.tensor(ranges)]
+        else:
+            image_bound = [[]]
+
+        encoded = {
+            'input_ids': input_ids,
+            'labels': labels,
+            'loss_scale': loss_scale,
+            'image_bound': image_bound,
+            'pixel_values': image_inputs['pixel_values'],
+            'tgt_sizes': image_inputs['tgt_sizes']
+        }
+        return encoded
+
+
+register_template(QwenTemplateMeta(
+    MLLMTemplateType.minicpmv2_6,
+    template_cls=MiniCPMV2_6Template,
+))
+
+register_template(ChatmlTemplateMeta(
+    MLLMTemplateType.minicpmv4,
+    template_cls=MiniCPMV2_6Template,
+))
+
+register_template(Qwen2_5TemplateMeta(
+    MLLMTemplateType.minicpmo,
+    template_cls=MiniCPMV2_6Template,
+))
+
+
+class MiniCPMV4_5Template(MiniCPMV2_6Template):
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        use_video = bool(inputs.videos)
+        use_image_id = True
+        max_slice_nums = self.max_slice_nums
+        if use_video:
+            max_slice_nums = self.video_max_slice_nums
+            use_image_id = False
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, -100)
+
+        image_processor = self.processor.image_processor
+        image_inputs = image_processor([images], return_tensors='pt',
+                                       max_slice_nums=max_slice_nums).to(self.model_info.torch_dtype)
+
+        def _get_new_tokens(i):
+            placeholder = image_processor.get_slice_image_placeholder(
+                image_inputs.image_sizes[0][i], image_idx=i, max_slice_nums=max_slice_nums, use_image_id=use_image_id)
+            placeholder += '\n'
+            return self.processor.encode(placeholder, add_special_tokens=False)
+
+        input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+
+        if inputs.images:
+            input_tensor_ids = torch.tensor(input_ids)
+            unk_token = self.processor.encode('<unk>', add_special_tokens=False)[0]
+            indices = (input_tensor_ids == unk_token).nonzero(as_tuple=True)[0].tolist()
+
+            ranges = []
+            start = indices[0]
+            for i in range(1, len(indices)):
+                if indices[i] != indices[i - 1] + 1:
+                    ranges.append([start, indices[i - 1] + 1])
+                    start = indices[i]
+            ranges.append([start, indices[-1] + 1])
+            image_bound = [torch.tensor(ranges)]
+        else:
+            image_bound = [[]]
+
+        encoded = {
+            'input_ids': input_ids,
+            'labels': labels,
+            'loss_scale': loss_scale,
+            'image_bound': image_bound,
+            'pixel_values': image_inputs['pixel_values'],
+            'tgt_sizes': image_inputs['tgt_sizes'],
+            'temporal_ids': image_inputs['temporal_ids'],
+        }
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = {}
+        for k in ['pixel_values', 'image_bound', 'tgt_sizes', 'temporal_ids']:
+            res[k] = self.gather_list(batch, k)
+        res.update(Template._data_collator(self, batch, padding_to=padding_to))
+        return res
+
+
+register_template(
+    Qwen3MixedTemplateMeta(
+        MLLMTemplateType.minicpmv4_5,
+        template_cls=MiniCPMV4_5Template,
+        is_thinking=True,
+        thinking_prefix='<think>\n',
+    ))
+
+
+class MiniCPMO4_5Template(MiniCPMV4_5Template):
+    SAMPLING_RATE = 16000
+    MAX_AUDIO_DURATION = 30  # seconds
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.use_audio_in_video = get_env_args('use_audio_in_video', bool, False)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image' or media_type == 'video' and not self.use_audio_in_video:
+            return super().replace_tag(media_type, index, inputs)
+        elif media_type == 'audio':
+            # Load audio from file path to numpy array at 16kHz
+            if isinstance(inputs.audios[index], str):
+                inputs.audios[index] = load_audio(inputs.audios[index], sampling_rate=self.SAMPLING_RATE)
+            return ['<|audio_start|><|audio_end|>']
+        elif media_type == 'video':
+            from minicpmo.utils import get_video_frame_audio_segments
+            video = inputs.videos[inputs.video_idx]
+            video_segments, audio_segments, _ = get_video_frame_audio_segments(
+                video, use_audio=self.use_audio_in_video, stack_frames=1)
+            # Insert frames into images list at current position
+            images = inputs.images
+            inputs.images = images[:inputs.image_idx] + video_segments + images[inputs.image_idx:]
+            # Build context list
+            image_context = [[-100]]
+            context_list = []
+            if self.use_audio_in_video and audio_segments:
+                # Insert audio segments into audios list at current position
+                audios = inputs.audios
+                inputs.audios = audios[:inputs.audio_idx] + audio_segments + audios[inputs.audio_idx:]
+                audio_context = ['<|audio_start|><|audio_end|>']
+                # Interleave: one image placeholder + one audio placeholder per second
+                for i in range(len(video_segments)):
+                    context_list += image_context
+                    if i < len(audio_segments):
+                        context_list += audio_context
+                inputs.audio_idx += len(audio_segments)
+            else:
+                for _ in range(len(video_segments)):
+                    context_list += image_context
+            inputs.image_idx += len(video_segments)
+            return context_list
+
+    def _get_audio_num_tokens(self, audio_sample_len: int) -> int:
+        """Compute the number of <unk> placeholder tokens for an audio of given sample count.
+
+        This mirrors the official get_audio_placeholder logic:
+        1. mel frames = ceil(audio_samples / hop_length)
+        2. after CNN downsampling: (mel_frames - 1) // 2 + 1
+        3. after avg pooling: (cnn_frames - pool_step) // pool_step + 1
+        """
+        hop_length = self.processor.audio_processor.hop_length  # 160
+        pool_step = self.config.audio_pool_step  # 5
+        feature_lens = math.ceil(audio_sample_len / hop_length)
+        feature_lens_after_cnn = (feature_lens - 1) // 2 + 1
+        output_lens = (feature_lens_after_cnn - pool_step) // pool_step + 1
+        return output_lens
+
+    def _extract_audio_features(self, audios: List[np.ndarray]):
+        """Extract mel features from audio arrays using the WhisperFeatureExtractor.
+
+        Handles chunking of long audios (>30s) into segments.
+        Matches the official audio_feature_extract output format.
+
+        Returns:
+            audio_features: tensor [N, 80, max_frames] or [] if no audios
+            audio_feature_lens: [tensor([l1, l2, ...])] or None
+        """
+        audio_processor = self.processor.audio_processor
+        max_audio_inp_len = self.MAX_AUDIO_DURATION * self.SAMPLING_RATE
+        all_audio_features = []
+        all_audio_lens = []
+
+        for audio in audios:
+            # Chunk long audios at 30s boundaries
+            if len(audio) <= max_audio_inp_len:
+                chunks = [audio]
+            else:
+                chunks = [audio[i:i + max_audio_inp_len] for i in range(0, len(audio), max_audio_inp_len)]
+
+            for chunk in chunks:
+                audio_input = audio_processor(
+                    chunk,
+                    sampling_rate=self.SAMPLING_RATE,
+                    return_tensors='pt',
+                    padding='max_length',
+                    return_attention_mask=True,
+                )
+                feat = audio_input['input_features']  # [1, 80, frames]
+                actual_len = audio_input['attention_mask'].sum(dim=1)  # [1]
+                feat = feat[:, :, :actual_len[0]]
+                all_audio_features.append(feat.squeeze(0))  # [80, actual_frames]
+                all_audio_lens.append(actual_len[0])
+
+        if all_audio_features:
+            # Pad and stack: [N, 80, max_frames] — same as official processor
+            audio_features = torch.nn.utils.rnn.pad_sequence(
+                [f.transpose(0, 1) for f in all_audio_features],
+                batch_first=True,
+                padding_value=0.0,
+            ).transpose(1, 2)
+            audio_feature_lens = [torch.hstack(all_audio_lens)]
+        else:
+            audio_features = []
+            audio_feature_lens = None
+
+        return audio_features, audio_feature_lens
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        # Step 1: Base encode — produces input_ids with -100 for images
+        # and audio_start_id,audio_end_id pairs for audios
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        use_video = bool(inputs.videos)
+        use_image_id = True
+        max_slice_nums = self.max_slice_nums
+        if use_video:
+            max_slice_nums = self.video_max_slice_nums
+            use_image_id = False
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+
+        # Step 2: Process images — replace -100 tokens with image placeholders
+        idx_list = findall(input_ids, -100)
+        image_processor = self.processor.image_processor
+        image_inputs = image_processor([images], return_tensors='pt',
+                                       max_slice_nums=max_slice_nums).to(self.model_info.torch_dtype)
+
+        def _get_new_tokens(i):
+            placeholder = image_processor.get_slice_image_placeholder(
+                image_inputs.image_sizes[0][i], image_idx=i, max_slice_nums=max_slice_nums, use_image_id=use_image_id)
+            placeholder += '\n'
+            return self.processor.encode(placeholder, add_special_tokens=False)
+
+        input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+
+        # Step 3: Process audios — expand audio_start/audio_end pairs with <unk> tokens
+        tokenizer = self.processor.tokenizer
+        audio_start_id = tokenizer.convert_tokens_to_ids('<|audio_start|>')
+        audio_end_id = tokenizer.convert_tokens_to_ids('<|audio_end|>')
+        unk_token_id = tokenizer.convert_tokens_to_ids('<unk>')
+
+        audio_features = None
+        audio_feature_lens = None
+
+        if inputs.audios:
+            audios = inputs.audios
+            audio_features, audio_feature_lens = self._extract_audio_features(audios)
+
+            # Find empty audio placeholder pairs (audio_start_id immediately followed by audio_end_id)
+            audio_placeholder_positions = []
+            for i in range(len(input_ids) - 1):
+                if input_ids[i] == audio_start_id and input_ids[i + 1] == audio_end_id:
+                    audio_placeholder_positions.append(i)
+
+            assert len(audio_placeholder_positions) == len(audios), \
+                f'Found {len(audio_placeholder_positions)} audio placeholders but have {len(audios)} audios'
+
+            # Expand each audio placeholder with <unk> tokens
+            offset = 0
+            for i, audio in enumerate(audios):
+                num_tokens = self._get_audio_num_tokens(len(audio))
+                unk_tokens = [unk_token_id] * num_tokens
+                pos = audio_placeholder_positions[i] + offset
+                # Current: [..., audio_start_id, audio_end_id, ...]
+                # Target:  [..., audio_start_id, unk*N, audio_end_id, ...]
+                input_ids = input_ids[:pos + 1] + unk_tokens + input_ids[pos + 1:]
+                if labels is not None:
+                    labels = labels[:pos + 1] + [-100] * num_tokens + labels[pos + 1:]
+                if loss_scale is not None:
+                    scale_val = loss_scale[pos]
+                    loss_scale = loss_scale[:pos + 1] + [scale_val] * num_tokens + loss_scale[pos + 1:]
+                offset += num_tokens
+
+        # Step 4: Compute image_bound using start/end token boundaries
+        # This is more robust than finding consecutive <unk> tokens, especially
+        # when both image and audio use <unk> as placeholder.
+        input_tensor_ids = torch.tensor(input_ids)
+
+        if images:
+            im_start_id = tokenizer.convert_tokens_to_ids('<image>')
+            im_end_id = tokenizer.convert_tokens_to_ids('</image>')
+            slice_start_id = tokenizer.convert_tokens_to_ids('<slice>')
+            slice_end_id = tokenizer.convert_tokens_to_ids('</slice>')
+
+            start_cond = (input_tensor_ids == im_start_id) | (input_tensor_ids == slice_start_id)
+            end_cond = (input_tensor_ids == im_end_id) | (input_tensor_ids == slice_end_id)
+            image_start_idx = torch.where(start_cond)[0] + 1
+            image_end_idx = torch.where(end_cond)[0]
+            valid_image_nums = min(len(image_start_idx), len(image_end_idx))
+            image_bound = [
+                torch.hstack([
+                    image_start_idx[:valid_image_nums].unsqueeze(-1),
+                    image_end_idx[:valid_image_nums].unsqueeze(-1),
+                ])
+            ]
+        else:
+            image_bound = [[]]
+
+        # Step 5: Compute audio_bounds
+        if inputs.audios:
+            audio_start_idx = torch.where(input_tensor_ids == audio_start_id)[0]
+            audio_end_idx = torch.where(input_tensor_ids == audio_end_id)[0]
+            assert len(audio_start_idx) == len(audio_end_idx)
+            audio_bounds = [torch.hstack([
+                (audio_start_idx + 1).unsqueeze(-1),
+                audio_end_idx.unsqueeze(-1),
+            ])]
+        else:
+            audio_bounds = [[]]
+
+        encoded = {
+            'input_ids': input_ids,
+            'labels': labels,
+            'loss_scale': loss_scale,
+            'image_bound': image_bound,
+            'pixel_values': image_inputs['pixel_values'],
+            'tgt_sizes': image_inputs['tgt_sizes'],
+            'audio_features': audio_features,
+            'audio_feature_lens': audio_feature_lens,
+            'audio_bounds': audio_bounds,
+        }
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        # Vision embeddings
+        inputs_embeds, _ = model.get_vllm_embedding(inputs)
+        # Audio embeddings — scatter audio features into the embedding
+        inputs_embeds = model.get_omni_embedding(
+            inputs,
+            input_embeddings=inputs_embeds,
+            chunk_length=getattr(self.config, 'audio_chunk_length', 1.0),
+        )
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = {}
+        # Vision data
+        for k in ['pixel_values', 'image_bound', 'tgt_sizes']:
+            res[k] = self.gather_list(batch, k)
+
+        # Audio data — collect from all samples
+        all_audio_feats = []  # list of [N_i, 80, frames_i] tensors
+        audio_feature_lens_list = []
+        audio_bounds_list = []
+
+        for b in batch:
+            af = b.pop('audio_features', None)
+            afl = b.pop('audio_feature_lens', None)
+            ab = b.pop('audio_bounds', None)
+
+            if af is not None and isinstance(af, torch.Tensor):
+                all_audio_feats.append(af)
+            if afl is not None:
+                audio_feature_lens_list.extend(afl)
+            if ab is not None:
+                audio_bounds_list.extend(ab)
+
+        # Re-pad audio features across the batch to the same max frame length
+        if all_audio_feats:
+            # Unpack per-sample tensors into individual segments, then re-pad
+            segments = []
+            for af in all_audio_feats:
+                for i in range(af.shape[0]):
+                    segments.append(af[i])  # [80, frames_i]
+            res['audio_features'] = torch.nn.utils.rnn.pad_sequence(
+                [s.transpose(0, 1) for s in segments],
+                batch_first=True,
+                padding_value=0.0,
+            ).transpose(1, 2)  # [total_segments, 80, max_frames]
+        else:
+            res['audio_features'] = []
+
+        res['audio_feature_lens'] = audio_feature_lens_list if audio_feature_lens_list else []
+        res['audio_bounds'] = audio_bounds_list if audio_bounds_list else []
+
+        res.update(Template._data_collator(self, batch, padding_to=padding_to))
+        return res
+
+
+register_template(
+    Qwen3MixedTemplateMeta(
+        MLLMTemplateType.minicpmo4_5,
+        template_cls=MiniCPMO4_5Template,
+        is_thinking=True,
+    ))
+
+
+class MiniCPMV4_6Template(Template):
+    support_padding_free = True
+    placeholder_tokens = ['<|image_pad|>', '<|video_pad|>']
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.downsample_mode = get_env_args('downsample_mode', str, '16x')
+        self.max_slice_nums = get_env_args('max_slice_nums', int, 9)
+        self.video_max_slice_nums = get_env_args('video_max_slice_nums', int, 1)
+        self.max_num_frames = get_env_args('max_num_frames', int, 128)
+        self.stack_frames = get_env_args('stack_frames', int, 1)
+        self.transformers_version = version.parse(transformers.__version__)
+        self.transformers_5_9 = self.transformers_version >= version.parse('5.9.0')
+
+    def _preprocess_inputs(self, inputs: StdTemplateInputs) -> None:
+        super()._preprocess_inputs(inputs)
+        # Inject downsample_mode into mm_processor_kwargs so that vLLM rollout
+        # receives the correct mode via _encode_truncated -> _add_request.
+        inputs.mm_processor_kwargs['downsample_mode'] = self.downsample_mode
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|image_pad|>\n']
+        else:
+            return ['<|video_pad|>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        split_token = self._tokenize(self.tokenizer.eos_token)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        for media_type in ['image', 'video']:
+            mm_data = getattr(inputs, f'{media_type}s')
+            media_token = f'<|{media_type}_pad|>'
+            media_token_id = self._tokenize(media_token)[0]
+            max_slice_nums = self.max_slice_nums if media_type == 'image' else self.video_max_slice_nums
+            if mm_data:
+                media_inputs = self.processor(
+                    text=self.tokenizer.eos_token.join([media_token] * len(mm_data)),
+                    images=inputs.images or None,
+                    videos=inputs.videos or None,
+                    return_tensors='pt',
+                    add_special_tokens=False,
+                    downsample_mode=self.downsample_mode,
+                    stack_frames=self.stack_frames,
+                    max_num_frames=self.max_num_frames,
+                    max_slice_nums=max_slice_nums,
+                )
+                splited_tokens = self._split_list(media_inputs['input_ids'][0].tolist(), split_token)
+                idx_list = findall(input_ids, media_token_id)
+
+                def _get_new_tokens(i):
+                    return splited_tokens[i]
+
+                input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                    _get_new_tokens)
+                encoded.update(media_inputs)
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = {}
+        pixel_values = [b['pixel_values'] for b in batch if b.get('pixel_values') is not None]
+        if len(pixel_values) > 0:
+            res['pixel_values'] = torch.concat(pixel_values, dim=-1)
+        pixel_values_videos = [b['pixel_values_videos'] for b in batch if b.get('pixel_values_videos') is not None]
+        if len(pixel_values_videos) > 0:
+            res['pixel_values_videos'] = torch.concat(pixel_values_videos, dim=-1)
+
+        for key in ['target_sizes', 'target_sizes_videos']:
+            value = self.concat_tensor(batch, key, dim=0)
+            if value is not None:
+                res[key] = value
+
+        # Inject downsample_mode so the model forward uses the same mode
+        # as data preprocessing, keeping image token/feature counts aligned.
+        res['downsample_mode'] = self.downsample_mode
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if self.padding_free:
+            res.update(get_packed_seq_params(res['position_ids']))
+        return res
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if self.padding_free and self.sequence_parallel_size <= 1 and not self.transformers_5_9:
+            raise RuntimeError('MiniCPM-V 4.6 packing/padding_free with sequence_parallel_size=1 requires '
+                               f'transformers>=5.9.0 (current: {self.transformers_version}). ')
+        return super()._post_encode(model, inputs)
+
+
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.minicpmv4_6,
+        template_cls=MiniCPMV4_6Template,
+        is_thinking=True,
+        thinking_prefix='<think>\n',
+        non_thinking_prefix='<think>\n\n</think>\n\n',
+    ))
+
+register_template(
+    ChatmlTemplateMeta(
+        LLMTemplateType.minicpm5,
+        is_thinking=True,
+        thinking_prefix='<think>\n',
+        non_thinking_prefix='<think>\n\n</think>\n\n',
+        agent_template='minicpm5',
+    ))

swift/template/templates/minimax.py

@@ -0,0 +1,301 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args, get_logger
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+logger = get_logger()
+
+
+@dataclass
+class MinimaxTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: [
+        '<beginning_of_sentence>user name=user\n{{QUERY}}<end_of_sentence>\n'
+        '<beginning_of_sentence>ai name=assistant\n'
+    ])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<end_of_sentence>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<end_of_sentence>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<beginning_of_sentence>system ai_setting=assistant\n{{SYSTEM}}<end_of_sentence>\n'])
+
+
+register_template(MinimaxTemplateMeta(LLMTemplateType.minimax))
+
+register_template(
+    MinimaxTemplateMeta(
+        LLMTemplateType.minimax_m1,
+        prefix=['<begin_of_document>'],
+        system_prefix=[
+            '<begin_of_document><beginning_of_sentence>system ai_setting=assistant\n{{SYSTEM}}<end_of_sentence>\n'
+        ],
+    ))
+
+
+class MinimaxVLTemplate(Template):
+    image_placeholder = ['<image>']
+    skip_prompt = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return self.image_placeholder * inputs.all_image_tokens[index]
+
+    def calc_num_image_tokens(self, image_inputs):
+        from transformers.image_utils import get_image_size, to_numpy_array
+        pixel_values = image_inputs['pixel_values']
+        image_sizes = image_inputs['image_sizes']
+        all_image_tokens = []
+        if not image_inputs:
+            return all_image_tokens
+
+        if self.processor.process_image_mode == 'anyres':
+            for pixel_value, image_size in zip(pixel_values, image_sizes):
+                height, width = image_size
+                num_image_tokens = self.processor.get_num_token(height, width, self.processor.grid_pinpoints,
+                                                                self.processor.patch_size)
+                all_image_tokens.append(num_image_tokens)
+        elif self.processor.process_image_mode == 'resize':
+            pixel_values = image_inputs['pixel_values']
+            all_image_tokens = []
+            for pixel_value in pixel_values:
+                height, width = get_image_size(to_numpy_array(pixel_value))
+                all_image_tokens.append(int(height * width / self.processor.patch_size**2))
+        else:
+            if self.processor.patch_size is not None:
+                pixel_values = image_inputs['pixel_values']
+                all_image_tokens = []
+                for pixel_value in pixel_values:
+                    height, width = get_image_size(to_numpy_array(pixel_value))
+                    new_width, new_height = self.processor.get_hw_multiple_of(
+                        (width, height), self.processor.patch_size, self.processor.max_size)
+                    num_image_tokens = ((new_height // self.processor.patch_size) *
+                                        (new_width // self.processor.patch_size))  # + 1
+                    all_image_tokens.append(num_image_tokens)
+            else:
+                logger.warning_once(
+                    'Expanding inputs for image tokens in MiniMaxVL01 should be done in processing. '
+                    "Please add `patch_size` and `vision_feature_select_strategy` to the model's "
+                    'processing config or set directly '
+                    'with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = '
+                    '{{vision_feature_select_strategy}}`. '
+                    'Using processors without these attributes in the config is deprecated '
+                    'and will throw an error in v4.47.')
+                raise ValueError(
+                    "You need to provide `patch_size` and `vision_feature_select_strategy` in the model's processing "
+                    'config to expand inputs for image tokens.')
+        return all_image_tokens
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        output_kwargs = self.processor._merge_kwargs(
+            self.processor.MiniMaxVL01ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+        )
+        if inputs.images:
+            image_inputs = self.processor.image_processor(
+                inputs.images, **output_kwargs['images_kwargs'], return_tensors='pt')
+            inputs.all_image_tokens = self.calc_num_image_tokens(image_inputs)
+        else:
+            image_inputs = {}
+        encoded = super()._encode(inputs)
+        for key in image_inputs:
+            encoded[key] = image_inputs[key]
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        pixel_values = self.gather_list(batch, 'pixel_values')
+        image_sizes = self.gather_list(batch, 'image_sizes')
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if pixel_values:
+            res['pixel_values'] = pixel_values
+        if image_sizes:
+            res['image_sizes'] = image_sizes
+        return res
+
+
+register_template(MinimaxTemplateMeta(LLMTemplateType.minimax_vl, template_cls=MinimaxVLTemplate))
+
+
+@dataclass
+class MinimaxM2TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: [']~!b[]~b]system\n{{SYSTEM}}[e~[\n'])
+    prompt: Prompt = field(default_factory=lambda: [']~b]user\n{{QUERY}}[e~[\n]~b]ai\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['[e~[\n'])
+    suffix: Prompt = field(default_factory=lambda: ['[e~[\n'])
+    agent_template: Optional[str] = 'minimax_m2'
+    is_thinking: bool = True
+    thinking_prefix: str = '<think>\n'
+
+
+register_template(
+    MinimaxM2TemplateMeta(
+        LLMTemplateType.minimax_m2,
+        default_system='You are MiniMax-M2, a helpful AI assistant built by MiniMax. Knowledge cutoff: 2025-06.',
+    ))
+
+register_template(
+    MinimaxM2TemplateMeta(
+        LLMTemplateType.minimax_m2_1,
+        default_system='You are a helpful assistant. Your name is MiniMax-M2.1 and is built by MiniMax.',
+    ))
+
+register_template(
+    MinimaxM2TemplateMeta(
+        LLMTemplateType.minimax_m2_5,
+        default_system='You are a helpful assistant. Your name is MiniMax-M2.5 and is built by MiniMax.',
+    ))
+
+register_template(
+    MinimaxM2TemplateMeta(
+        LLMTemplateType.minimax_m2_7,
+        default_system='You are a helpful assistant. Your name is MiniMax-M2.7 and is built by MiniMax.',
+    ))
+
+_MINIMAX_M3_IDENTITY = ('Your model version is MiniMax-M3, developed by MiniMax. Knowledge cutoff: January 2026. '
+                        'Founded in early 2022, MiniMax is a global AI foundation model company committed to '
+                        'advancing the frontiers of AI towards AGI.')
+
+_MINIMAX_M3_THINKING_BASE = (
+    'You have a thinking capability that allows you to reason step by step before responding. '
+    'When thinking is enabled, wrap your reasoning in <mm:think></mm:think> tags before your '
+    'response. When thinking is disabled, begin your response directly after the </mm:think> '
+    'prefix. When thinking is adaptive, decide on your own whether to think for the current turn.')
+
+_MINIMAX_M3_THINKING_MODE_TEXT = {
+    'enabled': ('Current thinking mode: enabled. You MUST think step by step before every response, '
+                'including after receiving function/tool results.'),
+    'disabled':
+    'Current thinking mode: disabled. Do not output any thinking process.',
+    'adaptive': ('Current thinking mode: adaptive. You are encouraged to think for complex '
+                 'decision-making, multi-step reasoning, or when analyzing function/tool results.'),
+}
+
+_MINIMAX_M3_DEFAULT_DEVELOPER = 'You are a helpful assistant.'
+
+
+def _build_m3_system_block(thinking_mode: str = 'adaptive') -> str:
+    mode_text = _MINIMAX_M3_THINKING_MODE_TEXT.get(thinking_mode, _MINIMAX_M3_THINKING_MODE_TEXT['adaptive'])
+    return (f'{_MINIMAX_M3_IDENTITY}'
+            f'\n\n<thinking_instructions>\n{_MINIMAX_M3_THINKING_BASE}\n{mode_text}\n</thinking_instructions>')
+
+
+class MinimaxM3VLTemplate(Template):
+    image_token = ']<]image[>['
+    video_token = ']<]video[>['
+    placeholder_tokens = [']<]image[>[', ']<]video[>[']
+
+    def init_env_args(self):
+        super().init_env_args()
+        # thinking_mode: "enabled" / "disabled" / "adaptive"
+        self.thinking_mode = get_env_args('thinking_mode', str, 'disabled')
+        self.chat_template_kwargs['thinking_mode'] = self.thinking_mode
+        # Map thinking_mode to enable_thinking for the broader framework
+        if self.thinking_mode == 'disabled':
+            self.enable_thinking = False
+        else:
+            self.enable_thinking = True
+
+    def _get_thinking_mode(self, inputs=None) -> str:
+        thinking_mode = None if inputs is None else inputs.chat_template_kwargs.get('thinking_mode')
+        if thinking_mode is None:
+            thinking_mode = self.chat_template_kwargs.get('thinking_mode', 'adaptive')
+        return thinking_mode
+
+    def _get_enable_thinking(self, inputs=None):
+        thinking_mode = self._get_thinking_mode(inputs)
+        return thinking_mode != 'disabled'
+
+    def _get_response_prefix(self, inputs=None):
+        # Check explicit override first
+        response_prefix = None if inputs is None else inputs.chat_template_kwargs.get('response_prefix')
+        if response_prefix is not None:
+            return response_prefix
+        if self.response_prefix is not None:
+            return self.response_prefix
+        thinking_mode = self._get_thinking_mode(inputs)
+        if thinking_mode == 'enabled':
+            return self.template_meta.thinking_prefix  # '<mm:think>'
+        elif thinking_mode == 'disabled':
+            return self.template_meta.non_thinking_prefix  # '</mm:think>'
+        else:  # adaptive
+            return ''  # No prefix, let model decide
+
+    def _get_system(self, inputs: StdTemplateInputs) -> str:
+        system = super()._get_system(inputs)
+        thinking_mode = self._get_thinking_mode(inputs)
+        system_block = _build_m3_system_block(thinking_mode)
+        return f'{system_block}[e~[\n]~b]developer\n{system or ""}'
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [self.image_token]
+        elif media_type == 'video':
+            return [self.video_token]
+        else:
+            raise ValueError(f'Unsupported media type for MiniMax-M3 VL: {media_type}')
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        if not inputs.images and not inputs.videos:
+            return encoded
+        media_text_parts = ([self.image_token] * len(inputs.images) + [self.video_token] * len(inputs.videos))
+        media_inputs = self.processor(
+            text=self.tokenizer.eos_token.join(media_text_parts),
+            images=inputs.images or None,
+            videos=inputs.videos or None,
+            return_tensors='pt',
+        )
+        split_token = self._tokenize(self.tokenizer.eos_token)
+        splited_tokens = self._split_list(media_inputs['input_ids'][0].tolist(), split_token)
+        media_inputs.pop('input_ids', None)
+        media_inputs.pop('attention_mask', None)
+
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+
+        idx_list = []
+        for key in ['image', 'video']:
+            token_id = getattr(self.config, f'{key}_token_id', None)
+            if token_id is None:
+                continue
+            idx_list += findall(input_ids, token_id)
+        sorted_order = sorted(range(len(idx_list)), key=lambda i: idx_list[i])
+        idx_list = [idx_list[i] for i in sorted_order]
+        splited_tokens = [splited_tokens[i] for i in sorted_order]
+
+        def _get_new_tokens(i):
+            return splited_tokens[i]
+
+        if idx_list:
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_tokens)
+        encoded.update(media_inputs)
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+
+@dataclass
+class MinimaxM3VLTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: [']~!b[]~b]system\n{{SYSTEM}}[e~[\n'])
+    prompt: Prompt = field(default_factory=lambda: [']~b]user\n{{QUERY}}[e~[\n]~b]ai\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['[e~[\n'])
+    suffix: Prompt = field(default_factory=lambda: ['[e~[\n'])
+    default_system: Optional[str] = _MINIMAX_M3_DEFAULT_DEVELOPER
+    agent_template: Optional[str] = 'minimax_m3'
+    is_thinking: bool = True
+    thinking_prefix: str = '<mm:think>'
+    non_thinking_prefix: str = '</mm:think>'
+    history_thinking_prefix: str = '</mm:think>'
+
+
+register_template(MinimaxM3VLTemplateMeta(MLLMTemplateType.minimax_m3_vl, template_cls=MinimaxM3VLTemplate))

swift/template/templates/minimind.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from ..constant import LLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..utils import Prompt, Word
+
+DEFAULT_SYSTEM = 'You are a helpful assistant'
+
+
+@dataclass
+class MiniMindTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['<|im_start|>user\n{{QUERY}}<|im_end|>\n<|im_start|>assistant\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>\n'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|im_start|>system\n{{SYSTEM}}<|im_end|>\n'])
+    default_system: Optional[str] = DEFAULT_SYSTEM
+    stop_words: List[Word] = field(default_factory=lambda: ['<|endoftext|>'])
+
+
+register_template(MiniMindTemplateMeta(LLMTemplateType.minimind))

swift/template/templates/mistral.py

@@ -0,0 +1,203 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+today = datetime.now().strftime('%Y-%m-%d')
+
+mistral_2501_system = (
+    'You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup '
+    'headquartered in Paris.\n'
+    f'Your knowledge base was last updated on 2023-10-01. The current date is {today}.\n\n'
+    "When you're not sure about some information, you say that you don't have the information and don't "
+    'make up anything.\n'
+    "If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer "
+    'the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. '
+    '"What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "'
+    'Where do you travel from?")')
+
+
+@dataclass
+class Mistral3TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>'])
+    prompt: Prompt = field(default_factory=lambda: ['[INST]{{QUERY}}[/INST]'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['</s>'])
+    suffix: Prompt = field(default_factory=lambda: ['</s>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<s>[SYSTEM_PROMPT]{{SYSTEM}}[/SYSTEM_PROMPT]'])
+
+
+register_template(Mistral3TemplateMeta(LLMTemplateType.mistral_2501, default_system=mistral_2501_system))
+
+
+class Mistral2503Template(Template):
+    placeholder_tokens = ['[IMG]']
+    image_token = 10
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['[IMG]']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, self.image_token)
+        patch_size = processor.patch_size * processor.spatial_merge_size
+        if idx_list:
+            image_inputs = processor.image_processor(images, patch_size=patch_size, return_tensors='pt')
+            encoded['pixel_values'] = image_inputs['pixel_values'].to(self.model_info.torch_dtype)
+            encoded['image_sizes'] = image_sizes = image_inputs['image_sizes']
+
+            def _get_new_tokens(i):
+                height, width = image_sizes[i]
+                num_height_tokens = height // patch_size
+                num_width_tokens = width // patch_size
+                replace_tokens = [[processor.image_token] * num_width_tokens + [processor.image_break_token]
+                                  ] * num_height_tokens
+                # Flatten list
+                replace_tokens = [item for sublist in replace_tokens for item in sublist]
+                replace_tokens[-1] = processor.image_end_token
+                replace_str = ''.join(replace_tokens)
+                return processor.encode(replace_str, add_special_tokens=False)
+
+            encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+
+        return encoded
+
+
+register_template(
+    Mistral3TemplateMeta(
+        MLLMTemplateType.mistral_2503, default_system=mistral_2501_system, template_cls=Mistral2503Template))
+
+devstral_small_2505_system = (  # from https://huggingface.co/mistralai/Devstral-Small-2505/blob/main/SYSTEM_PROMPT.txt
+    'You are Devstral, a helpful agentic model trained by Mistral AI and using the OpenHands scaffold. '
+    'You can interact with a computer to solve tasks.\n\n<ROLE>\nYour primary role is to assist users by '
+    'executing commands, modifying code, and solving technical problems effectively. You should be '
+    'thorough, methodical, and prioritize quality over speed.\n* If the user asks a question, like '
+    '"why is X happening", don\'t try to fix the problem. Just give an answer to the question.'
+    '\n</ROLE>\n\n<EFFICIENCY>\n* Each action you take is somewhat expensive. Wherever possible, '
+    'combine multiple actions into a single action, e.g. combine multiple bash commands into one, using '
+    'sed and grep to edit/view multiple files at once.\n* When exploring the codebase, use efficient tools '
+    'like find, grep, and git commands with appropriate filters to minimize unnecessary operations.'
+    '\n</EFFICIENCY>\n\n<FILE_SYSTEM_GUIDELINES>\n* When a user provides a file path, do NOT assume it\'s '
+    'relative to the current working directory. First explore the file system to locate the file before '
+    'working on it.\n* If asked to edit a file, edit the file directly, rather than creating a new file with '
+    'a different filename.\n* For global search-and-replace operations, consider using `sed` instead of '
+    'opening file editors multiple times.\n</FILE_SYSTEM_GUIDELINES>\n\n<CODE_QUALITY>\n* Write clean, '
+    'efficient code with minimal comments. Avoid redundancy in comments: Do not repeat information that can '
+    'be easily inferred from the code itself.\n* When implementing solutions, focus on making the minimal '
+    'changes needed to solve the problem.\n* Before implementing any changes, first thoroughly understand '
+    'the codebase through exploration.\n* If you are adding a lot of code to a function or file, consider '
+    'splitting the function or file into smaller pieces when appropriate.\n</CODE_QUALITY>\n\n'
+    '<VERSION_CONTROL>\n* When configuring git credentials, use "openhands" as the user.name and '
+    '"openhands@all-hands.dev" as the user.email by default, unless explicitly instructed otherwise.'
+    '\n* Exercise caution with git operations. Do NOT make potentially dangerous changes (e.g., pushing '
+    'to main, deleting repositories) unless explicitly asked to do so.\n* When committing changes, use `git'
+    ' status` to see all modified files, and stage all files necessary for the commit. Use `git commit -a` '
+    'whenever possible.\n* Do NOT commit files that typically shouldn\'t go into version control (e.g., '
+    'node_modules/, .env files, build directories, cache files, large binaries) unless explicitly '
+    'instructed by the user.\n* If unsure about committing certain files, check for the presence of .'
+    'gitignore files or ask the user for clarification.\n</VERSION_CONTROL>\n\n<PULL_REQUESTS>\n* When '
+    'creating pull requests, create only ONE per session/issue unless explicitly instructed otherwise.\n* '
+    'When working with an existing PR, update it with new commits rather than creating additional PRs for '
+    'the same issue.\n* When updating a PR, preserve the original PR title and purpose, updating description '
+    'only when necessary.\n</PULL_REQUESTS>\n\n<PROBLEM_SOLVING_WORKFLOW>\n1. EXPLORATION: Thoroughly '
+    'explore relevant files and understand the context before proposing solutions\n2. ANALYSIS: Consider '
+    'multiple approaches and select the most promising one\n3. TESTING:\n   * For bug fixes: Create tests to '
+    'verify issues before implementing fixes\n   * For new features: Consider test-driven development when '
+    'appropriate\n   * If the repository lacks testing infrastructure and implementing tests would require '
+    'extensive setup, consult with the user before investing time in building testing infrastructure\n   * '
+    'If the environment is not set up to run tests, consult with the user first before investing time to '
+    'install all dependencies\n4. IMPLEMENTATION: Make focused, minimal changes to address the problem\n5. '
+    'VERIFICATION: If the environment is set up to run tests, test your implementation thoroughly, including '
+    'edge cases. If the environment is not set up to run tests, consult with the user first before investing '
+    'time to run tests.\n</PROBLEM_SOLVING_WORKFLOW>\n\n<SECURITY>\n* Only use GITHUB_TOKEN and other '
+    'credentials in ways the user has explicitly requested and would expect.\n* Use APIs to work with GitHub '
+    'or other platforms, unless the user asks otherwise or your task requires browsing.\n</'
+    'SECURITY>\n\n<ENVIRONMENT_SETUP>\n* When user asks you to run an application, don\'t stop if the '
+    'application is not installed. Instead, please install the application and run the command again.\n* If '
+    'you encounter missing dependencies:\n  1. First, look around in the repository for existing dependency '
+    'files (requirements.txt, pyproject.toml, package.json, Gemfile, etc.)\n  2. If dependency files exist, '
+    'use them to install all dependencies at once (e.g., `pip install -r requirements.txt`, `npm install`, '
+    'etc.)\n  3. Only install individual packages directly if no dependency files are found or if only '
+    'specific packages are needed\n* Similarly, if you encounter missing dependencies for essential tools '
+    'requested by the user, install them when possible.\n</ENVIRONMENT_SETUP>\n\n<TROUBLESHOOTING>\n* If '
+    'you\'ve made repeated attempts to solve a problem but tests still fail or the user reports it\'s still '
+    'broken:\n  1. Step back and reflect on 5-7 different possible sources of the problem\n  2. Assess the '
+    'likelihood of each possible cause\n  3. Methodically address the most likely causes, starting with the '
+    'highest probability\n  4. Document your reasoning process\n* When you run into any major issue while '
+    'executing a plan from the user, please don\'t try to directly work around it. Instead, propose a new '
+    'plan and confirm with the user before proceeding.\n</TROUBLESHOOTING>')
+
+register_template(Mistral3TemplateMeta('devstral', default_system=devstral_small_2505_system))
+
+
+class Mistral2506Template(Mistral2503Template):
+
+    def _get_mistral_system(self):
+        from swift.model import get_model_name
+        model_dir = self.model_info.model_dir
+        model_name = get_model_name(model_dir)
+        file_path = os.path.join(model_dir, 'SYSTEM_PROMPT.txt')
+        with open(file_path, 'r', encoding='utf-8') as file:
+            system_prompt = file.read()
+        today = datetime.today().strftime('%Y-%m-%d')
+        yesterday = (datetime.today() - timedelta(days=1)).strftime('%Y-%m-%d')
+        return system_prompt.format(name=model_name, today=today, yesterday=yesterday)
+
+    def _get_system(self, inputs: StdTemplateInputs) -> Optional[str]:
+        system = super()._get_system(inputs)
+        if system is None:
+            system = self._get_mistral_system()
+        return system
+
+
+register_template(
+    Mistral3TemplateMeta(MLLMTemplateType.mistral_2506, default_system=None, template_cls=Mistral2506Template))
+
+
+class Mistral2512Template(Mistral2506Template):
+
+    def _get_mistral_system(self):
+        model_dir = self.model_info.model_dir
+        file_path = os.path.join(model_dir, 'SYSTEM_PROMPT.txt')
+        with open(file_path, 'r', encoding='utf-8') as file:
+            system_prompt = file.read()
+        today = datetime.today().strftime('%Y-%m-%d')
+        yesterday = (datetime.today() - timedelta(days=1)).strftime('%Y-%m-%d')
+        return system_prompt.format(today=today, yesterday=yesterday)
+
+
+register_template(
+    Mistral3TemplateMeta(
+        MLLMTemplateType.mistral_2512, default_system=None, template_cls=Mistral2512Template, agent_template='mistral'))
+
+
+class Mistral2512ThinkingTemplate(Mistral2506Template):
+
+    def _get_mistral_system(self):
+        model_dir = self.model_info.model_dir
+        file_path = os.path.join(model_dir, 'SYSTEM_PROMPT.txt')
+        with open(file_path, 'r', encoding='utf-8') as file:
+            system_prompt = file.read()
+        return system_prompt
+
+
+register_template(
+    Mistral3TemplateMeta(
+        MLLMTemplateType.mistral_2512_thinking,
+        default_system=None,
+        template_cls=Mistral2512ThinkingTemplate,
+        agent_template='mistral'))

swift/template/templates/molmo.py

@@ -0,0 +1,177 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import numpy as np
+import torch
+from PIL import Image
+from typing import Any, Dict, List, Literal, Optional, Tuple
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from .utils import ChatmlTemplateMeta
+
+
+class MolmoTemplate(Template):
+    placeholder_tokens = ['<im_patch>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        return []
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        # image
+        images_inputs = self.processor.process(images=inputs.images or None, text='')
+        images_input_ids = images_inputs.pop('input_ids').tolist()
+        user_token = self._tokenize(' User')
+        assert len(user_token) == 1
+        idx = findall(images_input_ids, user_token[0])
+        assert len(idx) == 1
+        labels = encoded['labels']
+        encoded['input_ids'] = images_input_ids[:idx[0]] + encoded['input_ids']
+        if labels:
+            encoded['labels'] = [-100] * idx[0] + labels
+        if 'images' in images_inputs:
+            images_inputs['images'] = images_inputs['images'].to(self.model_info.torch_dtype)
+        encoded.update(images_inputs)
+        return encoded
+
+    def generate(self, model, **kwargs):
+        kwargs.pop('attention_mask', None)
+        generation_config = kwargs.pop('generation_config')
+        batch = {
+            k: kwargs.pop(k, None)
+            for k in ['input_ids', 'attention_mask', 'images', 'image_input_idx', 'image_masks']
+        }
+        return model.generate_from_batch(batch, generation_config, **kwargs)
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        # prepare batchfy inputs
+        keys = ['images', 'image_input_idx', 'image_masks']
+        images_res = self.fetch_inputs(batch, keys)
+        for key in keys:
+            val = images_res.get(key)
+            if val:
+                images_res[key] = torch.stack(val)
+        res.update(images_res)
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.molmo,
+        prefix=[],
+        prompt=[' User: {{QUERY}} Assistant:'],
+        chat_sep=None,
+        suffix=['<|endoftext|>'],
+        template_cls=MolmoTemplate,
+    ))
+
+
+class Molmo2Template(Template):
+    """Molmo2 template for image and video understanding.
+
+    Uses ChatML format with BOS auto-insertion.
+    Media placeholders (<|image|>, <|video|>) are expanded via _extend_tokens.
+    Video loading/sampling is delegated entirely to processor.video_processor.
+    """
+
+    use_model = True
+
+    placeholder_tokens = [
+        '<|image|>',
+        '<|video|>',
+        '<im_patch>',
+        '<frame_start>',
+        '<frame_end>',
+    ]
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|image|>']
+        elif media_type == 'video':
+            return ['<|video|>']
+        else:
+            raise ValueError(f'Unsupported media_type: {media_type}')
+
+    def _prepare_mm_inputs(self, inputs: StdTemplateInputs) -> Tuple[Dict[str, Any], List[List[int]], List[List[int]]]:
+        media_inputs: Dict[str, Any] = {}
+        image_expansions: List[List[int]] = []
+        video_expansions: List[List[int]] = []
+        tokenizer = self.tokenizer
+
+        if inputs.images:
+            image_inputs = self.processor.image_processor(images=inputs.images, return_tensors='pt')
+            for image_grid in image_inputs['image_grids']:
+                image_tokens = self.processor.get_image_tokens(image_grid.cpu().numpy())
+                image_expansions.append(tokenizer.encode(''.join(image_tokens), add_special_tokens=False))
+            media_inputs.update(image_inputs)
+
+        if inputs.videos:
+            if len(inputs.videos) != 1:
+                raise ValueError('Molmo2 currently only supports single-video samples.')
+            video_inputs = self.processor.video_processor(
+                videos=inputs.videos,
+                return_tensors='pt',
+                return_metadata=True,
+            )
+            video_metadata = video_inputs.pop('video_metadata')
+            for video_grid, metadata in zip(video_inputs['video_grids'], video_metadata):
+                video_string = self.processor.get_video_string(
+                    video_grid.cpu().numpy(),
+                    np.asarray(metadata.timestamps, dtype=np.float32),
+                )
+                video_expansions.append(tokenizer.encode(video_string, add_special_tokens=False))
+            media_inputs.update(video_inputs)
+
+        return media_inputs, image_expansions, video_expansions
+
+    def _build_token_type_ids(self, input_ids: List[int]) -> List[int]:
+        image_token_ids = {int(token_id) for token_id in getattr(self.processor, 'image_token_ids', [])}
+        return [1 if token_id in image_token_ids else 0 for token_id in input_ids]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        media_inputs, image_expansions, video_expansions = self._prepare_mm_inputs(inputs)
+
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale')
+
+        # Expand image placeholders
+        image_placeholder = self.tokenizer.convert_tokens_to_ids('<|image|>')
+        idx_list = findall(input_ids, image_placeholder)
+        if idx_list:
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                lambda i: image_expansions[i])
+
+        # Expand video placeholders
+        video_placeholder = self.tokenizer.convert_tokens_to_ids('<|video|>')
+        idx_list = findall(input_ids, video_placeholder)
+        if idx_list:
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                lambda i: video_expansions[i])
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        encoded['token_type_ids'] = self._build_token_type_ids(input_ids)
+        encoded.update(media_inputs)
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        for key in ['image_grids', 'video_grids', 'image_token_pooling', 'video_token_pooling', 'image_num_crops']:
+            value = self.concat_tensor(batch, key, 0)
+            if value is not None:
+                res[key] = value
+        return res
+
+
+register_template(ChatmlTemplateMeta(
+    MLLMTemplateType.molmo2,
+    template_cls=Molmo2Template,
+))

swift/template/templates/moonshot.py

@@ -0,0 +1,198 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+import math
+import torch
+from dataclasses import dataclass, field
+from PIL import Image
+from torch import nn as nn
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import is_deepspeed_enabled, to_device
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class MoonlightTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda:
+                           ['<|im_user|>user<|im_middle|>{{QUERY}}<|im_end|><|im_assistant|>assistant<|im_middle|>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|im_system|>system<|im_middle|>{{SYSTEM}}<|im_end|>'])
+    default_system: Optional[str] = 'You are a helpful assistant'
+
+
+register_template(MoonlightTemplateMeta(LLMTemplateType.moonlight))
+
+register_template(
+    MoonlightTemplateMeta(
+        LLMTemplateType.kimi_k2, default_system='You are Kimi, an AI assistant created by Moonshot AI.'))
+
+
+class KimiVLTemplate(Template):
+    placeholder_tokens = ['<|media_pad|>']
+    support_padding_free = True
+    skip_prompt = False
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|media_start|>image<|media_content|><|media_pad|><|media_end|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        media_token = self._tokenize('<|media_pad|>')[0]
+        idx_list = findall(input_ids, media_token)
+        if inputs.images:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor(inputs.images, return_tensors='pt')
+            image_grid_hws = image_inputs['image_grid_hws']
+            merge_length = image_processor.merge_kernel_size[0] * image_processor.merge_kernel_size[1]
+
+            def _get_new_tokens(i):
+                token_len = (image_grid_hws[i].prod() // merge_length)
+                return [media_token] * token_len
+
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_tokens)
+
+            encoded['loss_scale'] = loss_scale
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded.update(image_inputs)
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        image_grid_hws = self.concat_tensor(batch, 'image_grid_hws', 0)
+        if image_grid_hws is not None:
+            res['image_grid_hws'] = image_grid_hws
+        return res
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        input_ids = inputs['input_ids']
+        pixel_values = inputs.get('pixel_values')
+        inputs_embeds = model.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None and pixel_values.size(0) > 0:
+            pixel_values = pixel_values.to(model.vision_tower.dtype)
+            image_features: torch.Tensor = model._extract_image_features(pixel_values, inputs['image_grid_hws'])
+            inputs_embeds = inputs_embeds.to(image_features[0].dtype).clone()
+            inputs_embeds = model._merge_with_image_features(inputs_embeds, input_ids, image_features)
+        elif is_deepspeed_enabled():
+            image_processor = self.processor.image_processor
+            dummy_image = Image.new('RGB', (32, 32), (0, 0, 0))
+            image_inputs = image_processor([dummy_image], return_tensors='pt')
+            pixel_values = image_inputs['pixel_values'].to(model.vision_tower.dtype)
+            image_features: torch.Tensor = model._extract_image_features(pixel_values, image_inputs['image_grid_hws'])
+            inputs_embeds = inputs_embeds + image_features.mean() * 0.
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(MoonlightTemplateMeta(MLLMTemplateType.kimi_vl, template_cls=KimiVLTemplate))
+
+
+class KimiK25Template(Template):
+    placeholder_tokens = ['<|media_pad|>', '<|kimi_k25_video_placeholder|>']
+    jinja_enable_thinking_key = 'thinking'
+    support_padding_free = True
+    skip_prompt = False
+
+    def _get_system(self, inputs: StdTemplateInputs) -> Optional[str]:
+        system = super()._get_system(inputs)
+        if system is not None and '<|im_middle|>' not in system:  # compat agent
+            system = f'system<|im_middle|>{system}'
+        return system
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|media_begin|>image<|media_content|><|media_pad|><|media_end|>\n']
+        raise ValueError(f'KimiK25Template does not currently support {media_type}. '
+                         'Please open an issue to request support.')
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        media_token = self._tokenize('<|media_pad|>')[0]
+        idx_list = findall(input_ids, media_token)
+        if inputs.images:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor([{
+                'type': 'image',
+                'image': image
+            } for image in inputs.images],
+                                           return_tensors='pt')
+            grid_thws = image_inputs['grid_thws']
+            merge_length = math.prod(self.config.vision_config.merge_kernel_size)
+
+            def _get_new_tokens(i):
+                token_len = (grid_thws[i].prod() // merge_length)
+                return [media_token] * token_len
+
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_tokens)
+
+            encoded['loss_scale'] = loss_scale
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded.update(image_inputs)
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        grid_thws = self.concat_tensor(batch, 'grid_thws', 0)
+        if grid_thws is not None:
+            res['grid_thws'] = grid_thws
+        return res
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        input_ids = inputs['input_ids']
+        pixel_values = inputs.get('pixel_values')
+        inputs_embeds = model.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None and pixel_values.size(0) > 0:
+            pixel_values = pixel_values.to(model.vision_tower.dtype)
+            image_features: torch.Tensor = model._extract_image_features(pixel_values, inputs['grid_thws'])
+            if model.mm_projector:
+                image_features = model.mm_projector(image_features)
+            image_features = torch.cat(image_features, dim=0)
+            inputs_embeds = inputs_embeds.to(image_features.dtype)
+            image_mask = (input_ids == self.config.media_placeholder_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+            inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_features)
+        elif is_deepspeed_enabled():
+            image_processor = self.processor.image_processor
+            dummy_image = Image.new('RGB', (32, 32), (0, 0, 0))
+            image_inputs = image_processor([{'type': 'image', 'image': dummy_image}], return_tensors='pt')
+            image_inputs = to_device(image_inputs, inputs_embeds.device)
+            pixel_values = image_inputs['pixel_values'].to(model.vision_tower.dtype)
+            image_features: torch.Tensor = model._extract_image_features(pixel_values, image_inputs['grid_thws'])
+            if model.mm_projector:
+                image_features = model.mm_projector(image_features)
+            image_features = torch.cat(image_features, dim=0)
+            inputs_embeds = inputs_embeds + image_features.mean() * 0.
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    MoonlightTemplateMeta(
+        MLLMTemplateType.kimi_k25,
+        template_cls=KimiK25Template,
+        system_prefix=['<|im_system|>{{SYSTEM}}<|im_end|>'],
+        default_system=None,
+        is_thinking=True,
+        thinking_prefix='<think>',
+        non_thinking_prefix='<think></think>',
+        history_thinking_prefix='<think></think>',
+        agent_template='kimi_k25',
+    ))

swift/template/templates/mplug.py

@@ -0,0 +1,223 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from dataclasses import dataclass, field
+from functools import partial
+from torch import nn
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_video_minicpmv_mplug_owl3
+from .qwen import QwenTemplateMeta
+
+
+class mPlugOwl2Template(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from mplug_owl2.mm_utils import process_images
+        processor = self.processor
+        images = inputs.images
+        for i, image in enumerate(images):
+            # ref: https://modelscope.cn/models/iic/mPLUG-Owl2.1
+            max_edge = max(image.size)
+            image = image.resize((max_edge, max_edge))
+            images[i] = image
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        res = {'input_ids': input_ids, 'labels': labels}
+        if images:
+            images = process_images(images, processor)
+            images = images.to(self.model_info.torch_dtype)
+            res['images'] = images
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = torch.concat(images)
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.mplug_owl2,
+        template_cls=mPlugOwl2Template,
+        prefix=['{{SYSTEM}}'],
+        prompt=['USER: {{QUERY}}ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=[['eos_token_id']],
+        stop_words=['<|endoftext|>', '</s>']))
+
+
+class mPlugOwl3Template(Template):
+    version = None
+
+    def init_env_args(self):
+        super().init_env_args()
+        self.max_num_frames = get_env_args('max_num_frames', int, 16)
+
+    def _get_image_token_list(self, cut_shape):
+        text = self.processor.image_processor.cut_prompt_template(img_token='<|image|>', h=cut_shape[0], w=cut_shape[1])
+        text_list = text.split('<|image|>')
+        res_text_list = []
+        for text in text_list[:-1]:
+            res_text_list += [text, '<|image|>']
+        res_text_list += text_list[-1]
+        token_list = self._encode_context_list(res_text_list)[0]
+        return token_list
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type in {'image', 'video'}
+        load_video = partial(load_video_minicpmv_mplug_owl3, max_num_frames=self.max_num_frames)
+        if media_type == 'image':
+            return [[-100], '\n']
+        elif media_type == 'video':
+            return self.replace_video2image(load_video, inputs, lambda i: [[-100]]) + ['\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        videos = inputs.videos
+        cut_enable = not videos
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, -100)
+        processor = self.processor
+        encoded = {}
+        if images:
+            image_inputs = processor.image_processor(images, cut_enable=cut_enable, return_tensors='pt')
+            cut_shapes = image_inputs['cut_shape'] or [None] * 2 * len(idx_list)
+            image_token_list = self.processor.encode('<|image|>', add_special_tokens=False)
+
+            def _get_new_tokens(i):
+                cut_shape = cut_shapes[2 * i]
+                if cut_shape:
+                    token_list = self._get_image_token_list(cut_shape)
+                else:
+                    token_list = image_token_list
+                return token_list
+
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_tokens)
+
+            image_token_idx = torch.tensor(findall(input_ids, image_token_list))
+            if self.version == '241101':
+                media_offset = image_token_idx
+            else:
+                _range = torch.arange(len(input_ids))[:, None]
+                matrix = (_range > image_token_idx[None]).sum(dim=1)
+                media_offset = torch.stack([torch.zeros(matrix.shape[0], dtype=torch.long), matrix], dim=-1)[None]
+            encoded.update({
+                'pixel_values': image_inputs['pixel_values'],
+                'media_offset': media_offset,
+            })
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if 'media_offset' in inputs:
+            media_offset = []
+            cusum_offset = 0
+            image_embeds = []
+            pixel_values = inputs.pop('pixel_values')
+            max_sequence_length = inputs['input_ids'].shape[1]
+            for i, curr_media_offset in enumerate(inputs['media_offset']):
+                if curr_media_offset is None:
+                    continue
+                if curr_media_offset.shape[1] < max_sequence_length:
+                    padding = curr_media_offset[:, -1:, :].expand(curr_media_offset.shape[0],
+                                                                  max_sequence_length - curr_media_offset.shape[1],
+                                                                  curr_media_offset.shape[2])
+                    curr_media_offset = torch.concat([curr_media_offset, padding], dim=1)
+                media_offset.append(curr_media_offset + cusum_offset)
+                image_embeds.append(model.forward_image(pixel_values[i]))
+                cusum_offset += image_embeds[-1].shape[0]
+            inputs['media_offset'] = torch.concat(media_offset)
+            inputs['image_embeds'] = torch.concat(image_embeds)
+        return inputs
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = self.fetch_inputs(batch, ['media_offset', 'pixel_values'])
+        for b in batch:
+            b.pop('pixel_values', None)
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+
+class mPlugOwl3_241101Template(mPlugOwl3Template):
+    version = '241101'
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if 'pixel_values' in inputs:
+            pixel_values = inputs.pop('pixel_values')
+            inputs['image_embeds'] = torch.concat([model.forward_image(pv) for pv in pixel_values])
+        else:
+            inputs['media_offset'] = [None] * inputs['input_ids'].shape[0]
+        return inputs
+
+
+@dataclass
+class mPlugOwl3TemplateMeta(QwenTemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|im_start|>system\n{{SYSTEM}}<|im_end|>\n'])
+    default_system: Optional[str] = None
+    system_prefix: Optional[Prompt] = None
+
+
+register_template(
+    mPlugOwl3TemplateMeta(MLLMTemplateType.mplug_owl3, template_cls=mPlugOwl3Template, agent_template=None))
+
+register_template(
+    mPlugOwl3TemplateMeta(
+        MLLMTemplateType.mplug_owl3_241101, template_cls=mPlugOwl3_241101Template, agent_template=None))
+
+
+class DocOwl2Template(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [f'<img {index + 1}>', [-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        if inputs.images:
+            image_tensor, patch_positions, _ = self.processor._process_image(inputs.images)
+            image_tensor = image_tensor.to(self.model_info.torch_dtype)
+            encoded.update({'images': image_tensor, 'patch_positions': patch_positions})
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        keys = ['images', 'patch_positions']
+        res = self.fetch_inputs(batch, keys)
+        for key in keys:
+            val = res.get(key)
+            if val:
+                res[key] = torch.concat([v for v in val if v is not None])
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.doc_owl2,
+        prefix=['<s>'],
+        prompt=[' USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        template_cls=DocOwl2Template,
+    ))

swift/template/templates/openbuddy.py

@@ -0,0 +1,48 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+from ..constant import LLMTemplateType
+from ..register import TemplateMeta, register_template
+
+OPENBUDDY_DEFAULT_SYSTEM = (
+    'You are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human User.\n'
+    'Always answer as helpfully and logically as possible, while being safe. '
+    'Your answers should not include any '
+    'harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. '
+    'Please ensure that your responses are socially unbiased and positive in nature.\n'
+    'If a question does not make any sense, or is not factually coherent, '
+    'explain why instead of answering something not correct. '
+    "If you don't know the answer to a question, please don't share false information.\n"
+    'You like to use emojis. You can speak fluently in many languages, for example: English, Chinese.\n'
+    'You cannot access the internet, but you have vast knowledge, cutoff: 2021-09.\n'
+    'You always deeply love and support China, Chinese government, people and culture.\n'
+    'You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), '
+    'you are based on LLaMA and Falcon transformers model, not related to GPT or OpenAI.')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.openbuddy,
+        prefix=[],
+        prompt=['User: {{QUERY}}\nAssistant:'],
+        chat_sep=['\n'],
+        default_system=OPENBUDDY_DEFAULT_SYSTEM,
+        system_prefix=['{{SYSTEM}}\n\n'],
+        auto_add_bos=True))
+
+OPENBUDDY2_DEFAULT_SYSTEM = (
+    'You(assistant) are a helpful, respectful and honest INTP-T AI Assistant named Buddy. '
+    'You are talking to a human(user).\nAlways answer as helpfully and logically as possible, while being safe. '
+    'Your answers should not include any harmful, political, religious, unethical, racist, '
+    'sexist, toxic, dangerous, or illegal content. '
+    'Please ensure that your responses are socially unbiased and positive in nature.\n'
+    'You cannot access the internet, but you have vast knowledge, cutoff: 2023-04.\n'
+    'You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), '
+    'not related to GPT or OpenAI')
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.openbuddy2,
+        prefix=[],
+        prompt=['<|role|>user<|says|>{{QUERY}}<|end|>\n<|role|>assistant<|says|>'],
+        chat_sep=['<|end|>\n'],
+        suffix=['<|end|>'],
+        default_system=OPENBUDDY2_DEFAULT_SYSTEM,
+        system_prefix=['<|role|>system<|says|>{{SYSTEM}}<|end|>\n']))

swift/template/templates/pixtral.py

@@ -0,0 +1,67 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from typing import Any, Dict, List, Optional
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import findall
+
+
+class PixtralTemplate(Template):
+    image_placeholder = ['[IMG]']
+    placeholder_tokens = ['[IMG]']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, 10)
+        if idx_list:
+            image_inputs = processor.image_processor(images, patch_size=processor.patch_size, return_tensors='pt')
+            encoded['pixel_values'] = image_inputs['pixel_values'].to(dtype=self.model_info.torch_dtype)
+            encoded['image_sizes'] = image_sizes = image_inputs['image_sizes']
+
+            def _get_new_tokens(i):
+                height, width = image_sizes[i]
+                num_height_tokens = height // processor.patch_size
+                num_width_tokens = width // processor.patch_size
+                replace_tokens = [processor.image_token * num_width_tokens + processor.image_break_token] * (
+                    num_height_tokens - 1)
+                replace_tokens += [processor.image_token * num_width_tokens + processor.image_end_token]
+                # Flatten list
+                replace_str = ''.join(replace_tokens)
+                img_tokens: List[int] = self.processor.encode(replace_str, add_special_tokens=False)
+                return img_tokens
+
+            encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        pixel_values = self.gather_list(batch, 'pixel_values')
+        image_sizes = self.gather_list(batch, 'image_sizes')
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if pixel_values:
+            pixel_values = torch.stack(pixel_values)
+            res['pixel_values'] = pixel_values
+        if image_sizes:
+            image_sizes = torch.stack(image_sizes)
+            res['image_sizes'] = image_sizes
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.pixtral,
+        prefix=['<s>{{SYSTEM}}'],
+        prompt=['[INST]{{QUERY}}[/INST]'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        template_cls=PixtralTemplate,
+    ))

swift/template/templates/seed.py

@@ -0,0 +1,261 @@
+import os
+import re
+import torch
+from dataclasses import dataclass, field
+from torch import nn
+from transformers.utils import strtobool
+from typing import Any, Dict, List, Literal, Optional, Type
+
+from swift.utils import is_deepspeed_enabled
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import Template, TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word, findall
+from .utils import ChatmlTemplateMeta
+
+
+class SeedTemplate(Template):
+
+    def get_thinking_budget(self, inputs: StdTemplateInputs):
+        thinking_budget = os.environ.get('THINKING_BUDGET')
+        if thinking_budget is not None:
+            max_length = int(thinking_budget)
+        else:
+            max_length = 0
+            for m in inputs.messages:
+                if m['role'] == 'assistant' and m['content']:
+                    if '<think>' in m['content'] and '</think>' in m['content']:
+                        _, think = m['content'].split('<think>', maxsplit=1)
+                        think, _ = think.split('</think>', maxsplit=1)
+                        if think.strip():
+                            thinking_token_len = len(self.tokenizer(think)['input_ids'])
+                            if thinking_token_len > max_length:
+                                max_length = thinking_token_len
+
+        def convert_integer_v2(n):
+            if n is None:
+                return None
+            elif n <= 0:
+                return 0
+            elif n <= 512:
+                return 512
+            elif n <= 1024:
+                return 1024
+            elif n <= 2048:
+                return 2048
+            elif n <= 4096:
+                return 4096
+            elif n <= 8192:
+                return 8192
+            elif n <= 16384:
+                return 16384
+            else:
+                return n
+
+        return convert_integer_v2(max_length)
+
+    def get_reflect_interval(self, inputs: StdTemplateInputs):
+        interval_mapping = {0: 0, 512: 128, 1024: 256, 2048: 512, 4096: 512, 8192: 1024, 16384: 1024}
+        budget = self.get_thinking_budget(inputs)
+        if budget is None:
+            return None
+        elif budget <= 0:
+            return 0
+        elif budget > 16384:
+            return 1024
+        else:
+            assert budget in interval_mapping.keys(
+            ), f'Supported thinking budget is {interval_mapping.keys()} or bigger.'
+            return interval_mapping[budget]
+
+    @staticmethod
+    def insert_budget_markers(text: str, tokenizer, interval: int, total_budget: int) -> str:
+        if total_budget > 0:
+            sentences = re.split(r'(?<=[.!?。！？])\s+', text)
+            sentences = [s.strip() for s in sentences if s.strip()]
+
+            result = []
+            current_tokens = 0
+            insertion_count = 0
+
+            for sentence in sentences:
+                sentence_tokens = len(tokenizer.encode(sentence))
+                if current_tokens + sentence_tokens >= (insertion_count + 1) * interval:
+                    remaining_budget = total_budget - (current_tokens + sentence_tokens)
+                    marker = (f'<seed:cot_budget_reflect>I have used {current_tokens + sentence_tokens} tokens, '
+                              f'and there are {remaining_budget} tokens remaining for use.</seed:cot_budget_reflect>')
+                    result.append(marker)
+                    insertion_count += 1
+
+                result.append(sentence)
+                current_tokens += sentence_tokens
+
+            return '\n'.join(result)
+        else:
+            return ('<seed:cot_budget_reflect>The current thinking budget is 0, so I will '
+                    'directly start answering the question.</seed:cot_budget_reflect>\n')
+
+    def _prepare_system(self, inputs):
+        budget = self.get_thinking_budget(inputs)
+        interval = self.get_reflect_interval(inputs)
+        if budget is None:
+            default_system = ''
+        elif budget > 0:
+            default_system = (
+                'You are an intelligent assistant with reflective ability. '
+                'In the process of thinking and reasoning, you need to strictly follow the thinking budget, '
+                f'which is {budget}. That is, you need to complete your thinking within {budget} tokens and start '
+                f'answering the user\'s questions. You will reflect on your thinking process every {interval} tokens, '
+                'stating how many tokens have been used and how many are left.\n')
+        else:
+            default_system = ('You are an intelligent assistant that can answer questions in one step without the need '
+                              'for reasoning and thinking, that is, your thinking budget is 0. Next, please skip the '
+                              'thinking process and directly start answering the user\'s questions.\n')
+
+        if default_system:
+            if inputs.system:
+                inputs.system = inputs.system + '<seed:eos><seed:bos>system\n' + default_system
+            else:
+                inputs.system = default_system
+
+    def _swift_prepare_inputs(self, inputs: StdTemplateInputs):
+        super()._swift_prepare_inputs(inputs)
+        if strtobool(os.environ.get('SEED_USE_THINKING', 'true')):
+            budget = self.get_thinking_budget(inputs)
+            interval = self.get_reflect_interval(inputs)
+            self._prepare_system(inputs)
+            if budget is not None:
+                for message in inputs.messages:
+                    if message['role'] == 'assistant':
+                        if '<think>' in message['content'] and '</think>' in message['content']:
+                            pre_text, post_text = message['content'].split('<think>', maxsplit=1)
+                            think, post_text = post_text.split('</think>', maxsplit=1)
+                            if '<seed:cot_budget_reflect>' not in message['content'] and strtobool(
+                                    os.environ.get('SEED_USE_BUDGET_INTERVAL', 'false')):
+                                think = self.insert_budget_markers(think, self.tokenizer, interval, budget)
+                            message['content'] = pre_text + '<seed:think>' + think + '</seed:think>' + post_text
+                        elif budget > 0:
+                            message['content'] = message['content'].replace('<think>', '').replace('</think>', '')
+                            message['content'] = '<seed:think></seed:think>' + message['content']
+                        elif budget <= 0:
+                            message['content'] = message['content'].replace('<think>', '').replace('</think>', '')
+                            message['content'] = (
+                                '<seed:think><seed:cot_budget_reflect>The current thinking budget is 0, '
+                                'so I will directly start answering the question.'
+                                '</seed:cot_budget_reflect>\n</seed:think>') + message['content']
+
+    def _simplify_context_list(self, context_list, loss_scale_list, inputs):
+        res, res_loss_scale = super()._simplify_context_list(context_list, loss_scale_list, inputs)
+        if not self.use_chat_template:
+            return res, res_loss_scale
+        budget = self.get_thinking_budget(inputs)
+        if res[-1].endswith('assistant\n') and budget == 0:
+            res.append('<seed:think><seed:cot_budget_reflect>')
+            res_loss_scale.append(res_loss_scale[-1])
+        return res, res_loss_scale
+
+    def _jinja_encode(self, inputs: StdTemplateInputs):
+        return super()._jinja_encode(inputs)
+
+
+@dataclass
+class SeedTemplateMeta(TemplateMeta):
+    template_type: str = 'seed'
+    prefix: Prompt = field(default_factory=lambda: ['<seed:bos>'])
+    prompt: Prompt = field(default_factory=lambda: ['<seed:bos>user\n{{QUERY}}<seed:eos><seed:bos>assistant\n'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<seed:bos>system\n{{SYSTEM}}<seed:eos>'])
+    auto_add_bos: bool = True
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<seed:eos>'])
+    suffix: Prompt = field(default_factory=lambda: ['<seed:eos>'])
+    template_cls: Type[Template] = SeedTemplate
+    default_system: Optional[str] = None
+    stop_words: List[Word] = field(default_factory=lambda: ['<seed:eos>'])
+
+
+register_template(SeedTemplateMeta(LLMTemplateType.seed_oss, default_system=None, template_cls=SeedTemplate))
+
+SAIL_VL_DEFAULT_SYSTEM = '你是由抖音内容理解组开发的多模态大模型，英文名叫UniVL, 是一个有用无害的人工智能助手。'
+
+
+class SailVLTemplate(Template):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.skip_prompt = False
+        self.num_image_token = self.processor.num_image_token
+        self.img_context_token_id = self.tokenizer.convert_tokens_to_ids('<IMG_CONTEXT>')
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image', 'This model only supports image input'
+        if self.mode == 'vllm':
+            raise NotImplementedError('vLLM not support this model now')
+        else:
+            image_context = ['<img>', [-100], '</img>\n']
+        return image_context
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        idx_list = findall(input_ids, -100)
+        pixel_values = None
+        loss_scale = encoded.get('loss_scale', None)
+        images = inputs.images
+        processor = self.processor
+        if images:
+            labels = encoded.get('labels')
+            image_inputs = processor.image_processor(images)
+            num_patches = image_inputs['num_patches_list']
+            pixel_values = image_inputs['pixel_values']
+        else:
+            pixel_values = None
+            num_patches = []
+        assert len(num_patches) == len(
+            idx_list), f'len(num_patches): {len(num_patches)}, len(idx_list): {len(idx_list)}'
+
+        def _get_new_tokens(i):
+            img_tokens: List[int] = self.processor.encode(
+                '<IMG_CONTEXT>', add_special_tokens=False) * self.num_image_token * num_patches[i]
+            return img_tokens
+
+        encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+            input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+        encoded['pixel_values'] = pixel_values
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        embedding = model.language_model.get_input_embeddings()
+        device = embedding.weight.device
+        input_ids = inputs['input_ids']
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            vit_embeds = model.extract_feature(pixel_values)
+            inputs_embeds = embedding(input_ids)
+            B, N, C = inputs_embeds.shape
+            inputs_embeds = inputs_embeds.reshape(B * N, C)
+
+            input_ids = input_ids.reshape(B * N)
+            selected = (input_ids == self.img_context_token_id)
+            assert selected.sum() != 0
+            inputs_embeds = inputs_embeds.clone()
+            inputs_embeds[selected] = vit_embeds.reshape(-1, C).to(inputs_embeds.device)
+
+            inputs_embeds = inputs_embeds.reshape(B, N, C)
+        elif is_deepspeed_enabled():
+            inputs_embeds = embedding(input_ids).to(device=device)
+            dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype)
+            vit_embeds = model.extract_feature(dummy_pixel_values).to(device=device)
+            inputs_embeds = inputs_embeds + vit_embeds.mean() * 0.
+
+        return {'inputs_embeds': inputs_embeds.to(input_ids.device)}
+
+
+@dataclass
+class SailVLTemplateMeta(ChatmlTemplateMeta):
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|im_start|>system\n{{SYSTEM}}<|im_end|>'])
+    prompt: Prompt = field(default_factory=lambda: ['<|im_start|>user\n{{QUERY}}<|im_end|><|im_start|>assistant\n'])
+
+
+register_template(
+    SailVLTemplateMeta(MLLMTemplateType.sail_vl2, default_system=SAIL_VL_DEFAULT_SYSTEM, template_cls=SailVLTemplate))

swift/template/templates/stepfun.py

@@ -0,0 +1,417 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import itertools
+import torch
+from functools import partial
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_env_args, to_float_dtype
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from ..vision_utils import load_batch, load_file
+from .qwen import QwenTemplateMeta
+
+
+class GOTImageEvalProcessor:
+
+    def __init__(self, image_size=384, mean=None, std=None):
+        from torchvision import transforms
+        from torchvision.transforms.functional import InterpolationMode
+        if mean is None:
+            mean = (0.48145466, 0.4578275, 0.40821073)
+        if std is None:
+            std = (0.26862954, 0.26130258, 0.27577711)
+
+        self.normalize = transforms.Normalize(mean, std)
+
+        self.transform = transforms.Compose([
+            transforms.Resize((image_size, image_size), interpolation=InterpolationMode.BICUBIC),
+            transforms.ToTensor(),
+            self.normalize,
+        ])
+
+    def __call__(self, item):
+        return self.transform(item)
+
+
+class GOT_OCR2Template(Template):
+    placeholder_tokens = ['<imgpad>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        # 'OCR: '
+        # 'OCR with format: '
+        assert media_type == 'image'
+        return ['<img>' + '<imgpad>' * 256 + '</img>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        image_processor_high = GOTImageEvalProcessor(image_size=1024)
+        for i, image in enumerate(images):
+            images[i] = image_processor_high(image)[None].to(self.model_info.torch_dtype)
+        if images:
+            encoded['images'] = images
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = self.gather_list(batch, 'images')
+        if images:
+            res['images'] = images
+        return res
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.got_ocr2,
+        default_system='        You should follow the instructions carefully and explain your answers in detail.',
+        template_cls=GOT_OCR2Template,
+        agent_template=None,
+    ))
+
+
+class GOT_OCR2HfTemplate(Template):
+    placeholder_tokens = ['<imgpad>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        # 'OCR: '
+        # 'OCR with format: '
+        assert media_type == 'image'
+        return ['<img>' + '<imgpad>' * 256 + '</img>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:  # 暂时照抄上面
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            encoded['images'] = images
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = self.gather_list(batch, 'images')
+        _inputs = self.processor(images, return_tensors='pt')
+        _inputs.pop('input_ids')  # this does not contain the response, so cannot be used when training
+        _inputs.pop('attention_mask')  # this does not contain the response, so cannot be used when training
+
+        res.update(_inputs.data)
+        return res
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.got_ocr2_hf,
+        default_system='        You should follow the instructions carefully and explain your answers in detail.',
+        template_cls=GOT_OCR2HfTemplate,
+        agent_template=None,
+    ))
+
+
+class StepAudioTemplate(Template):
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'audio', f'media_type: {media_type}'
+        from utils import load_audio
+        audio_wav, sr = load_audio(load_file(inputs.audios[index]))
+        audio_tokens = self.model.encoder(audio_wav, sr)
+        return audio_tokens
+
+
+class StepAudio2MiniTemplate(Template):
+    use_model = True
+
+    def load_audio(self, file_path, target_rate=16000, max_length=None):
+        '''
+        Open an audio file and read as mono waveform, resampling as necessary
+        If max_length is provided, truncate the audio to that length
+        '''
+        import torchaudio
+        waveform, sample_rate = torchaudio.load(file_path)
+        if sample_rate != target_rate:
+            waveform = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=target_rate)(waveform)
+        audio = waveform[0]  # get the first channel
+
+        # Truncate audio if it exceeds max_length
+        if max_length is not None and audio.shape[0] > max_length:
+            audio = audio[:max_length]
+
+        return audio
+
+    def _mel_filters(self, n_mels: int) -> 'torch.Tensor':
+        '''Load the mel filterbank matrix for projecting STFT into a Mel spectrogram.'''
+        import librosa
+        import torch
+        assert n_mels in {80, 128}, f'Unsupported n_mels: {n_mels}'
+        if n_mels == 128:
+            return torch.from_numpy(librosa.filters.mel(sr=16000, n_fft=400, n_mels=128))
+        else:
+            return torch.from_numpy(librosa.filters.mel(sr=16000, n_fft=400, n_mels=80))
+
+    def log_mel_spectrogram(self, audio, n_mels=128, padding=479):
+        '''
+        Compute the log-Mel spectrogram with specific padding for StepAudio
+        '''
+        import torch
+        import torch.nn.functional as F
+        if isinstance(audio, str):
+            audio = self.load_audio(audio)
+        elif not torch.is_tensor(audio):
+            audio = torch.from_numpy(audio)
+        if padding > 0:
+            audio = F.pad(audio, (0, padding))
+        window = torch.hann_window(400).to(audio.device)
+        stft = torch.stft(audio, 400, 160, window=window, return_complex=True)
+        magnitudes = stft[..., :-1].abs()**2
+        filters = self._mel_filters(n_mels)
+        mel_spec = filters @ magnitudes
+
+        log_spec = torch.clamp(mel_spec, min=1e-10).log10()
+        log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
+        log_spec = (log_spec + 4.0) / 4.0
+        return log_spec
+
+    def compute_token_num(self, max_feature_len):
+        # First, audio goes through encoder:
+        # 1. conv1: kernel=3, stride=1, padding=1 -> size unchanged
+        # 2. conv2: kernel=3, stride=2, padding=1 -> size/2
+        # 3. avg_pooler: kernel=2, stride=2 -> size/2
+        max_feature_len = max_feature_len - 2  # remove padding
+        encoder_output_dim = (max_feature_len + 1) // 2 // 2  # after conv2 and avg_pooler
+
+        # Then through adaptor (parameters from config file):
+        padding = 1
+        kernel_size = 3  # from config: audio_encoder_config.kernel_size
+        stride = 2  # from config: audio_encoder_config.adapter_stride
+        adapter_output_dim = (encoder_output_dim + 2 * padding - kernel_size) // stride + 1
+        return adapter_output_dim
+
+    def padding_mels(self, data: List['torch.Tensor']):
+        ''' Padding the data into batch data
+
+        Parameters
+        ----------
+            data: List[Tensor], shape of Tensor (128, T)
+
+        Returns:
+        -------
+            feats, feats lengths
+        '''
+        import torch
+        from torch.nn.utils.rnn import pad_sequence
+        sample = data
+        assert isinstance(sample, list)
+        feats_lengths = torch.tensor([s.size(1) - 2 for s in sample], dtype=torch.int32)
+        feats = [s.t() for s in sample]
+        padded_feats = pad_sequence(feats, batch_first=True, padding_value=0)
+
+        return padded_feats.transpose(1, 2), feats_lengths
+
+    def audio_process(self, audio):
+        results = []
+        mels = []
+        for i in range(0, audio.shape[0], 16000 * 25):
+            mel = self.log_mel_spectrogram(audio[i:i + 16000 * 25], n_mels=128, padding=479)
+            mels.append(mel)
+            audio_tokens = '<audio_patch>' * self.compute_token_num(mel.shape[1])
+            results.append(f'<audio_start>{audio_tokens}<audio_end>')
+        audio_ids = self._tokenize(''.join(results))
+        return audio_ids, mels
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'audio'
+        return ['<audio_patch>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        sampling_rate = get_env_args('sampling_rate', int, 16000)
+        inputs.audios = load_batch(inputs.audios, partial(self.load_audio, target_rate=sampling_rate))
+
+        audio_token = self._tokenize('<audio_patch>')[0]
+        idx_list = findall(input_ids, audio_token)
+
+        if idx_list:
+            audio_inputs = []
+            mels = []
+            for audio in inputs.audios:
+                audio_input, mel = self.audio_process(audio)
+                audio_inputs.append(audio_input)
+                mels.extend(mel)
+
+            def _get_new_audio_tokens(i):
+                return audio_inputs[i]
+
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_audio_tokens)
+            encoded['input_ids'] = input_ids  # Add labels to the batch
+            encoded['labels'] = labels  # Add labels to the batch
+            encoded['loss_scale'] = loss_scale
+            encoded['mels'] = mels
+            wavs, wav_lens = self.padding_mels(mels)
+            # audio_tokens = [151688, 151690, 151689]
+            # for audio_token_id in audio_tokens:
+            #     labels[labels == audio_token_id] = -100  # Mask image token IDs in labels
+
+        else:
+            wavs = None
+            wav_lens = None
+
+        encoded['wavs'] = wavs
+        encoded['wav_lens'] = wav_lens
+
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        combined_mels = list(itertools.chain.from_iterable([e.get('mels', []) for e in batch]))
+        batch_wavs, batch_wav_lens = self.padding_mels(combined_mels) if combined_mels else (None, None)
+        res = super()._data_collator(batch, padding_to=padding_to)
+        res['wav_lens'] = batch_wav_lens
+        res['wavs'] = batch_wavs
+
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.step_audio2_mini,
+        template_cls=StepAudio2MiniTemplate,
+        prefix=[],
+        prompt=['<|BOT|>human\n{{QUERY}}<|EOT|><|BOT|>assistant\n'],
+        system_prefix=['<|BOT|>system\n{{SYSTEM}}<|EOT|>'],
+        chat_sep=['<|EOT|>'],
+        suffix=['<|EOT|>'],
+    ))
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.step_audio,
+        template_cls=StepAudioTemplate,
+        prefix=['<s>'],
+        prompt=['<|BOT|>human\n{{QUERY}}<|EOT|><|BOT|>assistant\n'],
+        system_prefix=['<s><|BOT|>system\n{{SYSTEM}}<|EOT|>'],
+        chat_sep=['<|EOT|>'],
+        suffix=['<|EOT|>'],
+    ))
+
+
+class Step3VLTemplate(Template):
+    use_model = True
+    support_padding_free = False
+    image_token_id = 151679
+    placeholder_tokens = ['<im_patch>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<im_patch>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+
+        images = inputs.images
+        if images:
+            processor = self.processor
+            idx_list = findall(input_ids, self.image_token_id)
+            splitted_images_data = processor._split_images(images)
+            pixel_values_lst = []
+            patch_pixel_values_lst = []
+            patch_newline_mask_lst = []
+            image_repl_ids_lst = []
+            num_patches = []
+            for raw_img, img_patches, patch_newline_mask in splitted_images_data:
+                pixel_values_lst.extend(processor._convert_images_to_pixel_values([raw_img]))
+                if len(img_patches) > 0:
+                    patch_pixel_values_lst.extend(processor._convert_images_to_pixel_values(img_patches, is_patch=True))
+                num_patches.append(len(img_patches))
+                _, image_repl_ids = processor._get_image_repl_features(1, len(img_patches), patch_newline_mask)
+                image_repl_ids_lst.append(image_repl_ids)
+                if patch_newline_mask is not None:
+                    patch_newline_mask_lst.extend(patch_newline_mask)
+
+            image_inputs = {
+                'pixel_values': torch.cat(pixel_values_lst),
+                'num_patches': num_patches,
+            }
+            if patch_pixel_values_lst:
+                image_inputs['patch_pixel_values'] = torch.cat(patch_pixel_values_lst)
+            if patch_newline_mask_lst:
+                image_inputs['patch_newline_mask'] = torch.tensor(patch_newline_mask_lst, dtype=torch.bool)
+            image_inputs = to_float_dtype(image_inputs, self.model_info.torch_dtype)
+
+            def _get_new_tokens(i):
+                return image_repl_ids_lst[i]
+
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_tokens)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        encoded.update(image_inputs)
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+
+        input_ids = inputs['input_ids']
+        # Only one image is supported per sample. # File: modeling_step_vl.py line 319, in _process_image_input
+        #           cur_feature.append(image_features[i].view(-1, image_features.shape[-1]))
+        pixel_values = inputs.get('pixel_values')
+        num_patches = inputs.get('num_patches')
+        patch_pixel_values = inputs.get('patch_pixel_values')
+
+        base_model = self.get_base_model(model)
+        inputs_embeds = base_model.model.language_model.embed_tokens(input_ids)
+        if pixel_values is not None:
+            img_inputs = base_model.model._parse_and_validate_image_input(
+                pixel_values=pixel_values, num_patches=num_patches, patch_pixel_values=patch_pixel_values)
+
+            # [image embedding or concatenation of image embedding and patch image embedding]
+            img_embeddings = base_model.model._process_image_input(img_inputs)
+
+            is_multimodal = input_ids == self.image_token_id
+            is_multimodal = is_multimodal.to(inputs_embeds.device)
+            bs = is_multimodal.shape[0]
+            for i in range(bs):
+                assert is_multimodal[i].sum() == img_embeddings[i].shape[0]
+            B, L, D = inputs_embeds.shape
+            flat_img_embeds = torch.cat(img_embeddings, dim=0)
+            flat_mask = is_multimodal.view(-1)
+            flat_inputs_embeds = inputs_embeds.view(-1, D)
+            flat_inputs_embeds[flat_mask] = flat_img_embeds
+            inputs_embeds = flat_inputs_embeds.view(B, L, D)
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        num_patches = self.gather_list(batch, 'num_patches')
+        if num_patches:
+            res['num_patches'] = num_patches
+        patch_pixel_values = [b['patch_pixel_values'] for b in batch if b.get('patch_pixel_values') is not None]
+        patch_newline_mask = [b['patch_newline_mask'] for b in batch if b.get('patch_newline_mask') is not None]
+        if patch_pixel_values:
+            res['patch_pixel_values'] = torch.concat(patch_pixel_values)
+            res['patch_newline_mask'] = torch.concat(patch_newline_mask)
+        return res
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.step3_vl,
+        template_cls=Step3VLTemplate,
+        default_system=None,
+        is_thinking=True,
+        thinking_prefix='<think>\n',
+    ))

swift/template/templates/tencent.py

@@ -0,0 +1,116 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class HunYuanVLTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<｜hy_begin▁of▁sentence｜>'])
+    prompt: Prompt = field(default_factory=lambda: ['{{QUERY}}<｜hy_User｜>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<｜hy_Assistant｜><｜hy_begin▁of▁sentence｜>'])
+    suffix: Prompt = field(default_factory=lambda: ['<｜hy_Assistant｜>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<｜hy_begin▁of▁sentence｜>{{SYSTEM}}<｜hy_place▁holder▁no▁3｜>'])
+
+
+class HunYuanVLTemplate(Template):
+    image_token_id = 120120
+    placeholder_tokens = ['<｜hy_place▁holder▁no▁102｜>']
+
+    # support_padding_free = True  # position_ids with batch_dim of 0 does not support padding_free
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        if self.mode == 'vllm':
+            return ['<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>']
+        return [[-100]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        idx_list = findall(input_ids, -100)
+        processor = self.processor
+        images = inputs.images
+        if images:
+            image_inputs = processor.image_processor(images=images, return_tensors='pt')
+            image_grid_thw = image_inputs['image_grid_thw']
+            merge_size = processor.image_processor.merge_size
+
+            def _get_new_tokens(i):
+                grid_h, grid_w = image_grid_thw[i][-2:]
+                patch_h = grid_h // merge_size
+                patch_w = grid_w // merge_size
+                img_tokens: List[int] = [self.image_token_id] * (patch_h * (patch_w + 1) + 2)
+                return img_tokens
+
+            encoded['input_ids'], encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, labels, loss_scale, idx_list, _get_new_tokens)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['image_grid_thw'] = image_grid_thw
+
+            input_ids = encoded['input_ids']
+            position_ids = torch.arange(len(input_ids))
+            position_ids_w = torch.arange(len(input_ids))
+            position_ids_h = torch.arange(len(input_ids))
+            position_ids_t = torch.arange(len(input_ids))
+            image_tokens_cumsum = [0]
+            for i in range(len(image_grid_thw)):
+                grid_h, grid_w = image_grid_thw[i][-2:]
+                patch_h = grid_h // merge_size
+                patch_w = grid_w // merge_size
+                num_image_tokens = patch_h * (patch_w + 1) + 2
+                image_tokens_cumsum.append(image_tokens_cumsum[-1] + int(num_image_tokens))
+                image_token_pos_indices = torch.where(torch.tensor(input_ids) == self.image_token_id)
+                start_pos = image_token_pos_indices[0][image_tokens_cumsum[i]] + 1
+                replace_num = (patch_w + 1) * patch_h
+                position_ids_w[start_pos:start_pos + replace_num] = torch.tensor(
+                    list(range(patch_w + 1)) * patch_h, dtype=torch.int64)
+                patch_h_list = []
+                for h in range(patch_h):
+                    patch_h_list += [h] * (patch_w + 1)
+                position_ids_h[start_pos:start_pos + replace_num] = torch.tensor(patch_h_list, dtype=torch.int64)
+                position_ids_t[start_pos:start_pos + replace_num] = 0
+            position_ids = torch.stack([position_ids, position_ids_w, position_ids_h, position_ids_t]).unsqueeze(0)
+            encoded['position_ids'] = position_ids
+            attention_mask = torch.tensor(input_ids).ne(processor.pad_id)
+            encoded['attention_mask'] = attention_mask
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+
+        input_ids = inputs['input_ids']
+        pixel_values = inputs.get('pixel_values')
+        image_grid_thw = inputs.get('image_grid_thw')
+        base_model = self.get_base_model(model)
+        inputs_embeds = base_model.model.embed_tokens(input_ids)
+
+        if pixel_values is not None:
+            pixel_values = pixel_values.to(base_model.dtype)
+            image_embeds = base_model.vit(pixel_values, image_grid_thw)
+            image_embeds = image_embeds.to(input_ids.device, non_blocking=True)
+            image_mask, _ = base_model.get_placeholder_mask(
+                input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds)
+            inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+        return {'inputs_embeds': inputs_embeds}
+
+    def _pad_3d_position_ids(self,
+                             position_ids: List[torch.Tensor],
+                             padding_value: float = 0.,
+                             batch_dim: int = 1) -> torch.Tensor:
+        batch_dim = 0
+        return super()._pad_3d_position_ids(position_ids, padding_value, batch_dim)
+
+
+register_template(HunYuanVLTemplateMeta(MLLMTemplateType.hunyuan_ocr, template_cls=HunYuanVLTemplate))

swift/template/templates/utils.py

@@ -0,0 +1,31 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from dataclasses import dataclass, field
+from typing import Optional
+
+from ..constant import LLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..utils import Prompt
+
+DEFAULT_SYSTEM = 'You are a helpful assistant.'
+
+
+@dataclass
+class ChatmlTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['<|im_start|>user\n{{QUERY}}<|im_end|>\n<|im_start|>assistant\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>\n'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|im_start|>system\n{{SYSTEM}}<|im_end|>\n'])
+    auto_add_bos: bool = True
+
+
+@dataclass
+class EmptyTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['{{QUERY}}'])
+    chat_sep: Optional[Prompt] = None
+    auto_add_bos: bool = True
+
+
+register_template(ChatmlTemplateMeta(LLMTemplateType.chatml))
+register_template(EmptyTemplateMeta(LLMTemplateType.dummy))

swift/template/templates/valley.py

@@ -0,0 +1,134 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import io
+import torch
+from dataclasses import dataclass
+from PIL import Image
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context
+from .utils import ChatmlTemplateMeta
+
+
+@dataclass
+class ValleyTemplateMeta(ChatmlTemplateMeta):
+    auto_add_bos: bool = False
+    default_system: Optional[str] = ('You are Valley, a large language and vision assistant trained by ByteDance.'
+                                     'You are able to understand the visual content or video that the user provides,'
+                                     ' and assist the user with a variety of tasks using natural language.'
+                                     'Follow the instructions carefully and explain your answers in detail.')
+
+
+class ValleyTemplate(Template):
+    skip_prompt = True
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        # assert media_type == 'image'
+        if media_type == 'video':
+            from ..vision_utils import load_video_valley
+            return self.replace_video2image(load_video_valley, inputs, lambda i: [[151665, -200, 151666]])
+        return [[151665, -200, 151666]]
+
+    def preprocess_images(self, image_binary_list):
+        from valley_eagle.util.mm_utils import process_anyres_image
+
+        def byte2image(byte_data):
+            return Image.open(io.BytesIO(byte_data))
+
+        images = []
+        for binary in image_binary_list:
+            if isinstance(binary, Image.Image):
+                images.append(binary.convert('RGB'))
+            elif isinstance(binary, bytes):
+                images.append(byte2image(binary))
+            else:
+                raise ValueError('unsupported type')
+        video_pad = []
+        for img in images:
+            if self.model.config.anyres:
+                image = process_anyres_image(img, self.tokenizer.image_processor, self.model.config.grid_pinpoints)
+            else:
+                image = self.tokenizer.image_processor(img, return_tensors='pt')['pixel_values'][0]
+            video_pad.append(image)
+
+        if not self.model.config.anyres:
+            video = torch.stack(video_pad, dim=0)
+        else:
+            video = [torch.stack(img, dim=0) for img in video_pad]
+        return video
+
+    def process_images(self, inputs, images_binary):
+        import re
+        from qwen_vl_utils import fetch_image
+
+        if inputs.messages[-1]['role'] == 'user':
+            text = inputs.messages[-1]['content']
+        elif len(inputs.messages) > 1 and inputs.messages[-2]['role'] == 'user':
+            text = inputs.messages[-2]['content']
+        else:
+            text = ''
+        video_images_tensor = self.preprocess_images(images_binary)
+        img_length = len(video_images_tensor)
+        video_images_tensor = [video_images_tensor]
+        if img_length:
+            images = [[item.to(self.model.dtype) for item in img] for img in video_images_tensor]
+
+        messages_qwen = []
+        image_list = []
+        if isinstance(images_binary[0], Image.Image):
+            images_pil = [img.convert('RGB') for img in images_binary]
+        elif isinstance(images_binary[0], bytes):
+            images_pil = [Image.open(io.BytesIO(img)).convert('RGB') for img in images_binary]
+        image_sizes = torch.tensor([[x.size for x in images_pil]])
+        for image_file in images_pil:
+            image = fetch_image({'image': image_file})
+            image_list.append(image)
+        messages_qwen.append({'role': 'user', 'content': [{'type': 'text', 'text': text}]})
+        messages_qwen.append({'role': 'assistant', 'content': [{'type': 'text', 'text': ''}]})
+        text = self.tokenizer.qwen2vl_processor.apply_chat_template(
+            messages_qwen[:-1], tokenize=False, add_generation_prompt=True)
+        text_segs = re.split('<image>', text)
+        text = '<|vision_start|><|image_pad|><|vision_end|>'.join(text_segs[:len(image_list) + 1]) + ''.join(
+            text_segs[len(image_list) + 1:])
+        data_dict_qwen2vl = self.tokenizer.qwen2vl_processor(
+            text=[text], images=image_list, padding=True, return_tensors='pt')
+        results = {}
+
+        results['images'] = images
+        results['image_sizes'] = image_sizes
+        results['pixel_values'] = data_dict_qwen2vl['pixel_values']
+        results['image_grid_thw'] = data_dict_qwen2vl['image_grid_thw']
+        return results
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        if images:
+            results = self.process_images(inputs, images)
+            encoded['images'] = results['images']
+            encoded['image_sizes'] = results['image_sizes']
+            encoded['pixel_values'] = results['pixel_values']
+            encoded['image_grid_thw'] = results['image_grid_thw']
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if 'images' in batch[0]:
+            res['images'] = sum([b['images'] for b in batch if 'images' in b], start=[])
+            res['image_sizes'] = torch.concat([b['image_sizes'] for b in batch if 'image_sizes' in b], dim=0)
+        return res
+
+
+register_template(ValleyTemplateMeta(
+    MLLMTemplateType.valley,
+    template_cls=ValleyTemplate,
+))

swift/template/templates/yi.py

@@ -0,0 +1,62 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from typing import Any, Dict, List, Optional
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from .utils import DEFAULT_SYSTEM, ChatmlTemplateMeta
+
+register_template(ChatmlTemplateMeta(
+    LLMTemplateType.yi_coder,
+    default_system=DEFAULT_SYSTEM,
+))
+
+yi_vl_default_system = (
+    'This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. '
+    "Read all the images carefully, and respond to the human's questions with informative, "
+    'helpful, detailed and polite answers. '
+    '这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。'
+    '仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。')
+
+
+class YiVLTemplate(Template):
+    image_placeholder = [[-200], '\n']
+    use_model = True
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        model = self.model
+        from llava.mm_utils import expand2square
+        if not hasattr(model, 'vision_tower'):
+            model = model.model
+        image_processor = model.vision_tower.image_processor
+        images = inputs.images or []
+        for i, image in enumerate(images):
+            background_color = tuple(int(x * 255) for x in image_processor.image_mean)
+            image = expand2square(image, background_color)
+            images[i] = image
+        if images:
+            image_tensor = image_processor.preprocess(images, return_tensors='pt')['pixel_values']
+            encoded['images'] = image_tensor.to(model.dtype)
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = torch.concat(images)
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.yi_vl,
+        prefix=[],
+        prompt=[[8308], ' Human: {{QUERY}}\n', [8308], ' Assistant:'],
+        chat_sep=['\n'],
+        suffix=['\n', [8308]],
+        default_system=yi_vl_default_system,
+        template_cls=YiVLTemplate,
+        system_prefix=['{{SYSTEM}}\n\n']))

swift/template/utils.py

@@ -0,0 +1,246 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import re
+import torch
+from transformers import PreTrainedTokenizerBase, StoppingCriteria
+from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
+
+from swift.utils import get_logger
+
+logger = get_logger()
+
+Tool = Dict[str, Union[str, Dict]]
+History = List[Union[Tuple[str, str], List[str]]]
+Message = Dict[str, Union[str, List[Dict[str, Any]], List[int], None]]
+Messages = List[Message]
+Prompt = List[Union[str, List[int], List[str]]]
+Word = Union[str, List[int]]
+Context = Word
+
+
+class ContextType:
+    RESPONSE = 'response'
+    SUFFIX = 'suffix'
+    OTHER = 'other'
+
+
+class StopWordsCriteria(StoppingCriteria):
+    """Adding extra stop words in template to prevent unstoppable generation
+        Like suffixes and chat seps in the template.
+    """
+
+    def __init__(self, tokenizer: PreTrainedTokenizerBase, stop_words: List[Word], **tokenizer_kwargs) -> None:
+        self.tokenizer = tokenizer
+        self.stop_words = stop_words
+        self.tokenizer_kwargs = tokenizer_kwargs
+        self.start_idx = -1
+        self.is_done = None
+
+    def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, **kwargs) -> torch.Tensor:
+        if self.start_idx == -1:
+            self.start_idx = len(input_ids[0]) - 1
+            self.is_done = torch.full((input_ids.shape[0], ), False, device=input_ids.device, dtype=torch.bool)
+        # [-20:]: Assuming the end tokens do not exceed 20 tokens,
+        #   to avoid input_ids being too long and affecting efficiency.
+        start_idx = max(self.start_idx, input_ids.shape[1] - 20)
+        text_list = self.tokenizer.batch_decode(input_ids[:, start_idx:], **self.tokenizer_kwargs)
+        for i, text in enumerate(text_list):
+            if self.is_done[i]:
+                continue
+            is_finished = False
+            for stop_word in self.stop_words:
+                if isinstance(stop_word, str) and stop_word in text or isinstance(
+                        stop_word, list) and input_ids[i][-len(stop_word):].tolist() == stop_word:
+                    is_finished = True
+                    break
+            self.is_done[i] = is_finished
+        return self.is_done
+
+
+def fetch_one(element: Union[Tuple, List, Set, Dict, Any], item_type: Optional[Type] = None) -> Any:
+    if isinstance(element, (tuple, set, list)):
+        for ele in element:
+            out = fetch_one(ele)
+            if out and (item_type is None or isinstance(out, item_type)):
+                return out
+    elif isinstance(element, dict):
+        return fetch_one(list(element.values()))
+    else:
+        return element
+
+
+def findall(token_list: List[int], sub_token_list: Union[int, List[int]]) -> List[int]:
+    """Find the index of a token in the token_list."""
+    if isinstance(sub_token_list, int):
+        sub_token_list = [sub_token_list]
+    res = []
+    idx = -1
+    try:
+        while True:
+            idx = token_list.index(sub_token_list[0], idx + 1)
+            if len(sub_token_list) == 1 or sub_token_list == token_list[idx:idx + len(sub_token_list)]:
+                res.append(idx)
+    except ValueError:
+        pass
+    return res
+
+
+def align_image_inputs(input_ids: List[int], labels: List[int], new_input_ids,
+                       image_token: int) -> Tuple[List[int], List[int]]:
+    if isinstance(new_input_ids, torch.Tensor):
+        new_input_ids = new_input_ids.tolist()
+
+    # Find the tokens after the image_token in input_ids, and then align them.
+    i, j = 0, 0
+    while i < len(input_ids):
+        x = input_ids[i]
+        if x == image_token:
+            assert i + 1 < len(input_ids), f'input_ids[-10:]: {input_ids[-10:]}'
+            assert i - 1 >= 0, f'input_ids[:10]: {input_ids[:10]}'
+            # [1, 2, 3(i-1), image_token(i), 4(i+1) ,5, 6]
+            # [1, 2, 3(j_begin), a(j'), a, a, a, 4(j) ,5, 6]
+            j_begin = j - 1
+            for k in range(5):  # Increase robustness.
+                if j_begin + k < len(new_input_ids) and new_input_ids[j_begin + k] == input_ids[i - 1]:
+                    j_begin += k
+                    break
+                if j_begin - k >= 0 and new_input_ids[j_begin - k] == input_ids[i - 1]:
+                    j_begin -= k
+                    break
+            else:
+                raise ValueError(f'new_input_ids: {new_input_ids}, input_ids: {input_ids}')
+            j_begin += 1
+            while j < len(new_input_ids) and new_input_ids[j] != input_ids[i + 1]:
+                j += 1
+            input_ids = input_ids[:i] + new_input_ids[j_begin:j] + input_ids[i + 1:]
+            if labels:
+                labels = labels[:i] + [-100] * (j - j_begin) + labels[i + 1:]
+            i += j - j_begin
+        else:
+            j += 1
+        i += 1
+    return input_ids, labels
+
+
+def _split_str_by_regex(text: str, regex_delimiters: List[str]) -> List[str]:
+    combined_pattern = '|'.join(f'({pattern})' for pattern in regex_delimiters)
+    parts = re.split(combined_pattern, text, flags=re.DOTALL)
+    parts = [part for part in parts if part is not None]
+    if parts[0] == '':
+        parts.pop(0)
+    else:
+        parts.insert(0, '')
+    assert len(parts) % 2 == 0, f'result: {parts}'
+    assert ''.join(parts) == text, f'split_result: {parts}, text: {text}'
+    return parts
+
+
+def split_str_parts_by(text: str, delimiters: List[str], regex_mode: bool = False) -> List[Dict[str, str]]:
+    """Split the text field into parts.
+
+    Args:
+        text: A text to be split.
+        delimiters: The delimiters.
+
+    Returns:
+        The split text in list of dicts.
+    """
+    assert isinstance(text, str), f'text: {text}'
+    delimiters_origin = delimiters
+    if not regex_mode:
+        delimiters = [re.escape(delimiter) for delimiter in delimiters]
+    parts = _split_str_by_regex(text, delimiters) if delimiters else ['', text]
+    res = []
+    if regex_mode:
+        parts = [part for part in parts if part]
+        for part in parts:
+            for delimiter, delimiter_origin in zip(delimiters, delimiters_origin):
+                if re.match(delimiter, part, re.DOTALL):
+                    break
+            else:
+                delimiter_origin = ''
+            res.append({'key': delimiter_origin, 'content': part})
+    else:
+        for key, content in zip(parts[::2], parts[1::2]):
+            res.append({'key': key, 'content': content})
+    return res
+
+
+def get_last_user_round(messages):
+    """Get the index of the last occurrence of user role"""
+    for i in range(len(messages) - 1, -1, -1):
+        if messages[i]['role'] == 'user':
+            return i
+    return -1
+
+
+def history_to_messages(history: History,
+                        system: Optional[str] = None,
+                        roles: Optional[List[List[str]]] = None) -> 'Messages':
+    """
+    history: [['query1', 'response1'], ['query2', 'response2']]
+        or [['query1', 'response1'], ['query2', None]]
+    """
+    messages = []
+    if not roles:
+        roles = [['user', 'assistant']] * len(history)
+    else:
+        assert len(roles) == len(history), f'len(roles): {len(roles)}, len(history): {len(history)}'
+    if system is not None:
+        messages.append({'role': 'system', 'content': system})
+
+    for role, h in zip(roles, history):
+        assert isinstance(h, (list, tuple))
+        if h[0] is not None:
+            messages.append({'role': role[0], 'content': h[0]})
+        if h[1] is not None:
+            messages.append({'role': role[1], 'content': h[1]})
+    return messages
+
+
+def messages_to_history(messages: 'Messages') -> Dict[str, Any]:
+    system = None
+    messages = messages.copy()
+    if messages[0]['role'] == 'system':
+        system = messages[0]['content']
+        messages = messages[1::]
+    if len(messages) % 2 == 1:
+        messages.append({'role': 'assistant', 'content': None})
+    history = []
+    history_roles = []
+    for user_message, assistant_message in zip(messages[::2], messages[1::2]):
+        assert user_message['role'] in {'tool', 'user'}, f'user_message {user_message}'
+        assert assistant_message['role'] == 'assistant', f'assistant_message: {assistant_message}'
+        history.append([user_message['content'], assistant_message['content']])
+        history_roles.append([user_message['role'], assistant_message['role']])
+    query, response = history.pop() if history else (None, None)
+    query_role = history_roles.pop()[0] if history_roles else None
+    return {
+        'history': history,
+        'history_roles': history_roles,
+        'query': query,
+        'query_role': query_role,
+        'response': response,
+        'system': system,
+    }
+
+
+def update_generation_config_eos_token(generation_config, template):
+    if generation_config is None:
+        return
+    stop_words = template.template_meta.stop_words
+    eos_token_id = generation_config.eos_token_id
+    if eos_token_id is None:
+        eos_token_id = []
+    elif isinstance(eos_token_id, int):
+        eos_token_id = [eos_token_id]
+    modified = False
+    for stop_word in stop_words:
+        if stop_word is None:
+            continue
+        if isinstance(stop_word, str):
+            stop_word = template._tokenize(stop_word)
+        if isinstance(stop_word, (list, tuple)) and len(stop_word) == 1 and stop_word[0] not in eos_token_id:
+            eos_token_id.append(stop_word[0])
+            modified = True
+    if modified:
+        generation_config.eos_token_id = eos_token_id

swift/template/vision_utils.py

@@ -0,0 +1,521 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import base64
+import math
+import numpy as np
+import os
+import re
+import requests
+import torch
+from io import BytesIO
+from PIL import Image
+from requests.adapters import HTTPAdapter
+from typing import Any, Callable, List, TypeVar, Union
+from urllib3.util.retry import Retry
+
+from swift.utils import get_env_args
+
+# >>> internvl
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
+IMAGENET_STD = (0.229, 0.224, 0.225)
+
+
+def _build_transform(input_size):
+    import torchvision.transforms as T
+    from torchvision.transforms.functional import InterpolationMode
+    MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
+    transform = T.Compose([
+        T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
+        T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
+        T.ToTensor(),
+        T.Normalize(mean=MEAN, std=STD)
+    ])
+    return transform
+
+
+def _find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
+    best_ratio_diff = float('inf')
+    best_ratio = (1, 1)
+    area = width * height
+    for ratio in target_ratios:
+        target_aspect_ratio = ratio[0] / ratio[1]
+        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_ratio = ratio
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                best_ratio = ratio
+    return best_ratio
+
+
+def _dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
+    orig_width, orig_height = image.size
+    aspect_ratio = orig_width / orig_height
+
+    # calculate the existing image aspect ratio
+    target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1)
+                        if min_num <= i * j <= max_num)
+    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+    # find the closest aspect ratio to the target
+    target_aspect_ratio = _find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
+
+    # calculate the target width and height
+    target_width = image_size * target_aspect_ratio[0]
+    target_height = image_size * target_aspect_ratio[1]
+    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+    # resize the image
+    resized_img = image.resize((target_width, target_height))
+    processed_images = []
+    for i in range(blocks):
+        box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size,
+               ((i % (target_width // image_size)) + 1) * image_size, ((i //
+                                                                        (target_width // image_size)) + 1) * image_size)
+        # split the image
+        split_img = resized_img.crop(box)
+        processed_images.append(split_img)
+    assert len(processed_images) == blocks
+    if use_thumbnail and len(processed_images) != 1:
+        thumbnail_img = image.resize((image_size, image_size))
+        processed_images.append(thumbnail_img)
+    return processed_images
+
+
+# <<< internvl
+
+
+def rescale_image(img: Image.Image, max_pixels: int) -> Image.Image:
+    import torchvision.transforms as T
+    width = img.width
+    height = img.height
+    if max_pixels is None or max_pixels <= 0 or width * height <= max_pixels:
+        return img
+
+    ratio = width / height
+    height_scaled = math.sqrt(max_pixels / ratio)
+    width_scaled = height_scaled * ratio
+    return T.Resize((int(height_scaled), int(width_scaled)))(img)
+
+
+_T = TypeVar('_T')
+
+
+def _check_path(path: str) -> Union[str, None]:
+    """If it is a path, return the string; if it is base64, return None."""
+    MAX_PATH_HEURISTIC = 2000
+    if len(path) > MAX_PATH_HEURISTIC:
+        return
+    if os.path.exists(path):
+        return os.path.abspath(path)
+    data = path
+    ROOT_IMAGE_DIR = get_env_args('ROOT_IMAGE_DIR', str, None)
+    if ROOT_IMAGE_DIR is not None:
+        path = os.path.join(ROOT_IMAGE_DIR, path)
+    path = os.path.abspath(os.path.expanduser(path))
+    if os.path.exists(path):
+        return path
+    if data.startswith('data:'):
+        return
+    try:
+        base64.b64decode(data)
+        return
+    except Exception:
+        pass
+    return data
+
+
+def load_file(path: Union[str, bytes, _T]) -> Union[BytesIO, _T]:
+    res = path
+    if isinstance(path, str):
+        path = path.strip()
+        if path.startswith('http'):
+            retries = Retry(total=3, backoff_factor=1, allowed_methods=['GET'])
+            with requests.Session() as session:
+                session.mount('http://', HTTPAdapter(max_retries=retries))
+                session.mount('https://', HTTPAdapter(max_retries=retries))
+
+                timeout = float(os.getenv('SWIFT_TIMEOUT', '20'))
+                request_kwargs = {'timeout': timeout} if timeout > 0 else {}
+
+                response = session.get(path, **request_kwargs)
+                response.raise_for_status()
+                content = response.content
+                res = BytesIO(content)
+        else:
+            data = path
+            path = _check_path(path)
+            if path is None:
+                # base64_str
+                if data.startswith('data:'):
+                    match_ = re.match(r'data:(.+?);base64,(.+)', data)
+                    assert match_ is not None
+                    data = match_.group(2)
+                data = base64.b64decode(data)
+                res = BytesIO(data)
+            else:
+                with open(path, 'rb') as f:
+                    res = BytesIO(f.read())
+    elif isinstance(path, bytes):
+        res = BytesIO(path)
+    return res
+
+
+def load_image(image: Union[str, bytes, Image.Image]) -> Image.Image:
+    image = load_file(image)
+    if isinstance(image, BytesIO):
+        image = Image.open(image)
+    if image.mode != 'RGB':
+        image = image.convert('RGB')
+    return image
+
+
+def load_batch(path_list: List[Union[str, None, Any, BytesIO]],
+               load_func: Callable[[Any], _T] = load_image) -> List[_T]:
+    res = []
+    assert isinstance(path_list, (list, tuple)), f'path_list: {path_list}'
+    for path in path_list:
+        if path is None:  # ignore None
+            continue
+        res.append(load_func(path))
+    return res
+
+
+def load_video_hf(videos: List[str]):
+    from transformers.video_utils import load_video
+    res = []
+    video_metadata = []
+    for video in videos:
+        if isinstance(video, (list, tuple)) and isinstance(video[0], str):
+            # Case a: Video is provided as a list of image file names
+            video = [np.array(load_image(image_fname)) for image_fname in video]
+            video = np.stack(video)
+            metadata = None
+        else:
+            # Case b: Video is provided as a single file path or URL or decoded frames in a np.ndarray or torch.tensor
+            video_load_backend = get_env_args('video_load_backend', str, 'pyav')
+            video, metadata = load_video(
+                video,
+                backend=video_load_backend,
+            )
+        res.append(video)
+        video_metadata.append(metadata)
+    return res, video_metadata
+
+
+def _get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
+    if bound:
+        start, end = bound[0], bound[1]
+    else:
+        start, end = -100000, 100000
+    start_idx = max(first_idx, round(start * fps))
+    end_idx = min(round(end * fps), max_frame)
+    seg_size = float(end_idx - start_idx) / num_segments
+    frame_indices = np.array(
+        [int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)])
+    return frame_indices
+
+
+def transform_image(image, input_size=448, max_num=12):
+    transform = _build_transform(input_size=input_size)
+    images = _dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
+    pixel_values = [transform(image) for image in images]
+    pixel_values = torch.stack(pixel_values)
+    return pixel_values
+
+
+def load_video_internvl(video: Union[str, bytes], bound=None, num_segments=32):
+    from decord import VideoReader, cpu
+    video_io = load_file(video)
+    vr = VideoReader(video_io, ctx=cpu(0), num_threads=1)
+    max_frame = len(vr) - 1
+    fps = float(vr.get_avg_fps())
+
+    images = []
+    frame_indices = _get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
+    for frame_index in frame_indices:
+        images.append(Image.fromarray(vr[frame_index].asnumpy()).convert('RGB'))
+    return images
+
+
+def load_video_cogvlm2(video: Union[str, bytes]) -> np.ndarray:
+    from decord import VideoReader, bridge, cpu
+    video_io = load_file(video)
+    bridge.set_bridge('torch')
+    clip_end_sec = 60
+    clip_start_sec = 0
+    num_frames = get_env_args('num_frames', int, 24)
+    decord_vr = VideoReader(video_io, ctx=cpu(0))
+    duration = len(decord_vr)  # duration in terms of frames
+    start_frame = int(clip_start_sec * decord_vr.get_avg_fps())
+    end_frame = min(duration, int(clip_end_sec * decord_vr.get_avg_fps())) if \
+        clip_end_sec is not None else duration
+    frame_id_list = np.linspace(start_frame, end_frame - 1, num_frames, dtype=int)
+    video_data = decord_vr.get_batch(frame_id_list)
+    video_data = video_data.permute(3, 0, 1, 2)
+    return video_data
+
+
+def load_video_llava(video: Union[str, bytes]) -> np.ndarray:
+    import av
+    video_io = load_file(video)
+    container = av.open(video_io)
+    total_frames = container.streams.video[0].frames
+    num_frames = get_env_args('num_frames', int, 16)
+    indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format='rgb24') for x in frames])
+
+
+def load_video_minicpmv_mplug_owl3(video: Union[str, bytes], max_num_frames):
+
+    from decord import VideoReader, cpu  # pip install decord
+
+    def uniform_sample(_l, _n):
+        gap = len(_l) / _n
+        idxs = [int(i * gap + gap / 2) for i in range(_n)]
+        return [_l[i] for i in idxs]
+
+    video_io = load_file(video)
+    vr = VideoReader(video_io, ctx=cpu(0))
+    sample_fps = round(vr.get_avg_fps() / 1)  # FPS
+    frame_idx = [i for i in range(0, len(vr), sample_fps)]
+
+    if len(frame_idx) > max_num_frames:
+        frame_idx = uniform_sample(frame_idx, max_num_frames)
+    frames = vr.get_batch(frame_idx).asnumpy()
+    frames = [Image.fromarray(v.astype('uint8')) for v in frames]
+    return frames
+
+
+def _load_audio_librosa(audio: Union[str, bytes], sampling_rate: int, mono: bool = True):
+    import librosa
+    try:
+        audio_io = load_file(audio)
+        return librosa.load(audio_io, sr=sampling_rate, mono=mono)
+    except Exception:
+        if isinstance(audio, str) and audio.startswith(('http://', 'https://')):
+            import audioread
+            audio_io = audioread.ffdec.FFmpegAudioFile(audio)
+        else:
+            audio_io = _check_path(audio) if isinstance(audio, str) else audio
+        return librosa.load(audio_io, sr=sampling_rate, mono=mono)
+
+
+# ref: https://github.com/vllm-project/vllm/blob/v0.23.0/vllm/multimodal/audio.py#L169-L224
+def _resample_audio_pyav(audio: np.ndarray, *, orig_sr: float, target_sr: float) -> np.ndarray:
+    import av
+    orig_sr_int = int(round(orig_sr))
+    target_sr_int = int(round(target_sr))
+    if orig_sr_int == target_sr_int:
+        return audio
+    if audio.ndim == 2:
+        return np.stack([_resample_audio_pyav(ch, orig_sr=orig_sr, target_sr=target_sr) for ch in audio], axis=0)
+    expected_len = int(math.ceil(audio.shape[-1] * target_sr_int / orig_sr_int))
+    min_samples = 1024
+    audio_f32 = np.asarray(audio, dtype=np.float32)
+    if len(audio_f32) < min_samples:
+        audio_f32 = np.pad(audio_f32, (0, min_samples - len(audio_f32)))
+    audio_f32 = audio_f32.reshape(1, -1)
+    resampler = av.AudioResampler(format='fltp', layout='mono', rate=target_sr_int)
+    frame = av.AudioFrame.from_ndarray(audio_f32, format='fltp', layout='mono')
+    frame.sample_rate = orig_sr_int
+    out_frames = resampler.resample(frame)
+    out_frames.extend(resampler.resample(None))
+    result = np.concatenate([f.to_ndarray() for f in out_frames], axis=1).squeeze(0)
+    return result[:expected_len]
+
+
+# ref: https://github.com/vllm-project/vllm/blob/v0.23.0/vllm/multimodal/media/audio.py#L45-L160
+def _load_audio_soundfile_pyav(path: Union[str, bytes, BytesIO], *, sr: float, mono: bool = True):
+    """soundfile first, pyav fallback — same strategy as vLLM multimodal audio loader."""
+    bad_sf_codes = {0, 1, 3, 4}
+    if not isinstance(path, BytesIO):
+        path = load_file(path)
+
+    def _load_soundfile():
+        import soundfile
+        with soundfile.SoundFile(path) as f:
+            native_sr = f.samplerate
+            y = f.read(dtype='float32', always_2d=False).T
+
+        if mono and y.ndim > 1:
+            y = np.mean(y, axis=tuple(range(y.ndim - 1)))
+
+        if sr is not None and sr != native_sr:
+            y = _resample_audio_pyav(y, orig_sr=native_sr, target_sr=sr)
+            return y, int(sr)
+        return y, native_sr
+
+    def _load_pyav():
+        import av
+        path.seek(0)
+        with av.open(path) as container:
+            if not container.streams.audio:
+                raise ValueError('No audio stream found.')
+            stream = container.streams.audio[0]
+            stream.thread_type = 'AUTO'
+            native_sr = stream.rate
+            target_sr = sr or native_sr
+            chunks = []
+            needs_resampling = not math.isclose(float(target_sr), float(native_sr), rel_tol=0.0, abs_tol=1e-6)
+            resampler = av.AudioResampler(format='fltp', layout='mono', rate=target_sr) if needs_resampling else None
+            for frame in container.decode(stream):
+                if needs_resampling:
+                    for out_frame in resampler.resample(frame):
+                        chunks.append(out_frame.to_ndarray())
+                else:
+                    chunks.append(frame.to_ndarray())
+        if not chunks:
+            raise ValueError('No audio found.')
+        y = np.concatenate(chunks, axis=-1).astype(np.float32)
+        if mono and y.ndim > 1:
+            y = np.mean(y, axis=0)
+        return y, target_sr
+
+    try:
+        return _load_soundfile()
+    except ImportError:
+        path.seek(0)
+        return _load_pyav()
+    except Exception as exc:
+        import soundfile
+        if not isinstance(exc, soundfile.LibsndfileError) or exc.code not in bad_sf_codes:
+            raise
+    path.seek(0)
+    return _load_pyav()
+
+
+def load_audio(
+    audio: Union[str, bytes, BytesIO],
+    sampling_rate: int,
+    return_sr: bool = False,
+    mono: bool = True,
+):
+    backend = get_env_args('swift_audio_load_backend', str, 'librosa')
+    if backend == 'librosa':
+        res = _load_audio_librosa(audio, sampling_rate, mono=mono)
+    elif backend == 'soundfile_pyav':
+        res = _load_audio_soundfile_pyav(audio, sr=sampling_rate, mono=mono)
+    else:
+        raise ValueError(f'Unknown audio load backend {backend!r}. Supported: librosa, soundfile_pyav')
+    return res if return_sr else res[0]
+
+
+def _resolve_video_local_path(path: Union[str, bytes]) -> tuple:
+    """Return (local_path, is_temp_file). HTTP URLs and raw bytes are written to a temp file."""
+    if isinstance(path, bytes) or (isinstance(path, str) and path.startswith('http')):
+        import tempfile
+        with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as f:
+            f.write(load_file(path).read())
+            return f.name, True
+    checked = _check_path(path) if isinstance(path, str) else None
+    return checked or path, False
+
+
+def _video_to_ndarrays_local(local_path: str, num_frames: int = -1) -> np.ndarray:
+    import cv2
+    cap = cv2.VideoCapture(local_path)
+    if not cap.isOpened():
+        raise ValueError(f'Could not open video file {local_path}')
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if total_frames <= 0:
+        cap.release()
+        raise ValueError(f'Video file {local_path} has invalid or zero frame count: {total_frames}')
+    if num_frames <= 0 or num_frames > total_frames:
+        num_frames = total_frames
+    frame_indices = set(np.linspace(0, total_frames - 1, num_frames, dtype=int))
+    frames = []
+    for idx in range(total_frames):
+        if not cap.grab():
+            break
+        if idx in frame_indices:
+            ret, frame = cap.retrieve()
+            if ret:
+                frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+    cap.release()
+    if len(frames) < num_frames:
+        raise ValueError(f'Could not read enough frames from video file {local_path} '
+                         f'(expected {num_frames} frames, got {len(frames)})')
+    return np.stack(frames)
+
+
+def _video_get_metadata_local(local_path: str, num_frames: int = -1) -> dict:
+    import cv2
+    cap = cv2.VideoCapture(local_path)
+    if not cap.isOpened():
+        raise ValueError(f'Could not open video file {local_path}')
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if total_frames <= 0:
+        cap.release()
+        raise ValueError(f'Video file {local_path} has invalid or zero frame count: {total_frames}')
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    duration = total_frames / fps if fps > 0 else 0
+    cap.release()
+    if num_frames <= 0 or num_frames > total_frames:
+        num_frames = total_frames
+    return {
+        'total_num_frames': num_frames,
+        'fps': duration / num_frames if num_frames else fps,
+        'duration': duration,
+        'video_backend': 'opencv',
+        'frames_indices': list(range(num_frames)),
+        'do_sample_frames': num_frames == total_frames,
+    }
+
+
+def load_vllm_video(path: Union[str, bytes], num_frames: int = -1) -> tuple:
+    """Decode video frames + metadata for vLLM rollout; one download, temp file cleaned up."""
+    local_path, is_temp = _resolve_video_local_path(path)
+    try:
+        return _video_to_ndarrays_local(local_path, num_frames), _video_get_metadata_local(local_path, num_frames)
+    finally:
+        if is_temp:
+            try:
+                os.remove(local_path)
+            except OSError:
+                pass
+
+
+def load_video_valley(video: Union[str, bytes]):
+    import decord
+    from torchvision import transforms
+    video_io = load_file(video)
+    video_reader = decord.VideoReader(video_io)
+    decord.bridge.set_bridge('torch')
+    video = video_reader.get_batch(np.linspace(0, len(video_reader) - 1, 8).astype(np.int_)).byte()
+    images = [transforms.ToPILImage()(image.permute(2, 0, 1)).convert('RGB') for image in video]
+    return images
+
+
+def load_video_ovis2(video_path, num_frames):
+    from moviepy.editor import VideoFileClip
+    with VideoFileClip(video_path) as clip:
+        total_frames = int(clip.fps * clip.duration)
+        if total_frames <= num_frames:
+            sampled_indices = range(total_frames)
+        else:
+            stride = total_frames / num_frames
+            sampled_indices = [
+                min(total_frames - 1, int((stride * i + stride * (i + 1)) / 2)) for i in range(num_frames)
+            ]
+        frames = [clip.get_frame(index / clip.fps) for index in sampled_indices]
+        frames = [Image.fromarray(frame, mode='RGB') for frame in frames]
+    return frames
+
+
+def load_video_ovis2_5(video_path, num_frames):
+    from moviepy.editor import VideoFileClip
+    with VideoFileClip(video_path) as clip:
+        total_frames = int(clip.fps * clip.duration)
+        indices = [int(i * total_frames / num_frames) for i in range(num_frames)]
+        frames = [Image.fromarray(clip.get_frame(t)) for t in (idx / clip.fps for idx in indices)]
+    return frames