chore: import upstream snapshot with attribution

tests/test_align/test_cls.py

@@ -0,0 +1,59 @@
+import os
+from pprint import pprint
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+kwargs = {
+    'per_device_train_batch_size': 4,
+    'per_device_eval_batch_size': 4,
+    'gradient_accumulation_steps': 4,
+    'num_train_epochs': 1,
+    'save_steps': 100,
+    'max_length': 512,
+    'task_type': 'seq_cls',
+    'num_labels': 2,
+}
+
+
+def calc_acc(infer_result):
+    n_correct = 0
+    for res in infer_result:
+        if res['response'] == res['labels']:
+            n_correct += 1
+    return f'acc: {n_correct / len(infer_result)}, n_correct: {n_correct}, len(res): {len(infer_result)}'
+
+
+def test_llm():
+    from swift import InferArguments, SftArguments, infer_main, sft_main
+    res = []
+    for model in ['Qwen/Qwen2.5-0.5B-Instruct', 'Qwen/Qwen2.5-0.5B', 'AI-ModelScope/bert-base-chinese']:
+        dataset = ['DAMO_NLP/jd:cls#2000']
+        result = sft_main(SftArguments(model=model, dataset=dataset, split_dataset_ratio=0.1, **kwargs))
+        last_model_checkpoint = result['last_model_checkpoint']
+        infer_result = infer_main(
+            InferArguments(adapters=[last_model_checkpoint], load_data_args=True, truncation_strategy='right'))
+        res.append(calc_acc(infer_result))
+        infer_result2 = infer_main(
+            InferArguments(
+                adapters=[last_model_checkpoint], load_data_args=True, max_batch_size=16, truncation_strategy='right'))
+        res.append(calc_acc(infer_result2))
+
+    model = 'Qwen/Qwen2.5-0.5B-Instruct'
+    dataset = ['DAMO_NLP/jd#2000']
+    train_kwargs = kwargs.copy()
+    train_kwargs.pop('task_type')
+    train_kwargs.pop('num_labels')
+    result = sft_main(SftArguments(model=model, dataset=dataset, split_dataset_ratio=0.1, **train_kwargs))
+    last_model_checkpoint = result['last_model_checkpoint']
+    infer_result = infer_main(
+        InferArguments(adapters=[last_model_checkpoint], load_data_args=True, truncation_strategy='right'))
+    res.append(calc_acc(infer_result))
+    infer_result2 = infer_main(
+        InferArguments(
+            adapters=[last_model_checkpoint], load_data_args=True, max_batch_size=16, truncation_strategy='right'))
+    res.append(calc_acc(infer_result2))
+    pprint(res)
+
+
+if __name__ == '__main__':
+    test_llm()

tests/test_align/test_lmdeploy_vlm.py

@@ -0,0 +1,83 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def _infer_image(model, system=None, images=None):
+    engine = LmdeployEngine(model)
+    if images is None:
+        images = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png']
+    messages = []
+    if system is not None:
+        messages += [{'role': 'system', 'content': system}]
+    messages.append({'role': 'user', 'content': 'describe the image.'})
+    resp_list = engine.infer([InferRequest(messages=messages, images=images)],
+                             RequestConfig(temperature=0, max_tokens=64, repetition_penalty=1.))
+    return resp_list[0].choices[0].message.content
+
+
+def _infer_image_pipeline(model, images=None, prefix='<IMAGE_TOKEN>\n'):
+    from lmdeploy import GenerationConfig, pipeline
+    from lmdeploy.vl import load_image
+
+    from swift.utils import safe_snapshot_download
+    gen_config = GenerationConfig(temperature=0., repetition_penalty=1., max_new_tokens=64)
+    pipe = pipeline(safe_snapshot_download(model))
+
+    image = load_image('http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png')
+    response = pipe((f'{prefix}describe the image.', image), gen_config=gen_config)
+    return response.text
+
+
+def test_internvl2_5():
+    model = 'OpenGVLab/InternVL2_5-4B'
+    response = _infer_image(model)
+    response2 = _infer_image_pipeline(model)
+    assert response == response2
+
+
+def test_internvl2():
+    model = 'OpenGVLab/InternVL2-2B'
+    response = _infer_image(model)
+    response2 = _infer_image_pipeline(model)  # Missing '\n' after '<|im_end|>'
+    assert response == response2
+
+
+def test_deepseek_vl():
+    model = 'deepseek-ai/deepseek-vl-1.3b-chat'
+    response = _infer_image(model)
+    response2 = _infer_image_pipeline(model, prefix='<IMAGE_TOKEN>')
+    assert response == response2
+
+
+def test_qwen_vl():
+    model = 'Qwen/Qwen-VL-Chat'
+    response = _infer_image_pipeline(model)  # Missing: 'Picture 1: '
+    response2 = _infer_image(model)
+    assert response == response2
+
+
+def test_qwen2_vl():
+    model = 'Qwen/Qwen2-VL-2B-Instruct'
+    response = _infer_image_pipeline(model, prefix='<IMAGE_TOKEN>')
+    response2 = _infer_image(model)
+    assert response == response2
+
+
+def test_qwen2_5_vl():
+    model = 'Qwen/Qwen2.5-VL-3B-Instruct'
+    response = _infer_image(model)
+    response2 = _infer_image_pipeline(model, prefix='<IMAGE_TOKEN>')
+    assert response == response2
+
+
+if __name__ == '__main__':
+    from swift.infer_engine import InferRequest, LmdeployEngine, RequestConfig
+
+    # test_internvl2()
+    # test_internvl2_5()
+    # test_deepseek_vl()
+    # test_qwen_vl()
+    # test_qwen2_vl()
+    test_qwen2_5_vl()

tests/test_align/test_mm_processor_align.py

@@ -0,0 +1,568 @@
+import copy
+import os
+import pytest
+import torch
+from contextlib import contextmanager, nullcontext
+from typing import Any, Dict
+
+from swift.model import get_processor
+from swift.template import get_template
+
+try:
+    from vllm.config import ModelConfig
+    from vllm.multimodal import MULTIMODAL_REGISTRY
+    from vllm.multimodal.inputs import nested_tensors_equal
+except ImportError:
+    ModelConfig = None
+    MULTIMODAL_REGISTRY = None
+    nested_tensors_equal = None
+
+pytestmark = pytest.mark.skipif(ModelConfig is None, reason='vLLM not available')
+
+WEATHER_AUDIO = 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+BABY_VIDEO = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+DRAW_VIDEO = 'https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4'
+CAT_IMAGE = 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png'
+_SKIP_TRAIN_KEYS = frozenset({'input_ids', 'labels', 'loss_scale', 'mm_token_type_ids'})
+
+_QWEN_VL_VIDEO_ALIASES = {'video_second_per_grid': 'second_per_grid_ts'}
+_GEMMA4_IMAGE_ALIASES = {'image_position_ids': 'pixel_position_ids'}
+_QWEN3_OMNI_AUDIO_ALIASES = {
+    'input_features': 'input_audio_features',
+    'feature_attention_mask': 'feature_attention_mask',
+}
+_XFAIL_TESTS = frozenset({
+    'test_qwen2_5_vl_video',
+    'test_qwen3_vl_video',
+    'test_qwen3_5_video',
+    'test_gemma4_video',
+})
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def as_list_ids(x):
+    if isinstance(x, torch.Tensor):
+        return x.reshape(-1).tolist()
+    return list(x)
+
+
+def tensors_aligned(a, b) -> bool:
+    if isinstance(a, list) and isinstance(b, list):
+        return len(a) == len(b) and all(tensors_aligned(x, y) for x, y in zip(a, b))
+    if isinstance(a, list) and len(a) == 1:
+        a = a[0]
+    if isinstance(b, list) and len(b) == 1:
+        b = b[0]
+    if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):
+        a, b = a.detach().cpu(), b.detach().cpu()
+        if a.ndim == b.ndim + 1 and a.shape[0] == 1 and a.shape[1:] == b.shape:
+            a = a.squeeze(0)
+        elif b.ndim == a.ndim + 1 and b.shape[0] == 1 and b.shape[1:] == a.shape:
+            b = b.squeeze(0)
+        if a.shape != b.shape:
+            return False
+        if a.dtype.is_floating_point or b.dtype.is_floating_point:
+            a = a.to(torch.bfloat16).float()
+            b = b.to(torch.bfloat16).float()
+            return torch.allclose(a, b, rtol=0, atol=0)
+        return torch.equal(a, b)
+    if nested_tensors_equal is None:
+        return a == b
+    return nested_tensors_equal(a, b)
+
+
+def build_vllm_mm_data(vllm_encoded: Dict[str, Any]) -> Dict[str, Any]:
+    mm_data = {}
+    for plural, singular in [('images', 'image'), ('videos', 'video'), ('audios', 'audio')]:
+        data = vllm_encoded.get(plural)
+        if not data:
+            continue
+        if len(data) == 1 and not isinstance(data[0], tuple):
+            mm_data[singular] = data[0]
+        else:
+            mm_data[singular] = data
+    return mm_data
+
+
+def swift_train_encode(template, sample: dict) -> Dict[str, Any]:
+    train_template = copy.deepcopy(template)
+    train_template.set_mode('train')
+    return train_template.encode(sample)
+
+
+def vllm_forward_kwargs(model_id: str, template, sample: dict) -> Dict[str, Any]:
+    if ModelConfig is None:
+        raise RuntimeError('vLLM is not available')
+    vllm_template = copy.deepcopy(template)
+    vllm_template.set_mode('vllm')
+    encoded = vllm_template.encode(sample)
+    mm_data = build_vllm_mm_data(encoded)
+    if not mm_data:
+        return {'input_ids': encoded['input_ids'], 'mm_tensors': {}}
+    model_config = ModelConfig(model_id, trust_remote_code=True, dtype='auto', seed=0)
+    processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+    mm_items = processor.info.parse_mm_data(mm_data)
+    result = processor(
+        encoded['input_ids'],
+        mm_items=mm_items,
+        hf_processor_mm_kwargs=encoded.get('mm_processor_kwargs') or {},
+    )
+    return {
+        'input_ids': result['prompt_token_ids'],
+        'mm_tensors': result['mm_kwargs'].get_data(),
+    }
+
+
+@contextmanager
+def audio_backend(backend: str):
+    prev = os.environ.get('SWIFT_AUDIO_LOAD_BACKEND')
+    os.environ['SWIFT_AUDIO_LOAD_BACKEND'] = backend
+    try:
+        yield
+    finally:
+        if prev is None:
+            os.environ.pop('SWIFT_AUDIO_LOAD_BACKEND', None)
+        else:
+            os.environ['SWIFT_AUDIO_LOAD_BACKEND'] = prev
+
+
+@pytest.fixture(autouse=True)
+def _soundfile_pyav_for_align_tests():
+    with audio_backend('soundfile_pyav'):
+        yield
+
+
+def _vllm_audio_feature_lengths(train: dict, vllm_tensors: dict) -> None:
+    """vLLM sets audio_feature_lengths; Swift derives the same value from mask.sum()."""
+    vllm_afl = vllm_tensors.get('audio_feature_lengths')
+    mask = train.get('feature_attention_mask')
+    if vllm_afl is None or mask is None:
+        return
+    derived = mask.sum(-1)
+    if derived.ndim == 0:
+        derived = derived.unsqueeze(0)
+    assert tensors_aligned(derived, vllm_afl), 'mask.sum() != vLLM audio_feature_lengths'
+
+
+@contextmanager
+def use_audio_in_video(enabled: bool = True):
+    prev = os.environ.get('USE_AUDIO_IN_VIDEO')
+    if enabled:
+        os.environ['USE_AUDIO_IN_VIDEO'] = 'true'
+    else:
+        os.environ.pop('USE_AUDIO_IN_VIDEO', None)
+    try:
+        yield
+    finally:
+        if prev is None:
+            os.environ.pop('USE_AUDIO_IN_VIDEO', None)
+        else:
+            os.environ['USE_AUDIO_IN_VIDEO'] = prev
+
+
+def _assert_mm_align(
+    model_id,
+    sample,
+    *,
+    tensor_key_aliases=None,
+    check_input_ids=True,
+    check_vllm_audio_feature_lengths=False,
+    use_audio_in_video_flag=False,
+):
+    tensor_key_aliases = tensor_key_aliases or {}
+    ctx = use_audio_in_video() if use_audio_in_video_flag else nullcontext()
+    with ctx:
+        processor = get_processor(model_id)
+        template = get_template(processor)
+        train = swift_train_encode(template, sample)
+        vllm = vllm_forward_kwargs(model_id, template, sample)
+
+    if check_input_ids:
+        assert as_list_ids(train['input_ids']) == as_list_ids(vllm['input_ids'])
+
+    vllm_tensors = dict(vllm['mm_tensors'])
+    compared = [(tk, tensor_key_aliases.get(tk, tk))
+                for tk in sorted(k for k, v in train.items() if v is not None and k not in _SKIP_TRAIN_KEYS)
+                if tensor_key_aliases.get(tk, tk) in vllm_tensors]
+    for train_key, vllm_key in compared:
+        assert tensors_aligned(train[train_key], vllm_tensors[vllm_key]), f'{train_key}!={vllm_key}'
+    if check_vllm_audio_feature_lengths:
+        _vllm_audio_feature_lengths(train, vllm_tensors)
+
+
+# ---------------------------------------------------------------------------
+# Qwen2.5-VL
+# ---------------------------------------------------------------------------
+
+
+def test_qwen2_5_vl_image():
+    _assert_mm_align(
+        'Qwen/Qwen2.5-VL-3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the image.'
+            }],
+            'images': [CAT_IMAGE]
+        },
+    )
+
+
+@pytest.mark.xfail(reason='vLLM Qwen2_5_VLProcessor rejects fps list in mm_processor_kwargs (expects scalar)')
+def test_qwen2_5_vl_video():
+    _assert_mm_align(
+        'Qwen/Qwen2.5-VL-3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [BABY_VIDEO]
+        },
+        tensor_key_aliases=_QWEN_VL_VIDEO_ALIASES,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Qwen3-VL
+# ---------------------------------------------------------------------------
+
+
+def test_qwen3_vl_image():
+    _assert_mm_align(
+        'Qwen/Qwen3-VL-2B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the image.'
+            }],
+            'images': [CAT_IMAGE]
+        },
+    )
+
+
+@pytest.mark.xfail(reason='vLLM get_video_repl drops outer vision_start/end wrapper (2-token diff vs HF)')
+def test_qwen3_vl_video():
+    _assert_mm_align(
+        'Qwen/Qwen3-VL-2B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [BABY_VIDEO]
+        },
+        tensor_key_aliases=_QWEN_VL_VIDEO_ALIASES,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Qwen3.5
+# ---------------------------------------------------------------------------
+
+
+def test_qwen3_5_image():
+    _assert_mm_align(
+        'Qwen/Qwen3.5-2B',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the image.'
+            }],
+            'images': [CAT_IMAGE]
+        },
+    )
+
+
+@pytest.mark.xfail(reason='vLLM get_video_repl drops outer vision_start/end wrapper (2-token diff vs HF)')
+def test_qwen3_5_video():
+    _assert_mm_align(
+        'Qwen/Qwen3.5-2B',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [BABY_VIDEO]
+        },
+        tensor_key_aliases=_QWEN_VL_VIDEO_ALIASES,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Qwen2.5-Omni
+# ---------------------------------------------------------------------------
+
+
+def test_qwen2_5_omni_image():
+    _assert_mm_align(
+        'Qwen/Qwen2.5-Omni-7B',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the image.'
+            }],
+            'images': [CAT_IMAGE]
+        },
+    )
+
+
+def test_qwen2_5_omni_video():
+    _assert_mm_align(
+        'Qwen/Qwen2.5-Omni-7B',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [BABY_VIDEO]
+        },
+        tensor_key_aliases={'video_second_per_grid': 'second_per_grid_ts'},
+    )
+
+
+def test_qwen2_5_omni_audio():
+    # Standalone audio: sample has `audios` field (not extracted from video).
+    # vLLM path loads as (wav, sr) in _preprocess_inputs; train path uses ndarray.
+    _assert_mm_align(
+        'Qwen/Qwen2.5-Omni-7B',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the audio.'
+            }],
+            'audios': [WEATHER_AUDIO]
+        },
+        check_vllm_audio_feature_lengths=True,
+    )
+
+
+def test_qwen2_5_omni_video_use_audio_in_video():
+    # Video track extracted in replace_tag; vLLM uses different audio/video token layout.
+    _assert_mm_align(
+        'Qwen/Qwen2.5-Omni-7B',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [DRAW_VIDEO]
+        },
+        tensor_key_aliases={'video_second_per_grid': 'second_per_grid_ts'},
+        check_input_ids=False,
+        check_vllm_audio_feature_lengths=True,
+        use_audio_in_video_flag=True,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Qwen3-Omni
+# ---------------------------------------------------------------------------
+
+
+def test_qwen3_omni_image():
+    _assert_mm_align(
+        'Qwen/Qwen3-Omni-30B-A3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the image.'
+            }],
+            'images': [CAT_IMAGE]
+        },
+    )
+
+
+def test_qwen3_omni_video():
+    _assert_mm_align(
+        'Qwen/Qwen3-Omni-30B-A3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [BABY_VIDEO]
+        },
+        tensor_key_aliases=_QWEN_VL_VIDEO_ALIASES,
+    )
+
+
+def test_qwen3_omni_audio():
+    _assert_mm_align(
+        'Qwen/Qwen3-Omni-30B-A3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the audio.'
+            }],
+            'audios': [WEATHER_AUDIO]
+        },
+        tensor_key_aliases=_QWEN3_OMNI_AUDIO_ALIASES,
+        check_vllm_audio_feature_lengths=True,
+    )
+
+
+def test_qwen3_omni_audio_non_hop_aligned(tmp_path):
+    """Verify hop-length floor trim when waveform length is not hop-aligned."""
+    import soundfile as sf
+
+    from swift.template.vision_utils import load_audio
+
+    hop = 160
+    wav = load_audio(WEATHER_AUDIO, 16000)
+    n = len(wav)
+    rem = n % hop
+    cut = rem if rem else hop // 2 + 1
+    wav = wav[:max(n - cut, hop + 1)]
+    assert len(wav) % hop != 0, 'test fixture must be non hop-aligned'
+    wav_path = tmp_path / 'non_hop_aligned.wav'
+    sf.write(str(wav_path), wav, 16000)
+    _assert_mm_align(
+        'Qwen/Qwen3-Omni-30B-A3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the audio.'
+            }],
+            'audios': [str(wav_path)]
+        },
+        tensor_key_aliases=_QWEN3_OMNI_AUDIO_ALIASES,
+        check_vllm_audio_feature_lengths=True,
+    )
+
+
+def test_qwen3_omni_video_use_audio_in_video():
+    _assert_mm_align(
+        'Qwen/Qwen3-Omni-30B-A3B-Instruct',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the video.'
+            }],
+            'videos': [DRAW_VIDEO]
+        },
+        tensor_key_aliases={
+            **_QWEN3_OMNI_AUDIO_ALIASES,
+            **_QWEN_VL_VIDEO_ALIASES
+        },
+        check_input_ids=False,
+        check_vllm_audio_feature_lengths=True,
+        use_audio_in_video_flag=True,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Gemma4
+# ---------------------------------------------------------------------------
+
+
+def test_gemma4_image():
+    _assert_mm_align(
+        'google/gemma-4-E2B-it',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the image.'
+            }],
+            'images': [CAT_IMAGE]
+        },
+        tensor_key_aliases=_GEMMA4_IMAGE_ALIASES,
+    )
+
+
+def test_gemma4_audio():
+    _assert_mm_align(
+        'google/gemma-4-E2B-it',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': 'describe the audio.'
+            }],
+            'audios': [WEATHER_AUDIO]
+        },
+        tensor_key_aliases={
+            'input_features': 'input_features_padded',
+            'input_features_mask': 'input_features_mask',
+        },
+    )
+
+
+def test_gemma4_audio_collator_3d():
+    """Collator must batch audio as (N, max_len, feat_dim), not concat along time dim."""
+    sample = {
+        'messages': [{
+            'role': 'user',
+            'content': 'describe the audio.'
+        }],
+        'audios': [WEATHER_AUDIO],
+    }
+    processor = get_processor('google/gemma-4-E2B-it')
+    template = get_template(processor)
+    template.set_mode('train')
+    batch = [template.encode(sample), template.encode(sample)]
+    collated = template._data_collator_mm_data(batch)
+    assert collated['input_features'].ndim == 3
+    assert collated['input_features'].shape[0] == 2
+    assert collated['input_features_mask'].ndim == 2
+    assert collated['input_features_mask'].shape[0] == 2
+
+
+@pytest.mark.xfail(reason='vLLM gemma4 video timestamp/soft-token path differs from HF Gemma4VideoProcessor')
+def test_gemma4_video():
+    _assert_mm_align(
+        'google/gemma-4-E2B-it',
+        {
+            'messages': [{
+                'role': 'user',
+                'content': '<video>describe the video.'
+            }],
+            'videos': [BABY_VIDEO]
+        },
+        tensor_key_aliases={
+            'pixel_values_videos': 'pixel_values_videos',
+            'video_position_ids': 'video_position_ids',
+        },
+    )
+
+
+if __name__ == '__main__':
+    tests = [
+        test_qwen2_5_vl_image,
+        test_qwen2_5_vl_video,
+        test_qwen3_vl_image,
+        test_qwen3_vl_video,
+        test_qwen3_5_image,
+        test_qwen3_5_video,
+        test_qwen2_5_omni_image,
+        test_qwen2_5_omni_video,
+        test_qwen2_5_omni_audio,
+        test_qwen2_5_omni_video_use_audio_in_video,
+        test_qwen3_omni_image,
+        test_qwen3_omni_video,
+        test_qwen3_omni_audio,
+        test_qwen3_omni_audio_non_hop_aligned,
+        test_qwen3_omni_video_use_audio_in_video,
+        test_gemma4_image,
+        test_gemma4_audio,
+        test_gemma4_audio_collator_3d,
+        test_gemma4_video,
+    ]
+    passed = xfailed = failed = 0
+    for fn in tests:
+        name = fn.__name__
+        try:
+            fn()
+            print(f'{name}: PASS')
+            passed += 1
+        except Exception:
+            if name in _XFAIL_TESTS:
+                print(f'{name}: XFAIL (expected upstream vLLM mismatch)')
+                xfailed += 1
+            else:
+                print(f'{name}: FAIL')
+                failed += 1
+                raise
+    print(f'all mm processor align tests finished: {passed} passed, {xfailed} xfailed, {failed} failed')

tests/test_align/test_padding_side.py

@@ -0,0 +1,73 @@
+import os
+from pprint import pprint
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+kwargs = {
+    'per_device_train_batch_size': 4,
+    'per_device_eval_batch_size': 4,
+    'gradient_accumulation_steps': 4,
+    'num_train_epochs': 1,
+    'save_steps': 100,
+    'max_length': 8192,
+}
+
+
+def calc_acc(infer_result):
+    n_correct = 0
+    for res in infer_result:
+        if res['response'] == res['labels']:
+            n_correct += 1
+    return f'acc: {n_correct / len(infer_result)}, n_correct: {n_correct}, len(res): {len(infer_result)}'
+
+
+def calc_diff(infer_result, infer_result2):
+    n_correct = 0
+    for x1, x2 in zip(infer_result, infer_result2):
+        if x1['response'] == x2['response']:
+            n_correct += 1
+    return f'acc: {n_correct / len(infer_result)}, n_correct: {n_correct}, len(res): {len(infer_result)}'
+
+
+def test_llm():
+    from swift import InferArguments, SftArguments, infer_main, sft_main
+    res = []
+    for padding_side in ['left', 'right']:
+        model = 'Qwen/Qwen2.5-0.5B-Instruct'
+        dataset = ['damo/zh_cls_fudan-news#2000']
+        result = sft_main(
+            SftArguments(model=model, dataset=dataset, split_dataset_ratio=0.1, padding_side=padding_side, **kwargs))
+        last_model_checkpoint = result['last_model_checkpoint']
+        infer_result = infer_main(InferArguments(adapters=[last_model_checkpoint], load_data_args=True))
+        res.append(calc_acc(infer_result))
+        infer_result2 = infer_main(
+            InferArguments(adapters=[last_model_checkpoint], load_data_args=True, max_batch_size=16))
+        res.append(calc_acc(infer_result2))
+    pprint(res)
+
+
+def test_mllm():
+    from swift import InferArguments, SftArguments, infer_main, sft_main
+    from swift.template import Template
+    res = []
+    for padding_side in ['left', 'right']:
+        model = 'Qwen/Qwen2-VL-2B-Instruct'
+        dataset = ['AI-ModelScope/LaTeX_OCR#2000']
+        result = sft_main(
+            SftArguments(model=model, dataset=dataset, split_dataset_ratio=0.01, padding_side=padding_side, **kwargs))
+        last_model_checkpoint = result['last_model_checkpoint']
+        infer_result = infer_main(InferArguments(adapters=[last_model_checkpoint], load_data_args=True))
+        res.append(infer_result)
+        infer_result2 = infer_main(
+            InferArguments(adapters=[last_model_checkpoint], load_data_args=True, max_batch_size=16))
+        res.append(infer_result2)
+    print(calc_diff(res[0], res[1]))
+    print(calc_diff(res[2], res[3]))
+    print(calc_diff(res[0], res[2]))
+    print(calc_diff(res[0], res[3]))
+    print(calc_diff(res[2], res[1]))
+
+
+if __name__ == '__main__':
+    test_llm()
+    test_mllm()

tests/test_align/test_template/test_agent.py

@@ -0,0 +1,748 @@
+import os
+
+os.environ['SWIFT_DEBUG'] = '1'
+os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+system = 'You are a helpful assistant.'
+
+tools = [{
+    'type': 'function',
+    'function': {
+        'name': 'get_current_weather',
+        'description': 'Get the current weather in a given location',
+        'parameters': {
+            'type': 'object',
+            'properties': {
+                'location': {
+                    'type': 'string',
+                    'description': 'The city and state, e.g. San Francisco, CA'
+                },
+                'unit': {
+                    'type': 'string',
+                    'enum': ['celsius', 'fahrenheit']
+                }
+            },
+            'required': ['location']
+        }
+    }
+}, {
+    'name_for_model': 'tool2',
+    'name_for_human': '工具2',
+    'description': 'Tool2的描述',
+}]
+
+glm4_tools = [{
+    'type': 'function',
+    'function': {
+        'name': 'realtime_aqi',
+        'description': '天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息',
+        'parameters': {
+            'type': 'object',
+            'properties': {
+                'city': {
+                    'description': '城市名'
+                }
+            },
+            'required': ['city']
+        }
+    }
+}]
+glm4_tool_messasges = [
+    {
+        'role': 'tool',
+        'content': '{"city": "北京", "aqi": "10", "unit": "celsius"}'
+    },
+    {
+        'role': 'tool',
+        'content': '{"city": "上海", "aqi": "72", "unit": "fahrenheit"}'
+    },
+]
+glm4_query = '北京和上海今天的天气情况'
+
+
+def _infer(engine, num_tools: int = 1, agent_tools=None, tool_messages=None, query=None):
+    if agent_tools is None:
+        agent_tools = tools
+    if tool_messages is None:
+        tool_messages = []
+        for _ in range(num_tools):
+            tool_messages.append({
+                'role': 'tool',
+                'content': '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+            })
+    stop = [engine.template.agent_template.keyword.observation]
+    query = query or "How's the weather in Beijing today?"
+    infer_request = InferRequest([{'role': 'user', 'content': query}], tools=agent_tools)
+    request_config = RequestConfig(max_tokens=512, stop=stop, temperature=0)
+    resp_list = engine.infer([infer_request], request_config=request_config)
+    response = resp_list[0].choices[0].message.content
+    toolcall = resp_list[0].choices[0].message.tool_calls[0].function
+    print(f'response: {response}')
+    print(f'toolcall: {toolcall}')
+    assert toolcall is not None
+    infer_request.messages.append({'role': 'assistant', 'content': response})
+    infer_request.messages += tool_messages
+    resp_list = engine.infer([infer_request], request_config=request_config)
+    response2 = resp_list[0].choices[0].message.content
+    print(f'response2: {response2}')
+    infer_request.messages.append({'role': 'assistant', 'content': response2})
+    return infer_request.messages
+
+
+def test_react_en():
+    agent_template = agent_template_map['react_en']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 1144
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'react_en'
+    messages = _infer(engine)
+    assert messages[-1]['content'] == (
+        'Thought: The current temperature in Beijing is 32 degrees Celsius, and the condition is sunny '
+        'with a humidity of 50%.\nFinal Answer: The current temperature in Beijing is 32 degrees Celsius,'
+        ' and the condition is sunny with a humidity of 50%.')
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+
+def test_react_zh():
+    agent_template = agent_template_map['react_zh']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 712
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'react_zh'
+    _infer(engine)
+
+
+def test_qwen_en():
+    agent_template = agent_template_map['qwen_en']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 879
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'qwen_en'
+    messages = _infer(engine)
+    assert messages[-1]['content'] == (
+        '✿RETURN✿: Today in Beijing, the temperature is 32°C with sunny conditions and the humidity '
+        'is at 50%. Enjoy the nice weather!')
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+
+def test_qwen_zh():
+    agent_template = agent_template_map['qwen_zh']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 577
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'qwen_zh'
+    _infer(engine)
+
+
+def test_qwen_en_parallel():
+    agent_template = agent_template_map['qwen_en_parallel']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 1012
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'qwen_en_parallel'
+    messages = _infer(engine, num_tools=2)
+    assert messages[-1]['content'] == (
+        '✿RETURN✿: Today in Beijing, the temperature is 32 degrees Celsius with sunny conditions '
+        'and the humidity is at 50%. Enjoy the nice weather!')
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+
+def test_qwen_zh_parallel():
+    agent_template = agent_template_map['qwen_zh_parallel']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 688
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'qwen_zh_parallel'
+    _infer(engine, num_tools=2)
+
+
+def test_hermes():
+    agent_template = agent_template_map['hermes']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 875
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'hermes'
+    messages = _infer(engine, num_tools=2)
+    template.template_backend = 'jinja'
+    messages2 = _infer(engine, num_tools=2)
+    assert messages[-1]['content'] == messages2[-1]['content'] == (
+        'Today in Beijing, the temperature is 32 degrees Celsius with sunny conditions '
+        'and the humidity is at 50%. Enjoy the nice weather!')
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'] == encoded2['input_ids']
+
+
+def test_toolbench():
+    agent_template = agent_template_map['toolbench']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 1833
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    template = engine.template
+    template._agent_template = 'toolbench'
+    _infer(engine)
+
+
+def test_chatglm4():
+    agent_template = agent_template_map['chatglm4']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 846
+    engine = TransformersEngine('ZhipuAI/glm-4-9b-chat')
+    template = engine.template
+    template._agent_template = 'chatglm4'
+    _infer(engine, agent_tools=glm4_tools, tool_messages=glm4_tool_messasges, query=glm4_query)
+
+
+def test_glm4():
+    agent_template = agent_template_map['glm4']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 769
+    engine = TransformersEngine('ZhipuAI/GLM-4-9B-0414')
+    template = engine.template
+    template._agent_template = 'glm4'
+    messages = _infer(engine, agent_tools=glm4_tools, tool_messages=glm4_tool_messasges, query=glm4_query)
+    assert messages[-1]['content'] == '根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。'
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+
+def test_llama3():
+    engine = TransformersEngine('LLM-Research/Llama-3.2-3B-Instruct')
+    template = engine.template
+    template._agent_template = 'llama3'
+    messages = _infer(engine)
+
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+
+def test_llama4():
+    engine = TransformersEngine('LLM-Research/Llama-4-Scout-17B-16E-Instruct')
+    template = engine.template
+    messages = _infer(engine)
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+
+def test_hunyuan():
+    engine = TransformersEngine('Tencent-Hunyuan/Hunyuan-1.8B-Instruct')
+    template = engine.template
+    template.template_backend = 'jinja'
+    _infer(engine, num_tools=2)
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'][:-1] == encoded2['input_ids']
+
+
+def test_glm4_5():
+    engine = TransformersEngine('ZhipuAI/GLM-4.5-Air')
+    template = engine.template
+    template.template_backend = 'jinja'
+    _infer(engine, num_tools=2)
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'][:-1] == encoded2['input_ids']
+
+
+def test_glm4_7():
+    engine = TransformersEngine('ZhipuAI/GLM-4.7-FP8', load_model=False)
+    template = engine.template
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'][:-1] == encoded2['input_ids']
+
+
+def test_qwen3_coder():
+    engine = TransformersEngine('Qwen/Qwen3-Coder-30B-A3B-Instruct')
+    template = engine.template
+    template.template_backend = 'jinja'
+    _infer(engine, num_tools=2)
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'] == encoded2['input_ids']
+
+
+def test_qwen3_5():
+    engine = TransformersEngine('Qwen/Qwen3.5-35B-A3B')
+    template = engine.template
+    template.template_backend = 'jinja'
+    _infer(engine, num_tools=2)
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    data['messages'].insert(0, {'role': 'system', 'content': 'You are a helpful assistant.'})
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'] == encoded2['input_ids']
+
+
+def test_deepseek_v3_1():
+    engine = TransformersEngine('deepseek-ai/DeepSeek-V3.1', load_model=False)
+    template = engine.template
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    # To test multiple tool calls and responses, we duplicate some messages.
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+
+    expected_input_ids = (
+        '<｜begin▁of▁sentence｜>\n\n## Tools\n'
+        'You have access to the following tools:\n\n'
+        '### convert_temperature\n'
+        'Description: Convert temperature from one unit to another\n\n'
+        "Parameters: {\"type\": \"object\", \"properties\": {\"temperature\": {\"type\": \"number\", "
+        "\"description\": \"The temperature value\"}, \"from_unit\": {\"type\": \"string\", \"description\": "
+        "\"The unit to convert from\"}, \"to_unit\": {\"type\": \"string\", \"description\": \"The unit "
+        "to convert to\"}}, \"required\": [\"temperature\", \"from_unit\", \"to_unit\"]}\n\n"
+        '### get_current_date\n'
+        'Description: Get the current date\n\n'
+        'Parameters: {}\n\n'
+        'IMPORTANT: ALWAYS adhere to this exact format for tool use:\n'
+        '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>tool_call_name<｜tool▁sep｜>tool_call_arguments<｜tool▁call▁end｜>'
+        '{additional_tool_calls}<｜tool▁calls▁end｜>\n\n'
+        'Where:\n'
+        '- `tool_call_name` must be an exact match to one of the available tools\n'
+        "- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n"
+        '- For multiple tool calls, chain them directly without separators or spaces<｜User｜>'
+        'Hi, I need to convert a temperature from Celsius to Fahrenheit. The temperature is 30 degrees Celsius.'
+        '<｜Assistant｜></think><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>convert_temperature<｜tool▁sep｜>'
+        "{\"temperature\": 30, \"from_unit\": \"Celsius\", \"to_unit\": \"Fahrenheit\"}<｜tool▁call▁end｜>"
+        '<｜tool▁call▁begin｜>convert_temperature<｜tool▁sep｜>'
+        "{\"temperature\": 30, \"from_unit\": \"Celsius\", \"to_unit\": \"Fahrenheit\"}<｜tool▁call▁end｜>"
+        '<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'
+        "<｜tool▁output▁begin｜>{\"converted_temperature\": 86}<｜tool▁output▁end｜>"
+        "<｜tool▁output▁begin｜>{\"converted_temperature\": 86}<｜tool▁output▁end｜>"
+        'The converted temperature from 30 degrees Celsius to Fahrenheit is 86 degrees Fahrenheit.<｜end▁of▁sentence｜>')
+
+    # Expected labels string
+    expected_labels = (
+        '[-100 * 239]</think><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>convert_temperature<｜tool▁sep｜>'
+        "{\"temperature\": 30, \"from_unit\": \"Celsius\", \"to_unit\": \"Fahrenheit\"}<｜tool▁call▁end｜>"
+        '<｜tool▁call▁begin｜>convert_temperature<｜tool▁sep｜>'
+        "{\"temperature\": 30, \"from_unit\": \"Celsius\", \"to_unit\": \"Fahrenheit\"}<｜tool▁call▁end｜>"
+        '<｜tool▁calls▁end｜><｜end▁of▁sentence｜>[-100 * 22]'
+        'The converted temperature from 30 degrees Celsius to Fahrenheit is 86 degrees Fahrenheit.<｜end▁of▁sentence｜>')
+
+    assert template.safe_decode(encoded['input_ids']) == expected_input_ids
+    assert template.safe_decode(encoded['labels']) == expected_labels
+    assert encoded['input_ids'][-122:] == encoded2['input_ids'][1:]
+
+
+def test_youtu():
+    agent_template = agent_template_map['youtu']()
+    new_system = agent_template._format_tools(tools, system)
+    assert len(new_system) == 883
+    engine = TransformersEngine('Tencent-YouTu-Research/Youtu-LLM-2B')
+    template = engine.template
+    template._agent_template = 'youtu'
+
+    stop = [template.agent_template.keyword.observation]
+    query = "How's the weather in Beijing today?"
+    tool_messages = [{'role': 'tool', 'content': '{"temperature": 32, "condition": "Sunny", "humidity": 50}'}]
+    infer_request = InferRequest([{'role': 'user', 'content': query}], tools=tools)
+    request_config = RequestConfig(max_tokens=2048, stop=stop, temperature=0)
+
+    # First inference: get tool call
+    resp_list = engine.infer([infer_request], request_config=request_config)
+    response = resp_list[0].choices[0].message.content
+    toolcall = resp_list[0].choices[0].message.tool_calls
+    print(f'response: {response}')
+    print(f'toolcall: {toolcall}')
+    assert toolcall is not None, 'No tool_call generated'
+    infer_request.messages.append({'role': 'assistant', 'content': response})
+    infer_request.messages += tool_messages
+
+    # Second inference: get final response
+    resp_list = engine.infer([infer_request], request_config=request_config)
+    response2 = resp_list[0].choices[0].message.content
+    print(f'response2: {response2}')
+    infer_request.messages.append({'role': 'assistant', 'content': response2})
+    messages = infer_request.messages
+
+    template.set_mode('train')
+    encoded = template.encode({'messages': messages})
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+    template.template_backend = 'swift'
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert encoded['input_ids'] == encoded2['input_ids']
+
+
+def test_deepseek_v4():
+    engine = TransformersEngine('deepseek-ai/DeepSeek-V4-Flash', load_model=False)
+    template = engine.template
+
+    tools = [{
+        'type': 'function',
+        'function': {
+            'name': 'get_weather',
+            'description': 'Get the weather for a specific location',
+            'parameters': {
+                'type': 'object',
+                'properties': {
+                    'location': {
+                        'type': 'string',
+                        'description': 'The city name'
+                    },
+                    'unit': {
+                        'type': 'string',
+                        'enum': ['celsius', 'fahrenheit'],
+                        'description': 'Temperature unit'
+                    }
+                },
+                'required': ['location']
+            }
+        }
+    }, {
+        'type': 'function',
+        'function': {
+            'name': 'search',
+            'description': 'Search the web for information',
+            'parameters': {
+                'type': 'object',
+                'properties': {
+                    'query': {
+                        'type': 'string',
+                        'description': 'Search query'
+                    },
+                    'num_results': {
+                        'type': 'integer',
+                        'description': 'Number of results to return'
+                    }
+                },
+                'required': ['query']
+            }
+        }
+    }]
+    data = {
+        'tools':
+        tools,
+        'messages': [{
+            'role': 'system',
+            'content': 'You are a helpful assistant.'
+        }, {
+            'role': 'user',
+            'content': "What's the weather in Beijing?"
+        }, {
+            'role':
+            'assistant',
+            'content':
+            '<think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>\n\n'
+        }, {
+            'role':
+            'tool_call',
+            'content':
+            '{"name": "get_weather", "arguments": "{\\"location\\": \\"Beijing\\", \\"unit\\": \\"celsius\\"}"}'
+        }, {
+            'role': 'tool_response',
+            'content': '{"temperature": 22, "condition": "sunny", "humidity": 45}'
+        }, {
+            'role':
+            'assistant',
+            'content': ('<think>Got the weather data. Let me format a nice response.</think>'
+                        'The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.')
+        }]
+    }
+
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+
+    expected_input_ids = (
+        '<｜begin▁of▁sentence｜>You are a helpful assistant.\n\n## Tools\n\n'
+        'You have access to a set of tools to help answer the user\'s question. '
+        'You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following:\n\n'
+        '<｜DSML｜tool_calls>\n'
+        '<｜DSML｜invoke name="$TOOL_NAME">\n'
+        '<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>\n'
+        '...\n'
+        '</｜DSML｜invoke>\n'
+        '<｜DSML｜invoke name="$TOOL_NAME2">\n'
+        '...\n'
+        '</｜DSML｜invoke>\n'
+        '</｜DSML｜tool_calls>\n\n'
+        'String parameters should be specified as is and set `string="true"`. '
+        'For all other types (numbers, booleans, arrays, objects), '
+        'pass the value in JSON format and set `string="false"`.\n\n'
+        'If thinking_mode is enabled (triggered by <think>), '
+        'you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.'
+        '\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n'
+        '### Available Tool Schemas\n\n'
+        '{"name": "get_weather", "description": "Get the weather for a specific location", '
+        '"parameters": {"type": "object", "properties": {"location": {"type": "string", '
+        '"description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], '
+        '"description": "Temperature unit"}}, "required": ["location"]}}\n'
+        '{"name": "search", "description": "Search the web for information", '
+        '"parameters": {"type": "object", "properties": {"query": {"type": "string", '
+        '"description": "Search query"}, "num_results": {"type": "integer", '
+        '"description": "Number of results to return"}}, "required": ["query"]}}\n\n'
+        'You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n'
+        '<｜User｜>What\'s the weather in Beijing?<｜Assistant｜>'
+        '<think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>\n\n'
+        '<｜DSML｜tool_calls>\n'
+        '<｜DSML｜invoke name="get_weather">\n'
+        '<｜DSML｜parameter name="location" string="true">Beijing</｜DSML｜parameter>\n'
+        '<｜DSML｜parameter name="unit" string="true">celsius</｜DSML｜parameter>\n'
+        '</｜DSML｜invoke>\n'
+        '</｜DSML｜tool_calls>'
+        '<｜end▁of▁sentence｜>'
+        '<｜User｜><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result>'
+        '<｜Assistant｜>'
+        '<think>Got the weather data. Let me format a nice response.</think>'
+        'The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.'
+        '<｜end▁of▁sentence｜>')
+
+    assert template.safe_decode(encoded['input_ids']) == expected_input_ids
+
+
+def test_seed_oss():
+    engine = TransformersEngine('ByteDance-Seed/Seed-OSS-36B-Instruct', load_model=False)
+
+    template = engine.template
+    dataset = load_dataset('AI-ModelScope/function-calling-chatml')[0]
+    data = dataset[6]
+    # To test multiple tool calls and responses, we duplicate some messages.
+    data['messages'].insert(1, data['messages'][1])
+    data['messages'].insert(3, data['messages'][3])
+
+    # Incomplete tool function will cause seed template to throw an error.
+    data['tools'] = [('{\n'
+                      '    "name": "convert_temperature",\n'
+                      '    "description": "Convert temperature from one unit to another",\n'
+                      '    "parameters": {\n'
+                      '        "type": "object",\n'
+                      '        "properties": {\n'
+                      '            "temperature": {\n'
+                      '                "type": "number",\n'
+                      '                "description": "The temperature value"\n'
+                      '            },\n'
+                      '            "from_unit": {\n'
+                      '                "type": "string",\n'
+                      '                "description": "The unit to convert from"\n'
+                      '            },\n'
+                      '            "to_unit": {\n'
+                      '                "type": "string",\n'
+                      '                "description": "The unit to convert to"\n'
+                      '            }\n'
+                      '        },\n'
+                      '        "required": [\n'
+                      '            "temperature",\n'
+                      '            "from_unit",\n'
+                      '            "to_unit"\n'
+                      '        ]\n'
+                      '    }\n'
+                      '}'),
+                     ('{\n'
+                      '    "name": "get_current_date",\n'
+                      '    "description": "Get the current date",\n'
+                      '    "parameters":  {\n'
+                      '        "type": "object",\n'
+                      '        "properties": {\n'
+                      '         "date": {\n'
+                      '                "type": "number",\n'
+                      '                "description": "The date value"}}}\n'
+                      '}')]
+
+    data['thinking_budget'] = 0
+
+    template.template_backend = 'swift'
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded["labels"])}')
+    import re
+    expected_input_ids = re.sub(
+        r'<seed:think>.*?</seed:think>', '', template.safe_decode(encoded['input_ids']), flags=re.DOTALL)
+    template.template_backend = 'jinja'
+    encoded2 = template.encode(data)
+    print(f'input_ids: {template.safe_decode(encoded2["input_ids"])}')
+    print(f'labels: {template.safe_decode(encoded2["labels"])}')
+    assert template.safe_decode(encoded2['input_ids']) == expected_input_ids
+
+
+if __name__ == '__main__':
+    from swift import InferRequest, RequestConfig, TransformersEngine, agent_template_map, load_dataset
+
+    # test_react_en()
+    # test_react_zh()
+    # test_qwen_en()
+    # test_qwen_zh()
+    # test_qwen_en_parallel()
+    # test_qwen_zh_parallel()
+    # test_hermes()
+    # test_toolbench()
+    # test_chatglm4()
+    # test_glm4()
+    # test_llama3()
+    # test_llama4()
+    # test_hunyuan()
+    # test_glm4_5()
+    # test_glm4_7()
+    # test_qwen3_coder()
+    # test_qwen3_5()
+    # test_deepseek_v3_1()
+    test_deepseek_v4()
+    # test_seed_oss()
+    # test_youtu()

tests/test_align/test_template/test_audio.py

@@ -0,0 +1,116 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+
+
+def _infer_model(engine, system=None, messages=None, audios=None):
+    seed_everything(42)
+    request_config = RequestConfig(max_tokens=128, temperature=0)
+    if messages is None:
+        messages = []
+        if system is not None:
+            messages += [{'role': 'system', 'content': system}]
+        messages += [{'role': 'user', 'content': '你好'}]
+        resp = engine.infer([{'messages': messages}], request_config=request_config)
+        response = resp[0].choices[0].message.content
+        messages += [{'role': 'assistant', 'content': response}]
+        messages += [{'role': 'user', 'content': '<audio>这段语音说了什么'}]
+    else:
+        messages = messages.copy()
+    if audios is None:
+        audios = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    resp = engine.infer([{'messages': messages, 'audios': audios}], request_config=request_config)
+    response = resp[0].choices[0].message.content
+    messages += [{'role': 'assistant', 'content': response}]
+    logger.info(f'model: {engine.model_info.model_name}, messages: {messages}')
+    return response
+
+
+def test_qwen_audio():
+    engine = TransformersEngine('Qwen/Qwen-Audio-Chat')
+    _infer_model(engine)
+
+
+def test_qwen2_audio():
+    # transformers==4.48.3
+    engine = TransformersEngine('Qwen/Qwen2-Audio-7B-Instruct')
+    messages = [{'role': 'user', 'content': '<audio>'}]
+    audios = ['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/guess_age_gender.wav']
+    response = _infer_model(engine, messages=messages, audios=audios)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, audios=audios)
+    assert response == response2 == 'Yes, the speaker is female and in her twenties.'
+
+
+def test_xcomposer2d5_ol():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:audio')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_step_audio_chat():
+    engine = TransformersEngine('stepfun-ai/Step-Audio-Chat')
+    response = _infer_model(engine, messages=[{'role': 'user', 'content': '<audio>'}])
+    assert response == ('是的呢，今天天气晴朗，阳光明媚，微风和煦，非常适合外出活动。天空湛蓝，白云朵朵，让人心情愉悦。希望你能好好享受这美好的一天！')
+
+
+def test_qwen2_5_omni():
+    USE_AUDIO_IN_VIDEO = True
+    os.environ['USE_AUDIO_IN_VIDEO'] = str(USE_AUDIO_IN_VIDEO)
+    engine = TransformersEngine('Qwen/Qwen2.5-Omni-7B')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_gemma3n():
+    engine = TransformersEngine('google/gemma-3n-E4B-it')
+    messages = [{'role': 'user', 'content': '<audio>Transcribe this audio and complete the statement'}]
+    audios = ['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/guess_age_gender.wav']
+    response = _infer_model(engine, messages=messages, audios=audios)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, audios=audios)
+    assert response == response2
+
+
+def test_midashenglm():
+    engine = TransformersEngine('mispeech/midashenglm-7b')
+    messages = [{'role': 'user', 'content': '<audio>Caption the audio.'}]
+    response = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages)
+    assert response == response2 == "The audio contains a male voice speaking the phrase '今天天气真好呀' in Mandarin."
+
+
+def test_step_audio2_mini():
+    engine = TransformersEngine('stepfun-ai/Step-Audio-2-mini')
+    messages = [{'role': 'user', 'content': '<audio>Caption the audio'}]
+    response = _infer_model(engine, messages=messages)
+    assert response == 'A woman says "今天天气真好呀" in Mandarin.'
+
+
+def test_qwen3_asr():
+    messages = [{'role': 'user', 'content': '<audio>'}]
+    audios = ['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav']
+    engine = TransformersEngine('Qwen/Qwen3-ASR-1.7B')
+    engine.template._response_prefix = 'language Chinese<asr_text>'
+    response = _infer_model(engine, messages=messages, audios=audios)
+    assert response == 'language Chinese<asr_text>甚至出现交易几乎停滞的情况。'
+
+
+if __name__ == '__main__':
+    from swift.infer_engine import RequestConfig, TransformersEngine
+    from swift.utils import get_logger, seed_everything
+    logger = get_logger()
+    # test_qwen_audio()
+    # test_qwen2_audio()
+    # test_xcomposer2d5_ol()
+    # test_step_audio_chat()
+    # test_qwen2_5_omni()
+    # test_gemma3n()
+    # test_midashenglm()
+    # test_step_audio2_mini()
+    test_qwen3_asr()

tests/test_align/test_template/test_gene.py

@@ -0,0 +1,28 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+os.environ['SWIFT_DEBUG'] = '1'
+
+
+def test_deepseek_janus_pro_gene():
+    from swift import InferArguments, infer_main
+    args = InferArguments(model='deepseek-ai/Janus-Pro-1B', infer_backend='transformers')
+    infer_main(args)
+
+
+def test_emu3_gen(infer_backend):
+    from swift import InferArguments, infer_main
+    args = InferArguments(
+        model='BAAI/Emu3-Gen',
+        infer_backend=infer_backend,
+        stream=False,
+        use_chat_template=False,
+        top_k=2048,
+        max_new_tokens=40960)
+    infer_main(args)
+
+
+if __name__ == '__main__':
+    # test_emu3_gen('transformers')
+    test_deepseek_janus_pro_gene()

tests/test_align/test_template/test_llm.py

@@ -0,0 +1,795 @@
+import json
+import os
+import torch
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['SWIFT_DEBUG'] = '1'
+
+
+def _infer_model(engine, system=None, messages=None):
+    seed_everything(42)
+    request_config = RequestConfig(max_tokens=128, temperature=0)
+    if messages is None:
+        messages = []
+        if system is not None:
+            messages += [{'role': 'system', 'content': system}]
+        messages += [{'role': 'user', 'content': '你好'}]
+        resp = engine.infer([{'messages': messages}], request_config=request_config)
+        response = resp[0].choices[0].message.content
+        messages += [{'role': 'assistant', 'content': response}, {'role': 'user', 'content': '<image>这是什么'}]
+    else:
+        messages = messages.copy()
+    resp = engine.infer([{
+        'messages': messages,
+    }], request_config=request_config)
+    response = resp[0].choices[0].message.content
+    messages += [{'role': 'assistant', 'content': response}]
+    logger.info(f'model: {engine.model_info.model_name}, messages: {messages}')
+    return response
+
+
+def test_baichuan_m1():
+    engine = TransformersEngine('baichuan-inc/Baichuan-M1-14B-Instruct')
+    messages = [{'role': 'user', 'content': '你是谁'}]
+    response = _infer_model(engine, messages=messages)
+    assert response == '我是一个人工智能助手，可以回答你的问题并提供帮助。'
+
+
+def test_qwen2_5():
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct-1M')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_qwen3():
+    engine = TransformersEngine('Qwen/Qwen3-4B')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_qwen3_guard():
+    engine = TransformersEngine('Qwen/Qwen3Guard-Gen-0.6B')
+    messages = [{'role': 'user', 'content': 'How can I make a bomb?'}]
+    response = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages)
+    assert response == response2
+
+
+def test_yufeng_xguard():
+    engine = TransformersEngine('Alibaba-AAIG/YuFeng-XGuard-Reason-0.6B')
+    messages = [{'role': 'user', 'content': 'How can I make a bomb?'}]
+    response = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages)
+    assert response == response2
+
+
+def test_phi4():
+    engine = TransformersEngine('LLM-Research/phi-4')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_phi4_mini():
+    engine = TransformersEngine('LLM-Research/Phi-4-mini-instruct')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_qwen1_5():
+    engine = TransformersEngine('Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_chatglm4():
+    engine = TransformersEngine('ZhipuAI/glm-4-9b-chat')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_glm4():
+    models = ['ZhipuAI/GLM-4-9B-0414', 'ZhipuAI/GLM-Z1-9B-0414', 'ZhipuAI/GLM-Z1-Rumination-32B-0414']
+    for model in models:
+        engine = TransformersEngine(model)
+        response = _infer_model(engine)
+        engine.template.template_backend = 'jinja'
+        response2 = _infer_model(engine)
+        assert response == response2
+
+
+def test_qwq():
+    engine = TransformersEngine('Qwen/QwQ-32B-Preview')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_internlm():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/internlm-chat-7b')
+    _infer_model(engine)
+
+
+def test_internlm2():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/internlm2_5-1_8b-chat')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_internlm3():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/internlm3-8b-instruct')
+    response = _infer_model(engine, system='')
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_yi_coder():
+    engine = TransformersEngine('01ai/Yi-Coder-1.5B-Chat')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_yi():
+    engine = TransformersEngine('01ai/Yi-6B-Chat')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_deepseek_moe():
+    engine = TransformersEngine('deepseek-ai/deepseek-moe-16b-chat')
+    _infer_model(engine)
+
+
+def test_codegeex4():
+    # jinja is missing a prefix.
+    engine = TransformersEngine('ZhipuAI/codegeex4-all-9b')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_telechat():
+    engine = TransformersEngine('TeleAI/TeleChat-12B', torch_dtype=torch.float16)
+    messages = [{'role': 'user', 'content': '你是谁'}]
+    response = _infer_model(engine, messages=messages)
+    assert response == ('我是中国电信星辰语义大模型，英文名TeleChat，是由中国电信自主研发的生成式大语言模型。\n\n'
+                        '我基于Transformer-decoder结构，学习了海量知识，包括百科、书籍、论坛、党政媒体、GitHub代码、专业领域知识等，'
+                        '具备自然语言处理、语义理解、内容创作和逻辑推理等能力，可以与人类进行对话互动和情感交流，还能提供知识问答、创作写作、'
+                        '代码生成等服务，希望能为人类带来更加智能、高效和便捷的工作与生活体验。')
+
+
+def test_telechat2():
+    engine = TransformersEngine('TeleAI/TeleChat2-7B-32K', torch_dtype=torch.float16)
+    messages = [{'role': 'system', 'content': '你是一个乐于助人的智能助手，请使用用户提问的语言进行有帮助的问答'}, {'role': 'user', 'content': '你好'}]
+    response = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages)
+    assert response == response2
+
+
+def test_glm_edge():
+    engine = TransformersEngine('ZhipuAI/glm-edge-1.5b-chat')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_llama():
+    from swift.infer_engine import VllmEngine
+
+    # engine = TransformersEngine('LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4')
+    # engine = TransformersEngine('LLM-Research/Meta-Llama-3.1-8B-Instruct')
+    # engine = TransformersEngine('LLM-Research/Meta-Llama-3-8B-Instruct')
+    engine = VllmEngine('LLM-Research/Llama-3.2-1B-Instruct')
+    # engine = TransformersEngine('AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF')
+    # engine = TransformersEngine('unsloth/Llama-3.3-70B-Instruct-bnb-4bit')
+
+    res = _infer_model(engine, system='')
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, system='')
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_openbuddy():
+    # engine = TransformersEngine('OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k')
+    engine = TransformersEngine('OpenBuddy/openbuddy-nemotron-70b-v23.2-131k')
+    # engine = TransformersEngine('OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k')
+    res = _infer_model(engine, system='')
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_megrez():
+    engine = TransformersEngine('InfiniAI/Megrez-3b-Instruct')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_skywork_o1():
+    engine = TransformersEngine('AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B')
+    res = _infer_model(
+        engine,
+        messages=[{
+            'role':
+            'user',
+            'content':
+            ('Jane has 12 apples. She gives 4 apples to her friend Mark, then buys 1 more apple, and finally splits '
+             'all her apples equally among herself and her 2 siblings. How many apples does each person get?')
+        }])
+    assert res == ("To solve the problem, let's break it down into a series of logical steps:\n\n1. **Initial Number "
+                   'of Apples**: Jane starts with 12 apples.\n2. **Apples Given Away**: Jane gives 4 apples to her '
+                   'friend Mark. So, the number of apples she has now is:\n   \\[\n   12 - 4 = 8\n   \\]\n3. **Apples '
+                   'Bought**: Jane then buys 1 more apple. So, the number of apples she has now is:\n   \\[\n   '
+                   '8 + 1 = 9\n   \\]\n4. **Apples Split Equally')
+
+
+def test_internlm2_reward():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/internlm2-1_8b-reward')
+    messages = [{
+        'role': 'user',
+        'content': "Hello! What's your name?"
+    }, {
+        'role': 'assistant',
+        'content': 'My name is InternLM2! A helpful AI assistant. What can I do for you?'
+    }]
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == res2 == '0.48681640625'
+
+
+def test_qwen2_reward():
+    engine = TransformersEngine('Qwen/Qwen2-Math-RM-72B')
+    messages = [{
+        'role':
+        'user',
+        'content': ('Suppose that a certain software product has a mean time between failures of 10,000 hours '
+                    'and has a mean time to repair of 20 hours. If the product is used by 100 customers, '
+                    'what is its availability?\nAnswer Choices: (A) 80% (B) 90% (C) 98% (D) 99.80%\nPlease '
+                    'reason step by step, and put your final answer within \\boxed{}.')
+    }, {
+        'role':
+        'assistant',
+        'content': ("To find the availability of the software product, we'll use the formula:\n\n\\[ \\text{ "
+                    'availability} = \\frac{\\text{Mean Time Between Failures (MTBF)}}{\\text{Mean Time Between '
+                    'Failures (MTBF) + Mean Time To Repair (MTTR)}} \\]\n\nGiven:\n- MTBF = 10,000 hours\n- MTTR '
+                    "= 20 hours\n\nLet's plug these values into the formula:\n\n\\[ \\text{availability} = "
+                    '\\frac{10,000}{10,000 + 20} = \\frac{10,000}{10,020} \\]\n\nTo simplify this fraction, '
+                    'we can divide both the numerator and the denominator by 10,000:\n\n\\[ \\text{availability} ='
+                    ' \\frac{10,000 \\div 10,000}{10,020 \\div 10,000} = \\frac{1}{1.002} \\]\n\nTo express this as'
+                    ' a percentage, we can calculate the decimal value of the fraction and then multiply by '
+                    '100:\n\n\\[ \\text{availability} \\approx 0.998002 \\times 100 = 99.80\\% \\]\n\nTherefore, '
+                    'the availability of the software product is approximately 99.80%.\n\nThe correct answer is '
+                    '\\boxed{D}')
+    }]
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == '1.84375' and res2 == '1.390625'  # \n diff
+
+
+def test_qwen2_5_math():
+    engine = TransformersEngine('Qwen/Qwen2.5-Math-1.5B-Instruct')
+    messages = [{'role': 'user', 'content': 'Find the value of $x$ that satisfies the equation $4x+5 = 6x+7$.'}]
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == res2
+
+
+def test_skywork_reward():
+    prompt = ('Jane has 12 apples. She gives 4 apples to her friend Mark, then buys 1 more apple, and finally splits '
+              'all her apples equally among herself and her 2 siblings. How many apples does each person get?')
+    response = ('1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys '
+                '1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among herself '
+                'and her 2 siblings (3 people in total). 9 ÷ 3 = 3 apples each. Each person gets 3 apples.')
+
+    engine = TransformersEngine('AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2')
+    messages = [{'role': 'user', 'content': prompt}, {'role': 'assistant', 'content': response}]
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == '14.25'
+    assert res2 == '13.8125'
+
+
+def test_deepseek_r1_distill():
+    engine = TransformersEngine('deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_deepseek_prover_v2():
+    engine = TransformersEngine('deepseek-ai/DeepSeek-Prover-V2-7B')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_qwen2_5_prm():
+    engine = TransformersEngine('Qwen/Qwen2.5-Math-7B-PRM800K')
+    data = {
+        'system':
+        'Please reason step by step, and put your final answer within \\boxed{}.',
+        'query': ('Sue lives in a fun neighborhood.  One weekend, the neighbors decided to play a prank on Sue.  '
+                  "On Friday morning, the neighbors placed 18 pink plastic flamingos out on Sue's front yard.  "
+                  'On Saturday morning, the neighbors took back one third of the flamingos, painted them white, and '
+                  "put these newly painted white flamingos back out on Sue's front yard.  Then, on Sunday morning, "
+                  'they added another 18 pink plastic flamingos to the collection. At noon on Sunday, how many more '
+                  'pink plastic flamingos were out than white plastic flamingos?'),
+        'response':
+        [('To find out how many more pink plastic flamingos were out than white plastic flamingos at noon on Sunday, '
+          'we can break down the problem into steps. First, on Friday, the neighbors start with 18 pink '
+          'plastic flamingos.'),
+         ('On Saturday, they take back one third of the flamingos. Since there were 18 flamingos, (1/3 \\times 18 = 6) '
+          'flamingos are taken back. So, they have (18 - 6 = 12) flamingos left in their possession. Then, they paint '
+          "these 6 flamingos white and put them back out on Sue's front yard. Now, Sue has the original 12 pink "
+          'flamingos plus the 6 new white ones. Thus, by the end of Saturday, Sue has (12 + 6 = 18) pink flamingos '
+          'and 6 white flamingos.'),
+         ("On Sunday, the neighbors add another 18 pink plastic flamingos to Sue's front yard. By the end of Sunday "
+          'morning, Sue has (18 + 18 = 36) pink flamingos and still 6 white flamingos.'),
+         ('To find the difference, subtract the number of white flamingos from the number of pink '
+          'flamingos: (36 - 6 = 30). Therefore, at noon on Sunday, there were 30 more pink plastic flamingos out '
+          'than white plastic flamingos. The answer is (\\boxed{30}).')]
+    }
+
+    messages = [
+        {
+            'role': 'system',
+            'content': data['system']
+        },
+        {
+            'role': 'user',
+            'content': data['query']
+        },
+        {
+            'role': 'assistant',
+            'content': '<extra_0>'.join(data['response']) + '<extra_0>'
+        },
+    ]
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == res2 == json.dumps([0.9921875, 0.2490234375, 0.70703125, 0.9375]), f'res: {res}, res2: {res2}'
+
+
+def test_mistral_small():
+    engine = TransformersEngine('mistralai/Mistral-Small-24B-Instruct-2501')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_moonlight():
+    engine = TransformersEngine('moonshotai/Moonlight-16B-A3B-Instruct')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_ling():
+    engine = TransformersEngine('inclusionAI/Ling-lite')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_gemma3():
+    engine = TransformersEngine('LLM-Research/gemma-3-1b-it')
+    res = _infer_model(engine, system='You are a helpful assistant')
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, system='You are a helpful assistant')
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_mimo():
+    engine = TransformersEngine('XiaomiMiMo/MiMo-7B-RL-0530')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_minicpm():
+    engine = TransformersEngine('OpenBMB/MiniCPM4-0.5B')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_minimax():
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3,4,5,6,7'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3,4,5,6,7'
+    from transformers import QuantoConfig
+    quantization_config = QuantoConfig(weights='int8')
+    messages = [{
+        'role': 'system',
+        'content': 'You are a helpful assistant.'
+    }, {
+        'role': 'user',
+        'content': 'who are you?'
+    }]
+    engine = TransformersEngine('MiniMax/MiniMax-M1-40k', quantization_config=quantization_config)
+    res = _infer_model(engine, messages=messages)
+    print(f'res: {res}')
+
+
+def test_kimi_dev():
+    engine = TransformersEngine('moonshotai/Kimi-Dev-72B')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_hunyuan():
+    # engine = TransformersEngine('Tencent-Hunyuan/Hunyuan-A13B-Instruct')
+    engine = TransformersEngine('Tencent-Hunyuan/Hunyuan-4B-Instruct')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_ernie():
+    engine = TransformersEngine('PaddlePaddle/ERNIE-4.5-0.3B-PT')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_devstral():
+    from swift.template.templates.mistral import devstral_small_2505_system
+
+    engine = TransformersEngine('mistralai/Devstral-Small-2505')
+    res = _infer_model(engine, system=devstral_small_2505_system)
+
+    engine.template.template_backend = 'jinja'
+    # taken from: https://github.com/vllm-project/vllm/blob/main/examples/tool_chat_template_mistral3.jinja
+    chat_template = (
+        '{%- set today = strftime_now("%Y-%m-%d") %}\n'
+        '{%- set default_system_message = "You are Mistral Small 3, a Large Language Model (LLM) '
+        'created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was '
+        'last updated on 2023-10-01. The current date is " + today + ".\\n\\nWhen you\'re not sure '
+        'about some information, you say that you don\'t have the information and don\'t make up '
+        'anything.\\nIf the user\'s question is not clear, ambiguous, or does not provide enough '
+        'context for you to accurately answer the question, you do not try to answer it right away '
+        'and you rather ask the user to clarify their request (e.g. \\"What are some good restaurants '
+        'around me?\\" => \\"Where are you?\\" or \\"When is the next flight to Tokyo\\" => '
+        '\\"Where do you travel from?\\")" %}\n\n'
+        '{{- bos_token }}\n\n'
+        '{%- if messages[0][\'role\'] == \'system\' %}\n'
+        '    {%- if messages[0][\'content\'] is string %}\n'
+        '        {%- set system_message = messages[0][\'content\'] %}\n'
+        '        {%- set loop_messages = messages[1:] %}\n'
+        '    {%- else %}\n'
+        '        {%- set system_message = messages[0][\'content\'][0][\'text\'] %}\n'
+        '        {%- set loop_messages = messages[1:] %}\n'
+        '    {%- endif %}\n'
+        '{%- else %}\n'
+        '    {%- set system_message = default_system_message %}\n'
+        '    {%- set loop_messages = messages %}\n'
+        '{%- endif %}\n'
+        '{%- if not tools is defined %}\n'
+        '    {%- set tools = none %}\n'
+        '{%- elif tools is not none %}\n'
+        '    {%- set parallel_tool_prompt = "You are a helpful assistant that can call tools. '
+        'If you call one or more tools, format them in a single JSON array or objects, where each '
+        'object is a tool call, not as separate objects outside of an array or multiple arrays. '
+        'Use the format [{\\"name\\": tool call name, \\"arguments\\": tool call arguments}, '
+        'additional tool calls] if you call more than one tool. If you call tools, do not attempt '
+        'to interpret them or otherwise provide a response until you receive a tool call result '
+        'that you can interpret for the user." %}\n'
+        '    {%- if system_message is defined %}\n'
+        '        {%- set system_message = parallel_tool_prompt + "\\n\\n" + system_message %}\n'
+        '    {%- else %}\n'
+        '        {%- set system_message = parallel_tool_prompt %}\n'
+        '    {%- endif %}\n'
+        '{%- endif %}\n'
+        '{{- \'[SYSTEM_PROMPT]\' + system_message + \'[/SYSTEM_PROMPT]\' }}\n\n'
+        '{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}\n\n'
+        '{%- set filtered_messages = [] %}\n'
+        '{%- for message in loop_messages %}\n'
+        '    {%- if message["role"] not in ["tool", "tool_results"] and not message.get("tool_calls") %}\n'
+        '        {%- set filtered_messages = filtered_messages + [message] %}\n'
+        '    {%- endif %}\n'
+        '{%- endfor %}\n\n'
+        '{%- for message in filtered_messages %}\n'
+        '    {%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}\n'
+        '        {{- raise_exception("After the optional system message, conversation roles must '
+        'alternate user/assistant/user/assistant/...") }}\n'
+        '    {%- endif %}\n'
+        '{%- endfor %}\n\n'
+        '{%- for message in loop_messages %}\n'
+        '    {%- if message["role"] == "user" %}\n'
+        '        {%- if tools is not none and (message == user_messages[-1]) %}\n'
+        '            {{- "[AVAILABLE_TOOLS] [" }}\n'
+        '            {%- for tool in tools %}\n'
+        '                {%- set tool = tool.function %}\n'
+        '                {{- \'{"type": "function", "function": {\' }}\n'
+        '                {%- for key, val in tool.items() if key != "return" %}\n'
+        '                    {%- if val is string %}\n'
+        '                        {{- \'"\' + key + \'": "\' + val + \'"\' }}\n'
+        '                    {%- else %}\n'
+        '                        {{- \'"\' + key + \'": \' + val|tojson }}\n'
+        '                    {%- endif %}\n'
+        '                    {%- if not loop.last %}\n'
+        '                        {{- ", " }}\n'
+        '                    {%- endif %}\n'
+        '                {%- endfor %}\n'
+        '                {{- "}}" }}\n'
+        '                {%- if not loop.last %}\n'
+        '                    {{- ", " }}\n'
+        '                {%- else %}\n'
+        '                    {{- "]" }}\n'
+        '                {%- endif %}\n'
+        '            {%- endfor %}\n'
+        '            {{- "[/AVAILABLE_TOOLS]" }}\n'
+        '        {%- endif %}\n'
+        '        {%- if message[\'content\'] is string %}\n'
+        '        {{- \'[INST]\' + message[\'content\'] + \'[/INST]\' }}\n'
+        '        {%- else %}\n'
+        '                {{- \'[INST]\' }}\n'
+        '                {%- for block in message[\'content\'] %}\n'
+        '                        {%- if block[\'type\'] == \'text\' %}\n'
+        '                                {{- block[\'text\'] }}\n'
+        '                        {%- elif block[\'type\'] == \'image\' or block[\'type\'] == \'image_url\' %}\n'
+        '                                {{- \'[IMG]\' }}\n'
+        '                            {%- else %}\n'
+        '                                {{- raise_exception(\'Only text and image blocks are supported '
+        'in message content!\') }}\n'
+        '                            {%- endif %}\n'
+        '                    {%- endfor %}\n'
+        '                {{- \'[/INST]\' }}\n'
+        '            {%- endif %}\n'
+        '    {%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}\n'
+        '        {%- if message.tool_calls is defined %}\n'
+        '            {%- set tool_calls = message.tool_calls %}\n'
+        '        {%- else %}\n'
+        '            {%- set tool_calls = message.content %}\n'
+        '        {%- endif %}\n'
+        '        {{- "[TOOL_CALLS] [" }}\n'
+        '        {%- for tool_call in tool_calls %}\n'
+        '            {%- set out = tool_call.function|tojson %}\n'
+        '            {{- out[:-1] }}\n'
+        '            {%- if not tool_call.id is defined or tool_call.id|length < 9 %}\n'
+        '                {{- raise_exception("Tool call IDs should be alphanumeric strings with '
+        'length >= 9! (1)" + tool_call.id) }}\n'
+        '            {%- endif %}\n'
+        '            {{- \', "id": "\' + tool_call.id[-9:] + \'"}\' }}\n'
+        '            {%- if not loop.last %}\n'
+        '                {{- ", " }}\n'
+        '            {%- else %}\n'
+        '                {{- "]" + eos_token }}\n'
+        '            {%- endif %}\n'
+        '        {%- endfor %}\n'
+        '    {%- elif message[\'role\'] == \'assistant\' %}\n'
+        '        {%- if message[\'content\'] is string %}\n'
+        '            {{- message[\'content\'] + eos_token }}\n'
+        '        {%- else %}\n'
+        '            {{- message[\'content\'][0][\'text\'] + eos_token }}\n'
+        '        {%- endif %}\n'
+        '    {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}\n'
+        '        {%- if message.content is defined and message.content.content is defined %}\n'
+        '            {%- set content = message.content.content %}\n'
+        '        {%- else %}\n'
+        '            {%- set content = message.content %}\n'
+        '        {%- endif %}\n'
+        '        {{- \'[TOOL_RESULTS] {"content": \' + content|string + ", " }}\n'
+        '        {%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}\n'
+        '            {{- raise_exception("Tool call IDs should be alphanumeric strings with '
+        'length >= 9! (2)" + message.tool_call_id) }}\n'
+        '        {%- endif %}\n'
+        '        {{- \'"call_id": "\' + message.tool_call_id[-9:] + \'"}[/TOOL_RESULTS]\' }}\n'
+        '    {%- else %}\n'
+        '        {{- raise_exception("Only user and assistant roles are supported, with the '
+        'exception of an initial optional system message!") }}\n'
+        '    {%- endif %}\n'
+        '{%- endfor %}')
+    # manually set chat_template, as we're using mistral-3.1-24b-instruct-2503 tokenizer which
+    # doesn't have the chat_template.json file
+    engine.processor.chat_template = chat_template
+    res2 = _infer_model(engine, system=devstral_small_2505_system)
+
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_glm4_5():
+    messages = [{'role': 'user', 'content': '浙江的省会在哪?'}]
+    engine = TransformersEngine('ZhipuAI/GLM-4.5-Air')
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_gpt_oss():
+    messages = [{
+        'role':
+        'system',
+        'content':
+        '<|start|>system<|message|>You are Qwen.\nKnowledge cutoff: 2024-06\n'
+        'Current date: 2025-08-08\n\nReasoning: medium\n\n'
+        '# Valid channels: analysis, commentary, final. '
+        'Channel must be included for every message.<|end|>'
+        '<|start|>developer<|message|># Instructions\n\nYou are ChatGPT<|end|>'
+    }, {
+        'role': 'user',
+        'content': 'who are you?'
+    }]
+    engine = TransformersEngine('openai-mirror/gpt-oss-20b')
+    res = _infer_model(engine, messages=messages)
+    assert 'm Qwen' in res.rsplit('<|message|>', 1)[-1]
+
+
+def test_qwen3_next():
+    engine = TransformersEngine('Qwen/Qwen3-Next-80B-A3B-Instruct')
+    res = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_ernie_thinking():
+    engine = TransformersEngine('PaddlePaddle/ERNIE-4.5-21B-A3B-Thinking')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_ring2():
+    engine = TransformersEngine('inclusionAI/Ring-mini-2.0')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_ling2():
+    engine = TransformersEngine('inclusionAI/Ling-mini-2.0')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_minimind():
+    engine = TransformersEngine('gongjy/MiniMind2', model_type='minimind')
+    swift_response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    jinja_response = _infer_model(engine)
+    assert swift_response == jinja_response
+
+
+def test_medgemma3():
+    engine = TransformersEngine('google/medgemma-27b-text-it')
+    system = 'You are a helpful medical assistant.'
+    messages = [{'role': 'user', 'content': 'How do you differentiate bacterial from viral pneumonia?'}]
+    res = _infer_model(engine, system=system, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, system=system, messages=messages)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_youtu_llm():
+    engine = TransformersEngine('Tencent-YouTu-Research/Youtu-LLM-2B')
+    messages = [{'role': 'user', 'content': '你好'}]
+    res = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    res2 = _infer_model(engine, messages=messages)
+    assert res == res2, f'res: {res}, res2: {res2}'
+
+
+def test_glm4_moe_lite():
+    engine = TransformersEngine('ZhipuAI/GLM-4.7-Flash')
+    swift_response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    jinja_response = _infer_model(engine)
+    assert swift_response == jinja_response
+
+
+def test_olmoe():
+    engine = TransformersEngine('allenai/OLMoE-1B-7B-0924-Instruct')
+    # engine = TransformersEngine('allenai/OLMoE-1B-7B-0125-Instruct')
+    swift_response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    jinja_response = _infer_model(engine)
+    assert swift_response == jinja_response
+
+
+def test_minicpm5():
+    engine = TransformersEngine('OpenBMB/MiniCPM5-1B')
+    swift_response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    jinja_response = _infer_model(engine)
+    assert swift_response == jinja_response
+
+
+if __name__ == '__main__':
+    from swift.infer_engine import RequestConfig, TransformersEngine
+    from swift.utils import get_logger, seed_everything
+    logger = get_logger()
+    # test_qwen2_5()
+    # test_qwen1_5()
+    # test_qwq()
+    # test_internlm()
+    # test_internlm2()
+    # test_yi_coder()
+    # test_yi()
+    # test_deepseek_moe()
+    # test_codegeex4()
+    # test_chatglm4()
+    # test_telechat()
+    # test_telechat2()
+    # test_glm_edge()
+    # test_llama()
+    # test_openbuddy()
+    # test_megrez()
+    # test_skywork_o1()
+    # test_internlm2_reward()
+    # test_qwen2_reward()
+    # test_qwen2_5_math()
+    # test_skywork_reward()
+    # test_phi4()
+    # test_phi4_mini()
+    # test_internlm3()
+    # test_deepseek_r1_distill()
+    # test_deepseek_prover_v2()
+    # test_qwen2_5_prm()
+    # test_mistral_small()
+    # test_baichuan_m1()
+    # test_moonlight()
+    # test_ling()
+    # test_gemma3()
+    # test_glm4()
+    # test_qwen3()
+    # test_qwen3_guard()
+    # test_mimo()
+    # test_minicpm()
+    # test_minimax()
+    # test_kimi_dev()
+    # test_hunyuan()
+    # test_ernie()
+    # test_glm4_5()
+    # test_devstral()
+    # test_gpt_oss()
+    # test_qwen3_next()
+    # test_ernie_thinking()
+    # test_ring2()
+    # test_ling2()
+    # test_minimind()
+    # test_medgemma3()
+    # test_youtu_llm()
+    # test_glm4_moe_lite()
+    # test_olmoe()
+    test_minicpm5()

tests/test_align/test_template/test_template.py

@@ -0,0 +1,168 @@
+from swift.model import get_processor
+from swift.template import TemplateInputs, get_template
+
+
+def test_deepseek_v2_5():
+    tokenizer = get_processor('deepseek-ai/DeepSeek-V2.5-1210')
+    template = get_template(tokenizer)
+    inputs = TemplateInputs({
+        'messages': [{
+            'role': 'system',
+            'content': '000'
+        }, {
+            'role': 'user',
+            'content': 'aaa'
+        }, {
+            'role': 'assistant',
+            'content': 'bbb'
+        }, {
+            'role': 'user',
+            'content': 'ccc'
+        }]
+    })
+    res = template.encode(inputs)
+    template.print_inputs(res)
+    template.template_backend = 'jinja'
+    res2 = template.encode(inputs)
+    template.print_inputs(res2)
+    assert res['input_ids'] == res2['input_ids']
+
+
+def test_qwen2_5_math_reward():
+    tokenizer = get_processor('Qwen/Qwen2.5-Math-RM-72B')
+    template = get_template(tokenizer)
+    inputs = TemplateInputs({
+        'messages': [{
+            'role':
+            'user',
+            'content':
+            'Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins '
+            "for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per "
+            "fresh duck egg. How much in dollars does she make every day at the farmers' market?"
+        }, {
+            'role':
+            'assistant',
+            'content':
+            "To determine how much Janet makes from selling the duck eggs at the farmers' market, we need to "
+            'follow these steps:\n\n1. Calculate the total number of eggs laid by the ducks each day.\n2. '
+            'Determine how many eggs Janet eats and bakes for herself each day.\n3. Find out how many eggs are '
+            "left to be sold.\n4. Calculate the revenue from selling the remaining eggs at $2 per egg.\n\nLet's "
+            "start with the first step:\n\n1. Janet's ducks lay 16 eggs per day.\n\nNext, we calculate how many "
+            'eggs Janet eats and bakes for herself each day:\n\n2. Janet eats 3 eggs for breakfast every morning.'
+            '\n3. Janet bakes 4 eggs for her friends every day.\n\nSo, the total number of eggs Janet eats and '
+            'bakes for herself each day is:\n\\[ 3 + 4 = 7 \\text{ eggs} \\]\n\nNow, we find out how many eggs '
+            'are left to be sold:\n\\[ 16 - 7 = 9 \\text{ eggs} \\]\n\nFinally, we calculate the revenue from '
+            'selling the remaining eggs at $2 per egg:\n\\[ 9 \\times 2 = 18 \\text{ dollars} \\]\n\nTherefore, '
+            "Janet makes \\(\\boxed{18}\\) dollars every day at the farmers' market."
+        }]
+    })
+    res = template.encode(inputs)
+    template.print_inputs(res)
+    template.template_backend = 'jinja'
+    res2 = template.encode(inputs)
+    template.print_inputs(res)
+    assert res['input_ids'] == res2['input_ids']
+    assert len(res['input_ids']) == 364
+
+
+def test_minimax():
+    tokenizer = get_processor('MiniMax/MiniMax-Text-01')
+    template = get_template(tokenizer)
+    inputs = TemplateInputs({
+        'messages': [{
+            'role': 'system',
+            'content': 'You are a helpful assistant created by MiniMax based on MiniMax-Text-01 model.'
+        }, {
+            'role': 'user',
+            'content': 'Hello!'
+        }]
+    })
+    res = template.encode(inputs)
+    template.print_inputs(res)
+    assert tokenizer.decode(res['input_ids']) == (
+        '<beginning_of_sentence>system ai_setting=assistant\nYou are a helpful assistant created by MiniMax based '
+        'on MiniMax-Text-01 model.<end_of_sentence>\n<beginning_of_sentence>user name=user\nHello!<end_of_sentence>\n'
+        '<beginning_of_sentence>ai name=assistant\n')
+
+
+def test_minimax_vl():
+    tokenizer = get_processor('MiniMax/MiniMax-VL-01')
+    template = get_template(tokenizer)
+    inputs = TemplateInputs({
+        'messages': [{
+            'role': 'system',
+            'content': 'You are a helpful assistant created by MiniMax based on MiniMax-VL-01 model.'
+        }, {
+            'role': 'user',
+            'content': '<image>Describe this image.'
+        }],
+        'images': ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png']
+    })
+    res = template.encode(inputs)
+    assert len(res['input_ids']) == 5877
+
+
+def test_deepseek_v3_1():
+    tokenizer = get_processor('deepseek-ai/DeepSeek-V3.1')
+    template = get_template(tokenizer)
+    inputs = {
+        'messages': [{
+            'role': 'system',
+            'content': '000'
+        }, {
+            'role': 'user',
+            'content': 'aaa'
+        }, {
+            'role': 'assistant',
+            'content': 'bbb'
+        }, {
+            'role': 'user',
+            'content': 'ccc'
+        }]
+    }
+    res = template.encode(inputs)
+    template.print_inputs(res)
+    template.template_backend = 'jinja'
+    res2 = template.encode(inputs)
+    template.print_inputs(res2)
+    assert res['input_ids'] == res2['input_ids']
+
+
+def test_preserve_thinking():
+    tokenizer = get_processor('Qwen/Qwen3.6-35B-A3B')
+    template = get_template(tokenizer, preserve_thinking=True)
+    template.set_mode('train')
+    inputs = {
+        'messages': [{
+            'role': 'system',
+            'content': '000'
+        }, {
+            'role': 'user',
+            'content': 'aaa'
+        }, {
+            'role': 'assistant',
+            'content': '<think>\nbbb\n</think>\n\nbbb'
+        }, {
+            'role': 'user',
+            'content': 'ccc'
+        }, {
+            'role': 'assistant',
+            'content': '<think>\nddd\n</think>\n\nddd'
+        }]
+    }
+    template.template_backend = 'swift'
+    res = template.encode(inputs)
+    template.print_inputs(res)
+    template.template_backend = 'jinja'
+    res2 = template.encode(inputs)
+    template.print_inputs(res2)
+    assert res['input_ids'] == res2['input_ids']
+
+
+if __name__ == '__main__':
+    # test_deepseek_v2_5()
+    # test_qwen2_5_math_reward()
+    # test_minimax()
+    # test_minimax_vl()
+    # test_deepseek_v3_1()
+    test_preserve_thinking()

tests/test_align/test_template/test_tool.py

@@ -0,0 +1,74 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['SWIFT_DEBUG'] = '1'
+
+tools = [{
+    'name': 'get_current_weather',
+    'description': 'Get the current weather in a given location',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'location': {
+                'type': 'string',
+                'description': 'The city and state, e.g. San Francisco, CA'
+            },
+            'unit': {
+                'type': 'string',
+                'enum': ['celsius', 'fahrenheit']
+            }
+        },
+        'required': ['location']
+    }
+}]
+
+
+def _test_tool(engine, system=None):
+    messages = [
+        {
+            'role': 'user',
+            'content': "How's the weather in Beijing today?"
+        },
+        {
+            'role':
+            'assistant',
+            'content': ('<tool_call>\n{"name": "get_current_weather", "arguments": '
+                        '{"location": "Beijing, China", "unit": "celsius"}}\n</tool_call>')
+        },
+        {
+            'role': 'tool',
+            'content': "{'temp': 25, 'description': 'Partly cloudy', 'status': 'success'}"
+        },
+    ]
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    response = engine.infer([InferRequest(messages=messages, tools=tools)], request_config=request_config)
+    return response[0].choices[0].message.content
+
+
+def test_qwen2_5():
+    engine = TransformersEngine('Qwen/Qwen2.5-7B-Instruct')
+    response = _test_tool(engine)
+    assert response == 'Today in Beijing, the temperature is 25 degrees Celsius with partly cloudy skies.'
+
+
+def test_qwq():
+    engine = TransformersEngine('Qwen/QwQ-32B')
+    response = _test_tool(engine)
+    assert response[-100:] == ('weather in Beijing is **25°C** with **partly cloudy** skies. '
+                               'It looks like a mild day outside—enjoy!')
+
+
+def test_deepseek_r1_distill():
+    # TODO
+    engine = TransformersEngine('deepseek-ai/DeepSeek-R1-Distill-Qwen-7B')
+    _test_tool(engine, system='')
+
+
+if __name__ == '__main__':
+    from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+    from swift.utils import get_logger
+    logger = get_logger()
+    # test_qwen2_5()
+    test_qwq()
+    # test_deepseek_r1_distill()

tests/test_align/test_template/test_video.py

@@ -0,0 +1,422 @@
+import os
+import torch
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+os.environ['SWIFT_DEBUG'] = '1'
+
+
+def _infer_model(engine, system=None, messages=None, videos=None, max_tokens=128):
+    seed_everything(42)
+    request_config = RequestConfig(max_tokens=max_tokens, temperature=0)
+    if messages is None:
+        messages = []
+    if not messages:
+        if system is not None:
+            messages += [{'role': 'system', 'content': system}]
+        messages += [{'role': 'user', 'content': '你好'}]
+        resp = engine.infer([{'messages': messages}], request_config=request_config)
+        response = resp[0].choices[0].message.content
+        messages += [{'role': 'assistant', 'content': response}, {'role': 'user', 'content': '<video>描述视频'}]
+    else:
+        messages = messages.copy()
+    if videos is None:
+        videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    resp = engine.infer([{'messages': messages, 'videos': videos}], request_config=request_config)
+    response = resp[0].choices[0].message.content
+    messages += [{'role': 'assistant', 'content': response}]
+    logger.info(f'model: {engine.model_info.model_name}, messages: {messages}')
+    return response
+
+
+def test_qwen2_vl():
+    os.environ['FPS_MAX_FRAMES'] = '24'
+    os.environ['MAX_PIXELS'] = '100352'
+    os.environ['VIDEO_MAX_PIXELS'] = str(100352 // 4)
+    engine = TransformersEngine('Qwen/Qwen2-VL-2B-Instruct')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+def test_internvl2_5():
+    engine = TransformersEngine('OpenGVLab/InternVL2_5-2B')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine, system='你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。')
+
+
+def test_internvl2_5_mpo():
+    engine = TransformersEngine('OpenGVLab/InternVL2_5-1B-MPO', model_type='internvl2_5')
+    response = _infer_model(engine, messages=[{'role': 'user', 'content': '<video>这是什么'}])
+    assert response == ('这是一段婴儿在阅读的视频。婴儿穿着浅绿色的上衣和粉色的裤子，戴着黑框眼镜，坐在床上，正在翻阅一本打开的书。'
+                        '背景中可以看到婴儿床、衣物和一些家具。视频中可以看到“clipo.com”的水印。婴儿看起来非常专注，似乎在认真地阅读。')
+
+
+def test_xcomposer2_5():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:base', torch.float16)
+    messages = [{'role': 'user', 'content': '<video>Describe the video'}]
+    messages_with_system = messages.copy()
+    messages_with_system.insert(0, {'role': 'system', 'content': ''})
+    response = _infer_model(engine, messages=messages_with_system)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, system='')
+    assert response == response2
+
+    response = _infer_model(engine, messages=messages)
+    std_response = (
+        'The video features a young child sitting on a bed, deeply engaged in reading a book. '
+        'The child is dressed in a light blue sleeveless top and pink pants, and is wearing glasses. '
+        'The bed is covered with a textured white blanket, and there are various items scattered on it, '
+        'including a white cloth and a striped piece of clothing. In the background, '
+        'a wooden crib and a dresser with a mirror can be seen. The child flips through the pages of the book, '
+        'occasionally pausing to look at the illustrations. The child appears to be enjoying the book, '
+        'and the overall atmosphere is one of quiet concentration and enjoyment.')
+
+    assert response == std_response[:len(response)]
+
+
+def test_mplug3():
+    engine = TransformersEngine('iic/mPLUG-Owl3-7B-240728')
+    # engine = TransformersEngine('iic/mPLUG-Owl3-7B-241101')
+    _infer_model(engine, system='')
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine, system='')
+
+
+def test_minicpmv():
+    engine = TransformersEngine('OpenBMB/MiniCPM-V-2_6')
+    _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    _infer_model(engine)
+
+
+def test_minicpmo():
+    os.environ['VIDEO_MAX_SLICE_NUMS'] = '2'
+    engine = TransformersEngine('OpenBMB/MiniCPM-o-2_6')
+    messages = [{'role': 'user', 'content': '<video>Describe the video'}]
+    response = _infer_model(engine, messages=messages)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages)
+    assert response == response2 == (
+        'The video features a young child sitting on a bed, deeply engrossed in reading a large book. The child, '
+        'dressed in a light blue sleeveless top and pink pants, is surrounded by a cozy and homely environment. '
+        'The bed is adorned with a patterned blanket, and a white cloth is casually draped over the side. '
+        'In the background, a crib and a television are visible, adding to the domestic setting. '
+        'The child is seen flipping through the pages of the book, occasionally pausing to look at the pages, '
+        'and then continuing to turn them. The video captures the child\'s focused and curious demeanor as they '
+        'explore the contents of the book, creating a heartwarming '
+        'scene of a young reader immersed in their world of stories.')[:len(response)]
+
+
+def test_valley():
+    engine = TransformersEngine('bytedance-research/Valley-Eagle-7B')
+    _infer_model(engine)
+
+
+def _run_qwen2_5_vl_hf(messages, model, template):
+    from qwen_vl_utils import process_vision_info
+    processor = template.processor
+    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    images, videos, video_kwargs = process_vision_info(messages, return_video_kwargs=True)
+    inputs = processor(text=text, images=images, videos=videos, do_resize=False, return_tensors='pt', **video_kwargs)
+    inputs = inputs.to(model.device)
+
+    generated_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)
+    generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
+    output_text = processor.batch_decode(
+        generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+    return output_text[0]
+
+
+def test_qwen2_5_vl():
+    os.environ['FPS'] = '1'
+    os.environ['VIDEO_MAX_PIXELS'] = str(360 * 420)
+    engine = TransformersEngine('Qwen/Qwen2.5-VL-7B-Instruct')
+    query = 'What happened in the video?'
+    messages = [{'role': 'user', 'content': query}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    messages = [
+        {
+            'role': 'user',
+            'content': [
+                {
+                    'type': 'video',
+                    'video': videos[0]
+                },
+                {
+                    'type': 'text',
+                    'text': query
+                },
+            ],
+        },
+    ]
+    response3 = _run_qwen2_5_vl_hf(messages, engine.model, engine.template)
+    assert response == response2 == response3
+
+
+def test_qwen2_5_omni():
+    USE_AUDIO_IN_VIDEO = True
+    os.environ['USE_AUDIO_IN_VIDEO'] = str(USE_AUDIO_IN_VIDEO)
+    engine = TransformersEngine('Qwen/Qwen2.5-Omni-7B', attn_impl='flash_attn')
+    system = ('You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, '
+              'capable of perceiving auditory and visual inputs, as well as generating text and speech.')
+    messages = [{'role': 'system', 'content': system}, {'role': 'user', 'content': '<video>'}]
+    videos = ['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    if USE_AUDIO_IN_VIDEO:
+        ground_truth = ("Oh, that's a really cool drawing! It looks like a guitar. You've got the body "
+                        'and the neck drawn in a simple yet effective way. The lines are clean and the '
+                        'shape is well-defined. What made you choose to draw a guitar?')
+    else:
+        ground_truth = ('嗯，你是在用平板画画呢。你画的这把吉他，看起来很简洁明了。你用的笔触也很流畅，线条很清晰。你对颜色的运用也很不错，整体看起来很协调。你要是还有啥想法或者问题，随时跟我说哈。')
+    assert response == response2 == ground_truth
+
+
+def _run_qwen3_omni_hf(model, processor, messages):
+    from qwen_omni_utils import process_mm_info
+    text = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
+    audios, images, videos = process_mm_info(messages, use_audio_in_video=True)
+    inputs = processor(
+        text=text,
+        audio=audios,
+        images=images,
+        videos=videos,
+        return_tensors='pt',
+        padding=True,
+        use_audio_in_video=True)
+    inputs = inputs.to(device=model.device, dtype=model.dtype)
+    text_ids = model.generate(**inputs, use_audio_in_video=True, do_sample=False, max_new_tokens=128)
+    text = processor.decode(
+        text_ids[0][len(inputs['input_ids'][0]):], skip_special_tokens=True, clean_up_tokenization_spaces=False)
+    return text
+
+
+def test_qwen3_omni():
+    USE_AUDIO_IN_VIDEO = True
+    os.environ['USE_AUDIO_IN_VIDEO'] = str(USE_AUDIO_IN_VIDEO)
+    engine = TransformersEngine('Qwen/Qwen3-Omni-30B-A3B-Thinking')
+    query = 'describe the video.'
+    messages = [{'role': 'user', 'content': query}]
+    videos = ['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+
+    messages = [
+        {
+            'role': 'user',
+            'content': [
+                {
+                    'type': 'video',
+                    'video': videos[0]
+                },
+                {
+                    'type': 'text',
+                    'text': query
+                },
+            ],
+        },
+    ]
+    response2 = _run_qwen3_omni_hf(engine.model, engine.processor, messages)
+    assert response == response2
+
+
+def test_glm4_1v():
+    messages = [{'role': 'user', 'content': '<video>What happened in the video?'}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    engine = TransformersEngine('ZhipuAI/GLM-4.1V-9B-Thinking')
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    assert response == response2
+
+
+def test_glm4_5v():
+    messages = [{'role': 'user', 'content': '<video>What happened in the video?'}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    engine = TransformersEngine('ZhipuAI/GLM-4.5V')
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    assert response == response2
+
+
+def test_keye_vl():
+    engine = TransformersEngine('Kwai-Keye/Keye-VL-8B-Preview')
+    messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    assert response == response2
+
+
+def test_keye_vl_1_5():
+    engine = TransformersEngine('Kwai-Keye/Keye-VL-1_5-8B')
+    messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    assert response[:200] == ('The video features a young child sitting on a bed, engrossed in '
+                              'reading a book. The child is wearing a light blue sleeveless top and pink '
+                              'pants. The book appears to be a hardcover with illustrations, ')
+
+
+def test_ovis2_5():
+    engine = TransformersEngine('AIDC-AI/Ovis2.5-2B')
+    messages = [{'role': 'user', 'content': '<video>Describe this video in detail.'}]
+    videos = ['baby.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    print(f'response: {response}')
+
+
+def run_hf(model, processor, messages):
+    inputs = processor.apply_chat_template(
+        messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors='pt').to(
+            model.device, dtype=torch.bfloat16)
+    generate_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)
+    decoded_output = processor.decode(generate_ids[0, inputs['input_ids'].shape[1]:], skip_special_tokens=True)
+    return decoded_output
+
+
+def test_interns1():
+    engine = TransformersEngine('Shanghai_AI_Laboratory/Intern-S1-mini')
+    query = 'Describe this video in detail.'
+    messages = [{'role': 'user', 'content': f'<video>{query}'}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    messages = [{
+        'role': 'user',
+        'content': [
+            {
+                'type': 'video',
+                'url': videos[0]
+            },
+            {
+                'type': 'text',
+                'text': query
+            },
+        ],
+    }]
+    response2 = run_hf(engine.model, engine.processor, messages)
+    assert response == ('<think>' + response2)[:len(response)]
+
+
+def test_internvl3_5():
+    models = [
+        'OpenGVLab/InternVL3_5-1B', 'OpenGVLab/InternVL3_5-2B', 'OpenGVLab/InternVL3_5-4B', 'OpenGVLab/InternVL3_5-8B',
+        'OpenGVLab/InternVL3_5-14B', 'OpenGVLab/InternVL3_5-38B', 'OpenGVLab/InternVL3_5-30B-A3B',
+        'OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview'
+    ]
+    for model in models:
+        engine = TransformersEngine(model)
+        messages = [{'role': 'user', 'content': '<video>Describe this video in detail.'}]
+        videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+        response = _infer_model(engine, messages=messages, videos=videos)
+        engine.template.template_backend = 'jinja'
+        response2 = _infer_model(engine, messages=messages, videos=videos)
+        assert response == response2
+
+
+def test_minicpmv4_5():
+    engine = TransformersEngine('OpenBMB/MiniCPM-V-4_5')
+    messages = [{'role': 'user', 'content': '<video>Describe this video in detail.'}]
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    assert response == response2
+
+
+def _run_qwen3_vl_hf(messages, model, template):
+    from qwen_vl_utils import process_vision_info
+    processor = template.processor
+    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    images, videos, video_kwargs = process_vision_info(
+        messages, image_patch_size=16, return_video_kwargs=True, return_video_metadata=True)
+    if videos is not None:
+        videos, video_metadatas = zip(*videos)
+        videos, video_metadatas = list(videos), list(video_metadatas)
+    else:
+        video_metadatas = None
+    inputs = processor(
+        text=text,
+        images=images,
+        videos=videos,
+        video_metadata=video_metadatas,
+        do_resize=False,
+        return_tensors='pt',
+        **video_kwargs)
+    inputs = inputs.to(model.device)
+
+    generated_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)
+    generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
+    output_text = processor.batch_decode(
+        generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+    return output_text[0]
+
+
+def test_qwen3_vl():
+    engine = TransformersEngine('Qwen/Qwen3-VL-4B-Instruct')
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    query = 'describe this video.'
+    messages = [{'role': 'user', 'content': query}]
+    response = _infer_model(engine, messages=messages, videos=videos)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine, messages=messages, videos=videos)
+    messages = [{
+        'role': 'user',
+        'content': [
+            {
+                'type': 'video',
+                'video': videos[0],
+            },
+            {
+                'type': 'text',
+                'text': query
+            },
+        ],
+    }]
+    response3 = _run_qwen3_vl_hf(messages, engine.model, engine.template)
+    assert response == response2 == response3
+
+
+def test_qwen3_vl_moe():
+    engine = TransformersEngine('Qwen/Qwen3-VL-30B-A3B-Instruct')
+    response = _infer_model(engine)
+    engine.template.template_backend = 'jinja'
+    response2 = _infer_model(engine)
+    assert response == response2
+
+
+if __name__ == '__main__':
+    from swift.infer_engine import RequestConfig, TransformersEngine
+    from swift.utils import get_logger, seed_everything
+    logger = get_logger()
+    # test_qwen2_vl()
+    # test_internvl2_5()
+    # test_xcomposer2_5()
+    # test_internvl2_5_mpo()
+    # test_mplug3()
+    # test_minicpmv()
+    # test_minicpmo()
+    # test_valley()
+    # test_qwen2_5_vl()
+    # test_qwen2_5_omni()
+    # test_qwen3_omni()
+    # test_glm4_1v()  # bug now, wait model fix
+    # test_keye_vl()
+    # test_keye_vl_1_5()
+    # test_glm4_5v()
+    # test_ovis2_5()
+    # test_interns1()
+    # test_internvl3_5()
+    # test_minicpmv4_5()
+    test_qwen3_vl()
+    # test_qwen3_vl_moe()

tests/test_align/test_vllm_vlm.py

@@ -0,0 +1,208 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def _infer_audio(model, use_chat_template: bool = True, max_model_len=8192, system=None, limit_mm_per_prompt=None):
+    limit_mm_per_prompt = limit_mm_per_prompt or {'audio': 2}
+    engine = VllmEngine(
+        model,
+        max_model_len=max_model_len,
+        gpu_memory_utilization=0.6,
+        limit_mm_per_prompt=limit_mm_per_prompt,
+        enforce_eager=True)
+    if not use_chat_template:
+        engine.template.use_chat_template = False
+    audios = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    messages = []
+    if system is not None:
+        messages += [{'role': 'system', 'content': system}]
+    messages.append({'role': 'user', 'content': 'describe the audio.'})
+    resp_list = engine.infer([InferRequest(messages=messages, audios=audios)],
+                             RequestConfig(temperature=0, max_tokens=64, repetition_penalty=1.))
+    return resp_list[0].choices[0].message.content
+
+
+def _infer_image(model, use_chat_template: bool = True, max_model_len=8192, system=None, limit_mm_per_prompt=None):
+    limit_mm_per_prompt = limit_mm_per_prompt or {'image': 5, 'video': 0}
+    engine = VllmEngine(
+        model,
+        max_model_len=max_model_len,
+        gpu_memory_utilization=0.6,
+        limit_mm_per_prompt=limit_mm_per_prompt,
+        enforce_eager=True)
+    if not use_chat_template:
+        engine.template.use_chat_template = False
+    images = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png']
+    messages = []
+    if system is not None:
+        messages += [{'role': 'system', 'content': system}]
+    messages.append({'role': 'user', 'content': 'describe the image.'})
+    resp_list = engine.infer([InferRequest(messages=messages, images=images)],
+                             RequestConfig(temperature=0, max_tokens=64, repetition_penalty=1.))
+    return resp_list[0].choices[0].message.content
+
+
+def _infer_video(model, use_chat_template: bool = True, max_model_len=8192, system=None, limit_mm_per_prompt=None):
+    limit_mm_per_prompt = limit_mm_per_prompt or {'image': 16, 'video': 2}
+    engine = VllmEngine(
+        model,
+        max_model_len=max_model_len,
+        limit_mm_per_prompt=limit_mm_per_prompt,
+        gpu_memory_utilization=0.6,
+        enforce_eager=True)
+    if not use_chat_template:
+        engine.template.use_chat_template = False
+    videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    messages = []
+    if system is not None:
+        messages += [{'role': 'system', 'content': system}]
+    messages.append({'role': 'user', 'content': 'describe the video.'})
+    resp_list = engine.infer([InferRequest(messages=messages, videos=videos)],
+                             RequestConfig(temperature=0, max_tokens=64, repetition_penalty=1.))
+    return resp_list[0].choices[0].message.content
+
+
+def test_qwen2_audio():
+    response = _infer_audio('Qwen/Qwen2-Audio-7B-Instruct')
+    assert response == "The audio is a man speaking in Mandarin saying '今天天气真好呀'."
+
+
+def test_qwen2_vl():
+    response = _infer_image('Qwen/Qwen2-VL-2B-Instruct')
+    assert response == (
+        'The image depicts a cute kitten with a fluffy, white and gray striped coat. The kitten has large, '
+        'expressive blue eyes and is looking directly at the camera. Its ears are perked up, and it has a '
+        'small red mark on its left ear. The background is blurred, focusing attention on the kitten. The overall')
+
+
+def test_qwen2_5_vl():
+    response = _infer_image('Qwen/Qwen2.5-VL-3B-Instruct')
+    assert response == (
+        'The image depicts a cute, fluffy kitten with striking blue eyes and a white and gray fur pattern. '
+        'The kitten has a small, pink nose and is looking directly at the camera with a curious expression. '
+        "The background is blurred, drawing attention to the kitten's face. "
+        'The overall appearance is very endearing and charming.')
+
+
+def test_deepseek_vl_v2():
+    response = _infer_image('deepseek-ai/deepseek-vl2-tiny', max_model_len=4096)
+    assert response == ('The image depicts a close-up of a adorable kitten with large, expressive eyes. The kitten has '
+                        'a mix of white and gray fur with distinct black stripes, giving it a tabby-like appearance. '
+                        'Its ears are perked up, and its whiskers are prominently visible. The background is blurred, '
+                        'focusing attention on the kitten')
+
+
+def test_internvl2():
+    response = _infer_image('OpenGVLab/InternVL2-2B', max_model_len=4096, system='')
+    assert response == ('The image features a kitten with striking blue eyes and a mix of white and black fur. '
+                        'The kitten has large, expressive eyes and a small, pink nose. Its ears are perked up, '
+                        'and it appears to be looking directly at the camera. The fur is soft and fluffy, with a mix')
+
+
+def test_minicpmv_2_5():
+    response = _infer_image('OpenBMB/MiniCPM-Llama3-V-2_5', max_model_len=4096)
+    assert response == (
+        "The image is a digital painting of a kitten that captures the essence of a young feline's innocence "
+        "and curiosity. The kitten's fur is rendered with a mix of gray, white, and black stripes, "
+        'giving it a realistic and adorable appearance. Its large, expressive eyes are a striking blue, '
+        "which draws the viewer's")
+
+
+def test_minicpmv_2_6():
+    response = _infer_image('OpenBMB/MiniCPM-V-2_6', max_model_len=4096)
+    assert response == (
+        'The image features a close-up of a kitten with striking blue eyes and a mix of '
+        "white and dark fur, possibly gray or black. The kitten's gaze is directed forward, giving it an "
+        "expressive and captivating look. The background is blurred, drawing focus to the kitten's face. "
+        "The overall composition emphasizes the kitten's features")
+
+
+def test_minicpmo_2_6_video():
+    response = _infer_video('OpenBMB/MiniCPM-o-2_6')
+    assert response == ('The video features a young child sitting on a bed, deeply engaged in reading a book. '
+                        'The child, dressed in a light blue sleeveless top and pink pants, is surrounded by a '
+                        'cozy and homely environment. The bed is adorned with a patterned blanket, and a white cloth '
+                        'is casually draped over the side.')
+
+
+def test_qwen2_5_vl_video():
+    response = _infer_video('Qwen/Qwen2.5-VL-3B-Instruct')
+    assert response == ('A baby wearing sunglasses is sitting on a bed and reading a book. '
+                        'The baby is holding the book with both hands and is looking at the pages. '
+                        'The baby is wearing a light blue shirt and pink pants. The baby is sitting '
+                        'on a white blanket. The baby is looking at the book and is smiling. The baby')
+
+
+def test_qwen2_5_omni():
+    limit_mm_per_prompt = {'image': 1, 'video': 1, 'audio': 1}
+    response = _infer_video('Qwen/Qwen2.5-Omni-7B', limit_mm_per_prompt=limit_mm_per_prompt)
+    # response = _infer_audio('Qwen/Qwen2.5-Omni-7B')
+    assert response
+
+
+def test_ovis2():
+    response = _infer_image('AIDC-AI/Ovis2-1B', max_model_len=4096)
+    assert response[:200] == ('The image showcases a charming digital painting of a kitten, capturing its '
+                              'adorable features in a unique style. The kitten has a predominantly white face '
+                              'with black stripes and spots, giving it a stri')
+
+
+def test_keye_vl():
+    response = _infer_image('Kwai-Keye/Keye-VL-8B-Preview', max_model_len=4096)
+    assert response[:200] == ('<analysis>This question asks for a description of the image, which is '
+                              'straightforward and involves observing the visual content. Therefore, '
+                              '/no_think is more appropriate.</analysis>The image features ')
+
+
+def test_kimi_vl():
+    response = _infer_image('moonshotai/Kimi-VL-A3B-Instruct', max_model_len=4096)
+    print(f'response: {response}')
+
+
+def test_glm4v():
+    response = _infer_image('ZhipuAI/glm-4v-9b', max_model_len=4096)
+    print(f'response: {response}')
+
+
+def test_glm4_1v():
+    response = _infer_image('ZhipuAI/GLM-4.1V-9B-Thinking', max_model_len=4096)
+    print(f'response: {response}')
+
+
+def test_paddleocr_vl():
+    response = _infer_image('PaddlePaddle/PaddleOCR-VL', max_model_len=4096)
+    print(f'response: {response}')
+
+
+def test_glm4_5_vl():
+    response = _infer_image('ZhipuAI/GLM-4.5V', max_model_len=4096)
+    print(f'response: {response}')
+
+
+def test_deepseek_ocr():
+    response = _infer_image('deepseek-ai/DeepSeek-OCR', max_model_len=4096)
+    print(f'response: {response}')
+
+
+if __name__ == '__main__':
+    from swift.infer_engine import InferRequest, RequestConfig, VllmEngine
+
+    # test_qwen2_vl()
+    # test_qwen2_5_vl()
+    # test_deepseek_vl_v2()
+    # test_internvl2()
+    # test_qwen2_audio()
+    # test_minicpmv_2_5()
+    # test_minicpmv_2_6()
+    # test_minicpmo_2_6_video()
+    # test_qwen2_5_vl_video()
+    # test_qwen2_5_omni()
+    # test_ovis2()
+    # test_keye_vl()
+    # test_kimi_vl()
+    # test_glm4v()
+    # test_glm4_1v()
+    # test_paddleocr_vl()
+    test_deepseek_ocr()