chore: import upstream snapshot with attribution

swift/model/models/__init__.py

@@ -0,0 +1,3 @@
+from . import (baai, baichuan, baidu, bert, codefuse, deepseek, gemma, glm, internlm, llama, llava, llm, mamba,
+               microsoft, minicpm, minimax, mistral, mllm, moonshot, mplug, openbuddy, qwen, seed, skywork, stepfun,
+               telechat, tencent, valley, yi)

swift/model/models/baai.py

@@ -0,0 +1,113 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+import sys
+from transformers import AutoModel, AutoModelForSequenceClassification, PretrainedConfig, PreTrainedModel
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_device, git_clone_github, safe_snapshot_download
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+
+class Emu3GenLoader(ModelLoader):
+
+    def get_processor(self, model_dir, config) -> Processor:
+        self.model_info.max_model_len = self.model_info.max_model_len + 40960
+        config.image_area = int(os.environ.get('image_area', config.image_area))
+        config.max_position_embeddings = int(os.environ.get('max_position_embeddings', config.max_position_embeddings))
+        tokenizer = super().get_processor(model_dir, config)
+        import sys
+        sys.path.append(model_dir)
+        from processing_emu3 import Emu3Processor
+        vq_hub = safe_snapshot_download('BAAI/Emu3-VisionTokenizer', check_local=True)
+        from transformers import AutoImageProcessor, AutoModel
+        image_processor = AutoImageProcessor.from_pretrained(vq_hub, trust_remote_code=True)
+        image_tokenizer = AutoModel.from_pretrained(vq_hub, trust_remote_code=True).eval().to(get_device())
+        processor = Emu3Processor(image_processor, image_tokenizer, tokenizer)
+        processor.image_area = config.image_area
+        return processor
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs):
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.generation_config.do_sample = True
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.emu3_gen,
+        [
+            ModelGroup([
+                Model('BAAI/Emu3-Gen', 'BAAI/Emu3-Gen'),
+            ]),
+        ],
+        Emu3GenLoader,
+        template=TemplateType.emu3_gen,
+        architectures=['Emu3ForCausalLM'],
+        model_arch=ModelArch.emu3_chat,
+        tags=['t2i'],
+    ))
+
+
+class Emu3ChatLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer = super().get_processor(model_dir, config)
+        # download and load vision tokenizer
+        from transformers import AutoImageProcessor
+        vq_model = safe_snapshot_download('BAAI/Emu3-VisionTokenizer', check_local=True)
+        image_processor = AutoImageProcessor.from_pretrained(vq_model, trust_remote_code=True)
+        image_tokenizer = AutoModel.from_pretrained(
+            vq_model, device_map=self.model_kwargs['device_map'], trust_remote_code=True)
+        image_tokenizer.requires_grad_(False)
+        image_tokenizer.to(get_device())
+        # load processor
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/baaivision/Emu3.git')
+        sys.path.append(local_repo_path)
+        from emu3.mllm.processing_emu3 import Emu3Processor
+        return Emu3Processor(image_processor, image_tokenizer, tokenizer)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.emu3_chat,
+        [
+            ModelGroup([
+                Model('BAAI/Emu3-Chat', 'BAAI/Emu3-Chat'),
+            ]),
+        ],
+        Emu3ChatLoader,
+        template=TemplateType.emu3_chat,
+        architectures=['Emu3ForCausalLM'],
+        model_arch=ModelArch.emu3_chat,
+        tags=['vision'],
+        requires=['transformers>=4.44.0'],
+    ))
+
+
+class BgeRerankerLoader(ModelLoader):
+
+    def get_model(self, *args, **kwargs) -> PreTrainedModel:
+        self.auto_model_cls = self.auto_model_cls or AutoModelForSequenceClassification
+        return super().get_model(*args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.bge_reranker,
+        [
+            ModelGroup([
+                Model('BAAI/bge-reranker-base', 'BAAI/bge-reranker-base'),
+                Model('BAAI/bge-reranker-v2-m3', 'BAAI/bge-reranker-v2-m3'),
+                Model('BAAI/bge-reranker-large', 'BAAI/bge-reranker-large'),
+            ]),
+        ],
+        BgeRerankerLoader,
+        template=TemplateType.bge_reranker,
+        task_type='reranker',
+        architectures=['XLMRobertaForSequenceClassification'],
+    ))

swift/model/models/baichuan.py

@@ -0,0 +1,131 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch.nn.functional as F
+from torch import Tensor
+from transformers import PreTrainedModel
+from types import MethodType
+
+from swift.template import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+
+class BaichuanLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        # baichuan-13b does not implement the `get_input_embeddings` function
+        # fix gradient_checkpointing bug
+        try:
+            model.get_input_embeddings()
+        except NotImplementedError:
+            model.__class__.get_input_embeddings = lambda self: self.model.embed_tokens
+        return model
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.baichuan, [
+            ModelGroup([
+                Model('baichuan-inc/Baichuan-13B-Chat', 'baichuan-inc/Baichuan-13B-Chat'),
+                Model('baichuan-inc/Baichuan-13B-Base', 'baichuan-inc/Baichuan-13B-Base'),
+                Model('baichuan-inc/baichuan-7B', 'baichuan-inc/Baichuan-7B'),
+            ]),
+        ],
+        BaichuanLoader,
+        template=TemplateType.baichuan,
+        architectures=['BaichuanForCausalLM', 'BaiChuanForCausalLM'],
+        model_arch=ModelArch.baichuan,
+        requires=['transformers<4.34']))
+
+
+class BaichuanM1Loader(BaichuanLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers.dynamic_module_utils import get_class_from_dynamic_module
+        rotary_embedding = get_class_from_dynamic_module('modeling_baichuan.RotaryEmbedding', model_dir)
+        _old_forward = rotary_embedding.forward
+
+        def _new_forward(self, q, k, seqlen_offset=None, cu_seqlens=None, max_seqlen=None):
+            q = q.to(k.dtype)
+            res = _old_forward(self, q, k, seqlen_offset, cu_seqlens, max_seqlen)
+            return res
+
+        rotary_embedding.forward = _new_forward
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.baichuan_m1, [
+            ModelGroup([
+                Model('baichuan-inc/Baichuan-M1-14B-Instruct', 'baichuan-inc/Baichuan-M1-14B-Instruct'),
+            ]),
+        ],
+        BaichuanM1Loader,
+        template=TemplateType.baichuan_m1,
+        architectures=['BaichuanM1ForCausalLM'],
+        model_arch=ModelArch.baichuan,
+        requires=['transformers>=4.48']))
+
+
+def patch_baichuan2_lm_head_forward(self, hidden_states: Tensor) -> Tensor:
+    # patch: baichuan2 lm_head (fp32 bug)
+    if self.training:
+        norm_weight = F.normalize(self.weight).to(self.weight.dtype)
+    elif self.first_flag:
+        self.first_flag = False
+        self.weight.data = F.normalize(self.weight).to(self.weight.dtype)
+        norm_weight = self.weight
+    else:
+        norm_weight = self.weight
+    return F.linear(hidden_states, norm_weight)
+
+
+class Baichuan2Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, *args, **kwargs) -> PreTrainedModel:
+        if not hasattr(config, 'z_loss_weight'):
+            config.z_loss_weight = 0
+        # patch: baichuan2_13b configuration_baichuan.py bug
+        if hasattr(config, 'gradient_checkpointing'):
+            gradient_checkpointing = config.gradient_checkpointing
+            if isinstance(gradient_checkpointing, (tuple, list)):
+                config.gradient_checkpointing = gradient_checkpointing[0]
+        model = super().get_model(model_dir, config, *args, **kwargs)
+        model_ori = model
+        if not hasattr(model, 'lm_head'):  # fix awq
+            model = model.model
+        new_forward = MethodType(patch_baichuan2_lm_head_forward, model.lm_head)
+        if hasattr(model, '_old_forward'):  # device_map
+            model.lm_head._old_forward = new_forward
+        else:
+            model.lm_head.forward = new_forward
+        return model_ori
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.baichuan2,
+        [
+            ModelGroup([
+                Model('baichuan-inc/Baichuan2-7B-Chat', 'baichuan-inc/Baichuan2-7B-Chat'),
+                Model('baichuan-inc/Baichuan2-7B-Base', 'baichuan-inc/Baichuan2-7B-Base'),
+                Model('baichuan-inc/Baichuan2-13B-Chat', 'baichuan-inc/Baichuan2-13B-Chat'),
+                Model('baichuan-inc/Baichuan2-13B-Base', 'baichuan-inc/Baichuan2-13B-Base'),
+            ]),
+            ModelGroup([
+                Model('baichuan-inc/Baichuan2-7B-Chat-4bits', 'baichuan-inc/Baichuan2-7B-Chat-4bits'),
+                Model('baichuan-inc/Baichuan2-13B-Chat-4bits', 'baichuan-inc/Baichuan2-13B-Chat-4bits'),
+            ],
+                       requires=['bitsandbytes<0.41.2', 'accelerate<0.26'])
+        ],
+        Baichuan2Loader,
+        template=TemplateType.baichuan,
+        architectures=['BaichuanForCausalLM', 'BaiChuanForCausalLM'],
+        model_arch=ModelArch.baichuan,
+    ))

swift/model/models/baidu.py

@@ -0,0 +1,112 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.template import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+register_model(
+    ModelMeta(
+        LLMModelType.ernie4_5,
+        [
+            ModelGroup([
+                Model('PaddlePaddle/ERNIE-4.5-0.3B-Base-PT', 'baidu/ERNIE-4.5-0.3B-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-0.3B-PT', 'baidu/ERNIE-4.5-0.3B-PT'),
+            ], TemplateType.ernie),
+        ],
+        architectures=['Ernie4_5_ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.ernie4_5_moe,
+        [
+            ModelGroup([
+                Model('PaddlePaddle/ERNIE-4.5-21B-A3B-Base-PT', 'baidu/ERNIE-4.5-21B-A3B-Base-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-21B-A3B-PT', 'baidu/ERNIE-4.5-21B-A3B-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-300B-A47B-Base-PT', 'baidu/ERNIE-4.5-300B-A47B-Base-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-300B-A47B-PT', 'baidu/ERNIE-4.5-300B-A47B-PT'),
+            ], TemplateType.ernie),
+            ModelGroup([
+                Model('PaddlePaddle/ERNIE-4.5-21B-A3B-Thinking', 'baidu/ERNIE-4.5-21B-A3B-Thinking'),
+            ], TemplateType.ernie_thinking),
+        ],
+        architectures=['Ernie4_5_MoeForCausalLM'],
+    ))
+
+
+class ErnieVLLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        MOEAllGatherLayerV2 = get_class_from_dynamic_module('modeling_ernie4_5_vl.MOEAllGatherLayerV2', model_dir)
+        self.leaf_modules = MOEAllGatherLayerV2
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.add_image_preprocess(processor)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ernie_vl,
+        [
+            ModelGroup([
+                Model('PaddlePaddle/ERNIE-4.5-VL-28B-A3B-PT', 'baidu/ERNIE-4.5-VL-28B-A3B-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-VL-424B-A47B-PT', 'baidu/ERNIE-4.5-VL-424B-A47B-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-PT', 'baidu/ERNIE-4.5-VL-28B-A3B-Base-PT'),
+                Model('PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-PT', 'baidu/ERNIE-4.5-VL-424B-A47B-Base-PT'),
+            ], TemplateType.ernie_vl),
+            ModelGroup([
+                Model('PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking', 'baidu/ERNIE-4.5-VL-28B-A3B-Thinking'),
+            ], TemplateType.ernie_vl_thinking),
+        ],
+        ErnieVLLoader,
+        model_arch=ModelArch.ernie_vl,
+        architectures=['Ernie4_5_VLMoeForConditionalGeneration'],
+        requires=['transformers>=4.52', 'moviepy'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.paddle_ocr,
+        [
+            ModelGroup([
+                Model('PaddlePaddle/PaddleOCR-VL', 'PaddlePaddle/PaddleOCR-VL'),
+            ]),
+        ],
+        template=TemplateType.paddle_ocr,
+        model_arch=ModelArch.keye_vl,
+        architectures=['PaddleOCRVLForConditionalGeneration'],
+        requires=['transformers<5.0'],
+    ))
+
+
+class PaddleOCR1_5Loader(ModelLoader):
+    default_trust_remote_code = False
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForImageTextToText
+        self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.paddleocr_vl,
+        [
+            ModelGroup([
+                Model('PaddlePaddle/PaddleOCR-VL-1.5', 'PaddlePaddle/PaddleOCR-VL-1.5'),
+                Model('PaddlePaddle/PaddleOCR-VL-1.6', 'PaddlePaddle/PaddleOCR-VL-1.6'),
+            ],
+                       template=TemplateType.paddle_ocr_1_5),
+        ],
+        PaddleOCR1_5Loader,
+        model_arch=ModelArch.paddleocr_vl,
+        requires=['transformers>=5.0'],
+        architectures=['PaddleOCRVLForConditionalGeneration'],
+    ))

swift/model/models/bert.py

@@ -0,0 +1,89 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch.nn.functional as F
+from transformers import AutoModel, AutoModelForSequenceClassification, PreTrainedModel
+
+from swift.template import TemplateType
+from swift.utils import get_logger
+from ..constant import BertModelType, LLMModelType
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+
+class ModernBertLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, *args, **kwargs) -> PreTrainedModel:
+        config.reference_compile = False
+        return super().get_model(model_dir, config, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        BertModelType.modern_bert, [
+            ModelGroup([
+                Model('answerdotai/ModernBERT-base', 'answerdotai/ModernBERT-base'),
+                Model('answerdotai/ModernBERT-large', 'answerdotai/ModernBERT-large'),
+            ])
+        ],
+        ModernBertLoader,
+        template=TemplateType.dummy,
+        requires=['transformers>=4.48'],
+        architectures=['ModernBertForMaskedLM'],
+        tags=['bert']))
+
+
+class GTEBertLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        self.auto_model_cls = self.auto_model_cls or AutoModel
+        model = super().get_model(model_dir, *args, **kwargs)
+
+        def _normalizer_hook(module, input, output):
+            output.last_hidden_state = F.normalize(output.last_hidden_state[:, 0], p=2, dim=1)
+            return output
+
+        model.register_forward_hook(_normalizer_hook)
+        return model
+
+
+register_model(
+    ModelMeta(
+        BertModelType.modern_bert_gte,
+        [ModelGroup([
+            Model('iic/gte-modernbert-base', 'Alibaba-NLP/gte-modernbert-base'),
+        ])],
+        GTEBertLoader,
+        template=TemplateType.dummy,
+        requires=['transformers>=4.48'],
+        architectures=['ModernBertModel'],
+        tags=['bert', 'embedding']))
+
+
+class GTEBertReranker(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        self.auto_model_cls = self.auto_model_cls or AutoModelForSequenceClassification
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.modern_bert_gte_reranker,
+        [ModelGroup([
+            Model('iic/gte-reranker-modernbert-base', 'Alibaba-NLP/gte-reranker-modernbert-base'),
+        ])],
+        GTEBertReranker,
+        template=TemplateType.bert,
+        requires=['transformers>=4.48'],
+        architectures=['ModernBertForSequenceClassification'],
+        task_type='reranker',
+        tags=['bert', 'reranker']))
+
+register_model(
+    ModelMeta(
+        BertModelType.bert, [ModelGroup([
+            Model('iic/nlp_structbert_backbone_base_std'),
+        ])],
+        template=TemplateType.dummy,
+        tags=['bert']))

swift/model/models/codefuse.py

@@ -0,0 +1,63 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import AutoTokenizer, PretrainedConfig
+
+from swift.template import TemplateType
+from swift.utils import Processor
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+from .glm import ChatGLMLoader
+from .qwen import QwenLoader
+
+register_model(
+    ModelMeta(
+        LLMModelType.codefuse_qwen, [
+            ModelGroup([
+                Model('codefuse-ai/CodeFuse-QWen-14B', 'codefuse-ai/CodeFuse-QWen-14B'),
+            ]),
+        ],
+        QwenLoader,
+        template=TemplateType.codefuse,
+        architectures=['QWenLMHeadModel'],
+        model_arch=ModelArch.qwen,
+        tags=['coding']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.codefuse_codegeex2,
+        [
+            ModelGroup([Model('codefuse-ai/CodeFuse-CodeGeeX2-6B', 'codefuse-ai/CodeFuse-CodeGeeX2-6B')], ),
+        ],
+        ChatGLMLoader,
+        template=TemplateType.codefuse,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm,
+        tags=['coding'],
+        requires=['transformers<4.34'],
+    ))
+
+
+class CodeLlamaLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False, legacy=False)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.codefuse_codellama,
+        [
+            ModelGroup(
+                [
+                    Model('codefuse-ai/CodeFuse-CodeLlama-34B', 'codefuse-ai/CodeFuse-CodeLlama-34B'),
+                ],
+                tags=['coding'],
+            ),
+        ],
+        CodeLlamaLoader,
+        template=TemplateType.codefuse_codellama,
+        model_arch=ModelArch.llama,
+        mcore_model_type='gpt',
+        architectures=['LlamaForCausalLM'],
+    ))

swift/model/models/deepseek.py

@@ -0,0 +1,509 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import sys
+import torch
+from transformers import AutoModel, PretrainedConfig, PreTrainedModel
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_logger, git_clone_github
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_output_clone, patch_output_to_input_device
+from ..register import ModelLoader, register_model
+from ..utils import use_submodel_func
+
+
+class DeepseekLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        # fix dtype bug
+        mlp_cls = model.model.layers[-1].mlp.__class__
+
+        for module in model.modules():
+            if isinstance(module, mlp_cls):
+                patch_output_to_input_device(module)
+        return model
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.deepseek,
+        [
+            ModelGroup([
+                Model('deepseek-ai/deepseek-moe-16b-chat', 'deepseek-ai/deepseek-moe-16b-chat'),
+                Model('deepseek-ai/deepseek-moe-16b-base', 'deepseek-ai/deepseek-moe-16b-base'),
+            ], ),
+        ],
+        DeepseekLoader,
+        template=TemplateType.deepseek,
+        architectures=['DeepseekForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.deepseek_v2,
+        [
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-Coder-V2-Instruct', 'deepseek-ai/DeepSeek-Coder-V2-Instruct'),
+                Model('deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct', 'deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct'),
+                Model('deepseek-ai/DeepSeek-Coder-V2-Base', 'deepseek-ai/DeepSeek-Coder-V2-Base'),
+                Model('deepseek-ai/DeepSeek-Coder-V2-Lite-Base', 'deepseek-ai/DeepSeek-Coder-V2-Lite-Base'),
+                Model('deepseek-ai/DeepSeek-V2-Lite', 'deepseek-ai/DeepSeek-V2-Lite'),
+                Model('deepseek-ai/DeepSeek-V2-Lite-Chat', 'deepseek-ai/DeepSeek-V2-Lite-Chat'),
+                Model('deepseek-ai/DeepSeek-V2', 'deepseek-ai/DeepSeek-V2'),
+                Model('deepseek-ai/DeepSeek-V2-Chat', 'deepseek-ai/DeepSeek-V2-Chat'),
+            ], TemplateType.deepseek),
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-V2.5', 'deepseek-ai/DeepSeek-V2.5'),
+                Model('deepseek-ai/DeepSeek-V2.5-1210', 'deepseek-ai/DeepSeek-V2.5-1210')
+            ], TemplateType.deepseek_v2_5)
+        ],
+        DeepseekLoader,
+        model_arch=ModelArch.deepseek_v2,
+        architectures=['DeepseekV2ForCausalLM'],
+        requires=['transformers>=4.39.3'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.deepseek_v3,
+        [
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-V3-Base', 'deepseek-ai/DeepSeek-V3-Base'),
+                Model('deepseek-ai/DeepSeek-V3', 'deepseek-ai/DeepSeek-V3'),
+                Model('deepseek-ai/DeepSeek-V3-0324', 'deepseek-ai/DeepSeek-V3-0324'),
+            ], TemplateType.deepseek_v2_5),
+            ModelGroup([
+                Model('cognitivecomputations/DeepSeek-V3-awq', 'cognitivecomputations/DeepSeek-V3-AWQ'),
+                Model('cognitivecomputations/DeepSeek-V3-0324-AWQ', 'cognitivecomputations/DeepSeek-V3-0324-AWQ')
+            ], TemplateType.deepseek_v2_5),
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-Prover-V2-7B', 'deepseek-ai/DeepSeek-Prover-V2-7B'),
+                Model('deepseek-ai/DeepSeek-Prover-V2-671B', 'deepseek-ai/DeepSeek-Prover-V2-671B'),
+            ], TemplateType.deepseek_v2_5),
+            ModelGroup([
+                Model('unsloth/DeepSeek-V3-bf16', 'unsloth/DeepSeek-V3-bf16'),
+                Model('unsloth/DeepSeek-V3-0324-BF16', 'unsloth/DeepSeek-V3-0324-BF16'),
+                Model('unsloth/DeepSeek-Prover-V2-671B-BF16', 'unsloth/DeepSeek-Prover-V2-671B-BF16'),
+            ], TemplateType.deepseek_v2_5),
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-R1', 'deepseek-ai/DeepSeek-R1'),
+                Model('deepseek-ai/DeepSeek-R1-Zero', 'deepseek-ai/DeepSeek-R1-Zero'),
+                Model('deepseek-ai/DeepSeek-R1-0528', 'deepseek-ai/DeepSeek-R1-0528'),
+            ], TemplateType.deepseek_r1),
+            ModelGroup([
+                Model('cognitivecomputations/DeepSeek-R1-awq', 'cognitivecomputations/DeepSeek-R1-AWQ'),
+                Model('cognitivecomputations/DeepSeek-R1-0528-AWQ', 'cognitivecomputations/DeepSeek-R1-0528-AWQ'),
+            ], TemplateType.deepseek_r1),
+            ModelGroup([
+                Model('unsloth/DeepSeek-R1-BF16', 'unsloth/DeepSeek-R1-BF16'),
+                Model('unsloth/DeepSeek-R1-Zero-BF16', 'unsloth/DeepSeek-R1-Zero-BF16'),
+                Model('unsloth/DeepSeek-R1-0528-BF16', 'unsloth/DeepSeek-R1-0528-BF16'),
+            ], TemplateType.deepseek_r1),
+            ModelGroup([
+                Model('moonshotai/Moonlight-16B-A3B', 'moonshotai/Moonlight-16B-A3B'),
+                Model('moonshotai/Moonlight-16B-A3B-Instruct', 'moonshotai/Moonlight-16B-A3B-Instruct'),
+            ],
+                       TemplateType.moonlight,
+                       requires=['transformers<4.49']),
+            ModelGroup([
+                Model('moonshotai/Kimi-K2-Base', 'moonshotai/Kimi-K2-Base'),
+                Model('moonshotai/Kimi-K2-Instruct', 'moonshotai/Kimi-K2-Instruct'),
+                Model('moonshotai/Kimi-K2-Instruct-0905', 'moonshotai/Kimi-K2-Instruct-0905'),
+                Model('moonshotai/Kimi-K2-Thinking', 'moonshotai/Kimi-K2-Thinking'),
+            ], TemplateType.kimi_k2),
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-V3.1-Base', 'deepseek-ai/DeepSeek-V3.1-Base'),
+                Model('deepseek-ai/DeepSeek-V3.1', 'deepseek-ai/DeepSeek-V3.1'),
+                Model('deepseek-ai/DeepSeek-V3.1-Terminus', 'deepseek-ai/DeepSeek-V3.1-Terminus'),
+            ], TemplateType.deepseek_v3_1),
+        ],
+        DeepseekLoader,
+        model_arch=ModelArch.deepseek_v2,
+        architectures=['DeepseekV3ForCausalLM'],
+        requires=['transformers>=4.39.3'],
+    ))
+
+
+class DeepseekV32Loader(ModelLoader):
+
+    def get_config(self, model_dir: str):
+        try:
+            from transformers.models.deepseek_v32 import DeepseekV32Config
+        except ImportError:
+            from transformers.models.deepseek_v3 import DeepseekV3Config as DeepseekV32Config
+        return DeepseekV32Config.from_pretrained(model_dir)
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        try:
+            from transformers.models.deepseek_v32 import DeepseekV32ForCausalLM
+        except ImportError:
+            # It’s only for compatibility with Megatron training or vllm/sglang infer,
+            # while we wait for Transformers to support deepseek_v32.
+            from transformers.models.deepseek_v3 import DeepseekV3ForCausalLM as DeepseekV32ForCausalLM
+            if not self.return_dummy_model:
+                raise ValueError('DeepSeek-V3.2 is not supported in transformers.')
+        self.auto_model_cls = DeepseekV32ForCausalLM
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.deepseek_v32,
+        [
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-V3.2', 'deepseek-ai/DeepSeek-V3.2'),
+                Model('deepseek-ai/DeepSeek-V3.2-Speciale', 'deepseek-ai/DeepSeek-V3.2-Speciale'),
+                Model('deepseek-ai/DeepSeek-V3.2-Exp', 'deepseek-ai/DeepSeek-V3.2-Exp'),
+                Model('deepseek-ai/DeepSeek-V3.2-Exp-Base', 'deepseek-ai/DeepSeek-V3.2-Exp-Base'),
+                Model('deepseek-ai/DeepSeek-Math-V2', 'deepseek-ai/DeepSeek-Math-V2'),
+            ]),
+        ],
+        DeepseekV32Loader,
+        template=TemplateType.deepseek_v3_1,
+        architectures=['DeepseekV32ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.deepseek_v4,
+        [
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-V4-Flash', 'deepseek-ai/DeepSeek-V4-Flash'),
+                Model('deepseek-ai/DeepSeek-V4-Flash-Base', 'deepseek-ai/DeepSeek-V4-Flash-Base'),
+            ]),
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-V4-Pro', 'deepseek-ai/DeepSeek-V4-Pro'),
+                Model('deepseek-ai/DeepSeek-V4-Pro-Base', 'deepseek-ai/DeepSeek-V4-Pro-Base'),
+            ]),
+        ],
+        template=TemplateType.deepseek_v4,
+        architectures=['DeepseekV4ForCausalLM'],
+    ))
+
+
+class DeepseekVLLoader(ModelLoader):
+
+    def get_config(self, model_dir: str):
+        # compat with python==3.10
+        if sys.version_info.minor >= 10:
+            import collections
+            import collections.abc
+            for type_name in collections.abc.__all__:
+                setattr(collections, type_name, getattr(collections.abc, type_name))
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/deepseek-ai/DeepSeek-VL')
+        sys.path.append(local_repo_path)
+        from deepseek_vl.models import VLChatProcessor
+        self.auto_tokenizer_cls = VLChatProcessor
+        return super().get_config(model_dir)
+
+    def _get_model(self, model_dir: str, llm_prefix, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        llm = getattr(model, llm_prefix)
+        patch_output_clone(llm.model.embed_tokens)
+        patch_output_to_input_device(llm.model.embed_tokens)
+        use_submodel_func(model, llm_prefix)
+        model.generation_config = llm.generation_config
+        return model
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        return self._get_model(model_dir, 'language_model', *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.deepseek_vl,
+        [
+            ModelGroup([
+                Model('deepseek-ai/deepseek-vl-1.3b-chat', 'deepseek-ai/deepseek-vl-1.3b-chat'),
+                Model('deepseek-ai/deepseek-vl-7b-chat', 'deepseek-ai/deepseek-vl-7b-chat'),
+            ], ),
+        ],
+        DeepseekVLLoader,
+        template=TemplateType.deepseek_vl,
+        architectures=['MultiModalityCausalLM'],
+        model_arch=ModelArch.deepseek_vl,
+        tags=['vision'],
+    ))
+
+
+class DeepseekJanusLoader(DeepseekVLLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        return self._get_model(model_dir, 'language_model', *args, **kwargs)
+
+    def get_config(self, model_dir: str):
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/deepseek-ai/Janus')
+        sys.path.append(local_repo_path)
+        from janus.models import VLChatProcessor
+        self.auto_tokenizer_cls = VLChatProcessor
+        return super(DeepseekVLLoader, self).get_config(model_dir)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.deepseek_janus,
+        [
+            ModelGroup([
+                Model('deepseek-ai/Janus-1.3B', 'deepseek-ai/Janus-1.3B'),
+            ]),
+        ],
+        DeepseekJanusLoader,
+        template=TemplateType.deepseek_janus,
+        model_arch=ModelArch.deepseek_janus,
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.deepseek_janus_pro,
+        [
+            ModelGroup([
+                Model('deepseek-ai/Janus-Pro-1B', 'deepseek-ai/Janus-Pro-1B'),
+                Model('deepseek-ai/Janus-Pro-7B', 'deepseek-ai/Janus-Pro-7B'),
+            ]),
+        ],
+        DeepseekJanusLoader,
+        template=TemplateType.deepseek_janus_pro,
+        model_arch=ModelArch.deepseek_janus,
+        tags=['vision'],
+    ))
+
+
+class DeepseekVL2Loader(DeepseekVLLoader):
+
+    def get_config(self, model_dir: str):
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/deepseek-ai/DeepSeek-VL2')
+        sys.path.append(local_repo_path)
+        try:
+            from deepseek_vl2.models import DeepseekVLV2Processor
+        except ImportError:
+            # compat transformers>=4.42
+            import transformers
+            transformers.models.llama.modeling_llama.LlamaFlashAttention2 = None
+            from deepseek_vl2.models import DeepseekVLV2Processor
+        self.auto_tokenizer_cls = DeepseekVLV2Processor
+        return super(DeepseekVLLoader, self).get_config(model_dir)
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        return super()._get_model(model_dir, 'language', *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.deepseek_vl2,
+        [
+            ModelGroup([
+                Model('deepseek-ai/deepseek-vl2-tiny', 'deepseek-ai/deepseek-vl2-tiny'),
+                Model('deepseek-ai/deepseek-vl2-small', 'deepseek-ai/deepseek-vl2-small'),
+                Model('deepseek-ai/deepseek-vl2', 'deepseek-ai/deepseek-vl2'),
+            ]),
+        ],
+        DeepseekVL2Loader,
+        template=TemplateType.deepseek_vl2,
+        model_arch=ModelArch.deepseek_vl2,
+        requires=['transformers<4.42'],
+        tags=['vision'],
+    ))
+
+
+class DeepseekOCRLoader(ModelLoader):
+    visual_name = 'vision_model'
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        self.auto_model_cls = self.auto_model_cls or AutoModel
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.model.embed_tokens)
+        patch_output_to_input_device(model.model.sam_model)
+        patch_output_to_input_device(getattr(model.model, self.visual_name))
+        patch_output_to_input_device(model.model.projector)
+        return model
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        from transformers import AutoProcessor, AutoTokenizer
+
+        # When not loading model (e.g., vllm backend), avoid triggering AutoConfig which would execute
+        # trust_remote_code and cause transformers version compatibility issues
+        # For vllm backend, we only need the processor/tokenizer
+        try:
+            processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+        except Exception:
+            # Fallback to AutoTokenizer if AutoProcessor is not available
+            processor = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
+        return processor
+
+
+class DeepseekOCR2Loader(DeepseekOCRLoader):
+    visual_name = 'qwen2_model'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.deepseek_ocr,
+        [
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-OCR', 'deepseek-ai/DeepSeek-OCR'),
+            ]),
+        ],
+        DeepseekOCRLoader,
+        template=TemplateType.deepseek_ocr,
+        model_arch=ModelArch.deepseek_ocr,
+        architectures=['DeepseekOCRForCausalLM'],
+        requires=['transformers==4.46.3', 'easydict'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.deepseek_ocr2,
+        [
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-OCR-2', 'deepseek-ai/DeepSeek-OCR-2'),
+            ]),
+        ],
+        DeepseekOCR2Loader,
+        template=TemplateType.deepseek_ocr2,
+        model_arch=ModelArch.deepseek_ocr2,
+        architectures=['DeepseekOCR2ForCausalLM'],
+        requires=['transformers==4.46.3', 'easydict'],
+        tags=['vision'],
+    ))
+
+
+class UnlimitedOCRLoader(DeepseekOCRLoader):
+    visual_name = 'vision_model'
+
+    @staticmethod
+    def _apply_multi_gpu_patch():
+        """
+        Fixed two bugs affecting `UnlimitedOCRModel` in multi-GPU scenarios using `device_map='auto'`:
+
+        Bug 1 - Device mismatch in `torch.cat`:
+            `image_newline` and `view_seperator` are `nn.Parameter`s;
+            under `device_map='auto'`, their device placement might not align
+            with the image features.
+
+        Bug 2 - Device mismatch in `masked_scatter_`:
+            Hard-coded `.cuda()` usage caused a conflict where `images_in_this_batch`
+            resided on the projector's device (e.g., `cuda:7`),
+            while `inputs_embeds` resided on the device hosting `embed_tokens` (e.g., `cuda:0`).
+
+        Fix strategy: Temporarily replace `torch.cat` and `torch.Tensor.masked_scatter_` during the forward pass
+        to handle device placement automatically, then restore the original methods after execution.
+        """
+        modeling_module = None
+        for mod_name, mod in sys.modules.items():
+            if 'modeling_unlimitedocr' in mod_name:
+                modeling_module = mod
+                break
+
+        if modeling_module is None:
+            return False
+
+        UnlimitedOCRModel = getattr(modeling_module, 'UnlimitedOCRModel', None)
+        if UnlimitedOCRModel is None:
+            return False
+
+        # Avoid redundant patching
+        if getattr(UnlimitedOCRModel, '_swift_multi_gpu_patched', False):
+            return True
+
+        _original_forward = UnlimitedOCRModel.forward
+
+        def _patched_forward(self, *args, **kwargs):
+            _orig_cat = torch.cat
+            _orig_masked_scatter_ = torch.Tensor.masked_scatter_
+
+            def _safe_cat(tensors, dim=0, **cat_kwargs):
+                # Using the device of the first tensor as the reference, the others are aligned to it.
+                ref_device = None
+                for t in tensors:
+                    if isinstance(t, torch.Tensor):
+                        ref_device = t.device
+                        break
+                if ref_device is None:
+                    return _orig_cat(tensors, dim, **cat_kwargs)
+                aligned = [
+                    t.to(ref_device) if isinstance(t, torch.Tensor) and t.device != ref_device else t for t in tensors
+                ]
+                return _orig_cat(aligned, dim, **cat_kwargs)
+
+            def _safe_masked_scatter_(tensor_self, mask, source):
+                # Use the device of tensor_self (inputs_embeds[idx]) as the reference.
+                dev = tensor_self.device
+                if mask.device != dev:
+                    mask = mask.to(dev)
+                if source.device != dev:
+                    source = source.to(dev)
+                return _orig_masked_scatter_(tensor_self, mask, source)
+
+            # Simultaneously replace the module namespace and the global scope (double insurance).
+            modeling_module.torch.cat = _safe_cat
+            torch.cat = _safe_cat
+            torch.Tensor.masked_scatter_ = _safe_masked_scatter_
+            try:
+                return _original_forward(self, *args, **kwargs)
+            finally:
+                # Restore the state to avoid contaminating other modules.
+                modeling_module.torch.cat = _orig_cat
+                torch.cat = _orig_cat
+                torch.Tensor.masked_scatter_ = _orig_masked_scatter_
+
+        UnlimitedOCRModel.forward = _patched_forward
+        UnlimitedOCRModel._swift_multi_gpu_patched = True
+        return True
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        logger = get_logger()
+
+        self.auto_model_cls = self.auto_model_cls or AutoModel
+        model = super(DeepseekOCRLoader, self).get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.model.embed_tokens)
+        patch_output_to_input_device(model.model.sam_model)
+        patch_output_to_input_device(getattr(model.model, self.visual_name))
+        patch_output_to_input_device(model.model.projector)
+        patch_output_to_input_device(model.model)
+
+        _orig_sw = (getattr(model.config, 'sliding_window_size', None) or getattr(model.config, 'sliding_window', None))
+        if _orig_sw is not None:
+            model.config._ring_window = _orig_sw
+            model.config.sliding_window = None
+            logger.info('[UnlimitedOCR] R-SWA enabled: ring_window=%d', _orig_sw)
+        else:
+            logger.warning('[UnlimitedOCR] sliding_window config not found, R-SWA may not work.')
+
+        n_devices = len(set(str(p.device) for p in model.parameters() if p.device.type == 'cuda'))
+        if n_devices > 1:
+            if self._apply_multi_gpu_patch():
+                logger.info('[UnlimitedOCR] Multi-GPU patch applied (%d GPUs).', n_devices)
+            else:
+                logger.warning('[UnlimitedOCR] Multi-GPU deployment failed to apply patch.'
+                               'If an inference error occurs, please check whether'
+                               ' `modeling_unlimitedocr` has been loaded correctly.')
+
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.unlimited_ocr,
+        [
+            ModelGroup([
+                Model('PaddlePaddle/Unlimited-OCR', 'PaddlePaddle/Unlimited-OCR'),
+            ]),
+        ],
+        UnlimitedOCRLoader,
+        template=TemplateType.unlimited_ocr,
+        model_arch=ModelArch.unlimited_ocr,
+        architectures=['UnlimitedOCRForCausalLM'],
+        requires=['transformers==4.46.3', 'easydict'],
+        tags=['vision'],
+    ))

swift/model/models/gemma.py

@@ -0,0 +1,508 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import inspect
+import torch
+import torch.distributed as dist
+import transformers
+from packaging import version
+from PIL import Image
+from transformers import PreTrainedModel
+from types import MethodType
+
+from swift.template import TemplateType
+from swift.utils import is_deepspeed_enabled, to_device
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_output_to_input_device
+from ..register import ModelLoader, SentenceTransformersLoader, register_model
+
+transformers_5_9 = version.parse(transformers.__version__) >= version.parse('5.9')
+
+
+class PaligemmaVisionLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import PaliGemmaForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or PaliGemmaForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.paligemma,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/paligemma-3b-pt-224', 'google/paligemma-3b-pt-224'),
+                Model('AI-ModelScope/paligemma-3b-pt-448', 'google/paligemma-3b-pt-448'),
+                Model('AI-ModelScope/paligemma-3b-pt-896', 'google/paligemma-3b-pt-896'),
+            ]),
+            ModelGroup([
+                Model('AI-ModelScope/paligemma-3b-mix-224', 'google/paligemma-3b-mix-224'),
+                Model('AI-ModelScope/paligemma-3b-mix-448', 'google/paligemma-3b-mix-448'),
+            ]),
+            ModelGroup([
+                Model('AI-ModelScope/paligemma2-3b-pt-224', 'google/paligemma2-3b-pt-224'),
+                Model('AI-ModelScope/paligemma2-3b-pt-448', 'google/paligemma2-3b-pt-448'),
+                Model('AI-ModelScope/paligemma2-3b-pt-896', 'google/paligemma2-3b-pt-896'),
+                Model('AI-ModelScope/paligemma2-10b-pt-224', 'google/paligemma2-10b-pt-224'),
+                Model('AI-ModelScope/paligemma2-10b-pt-448', 'google/paligemma2-10b-pt-448'),
+                Model('AI-ModelScope/paligemma2-10b-pt-896', 'google/paligemma2-10b-pt-896'),
+                Model('AI-ModelScope/paligemma2-28b-pt-224', 'google/paligemma2-28b-pt-224'),
+                Model('AI-ModelScope/paligemma2-28b-pt-448', 'google/paligemma2-28b-pt-448'),
+                Model('AI-ModelScope/paligemma2-28b-pt-896', 'google/paligemma2-28b-pt-896'),
+            ]),
+            ModelGroup([
+                Model('AI-ModelScope/paligemma2-3b-ft-docci-448', 'google/paligemma2-3b-ft-docci-448'),
+                Model('AI-ModelScope/paligemma2-10b-ft-docci-448', 'google/paligemma2-10b-ft-docci-448'),
+            ]),
+        ],
+        PaligemmaVisionLoader,
+        template=TemplateType.paligemma,
+        architectures=['PaliGemmaForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.41'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.gemma,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/gemma-2b-it', 'google/gemma-2b-it'),
+                Model('AI-ModelScope/gemma-2b', 'google/gemma-2b'),
+                Model('AI-ModelScope/gemma-7b', 'google/gemma-7b'),
+                Model('AI-ModelScope/gemma-7b-it', 'google/gemma-7b-it'),
+            ], ),
+        ],
+        template=TemplateType.gemma,
+        architectures=['GemmaForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.38'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.gemma2,
+        [
+            ModelGroup([
+                Model('LLM-Research/gemma-2-2b-it', 'google/gemma-2-2b-it'),
+                Model('LLM-Research/gemma-2-2b', 'google/gemma-2-2b'),
+                Model('LLM-Research/gemma-2-9b', 'google/gemma-2-9b'),
+                Model('LLM-Research/gemma-2-9b-it', 'google/gemma-2-9b-it'),
+                Model('LLM-Research/gemma-2-27b', 'google/gemma-2-27b'),
+                Model('LLM-Research/gemma-2-27b-it', 'google/gemma-2-27b-it'),
+            ], ),
+        ],
+        template=TemplateType.gemma,
+        architectures=['Gemma2ForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.42'],
+    ))
+
+
+class Gemma3TextLoader(ModelLoader):
+
+    def get_config(self, model_dir):
+        # It is strongly recommended to train Gemma3 models with the `eager` attention implementation instead of `sdpa`.
+        self.attn_impl = self.attn_impl or 'eager'
+        return super().get_config(model_dir)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.gemma3_text,
+        [
+            ModelGroup([
+                Model('LLM-Research/gemma-3-1b-pt', 'google/gemma-3-1b-pt'),
+                Model('LLM-Research/gemma-3-1b-it', 'google/gemma-3-1b-it'),
+                Model('google/gemma-3-270m', 'google/gemma-3-270m'),
+                Model('google/gemma-3-270m-it', 'google/gemma-3-270m-it'),
+                Model('google/medgemma-27b-text-it', 'google/medgemma-27b-text-it'),
+            ], ),
+        ],
+        Gemma3TextLoader,
+        template=TemplateType.gemma3_text,
+        architectures=['Gemma3ForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.49'],
+    ))
+
+
+class Gemma3VisionLoader(ModelLoader):
+
+    def get_config(self, model_dir):
+        # It is strongly recommended to train Gemma3 models with the `eager` attention implementation instead of `sdpa`.
+        self.attn_impl = self.attn_impl or 'eager'
+        return super().get_config(model_dir)
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import Gemma3ForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Gemma3ForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.gemma3_vision,
+        [
+            ModelGroup([
+                Model('LLM-Research/gemma-3-4b-pt', 'google/gemma-3-4b-pt'),
+                Model('LLM-Research/gemma-3-4b-it', 'google/gemma-3-4b-it'),
+                Model('LLM-Research/gemma-3-12b-pt', 'google/gemma-3-12b-pt'),
+                Model('LLM-Research/gemma-3-12b-it', 'google/gemma-3-12b-it'),
+                Model('LLM-Research/gemma-3-27b-pt', 'google/gemma-3-27b-pt'),
+                Model('LLM-Research/gemma-3-27b-it', 'google/gemma-3-27b-it'),
+                Model('google/medgemma-4b-pt', 'google/medgemma-4b-pt'),
+                Model('google/medgemma-4b-it', 'google/medgemma-4b-it'),
+                Model('google/medgemma-27b-it', 'google/medgemma-27b-it'),
+            ], ),
+        ],
+        Gemma3VisionLoader,
+        template=TemplateType.gemma3_vision,
+        architectures=['Gemma3ForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.49'],
+    ))
+
+
+class Gemma3nLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import Gemma3nForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Gemma3nForConditionalGeneration
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_to_input_device(model.model.embed_vision)
+        patch_output_to_input_device(model.model.embed_audio)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.gemma3n,
+        [
+            ModelGroup([
+                Model('google/gemma-3n-E2B', 'google/gemma-3n-E2B'),
+                Model('google/gemma-3n-E4B', 'google/gemma-3n-E4B'),
+                Model('google/gemma-3n-E2B-it', 'google/gemma-3n-E2B-it'),
+                Model('google/gemma-3n-E4B-it', 'google/gemma-3n-E4B-it'),
+            ], ),
+        ],
+        Gemma3nLoader,
+        template=TemplateType.gemma3n,
+        architectures=['Gemma3nForConditionalGeneration'],
+        model_arch=ModelArch.gemma3n,
+        requires=['transformers>=4.53.1'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.gemma_emb,
+        [
+            ModelGroup([
+                Model('google/embeddinggemma-300m', 'google/embeddinggemma-300m'),
+            ], ),
+        ],
+        SentenceTransformersLoader,
+        template=TemplateType.dummy,
+        architectures=['Gemma3TextModel'],
+    ))
+
+
+def _patch_gemma4_forward(model, processor, is_gemma4_unified: bool = False):
+    if is_gemma4_unified:
+        from transformers.models.gemma4_unified.modeling_gemma4_unified import \
+            Gemma4UnifiedModelOutputWithPast as Gemma4ModelOutputWithPast
+        from transformers.models.gemma4_unified.modeling_gemma4_unified import (create_masks_for_generate,
+                                                                                torch_compilable_check)
+    else:
+        from transformers.models.gemma4.modeling_gemma4 import (Gemma4ModelOutputWithPast, create_masks_for_generate,
+                                                                torch_compilable_check)
+    if hasattr(model, 'origin_forward'):
+        return
+
+    def _forward_dummy_image(model, inputs_embeds):
+        images = [Image.new('RGB', (32, 32), (0, 0, 0))]
+        image_inputs = processor.image_processor(images=images, return_tensors='pt')
+        image_inputs = to_device(image_inputs, inputs_embeds.device)
+        dummy_pixel = image_inputs['pixel_values'].to(model.dtype)
+        dummy_pos_ids = image_inputs.get('image_position_ids')
+        image_features = model.get_image_features(dummy_pixel, dummy_pos_ids, return_dict=True).pooler_output
+        inputs_embeds = inputs_embeds + image_features.mean() * 0.
+        return inputs_embeds
+
+    # transformers 5.6.2
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        pixel_values_videos: torch.FloatTensor | None = None,
+        input_features: torch.FloatTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        input_features_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values=None,
+        mm_token_type_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        image_position_ids: torch.LongTensor | None = None,
+        video_position_ids: torch.LongTensor | None = None,
+        per_layer_inputs: torch.Tensor | None = None,
+        **kwargs,
+    ) -> Gemma4ModelOutputWithPast:
+        r"""
+        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
+            The attention mask for the input audio.
+        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
+            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
+            Passed through to the vision encoder for positional embedding computation.
+        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
+            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
+            Passed through to the vision encoder for positional embedding computation.
+        """
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError('You must specify exactly one of input_ids or inputs_embeds')
+
+        image_mask, video_mask, audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds)
+        multimodal_mask = image_mask | video_mask | audio_mask
+
+        # Replace image id with PAD if the image token if OOV, to avoid index-errors
+        llm_input_ids = None
+        if inputs_embeds is None:
+            llm_input_ids = input_ids.clone()
+            llm_input_ids[multimodal_mask] = self.config.text_config.pad_token_id
+            inputs_embeds = self.get_input_embeddings()(llm_input_ids)
+
+        if per_layer_inputs is None and self.config.get_text_config().hidden_size_per_layer_input:
+            pad_embedding = self.language_model.embed_tokens.weight[self.config.text_config.pad_token_id, :]
+            pad_embedding = pad_embedding.to(device=multimodal_mask.device)
+            llm_inputs_embeds = torch.where(multimodal_mask[..., None], pad_embedding.view(1, 1, -1), inputs_embeds)
+            per_layer_inputs = self.language_model.get_per_layer_inputs(llm_input_ids, llm_inputs_embeds)
+        else:
+            per_layer_inputs = None
+
+        state = input_ids.new_tensor(
+            [pixel_values is not None or pixel_values_videos is not None, input_features is not None], dtype=torch.bool)
+        if dist.is_initialized() and is_deepspeed_enabled():
+            dist.all_reduce(state, dist.ReduceOp.MAX)
+        has_image, has_audio = state.tolist()
+
+        # Mixed modality training with both images and videos is not currently supported.
+        if pixel_values is None and pixel_values_videos is None and has_image:
+            inputs_embeds = _forward_dummy_image(self, inputs_embeds)
+
+        # Merge text and images
+        if pixel_values is not None:
+            image_features = self.get_image_features(pixel_values, image_position_ids, return_dict=True).pooler_output
+            image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
+
+            # Confirm the number of soft tokens from the vision tower matches the number of slots in the embeddings.
+            n_image_tokens = image_mask.sum()
+            image_mask = image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+            torch_compilable_check(
+                inputs_embeds[image_mask].numel() == image_features.numel(),
+                f'Image features and image tokens do not match, tokens: {n_image_tokens}, features:'
+                f' {image_features.shape[0]}',
+            )
+
+            inputs_embeds = inputs_embeds.masked_scatter(
+                image_mask.to(inputs_embeds.device), image_features.to(inputs_embeds.device))
+
+        if pixel_values_videos is not None:
+            video_features = self.get_video_features(
+                pixel_values_videos, video_position_ids, return_dict=True).pooler_output
+            video_features = video_features.to(inputs_embeds.device, inputs_embeds.dtype)
+
+            # Confirm the number of soft tokens from the vision tower matches the number of slots in the embeddings.
+            n_video_tokens = video_mask.sum()
+            video_mask = video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+            torch_compilable_check(
+                inputs_embeds[video_mask].numel() == video_features.numel(),
+                f'Video features and video tokens do not match, tokens: {n_video_tokens}, features:'
+                f' {video_features.shape[0]}',
+            )
+
+            inputs_embeds = inputs_embeds.masked_scatter(
+                video_mask.to(inputs_embeds.device), video_features.to(inputs_embeds.device))
+
+        # Merge text and audio
+        if input_features is not None and input_features_mask is not None:
+            audio_output = self.get_audio_features(input_features, input_features_mask, return_dict=True)
+            audio_features = audio_output.pooler_output
+            audio_mask_from_encoder = audio_output.attention_mask  # True = valid
+
+            # Strip padding tokens: only keep real (non-padding) audio soft tokens.
+            # audio_mask_from_encoder is True for valid positions, False for padding tokens.
+            # This mirrors the vision encoder's padding stripping (see Gemma4VisionEncoder.forward).
+            audio_features = audio_features[audio_mask_from_encoder]
+
+            n_audio_tokens = audio_mask.sum()
+            audio_mask = audio_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+            torch_compilable_check(
+                inputs_embeds[audio_mask].numel() == audio_features.numel(),
+                f'Audio features and audio tokens do not match, tokens: {n_audio_tokens}, features:'
+                f' {audio_features.shape[0] * audio_features.shape[1]}',
+            )
+
+            inputs_embeds = inputs_embeds.masked_scatter(
+                audio_mask.to(inputs_embeds.device), audio_features.to(inputs_embeds.device))
+        elif has_audio and self.audio_tower is not None:
+            feature_size = processor.feature_extractor.feature_size
+            dummy_features = input_ids.new_zeros([1, 128, feature_size], dtype=self.audio_tower.dtype)
+            dummy_mask = input_ids.new_ones([1, 128], dtype=torch.bool)
+            audio_output = self.get_audio_features(dummy_features, dummy_mask, return_dict=True)
+            audio_features = audio_output.pooler_output
+            inputs_embeds = inputs_embeds + audio_features.mean() * 0.
+
+        # It may already have been prepared by, e.g., `generate`
+        if position_ids is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
+            position_ids = position_ids.unsqueeze(0)
+
+        bi_vision_attn = self.config.get_text_config().use_bidirectional_attention == 'vision'
+        if not isinstance(causal_mask_mapping := attention_mask, dict):
+            if bi_vision_attn and not transformers_5_9:
+                from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping
+
+                # Larger Gemma 4 models use Gemma 3's bidirectional attention mask for vision inputs
+                causal_mask_mapping = create_causal_mask_mapping(
+                    self.config,
+                    inputs_embeds=inputs_embeds,
+                    attention_mask=attention_mask,
+                    past_key_values=past_key_values,
+                    position_ids=position_ids,
+                    mm_token_type_ids=mm_token_type_ids,
+                )
+            else:
+                mask_kwargs = {
+                    'config': self.config,
+                    'inputs_embeds': inputs_embeds,
+                    'attention_mask': attention_mask,
+                    'past_key_values': past_key_values,
+                    'position_ids': position_ids,
+                }
+                if bi_vision_attn:
+                    from transformers.models.gemma4.modeling_gemma4 import get_block_sequence_ids_for_mask
+                    block_sequence_ids = torch.full([*inputs_embeds.size()[:-1]], -1, device=inputs_embeds.device)
+                    if mm_token_type_ids is not None:
+                        kwargs = {
+                            'device': inputs_embeds.device
+                        } if 'device' in inspect.signature(get_block_sequence_ids_for_mask).parameters else {}
+                        block_sequence_ids = get_block_sequence_ids_for_mask(mm_token_type_ids, **kwargs)
+
+                    mask_kwargs['block_sequence_ids'] = block_sequence_ids
+
+                causal_mask_mapping = create_masks_for_generate(**mask_kwargs)
+        kwargs.pop('return_dict', None)
+        outputs = self.language_model(
+            per_layer_inputs=per_layer_inputs,
+            attention_mask=causal_mask_mapping,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            return_dict=True,
+            **kwargs,
+        )
+
+        return Gemma4ModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=image_features if pixel_values is not None else None,
+            audio_hidden_states=audio_features if input_features is not None else None,
+        )
+
+    model.origin_forward = model.forward
+    model.forward = MethodType(forward, model)
+
+
+class Gemma4Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        from transformers import Gemma4ForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Gemma4ForConditionalGeneration
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        _patch_gemma4_forward(model.model, processor)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.gemma4,
+        [
+            ModelGroup([
+                Model('google/gemma-4-E2B', 'google/gemma-4-E2B'),
+                Model('google/gemma-4-E2B-it', 'google/gemma-4-E2B-it'),
+                Model('google/gemma-4-E4B', 'google/gemma-4-E4B'),
+                Model('google/gemma-4-E4B-it', 'google/gemma-4-E4B-it'),
+            ],
+                       template=TemplateType.gemma4_nothinking),
+            ModelGroup([
+                Model('google/gemma-4-31B', 'google/gemma-4-31B'),
+                Model('google/gemma-4-31B-it', 'google/gemma-4-31B-it'),
+                Model('google/gemma-4-26B-A4B', 'google/gemma-4-26B-A4B'),
+                Model('google/gemma-4-26B-A4B-it', 'google/gemma-4-26B-A4B-it'),
+            ],
+                       template=TemplateType.gemma4),
+        ],
+        Gemma4Loader,
+        architectures=['Gemma4ForConditionalGeneration'],
+        model_arch=ModelArch.gemma3n,
+    ))
+
+
+class Gemma4UnifiedLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        from transformers import Gemma4UnifiedForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Gemma4UnifiedForConditionalGeneration
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        _patch_gemma4_forward(model.model, processor, is_gemma4_unified=True)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.gemma4_unified,
+        [
+            ModelGroup([
+                Model('google/gemma-4-12B', 'google/gemma-4-12B'),
+                Model('google/gemma-4-12B-it', 'google/gemma-4-12B-it'),
+            ],
+                       template=TemplateType.gemma4),
+        ],
+        Gemma4UnifiedLoader,
+        architectures=['Gemma4UnifiedForConditionalGeneration'],
+        model_arch=ModelArch.gemma4_unified,
+        requires=['transformers>=5.10.1'],
+    ))
+
+
+class DiffusionGemmaLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        from transformers import DiffusionGemmaForBlockDiffusion
+        self.auto_model_cls = self.auto_model_cls or DiffusionGemmaForBlockDiffusion
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.prepare_inputs_for_generation = None
+        model.config.use_cache = True
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.diffusion_gemma,
+        [
+            ModelGroup([
+                Model('google/diffusiongemma-26B-A4B-it', 'google/diffusiongemma-26B-A4B-it'),
+            ],
+                       template=TemplateType.diffusion_gemma),
+        ],
+        DiffusionGemmaLoader,
+        architectures=['DiffusionGemmaForBlockDiffusion'],
+        model_arch=ModelArch.diffusion_gemma,
+        requires=['transformers>=5.11'],
+    ))

swift/model/models/glm.py

@@ -0,0 +1,518 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import inspect
+import torch
+import transformers
+from packaging import version
+from transformers import AutoTokenizer, PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from transformers.models.auto.tokenization_auto import get_tokenizer_config
+from typing import Any, Dict, Type
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_device_count, get_dist_setting, get_logger, safe_snapshot_download
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_get_input_embeddings, patch_output_to_input_device
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+
+def remove_property(tokenizer_cls: Type[PreTrainedTokenizerBase], tokenizer_config: Dict[str, Any]) -> None:
+    for k, v in tokenizer_cls.__dict__.items():
+        if k.endswith('_token') and isinstance(v, property) and k in tokenizer_config:
+            setattr(tokenizer_cls, k, tokenizer_config[k])
+
+
+def _patch_tokenizer(tokenizer):
+    tokenizer_cls = tokenizer.__class__
+    if hasattr(tokenizer_cls, '_origin_pad'):
+        return
+    tokenizer_cls._origin_pad = tokenizer_cls._pad
+    parameters = inspect.signature(tokenizer_cls._origin_pad).parameters
+
+    def _pad(self, *args, **kwargs):
+        if 'padding_side' in kwargs and kwargs['padding_side'] is None and 'padding_side' not in parameters:
+            kwargs.pop('padding_side')
+        return tokenizer_cls._origin_pad(self, *args, **kwargs)
+
+    tokenizer_cls._pad = _pad
+
+
+class ChatGLMLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        if model_kwargs.get('quantization_config') is not None:
+            model_kwargs['quantization_config'].llm_int8_skip_modules = ['output_layer']
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        from torch.nn import CrossEntropyLoss
+        __old_forward = CrossEntropyLoss.forward
+
+        def cross_entropy_forward(self, inputs: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+            target = target.to(device=inputs.device)
+            return __old_forward(self, inputs, target)
+
+        CrossEntropyLoss.forward = cross_entropy_forward
+        return model
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        # fix transformers>=4.34 bug
+        if version.parse(transformers.__version__) >= version.parse('4.34'):
+            tokenizer_config = get_tokenizer_config(model_dir)
+            class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
+            tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
+            tokenizer_cls._auto_class = 'AutoTokenizer'
+            remove_property(tokenizer_cls, tokenizer_config)
+            tokenizer = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
+        else:
+            tokenizer = super().get_processor(model_dir, config)
+        _patch_tokenizer(tokenizer)
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.chatglm2, [
+            ModelGroup([
+                Model('ZhipuAI/chatglm2-6b', 'zai-org/chatglm2-6b'),
+                Model('ZhipuAI/chatglm2-6b-32k', 'zai-org/chatglm2-6b-32k')
+            ],
+                       requires=['transformers<4.42']),
+            ModelGroup(
+                [Model('ZhipuAI/codegeex2-6b', 'zai-org/codegeex2-6b')],
+                requires=['transformers<4.34'],
+                tags=['coding'],
+            ),
+        ],
+        ChatGLMLoader,
+        template=TemplateType.chatglm2,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm))
+
+register_model(
+    ModelMeta(
+        LLMModelType.chatglm3, [
+            ModelGroup([
+                Model('ZhipuAI/chatglm3-6b', 'zai-org/chatglm3-6b'),
+                Model('ZhipuAI/chatglm3-6b-base', 'zai-org/chatglm3-6b-base'),
+                Model('ZhipuAI/chatglm3-6b-32k', 'zai-org/chatglm3-6b-32k'),
+                Model('ZhipuAI/chatglm3-6b-128k', 'zai-org/chatglm3-6b-128k'),
+            ])
+        ],
+        ChatGLMLoader,
+        template=TemplateType.chatglm4,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.chatglm))
+
+
+class ChatGLM4Loader(ChatGLMLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer = super().get_processor(model_dir, config)
+        if len(tokenizer.encode('<|user|>', add_special_tokens=False)) > 1:
+            for k in tokenizer.special_tokens.keys():
+                tokenizer.add_tokens(k)
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.chatglm4,
+        [
+            ModelGroup([
+                Model('ZhipuAI/glm-4-9b-chat', 'zai-org/glm-4-9b-chat'),
+                Model('ZhipuAI/glm-4-9b', 'zai-org/glm-4-9b'),
+                Model('ZhipuAI/glm-4-9b-chat-1m', 'zai-org/glm-4-9b-chat-1m'),
+            ]),
+            ModelGroup([
+                Model('ZhipuAI/LongWriter-glm4-9b', 'zai-org/LongWriter-glm4-9b'),
+            ])
+        ],
+        ChatGLM4Loader,
+        template=TemplateType.chatglm4,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm,
+        requires=['transformers>=4.42'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm4,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-4-9B-0414', 'zai-org/GLM-4-9B-0414'),
+                Model('ZhipuAI/GLM-4-32B-0414', 'zai-org/GLM-4-32B-0414'),
+                Model('ZhipuAI/GLM-4-32B-Base-0414', 'zai-org/GLM-4-32B-Base-0414'),
+                Model('ZhipuAI/GLM-Z1-9B-0414', 'zai-org/GLM-Z1-9B-0414'),
+                Model('ZhipuAI/GLM-Z1-32B-0414', 'zai-org/GLM-Z1-32B-0414'),
+            ], TemplateType.glm4),
+            ModelGroup([
+                Model('ZhipuAI/GLM-Z1-Rumination-32B-0414', 'zai-org/GLM-Z1-Rumination-32B-0414'),
+            ], TemplateType.glm4_z1_rumination)
+        ],
+        requires=['transformers>=4.51'],
+        architectures=['Glm4ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.codegeex4,
+        [ModelGroup([
+            Model('ZhipuAI/codegeex4-all-9b', 'zai-org/codegeex4-all-9b'),
+        ])],
+        ChatGLM4Loader,
+        template=TemplateType.codegeex4,
+        requires=['transformers<4.42'],
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm,
+        tags=['coding'],
+    ))
+
+
+class ChatGLM4vLoader(ChatGLMLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        # fix device_map 4
+        n_gpu = get_device_count()
+        local_world_size = get_dist_setting()[3]
+        if n_gpu // local_world_size >= 4:
+            for layer in model.transformer.vision.transformer.layers:
+                patch_output_to_input_device(layer.mlp)
+                patch_output_to_input_device(layer.post_attention_layernorm)
+            device = next(model.transformer.vision.linear_proj.parameters()).device
+            model.transformer.vision.boi.data = model.transformer.vision.boi.to(device)
+            model.transformer.vision.eoi.data = model.transformer.vision.eoi.to(device)
+        return model
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        processor = super().get_processor(model_dir, config)
+        processor.init_kwargs['image_size'] = 1120
+        return processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.chatglm4v,
+        [
+            ModelGroup(
+                [
+                    Model('ZhipuAI/glm-4v-9b', 'zai-org/glm-4v-9b'),
+                ],
+                requires=['transformers>=4.42,<4.45'],
+            ),
+            ModelGroup(
+                [
+                    Model('ZhipuAI/cogagent-9b-20241220', 'zai-org/cogagent-9b-20241220'),
+                ],
+                requires=['transformers>=4.42'],
+            )
+        ],
+        ChatGLM4vLoader,
+        template=TemplateType.chatglm4v,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm4v,
+    ))
+
+
+class GLM4vLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import Glm4vForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Glm4vForConditionalGeneration
+        model = super().get_model(model_dir, *args, **kwargs)
+        if hasattr(model, 'visual'):
+            patch_get_input_embeddings(model.visual, 'patch_embed')
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.glm4v,
+        [
+            ModelGroup(
+                [
+                    Model('ZhipuAI/GLM-4.1V-9B-Base', 'zai-org/GLM-4.1V-9B-Base'),
+                    Model('ZhipuAI/GLM-4.1V-9B-Thinking', 'zai-org/GLM-4.1V-9B-Thinking'),
+                    Model('ZhipuAI/AutoGLM-Phone-9B', 'zai-org/AutoGLM-Phone-9B')
+                ],
+                template=TemplateType.glm4v,
+                requires=['transformers>=4.53'],
+            ),
+            ModelGroup(
+                [
+                    Model('ZhipuAI/Glyph', 'zai-org/Glyph'),
+                ],
+                template=TemplateType.glm4_5v,
+                requires=['transformers>=4.57'],
+            ),
+            ModelGroup(
+                [
+                    Model('ZhipuAI/GLM-4.6V-Flash', 'zai-org/GLM-4.6V-Flash'),
+                ],
+                template=TemplateType.glm4_5v,
+                requires=['transformers>=5.0.0.dev'],
+            ),
+        ],
+        GLM4vLoader,
+        model_arch=ModelArch.glm4v,
+        architectures=['Glm4vForConditionalGeneration'],
+    ))
+
+
+class CogVLMLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        logger.warning('CogAgent with FusedLayerNorm will cause an training loss of NAN, '
+                       'to avoid this, please uninstall apex.')
+        logger.info('Please ignore the unimported warning.')
+        return super().get_model(model_dir, *args, **kwargs)
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer_dir = safe_snapshot_download('AI-ModelScope/vicuna-7b-v1.5', download_model=False, check_local=True)
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, trust_remote_code=True)
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogvlm, [
+            ModelGroup([
+                Model('ZhipuAI/cogvlm-chat', 'zai-org/cogvlm-chat-hf'),
+            ]),
+        ],
+        CogVLMLoader,
+        template=TemplateType.cogvlm,
+        architectures=['CogVLMForCausalLM'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.cogvlm))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogagent_chat, [
+            ModelGroup([
+                Model('ZhipuAI/cogagent-chat', 'zai-org/cogagent-chat-hf'),
+            ]),
+        ],
+        CogVLMLoader,
+        template=TemplateType.cogagent_chat,
+        architectures=['CogAgentForCausalLM'],
+        requires=['transformers<4.42', 'timm'],
+        model_arch=ModelArch.cogvlm))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogagent_vqa, [ModelGroup([
+            Model('ZhipuAI/cogagent-vqa', 'zai-org/cogagent-vqa-hf'),
+        ])],
+        CogVLMLoader,
+        template=TemplateType.cogagent_vqa,
+        architectures=['CogAgentForCausalLM'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.cogvlm))
+
+
+class CogVLM2Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        # fix device map 4
+        for layer in model.model.vision.transformer.layers:
+            patch_output_to_input_device(layer.mlp)
+            patch_output_to_input_device(layer.post_attention_layernorm)
+
+        device = next(model.model.vision.linear_proj.parameters()).device
+        model.model.vision.boi.data = model.model.vision.boi.to(device)
+        model.model.vision.eoi.data = model.model.vision.eoi.to(device)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogvlm2, [
+            ModelGroup([
+                Model('ZhipuAI/cogvlm2-llama3-chat-19B', 'zai-org/cogvlm2-llama3-chat-19B'),
+                Model('ZhipuAI/cogvlm2-llama3-chinese-chat-19B', 'zai-org/cogvlm2-llama3-chinese-chat-19B'),
+            ]),
+        ],
+        CogVLM2Loader,
+        template=TemplateType.cogvlm2,
+        architectures=['CogVLMForCausalLM'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.cogvlm))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogvlm2_video,
+        [
+            ModelGroup([
+                Model('ZhipuAI/cogvlm2-video-llama3-chat', 'zai-org/cogvlm2-video-llama3-chat'),
+            ]),
+        ],
+        CogVLM2Loader,
+        template=TemplateType.cogvlm2_video,
+        architectures=['CogVLMVideoForCausalLM'],
+        requires=['decord', 'pytorchvideo', 'transformers>=4.42'],
+        model_arch=ModelArch.cogvlm,
+        tags=['video'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm_edge,
+        [
+            ModelGroup([
+                Model('ZhipuAI/glm-edge-1.5b-chat', 'zai-org/glm-edge-1.5b-chat'),
+                Model('ZhipuAI/glm-edge-4b-chat', 'zai-org/glm-edge-4b-chat'),
+            ]),
+        ],
+        template=TemplateType.chatglm4,
+        architectures=['GlmForCausalLM'],
+        requires=['transformers>=4.46'],
+    ))
+
+
+class GLMEdgeVLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        from transformers import AutoImageProcessor
+        self.auto_tokenizer_cls = AutoImageProcessor
+        return super().get_processor(model_dir, config)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.glm_edge_v,
+        [
+            ModelGroup([
+                Model('ZhipuAI/glm-edge-v-2b', 'zai-org/glm-edge-v-2b'),
+                Model('ZhipuAI/glm-edge-4b-chat', 'zai-org/glm-edge-4b-chat'),
+            ]),
+        ],
+        GLMEdgeVLoader,
+        template=TemplateType.glm_edge_v,
+        architectures=['GlmForCausalLM'],
+        requires=['transformers>=4.46'],
+        model_arch=ModelArch.glm_edge_v,
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm4_moe,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-4.5-Air-Base', 'zai-org/GLM-4.5-Air-Base'),
+                Model('ZhipuAI/GLM-4.5-Air', 'zai-org/GLM-4.5-Air'),
+                Model('ZhipuAI/GLM-4.5-Air-FP8', 'zai-org/GLM-4.5-Air-FP8'),
+                Model('ZhipuAI/GLM-4.5-Base', 'zai-org/GLM-4.5-Base'),
+                Model('ZhipuAI/GLM-4.5', 'zai-org/GLM-4.5'),
+                Model('ZhipuAI/GLM-4.5-FP8', 'zai-org/GLM-4.5-FP8'),
+            ], TemplateType.glm4_5),
+            ModelGroup([
+                Model('ZhipuAI/GLM-4.6', 'zai-org/GLM-4.6'),
+                Model('ZhipuAI/GLM-4.6-FP8', 'zai-org/GLM-4.6-FP8'),
+            ], TemplateType.glm4_5),
+            ModelGroup([
+                Model('ZhipuAI/GLM-4.7', 'zai-org/GLM-4.7'),
+                Model('ZhipuAI/GLM-4.7-FP8', 'zai-org/GLM-4.7-FP8'),
+            ], TemplateType.glm4_7),
+        ],
+        requires=['transformers>=4.54'],
+        architectures=['Glm4MoeForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm4_moe_lite,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-4.7-Flash', 'zai-org/GLM-4.7-Flash'),
+            ], TemplateType.glm4_7),
+        ],
+        requires=['transformers>=5.0.0.dev'],
+        architectures=['Glm4MoeLiteForCausalLM'],
+    ))
+
+
+class Glm4vMoeLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import Glm4vMoeForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Glm4vMoeForConditionalGeneration
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_get_input_embeddings(model.visual, 'patch_embed')
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.glm4v_moe,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-4.5V', 'zai-org/GLM-4.5V'),
+                Model('ZhipuAI/GLM-4.5V-FP8', 'zai-org/GLM-4.5V-FP8'),
+            ]),
+            ModelGroup([
+                Model('ZhipuAI/GLM-4.6V', 'zai-org/GLM-4.6V'),
+                Model('ZhipuAI/GLM-4.6V-FP8', 'zai-org/GLM-4.6V-FP8'),
+            ],
+                       requires=['transformers>=5.0.0.dev']),
+        ],
+        Glm4vMoeLoader,
+        template=TemplateType.glm4_5v,
+        model_arch=ModelArch.glm4v,
+        architectures=['Glm4vMoeForConditionalGeneration'],
+        requires=['transformers>=4.56'],
+    ))
+
+
+class GLMOCRLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForImageTextToText
+        self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
+        model = super().get_model(model_dir, *args, **kwargs)
+        if hasattr(model, 'visual'):
+            patch_get_input_embeddings(model.visual, 'patch_embed')
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.glm_ocr,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-OCR', 'zai-org/GLM-OCR'),
+            ]),
+        ],
+        GLMOCRLoader,
+        template=TemplateType.glm_ocr,
+        model_arch=ModelArch.glm4v,
+        architectures=['GlmOcrForConditionalGeneration'],
+        requires=['transformers>=5.0.1dev0'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm_moe_dsa,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-5', 'zai-org/GLM-5'),
+            ], template=TemplateType.glm4_7),
+            ModelGroup([
+                Model('ZhipuAI/GLM-5.1', 'zai-org/GLM-5.1'),
+                Model('ZhipuAI/GLM-5.1-FP8', 'ZhipuAI/GLM-5.1-FP8'),
+            ],
+                       template=TemplateType.glm5_1),
+            ModelGroup([
+                Model('ZhipuAI/GLM-5.2', 'ZhipuAI/GLM-5.2'),
+                Model('ZhipuAI/GLM-5.2-FP8', 'ZhipuAI/GLM-5.2-FP8'),
+            ],
+                       template=TemplateType.glm5_2),
+        ],
+        architectures=['GlmMoeDsaForCausalLM'],
+        requires=['transformers>=5.2.0'],
+    ))

swift/model/models/internlm.py

@@ -0,0 +1,507 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import AutoTokenizer, PretrainedConfig, PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, safe_snapshot_download
+from ..constant import LLMModelType, MLLMModelType, RMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_output_clone, patch_output_to_input_device
+from ..register import ModelLoader, RewardModelLoader, register_model
+from ..utils import use_submodel_func
+from .qwen import Qwen2AudioLoader
+
+register_model(
+    ModelMeta(
+        LLMModelType.internlm,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm-chat-7b', 'internlm/internlm-chat-7b'),
+                Model('Shanghai_AI_Laboratory/internlm-7b', 'internlm/internlm-7b'),
+                Model('Shanghai_AI_Laboratory/internlm-chat-7b-8k'),
+                Model('Shanghai_AI_Laboratory/internlm-20b', 'internlm/internlm-20b'),
+                Model('Shanghai_AI_Laboratory/internlm-chat-20b', 'internlm/internlm-chat-20b'),
+            ])
+        ],
+        template=TemplateType.internlm,
+        architectures=['InternLMForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.internlm2,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm2-chat-1_8b', 'internlm/internlm2-chat-1_8b'),
+                Model('Shanghai_AI_Laboratory/internlm2-1_8b', 'internlm/internlm2-1_8b'),
+                Model('Shanghai_AI_Laboratory/internlm2-chat-1_8b-sft', 'internlm/internlm2-chat-1_8b-sft'),
+                Model('Shanghai_AI_Laboratory/internlm2-base-7b', 'internlm/internlm2-base-7b'),
+                Model('Shanghai_AI_Laboratory/internlm2-7b', 'internlm/internlm2-7b'),
+                Model('Shanghai_AI_Laboratory/internlm2-chat-7b', 'internlm/internlm2-chat-7b'),
+                Model('Shanghai_AI_Laboratory/internlm2-chat-7b-sft', 'internlm/internlm2-chat-7b-sft'),
+                Model('Shanghai_AI_Laboratory/internlm2-base-20b', 'internlm/internlm2-base-20b'),
+                Model('Shanghai_AI_Laboratory/internlm2-20b', 'internlm/internlm2-20b'),
+                Model('Shanghai_AI_Laboratory/internlm2-chat-20b', 'internlm/internlm2-chat-20b'),
+                Model('Shanghai_AI_Laboratory/internlm2-chat-20b-sft', 'internlm/internlm2-chat-20b-sft'),
+            ]),
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm2-math-7b', 'internlm/internlm2-math-7b'),
+                Model('Shanghai_AI_Laboratory/internlm2-math-base-7b', 'internlm/internlm2-math-base-7b'),
+                Model('Shanghai_AI_Laboratory/internlm2-math-base-20b', 'internlm/internlm2-math-base-20b'),
+                Model('Shanghai_AI_Laboratory/internlm2-math-20b', 'internlm/internlm2-math-20b'),
+            ],
+                       tags=['math']),
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm2_5-1_8b-chat', 'internlm/internlm2_5-1_8b-chat'),
+                Model('Shanghai_AI_Laboratory/internlm2_5-1_8b', 'internlm/internlm2_5-1_8b'),
+                Model('Shanghai_AI_Laboratory/internlm2_5-7b', 'internlm/internlm2_5-7b'),
+                Model('Shanghai_AI_Laboratory/internlm2_5-7b-chat', 'internlm/internlm2_5-7b-chat'),
+                Model('Shanghai_AI_Laboratory/internlm2_5-7b-chat-1m', 'internlm/internlm2_5-7b-chat-1m'),
+                Model('Shanghai_AI_Laboratory/internlm2_5-20b', 'internlm/internlm2_5-20b'),
+                Model('Shanghai_AI_Laboratory/internlm2_5-20b-chat', 'internlm/internlm2_5-20b-chat'),
+            ])
+        ],
+        template=TemplateType.internlm2,
+        requires=['transformers>=4.38'],
+        architectures=['InternLM2ForCausalLM'],
+        model_arch=ModelArch.internlm2,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.internlm3,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm3-8b-instruct', 'internlm/internlm3-8b-instruct'),
+            ]),
+        ],
+        template=TemplateType.internlm2,
+        requires=['transformers>=4.48'],
+        architectures=['InternLM3ForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+
+class InternVLLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        self.auto_tokenizer_cls = AutoTokenizer
+        return super().get_processor(model_dir, config)
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        if self.model_info.quant_method == 'bnb':  # 'is_training'
+            # patch: bnb backward shape mismatch bug
+            if model is not None and model.language_model is not None:
+                model.language_model.output.state.force_no_igemmlt = True
+        use_submodel_func(model, 'language_model')
+        patch_output_clone(model.language_model.get_input_embeddings())
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.internvl_chat,
+        [
+            ModelGroup([
+                Model('OpenGVLab/Mini-InternVL-Chat-2B-V1-5', 'OpenGVLab/Mini-InternVL-Chat-2B-V1-5'),
+                Model('AI-ModelScope/InternVL-Chat-V1-5', 'OpenGVLab/InternVL-Chat-V1-5'),
+                Model('AI-ModelScope/InternVL-Chat-V1-5-int8', 'OpenGVLab/InternVL-Chat-V1-5-int8'),
+            ],
+                       template=TemplateType.internvl,
+                       requires=['transformers>=4.35', 'timm'],
+                       tags=['vision']),
+            ModelGroup([
+                Model('OpenGVLab/Mini-InternVL-Chat-4B-V1-5', 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'),
+            ],
+                       template=TemplateType.internvl_phi3,
+                       requires=['transformers>=4.35,<4.42', 'timm'],
+                       tags=['vision']),
+            ModelGroup(
+                [
+                    Model('OpenGVLab/InternVL2-1B', 'OpenGVLab/InternVL2-1B'),
+                    Model('OpenGVLab/InternVL2-2B', 'OpenGVLab/InternVL2-2B'),
+                    Model('OpenGVLab/InternVL2-8B', 'OpenGVLab/InternVL2-8B'),
+                    Model('OpenGVLab/InternVL2-26B', 'OpenGVLab/InternVL2-26B'),
+                    Model('OpenGVLab/InternVL2-40B', 'OpenGVLab/InternVL2-40B'),
+                    Model('OpenGVLab/InternVL2-Llama3-76B', 'OpenGVLab/InternVL2-Llama3-76B'),
+                    # (infer use lmdeploy)
+                    Model('OpenGVLab/InternVL2-2B-AWQ', 'OpenGVLab/InternVL2-2B-AWQ'),
+                    Model('OpenGVLab/InternVL2-8B-AWQ', 'OpenGVLab/InternVL2-8B-AWQ'),
+                    Model('OpenGVLab/InternVL2-26B-AWQ', 'OpenGVLab/InternVL2-26B-AWQ'),
+                    Model('OpenGVLab/InternVL2-40B-AWQ', 'OpenGVLab/InternVL2-40B-AWQ'),
+                    Model('OpenGVLab/InternVL2-Llama3-76B-AWQ', 'OpenGVLab/InternVL2-Llama3-76B-AWQ'),
+                    # mpo
+                    Model('OpenGVLab/InternVL2-8B-MPO', 'OpenGVLab/InternVL2-8B-MPO'),
+                    # pretrain
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-1B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-1B-Pretrain'),
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-2B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-2B-Pretrain'),
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-4B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-4B-Pretrain'),
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-8B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-8B-Pretrain'),
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-26B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-26B-Pretrain'),
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-40B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-40B-Pretrain'),
+                    Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-Llama3-76B-Pretrain',
+                          'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-Llama3-76B-Pretrain'),
+                ],
+                template=TemplateType.internvl2,
+                requires=['transformers>=4.36', 'timm'],
+                tags=['vision', 'video'],
+            ),
+            ModelGroup(
+                [
+                    Model('OpenGVLab/InternVL2-4B', 'OpenGVLab/InternVL2-4B'),
+                ],
+                template=TemplateType.internvl2_phi3,
+                requires=['transformers>=4.36,<4.42', 'timm'],
+                tags=['vision', 'video'],
+            ),
+            ModelGroup(
+                [
+                    Model('OpenGVLab/InternVL2_5-1B', 'OpenGVLab/InternVL2_5-1B'),
+                    Model('OpenGVLab/InternVL2_5-2B', 'OpenGVLab/InternVL2_5-2B'),
+                    Model('OpenGVLab/InternVL2_5-4B', 'OpenGVLab/InternVL2_5-4B'),
+                    Model('OpenGVLab/InternVL2_5-8B', 'OpenGVLab/InternVL2_5-8B'),
+                    Model('OpenGVLab/InternVL2_5-26B', 'OpenGVLab/InternVL2_5-26B'),
+                    Model('OpenGVLab/InternVL2_5-38B', 'OpenGVLab/InternVL2_5-38B'),
+                    Model('OpenGVLab/InternVL2_5-78B', 'OpenGVLab/InternVL2_5-78B'),
+                    # quant (infer use lmdeploy)
+                    Model('OpenGVLab/InternVL2_5-4B-AWQ', 'OpenGVLab/InternVL2_5-4B-AWQ'),
+                    Model('OpenGVLab/InternVL2_5-8B-AWQ', 'OpenGVLab/InternVL2_5-8B-AWQ'),
+                    Model('OpenGVLab/InternVL2_5-26B-AWQ', 'OpenGVLab/InternVL2_5-26B-AWQ'),
+                    Model('OpenGVLab/InternVL2_5-38B-AWQ', 'OpenGVLab/InternVL2_5-38B-AWQ'),
+                    Model('OpenGVLab/InternVL2_5-78B-AWQ', 'OpenGVLab/InternVL2_5-78B-AWQ'),
+                    # mpo
+                    Model('OpenGVLab/InternVL2_5-1B-MPO', 'OpenGVLab/InternVL2_5-1B-MPO'),
+                    Model('OpenGVLab/InternVL2_5-2B-MPO', 'OpenGVLab/InternVL2_5-2B-MPO'),
+                    Model('OpenGVLab/InternVL2_5-4B-MPO', 'OpenGVLab/InternVL2_5-4B-MPO'),
+                    Model('OpenGVLab/InternVL2_5-8B-MPO', 'OpenGVLab/InternVL2_5-8B-MPO'),
+                    Model('OpenGVLab/InternVL2_5-26B-MPO', 'OpenGVLab/InternVL2_5-26B-MPO'),
+                    Model('OpenGVLab/InternVL2_5-38B-MPO', 'OpenGVLab/InternVL2_5-38B-MPO'),
+                    Model('OpenGVLab/InternVL2_5-78B-MPO', 'OpenGVLab/InternVL2_5-78B-MPO'),
+                ],
+                template=TemplateType.internvl2_5,
+                requires=['transformers>=4.36', 'timm'],
+                tags=['vision', 'video'],
+            ),
+            ModelGroup(
+                [
+                    # pretrain
+                    Model('OpenGVLab/InternVL3-1B-Pretrained', 'OpenGVLab/InternVL3-1B-Pretrained'),
+                    Model('OpenGVLab/InternVL3-2B-Pretrained', 'OpenGVLab/InternVL3-2B-Pretrained'),
+                    Model('OpenGVLab/InternVL3-8B-Pretrained', 'OpenGVLab/InternVL3-8B-Pretrained'),
+                    Model('OpenGVLab/InternVL3-9B-Pretrained', 'OpenGVLab/InternVL3-9B-Pretrained'),
+                    Model('OpenGVLab/InternVL3-14B-Pretrained', 'OpenGVLab/InternVL3-14B-Pretrained'),
+                    Model('OpenGVLab/InternVL3-38B-Pretrained', 'OpenGVLab/InternVL3-38B-Pretrained'),
+                    Model('OpenGVLab/InternVL3-78B-Pretrained', 'OpenGVLab/InternVL3-78B-Pretrained'),
+                    # instruct
+                    Model('OpenGVLab/InternVL3-1B-Instruct', 'OpenGVLab/InternVL3-1B-Instruct'),
+                    Model('OpenGVLab/InternVL3-2B-Instruct', 'OpenGVLab/InternVL3-2B-Instruct'),
+                    Model('OpenGVLab/InternVL3-8B-Instruct', 'OpenGVLab/InternVL3-8B-Instruct'),
+                    Model('OpenGVLab/InternVL3-9B-Instruct', 'OpenGVLab/InternVL3-9B-Instruct'),
+                    Model('OpenGVLab/InternVL3-14B-Instruct', 'OpenGVLab/InternVL3-14B-Instruct'),
+                    Model('OpenGVLab/InternVL3-38B-Instruct', 'OpenGVLab/InternVL3-38B-Instruct'),
+                    Model('OpenGVLab/InternVL3-78B-Instruct', 'OpenGVLab/InternVL3-78B-Instruct'),
+                    # mpo
+                    Model('OpenGVLab/InternVL3-1B', 'OpenGVLab/InternVL3-1B'),
+                    Model('OpenGVLab/InternVL3-2B', 'OpenGVLab/InternVL3-2B'),
+                    Model('OpenGVLab/InternVL3-8B', 'OpenGVLab/InternVL3-8B'),
+                    Model('OpenGVLab/InternVL3-9B', 'OpenGVLab/InternVL3-9B'),
+                    Model('OpenGVLab/InternVL3-14B', 'OpenGVLab/InternVL3-14B'),
+                    Model('OpenGVLab/InternVL3-38B', 'OpenGVLab/InternVL3-38B'),
+                    Model('OpenGVLab/InternVL3-78B', 'OpenGVLab/InternVL3-78B'),
+                    # awq (Use lmdeploy for inference.)
+                    Model('OpenGVLab/InternVL3-1B-AWQ', 'OpenGVLab/InternVL3-1B-AWQ'),
+                    Model('OpenGVLab/InternVL3-2B-AWQ', 'OpenGVLab/InternVL3-2B-AWQ'),
+                    Model('OpenGVLab/InternVL3-8B-AWQ', 'OpenGVLab/InternVL3-8B-AWQ'),
+                    Model('OpenGVLab/InternVL3-9B-AWQ', 'OpenGVLab/InternVL3-9B-AWQ'),
+                    Model('OpenGVLab/InternVL3-14B-AWQ', 'OpenGVLab/InternVL3-14B-AWQ'),
+                    Model('OpenGVLab/InternVL3-38B-AWQ', 'OpenGVLab/InternVL3-38B-AWQ'),
+                    Model('OpenGVLab/InternVL3-78B-AWQ', 'OpenGVLab/InternVL3-78B-AWQ'),
+                    # SenseNova-SI
+                    Model('SenseNova/SenseNova-SI-InternVL3-2B', 'sensenova/SenseNova-SI-InternVL3-2B'),
+                    Model('SenseNova/SenseNova-SI-InternVL3-8B', 'sensenova/SenseNova-SI-InternVL3-8B'),
+                    Model('SenseNova/SenseNova-SI-1.1-InternVL3-2B', 'sensenova/SenseNova-SI-1.1-InternVL3-2B'),
+                    Model('SenseNova/SenseNova-SI-1.1-InternVL3-8B', 'sensenova/SenseNova-SI-1.1-InternVL3-8B'),
+                ],
+                template=TemplateType.internvl2_5,
+                requires=['transformers>=4.37.2', 'timm'],
+                tags=['vision', 'video'],
+            ),
+            ModelGroup(
+                [
+                    # pretrain
+                    Model('OpenGVLab/InternVL3_5-1B-Pretrained', 'OpenGVLab/InternVL3_5-1B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-2B-Pretrained', 'OpenGVLab/InternVL3_5-2B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-4B-Pretrained', 'OpenGVLab/InternVL3_5-4B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-8B-Pretrained', 'OpenGVLab/InternVL3_5-8B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-14B-Pretrained', 'OpenGVLab/InternVL3_5-14B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-38B-Pretrained', 'OpenGVLab/InternVL3_5-38B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-30B-A3B-Pretrained', 'OpenGVLab/InternVL3_5-30B-A3B-Pretrained'),
+                    Model('OpenGVLab/InternVL3_5-241B-A28B-Pretrained', 'OpenGVLab/InternVL3_5-241B-A28B-Pretrained'),
+                    # Instruct
+                    Model('OpenGVLab/InternVL3_5-1B-Instruct', 'OpenGVLab/InternVL3_5-1B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-2B-Instruct', 'OpenGVLab/InternVL3_5-2B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-4B-Instruct', 'OpenGVLab/InternVL3_5-4B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-8B-Instruct', 'OpenGVLab/InternVL3_5-8B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-14B-Instruct', 'OpenGVLab/InternVL3_5-14B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-38B-Instruct', 'OpenGVLab/InternVL3_5-38B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-30B-A3B-Instruct', 'OpenGVLab/InternVL3_5-30B-A3B-Instruct'),
+                    Model('OpenGVLab/InternVL3_5-241B-A28B-Instruct', 'OpenGVLab/InternVL3_5-241B-A28B-Instruct'),
+                    # MPO
+                    Model('OpenGVLab/InternVL3_5-1B-MPO', 'OpenGVLab/InternVL3_5-1B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-2B-MPO', 'OpenGVLab/InternVL3_5-2B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-4B-MPO', 'OpenGVLab/InternVL3_5-4B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-8B-MPO', 'OpenGVLab/InternVL3_5-8B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-14B-MPO', 'OpenGVLab/InternVL3_5-14B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-38B-MPO', 'OpenGVLab/InternVL3_5-38B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-30B-A3B-MPO', 'OpenGVLab/InternVL3_5-30B-A3B-MPO'),
+                    Model('OpenGVLab/InternVL3_5-241B-A28B-MPO', 'OpenGVLab/InternVL3_5-241B-A28B-MPO'),
+                    #
+                    Model('OpenGVLab/InternVL3_5-1B', 'OpenGVLab/InternVL3_5-1B'),
+                    Model('OpenGVLab/InternVL3_5-2B', 'OpenGVLab/InternVL3_5-2B'),
+                    Model('OpenGVLab/InternVL3_5-4B', 'OpenGVLab/InternVL3_5-4B'),
+                    Model('OpenGVLab/InternVL3_5-8B', 'OpenGVLab/InternVL3_5-8B'),
+                    Model('OpenGVLab/InternVL3_5-14B', 'OpenGVLab/InternVL3_5-14B'),
+                    Model('OpenGVLab/InternVL3_5-38B', 'OpenGVLab/InternVL3_5-38B'),
+                    Model('OpenGVLab/InternVL3_5-30B-A3B', 'OpenGVLab/InternVL3_5-30B-A3B'),
+                    Model('OpenGVLab/InternVL3_5-241B-A28B', 'OpenGVLab/InternVL3_5-241B-A28B'),
+                ],
+                template=TemplateType.internvl3_5,
+                requires=['transformers>=4.37.2', 'timm'],
+                tags=['vision', 'video'],
+            ),
+            ModelGroup(
+                [
+                    Model('OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview',
+                          'OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview'),
+                ],
+                template=TemplateType.internvl3_5_gpt,
+                requires=['transformers>=4.37.2', 'timm'],
+                tags=['vision', 'video'],
+            ),
+        ],
+        InternVLLoader,
+        architectures=['InternVLChatModel'],
+        model_arch=ModelArch.internvl,
+    ))
+
+
+class Interns1Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers.modeling_utils import PreTrainedModel
+        model = super().get_model(model_dir, *args, **kwargs)
+        if not hasattr(PreTrainedModel, '_old_enable_input_require_grads'):
+            old_enable_input_require_grads = PreTrainedModel.enable_input_require_grads
+
+            def patched_enable_input_require_grads(self):
+
+                def make_inputs_require_grads(module, input, output):
+                    if isinstance(output, tuple):
+                        output[0].requires_grad_(True)
+                    else:
+                        output.requires_grad_(True)
+
+                self._require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)
+
+            PreTrainedModel.enable_input_require_grads = patched_enable_input_require_grads
+            PreTrainedModel._old_enable_input_require_grads = old_enable_input_require_grads
+        return model
+
+
+class InternVLHfLoader(Interns1Loader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForImageTextToText
+        self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.internvl,
+        [
+            ModelGroup([
+                Model('OpenGVLab/InternVL3-1B-hf', 'OpenGVLab/InternVL3-1B-hf'),
+                Model('OpenGVLab/InternVL3-2B-hf', 'OpenGVLab/InternVL3-2B-hf'),
+                Model('OpenGVLab/InternVL3-8B-hf', 'OpenGVLab/InternVL3-8B-hf'),
+                Model('OpenGVLab/InternVL3-9B-hf', 'OpenGVLab/InternVL3-9B-hf'),
+                Model('OpenGVLab/InternVL3-14B-hf', 'OpenGVLab/InternVL3-14B-hf'),
+                Model('OpenGVLab/InternVL3-38B-hf', 'OpenGVLab/InternVL3-38B-hf'),
+                Model('OpenGVLab/InternVL3-78B-hf', 'OpenGVLab/InternVL3-78B-hf'),
+            ],
+                       template=TemplateType.internvl_hf,
+                       requires=['transformers>=4.52.1', 'timm']),
+            ModelGroup([
+                Model('OpenGVLab/InternVL3_5-1B-HF', 'OpenGVLab/InternVL3_5-1B-HF'),
+                Model('OpenGVLab/InternVL3_5-2B-HF', 'OpenGVLab/InternVL3_5-2B-HF'),
+                Model('OpenGVLab/InternVL3_5-4B-HF', 'OpenGVLab/InternVL3_5-4B-HF'),
+                Model('OpenGVLab/InternVL3_5-8B-HF', 'OpenGVLab/InternVL3_5-8B-HF'),
+                Model('OpenGVLab/InternVL3_5-14B-HF', 'OpenGVLab/InternVL3_5-14B-HF'),
+                Model('OpenGVLab/InternVL3_5-38B-HF', 'OpenGVLab/InternVL3_5-38B-HF'),
+                Model('OpenGVLab/InternVL3_5-30B-A3B-HF', 'OpenGVLab/InternVL3_5-30B-A3B-HF'),
+                Model('OpenGVLab/InternVL3_5-241B-A28B-HF', 'OpenGVLab/InternVL3_5-241B-A28B-HF'),
+            ],
+                       template=TemplateType.internvl_hf,
+                       requires=['transformers>=4.52.1', 'timm']),
+            ModelGroup([
+                Model('OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF',
+                      'OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF'),
+            ],
+                       template=TemplateType.internvl_hf,
+                       requires=['transformers>=4.55.0', 'timm']),
+        ],
+        InternVLHfLoader,
+        architectures=['InternVLForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        tags=['vision', 'video'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.interns1,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/Intern-S1-mini', 'internlm/Intern-S1-mini'),
+                Model('Shanghai_AI_Laboratory/Intern-S1', 'internlm/Intern-S1'),
+                Model('Shanghai_AI_Laboratory/Intern-S1-mini-FP8', 'internlm/Intern-S1-mini-FP8'),
+                Model('Shanghai_AI_Laboratory/Intern-S1-FP8', 'internlm/Intern-S1-FP8'),
+            ]),
+        ],
+        Interns1Loader,
+        template=TemplateType.interns1,
+        architectures=['InternS1ForConditionalGeneration'],
+        model_arch=ModelArch.interns1,
+        requires=['transformers>=4.55.2,<4.56'],
+        tags=['vision', 'video'],
+    ))
+
+
+class Xcomposer2Loader(ModelLoader):
+    version = 'v2'
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        if self.version == 'v2-4khd':
+            from transformers import CLIPVisionModel
+
+            def load_model(self):
+                self.vision_tower_name = safe_snapshot_download(
+                    'AI-ModelScope/clip-vit-large-patch14-336', check_local=True)
+                self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name)
+                self.vision_tower.requires_grad_(False)
+                self.is_loaded = True
+
+            CLIPVisionTower = get_class_from_dynamic_module('build_mlp.CLIPVisionTower', model_dir)
+            CLIPVisionTower.load_model = load_model
+        model = super().get_model(model_dir, *args, **kwargs)
+        model.vit.vision_tower.gradient_checkpointing_enable()
+        if self.version == 'v2':
+            # fix AttributeError: no attribute 'attention_dropout'
+            model.model.layers[0].attention.__class__.attention_dropout = 0.
+
+        if self.version == 'v2.5':
+            patch_output_to_input_device(model.vit)
+            patch_output_to_input_device(model.vision_proj)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.xcomposer2,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm-xcomposer2-7b', 'internlm/internlm-xcomposer2-7b'),
+            ], ),
+        ],
+        Xcomposer2Loader,
+        template=TemplateType.xcomposer2,
+        architectures=['InternLMXComposer2ForCausalLM'],
+        model_arch=ModelArch.xcomposer,
+        tags=['vision'],
+    ))
+
+
+class Xcomposer2_4khdLoader(Xcomposer2Loader):
+    version = 'v2-4khd'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.xcomposer2_4khd,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm-xcomposer2-4khd-7b', 'internlm/internlm-xcomposer2-4khd-7b'),
+            ], ),
+        ],
+        Xcomposer2_4khdLoader,
+        template=TemplateType.xcomposer2,
+        architectures=['InternLM2ForCausalLM', 'InternLMXComposer2ForCausalLM'],
+        model_arch=ModelArch.xcomposer,
+        tags=['vision'],
+    ))
+
+
+class Xcomposer2_5Loader(Xcomposer2Loader):
+    version = 'v2.5'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.xcomposer2_5,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm-xcomposer2d5-7b', 'internlm/internlm-xcomposer2d5-7b'),
+                Model('Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:base',
+                      'internlm/internlm-xcomposer2d5-ol-7b:base')
+            ]),
+        ],
+        Xcomposer2_5Loader,
+        template=TemplateType.xcomposer2_5,
+        architectures=['InternLMXComposer2ForCausalLM'],
+        model_arch=ModelArch.xcomposer,
+        tags=['vision'],
+        requires=['decord'],
+        # target_modules: attention.wqkv attention.wo feed_forward.w1 feed_forward.w2 feed_forward.w3
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.xcomposer2_5_ol_audio,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:audio',
+                      'internlm/internlm-xcomposer2d5-ol-7b:audio'),
+            ]),
+        ],
+        Qwen2AudioLoader,
+        template=TemplateType.qwen2_audio,
+        requires=['transformers>=4.45'],
+        architectures=['Qwen2AudioForConditionalGeneration'],
+        model_arch=ModelArch.qwen2_audio,
+        tags=['audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.internlm2_reward,
+        [
+            ModelGroup([
+                Model('Shanghai_AI_Laboratory/internlm2-1_8b-reward', 'internlm/internlm2-1_8b-reward'),
+                Model('Shanghai_AI_Laboratory/internlm2-7b-reward', 'internlm/internlm2-7b-reward'),
+                Model('Shanghai_AI_Laboratory/internlm2-20b-reward', 'internlm/internlm2-20b-reward'),
+            ]),
+        ],
+        RewardModelLoader,
+        template=TemplateType.internlm2_reward,
+        is_reward=True,
+        requires=['transformers>=4.38'],
+        architectures=['InternLM2ForRewardModel'],
+    ))

swift/model/models/llama.py

@@ -0,0 +1,348 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+import sys
+from transformers import PreTrainedModel
+
+from swift.template import TemplateType
+from swift.utils import get_device, git_clone_github
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+
+class LlamaLoader(ModelLoader):
+
+    def get_config(self, model_dir):
+        config = super().get_config(model_dir)
+        if getattr(config, 'pretraining_tp', 1) > 1:
+            config.pretraining_tp = 1
+        return config
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.llama,
+        [
+            # llama2
+            ModelGroup(
+                [
+                    # base
+                    Model('modelscope/Llama-2-7b-ms', 'meta-llama/Llama-2-7b-hf'),
+                    Model('modelscope/Llama-2-13b-ms', 'meta-llama/Llama-2-13b-hf'),
+                    Model('modelscope/Llama-2-70b-ms', 'meta-llama/Llama-2-70b-hf'),
+                    # chat
+                    Model('modelscope/Llama-2-7b-chat-ms', 'meta-llama/Llama-2-7b-chat-hf'),
+                    Model('modelscope/Llama-2-13b-chat-ms', 'meta-llama/Llama-2-13b-chat-hf'),
+                    Model('modelscope/Llama-2-70b-chat-ms', 'meta-llama/Llama-2-70b-chat-hf'),
+                ],
+                TemplateType.llama,
+                ignore_patterns=[r'.+\.bin$']),
+            # chinese-llama2
+            ModelGroup(
+                [
+                    # base
+                    Model('AI-ModelScope/chinese-llama-2-1.3b', 'hfl/chinese-llama-2-1.3b'),
+                    Model('AI-ModelScope/chinese-llama-2-7b', 'hfl/chinese-llama-2-7b'),
+                    Model('AI-ModelScope/chinese-llama-2-7b-16k', 'hfl/chinese-llama-2-7b-16k'),
+                    Model('AI-ModelScope/chinese-llama-2-7b-64k', 'hfl/chinese-llama-2-7b-64k'),
+                    Model('AI-ModelScope/chinese-llama-2-13b', 'hfl/chinese-llama-2-13b'),
+                    Model('AI-ModelScope/chinese-llama-2-13b-16k', 'hfl/chinese-llama-2-13b-16k'),
+                    # chat
+                    Model('AI-ModelScope/chinese-alpaca-2-1.3b', 'hfl/chinese-alpaca-2-1.3b'),
+                    Model('AI-ModelScope/chinese-alpaca-2-7b', 'hfl/chinese-alpaca-2-7b'),
+                    Model('AI-ModelScope/chinese-alpaca-2-7b-16k', 'hfl/chinese-alpaca-2-7b-16k'),
+                    Model('AI-ModelScope/chinese-alpaca-2-7b-64k', 'hfl/chinese-alpaca-2-7b-64k'),
+                    Model('AI-ModelScope/chinese-alpaca-2-13b', 'hfl/chinese-alpaca-2-13b'),
+                    Model('AI-ModelScope/chinese-alpaca-2-13b-16k', 'hfl/chinese-alpaca-2-13b-16k'),
+                ],
+                TemplateType.llama),
+            # base quant
+            ModelGroup([
+                Model('AI-ModelScope/Llama-2-7b-AQLM-2Bit-1x16-hf', 'ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf'),
+            ],
+                       TemplateType.llama,
+                       requires=['transformers>=4.38', 'aqlm', 'torch>=2.2.0']),
+            ModelGroup([
+                Model('FlagAlpha/Atom-7B', 'FlagAlpha/Atom-7B'),
+                Model('FlagAlpha/Atom-7B-Chat', 'FlagAlpha/Atom-7B-Chat'),
+            ],
+                       template=TemplateType.atom),
+            ModelGroup([
+                Model('langboat/Mengzi3-13B-Base', 'Langboat/Mengzi3-13B-Base'),
+            ],
+                       template=TemplateType.mengzi),
+            ModelGroup([
+                Model('AI-ModelScope/NuminaMath-7B-TIR', 'AI-MO/NuminaMath-7B-TIR'),
+            ],
+                       template=TemplateType.numina,
+                       tags=['math']),
+            ModelGroup([
+                Model('Fengshenbang/Ziya2-13B-Base', 'IDEA-CCNL/Ziya2-13B-Base'),
+                Model('Fengshenbang/Ziya2-13B-Chat', 'IDEA-CCNL/Ziya2-13B-Chat'),
+            ],
+                       template=TemplateType.ziya),
+            ModelGroup([
+                Model('InfiniAI/Megrez-3b-Instruct', 'Infinigence/Megrez-3B-Instruct'),
+            ], TemplateType.megrez),
+            # deepseek
+            ModelGroup([
+                Model('deepseek-ai/deepseek-llm-7b-base', 'deepseek-ai/deepseek-llm-7b-base'),
+                Model('deepseek-ai/deepseek-llm-7b-chat', 'deepseek-ai/deepseek-llm-7b-chat'),
+                Model('deepseek-ai/deepseek-llm-67b-base', 'deepseek-ai/deepseek-llm-67b-base'),
+                Model('deepseek-ai/deepseek-llm-67b-chat', 'deepseek-ai/deepseek-llm-67b-chat'),
+            ], TemplateType.deepseek),
+            ModelGroup(
+                [
+                    Model('deepseek-ai/deepseek-math-7b-base', 'deepseek-ai/deepseek-math-7b-base'),
+                    Model('deepseek-ai/deepseek-math-7b-instruct', 'deepseek-ai/deepseek-math-7b-instruct'),
+                    Model('deepseek-ai/deepseek-math-7b-rl', 'deepseek-ai/deepseek-math-7b-rl'),
+                ],
+                TemplateType.deepseek,
+                tags=['math'],
+            ),
+            ModelGroup(
+                [
+                    Model('deepseek-ai/deepseek-coder-1.3b-base', 'deepseek-ai/deepseek-coder-1.3b-base'),
+                    Model('deepseek-ai/deepseek-coder-1.3b-instruct', 'deepseek-ai/deepseek-coder-1.3b-instruct'),
+                    Model('deepseek-ai/deepseek-coder-6.7b-base', 'deepseek-ai/deepseek-coder-6.7b-base'),
+                    Model('deepseek-ai/deepseek-coder-6.7b-instruct', 'deepseek-ai/deepseek-coder-6.7b-instruct'),
+                    Model('deepseek-ai/deepseek-coder-33b-base', 'deepseek-ai/deepseek-coder-33b-base'),
+                    Model('deepseek-ai/deepseek-coder-33b-instruct', 'deepseek-ai/deepseek-coder-33b-instruct'),
+                ],
+                TemplateType.deepseek,
+                tags=['coding'],
+            ),
+            # MiniMind2
+            ModelGroup(
+                [
+                    # MiniMind2
+                    Model('gongjy/MiniMind2', 'jingyaogong/MiniMind2'),
+                    # MiniMind2-Small
+                    Model(None, 'jingyaogong/MiniMind2-Small'),
+                ],
+                TemplateType.minimind,
+                requires=['transformers>=4.57.1']),
+            # llama3
+            ModelGroup(
+                [
+                    # chat
+                    Model('LLM-Research/Meta-Llama-3-8B-Instruct', 'meta-llama/Meta-Llama-3-8B-Instruct'),
+                    Model('LLM-Research/Meta-Llama-3-70B-Instruct', 'meta-llama/Meta-Llama-3-70B-Instruct'),
+                    # base
+                    Model('LLM-Research/Meta-Llama-3-8B', 'meta-llama/Meta-Llama-3-8B'),
+                    Model('LLM-Research/Meta-Llama-3-70B', 'meta-llama/Meta-Llama-3-70B'),
+                ],
+                TemplateType.llama3),
+            # llama3-quant
+            ModelGroup([
+                Model('swift/Meta-Llama-3-8B-Instruct-GPTQ-Int4', 'study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int4'),
+                Model('swift/Meta-Llama-3-8B-Instruct-GPTQ-Int8', 'study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int8'),
+                Model('swift/Meta-Llama-3-8B-Instruct-AWQ', 'study-hjt/Meta-Llama-3-8B-Instruct-AWQ'),
+                Model('swift/Meta-Llama-3-70B-Instruct-GPTQ-Int4', 'study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int4'),
+                Model('swift/Meta-Llama-3-70B-Instruct-GPTQ-Int8', 'study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int8'),
+                Model('swift/Meta-Llama-3-70B-Instruct-AWQ', 'study-hjt/Meta-Llama-3-70B-Instruct-AWQ'),
+            ], TemplateType.llama3),
+            # chinese-llama3
+            ModelGroup([
+                Model('ChineseAlpacaGroup/llama-3-chinese-8b-instruct', 'hfl/llama-3-chinese-8b-instruct'),
+                Model('ChineseAlpacaGroup/llama-3-chinese-8b', 'hfl/llama-3-chinese-8b'),
+            ], TemplateType.llama3),
+            # llama3.1
+            ModelGroup(
+                [
+                    # chat
+                    Model('LLM-Research/Meta-Llama-3.1-8B-Instruct', 'meta-llama/Meta-Llama-3.1-8B-Instruct'),
+                    Model('LLM-Research/Meta-Llama-3.1-70B-Instruct', 'meta-llama/Meta-Llama-3.1-70B-Instruct'),
+                    Model('LLM-Research/Meta-Llama-3.1-405B-Instruct', 'meta-llama/Meta-Llama-3.1-405B-Instruct'),
+                    # base
+                    Model('LLM-Research/Meta-Llama-3.1-8B', 'meta-llama/Meta-Llama-3.1-8B'),
+                    Model('LLM-Research/Meta-Llama-3.1-70B', 'meta-llama/Meta-Llama-3.1-70B'),
+                    Model('LLM-Research/Meta-Llama-3.1-405B', 'meta-llama/Meta-Llama-3.1-405B'),
+                    # fp8
+                    Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', 'meta-llama/Meta-Llama-3.1-70B-Instruct-FP8'),
+                    Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8',
+                          'meta-llama/Meta-Llama-3.1-405B-Instruct-FP8'),
+                ],
+                TemplateType.llama3_2,
+                requires=['transformers>=4.43']),
+            # llama3.1-quant
+            ModelGroup(
+                [
+                    # bnb-nf4
+                    Model('LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4',
+                          'hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4'),
+                    Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit',
+                          'unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit'),
+                    Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4',
+                          'hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4'),
+                    # gptq-int4
+                    Model('LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4',
+                          'hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4'),
+                    Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4',
+                          'hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4'),
+                    Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4',
+                          'hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4'),
+                    # awq-int4
+                    Model('LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4',
+                          'hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4'),
+                    Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4',
+                          'hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4'),
+                    Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4',
+                          'hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4'),
+                ],
+                TemplateType.llama3_2,
+                requires=['transformers>=4.43']),
+            # nvidia Nemotron
+            ModelGroup([
+                Model('AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', 'nvidia/Llama-3.1-Nemotron-70B-Instruct-HF'),
+            ],
+                       TemplateType.llama3_2,
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B', 'Skywork/Skywork-o1-Open-Llama-3.1-8B'),
+            ],
+                       TemplateType.skywork_o1,
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('LLM-Research/Llama-3.2-1B', 'meta-llama/Llama-3.2-1B'),
+                Model('LLM-Research/Llama-3.2-3B', 'meta-llama/Llama-3.2-3B'),
+                Model('LLM-Research/Llama-3.2-1B-Instruct', 'meta-llama/Llama-3.2-1B-Instruct'),
+                Model('LLM-Research/Llama-3.2-3B-Instruct', 'meta-llama/Llama-3.2-3B-Instruct'),
+            ],
+                       template=TemplateType.llama3_2,
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('LLM-Research/Llama-3.3-70B-Instruct', 'meta-llama/Llama-3.3-70B-Instruct'),
+                Model('unsloth/Llama-3.3-70B-Instruct-bnb-4bit', 'unsloth/Llama-3.3-70B-Instruct-bnb-4bit'),
+            ],
+                       template=TemplateType.llama3_2,
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('ZhipuAI/LongWriter-llama3.1-8b', 'zai-org/LongWriter-llama3.1-8b'),
+            ],
+                       TemplateType.longwriter_llama,
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('deepseek-ai/DeepSeek-R1-Distill-Llama-8B', 'deepseek-ai/DeepSeek-R1-Distill-Llama-8B'),
+                Model('deepseek-ai/DeepSeek-R1-Distill-Llama-70B', 'deepseek-ai/DeepSeek-R1-Distill-Llama-70B'),
+            ], TemplateType.deepseek_r1),
+            # MiniCPM5
+            ModelGroup([
+                Model('OpenBMB/MiniCPM5-1B', 'openbmb/MiniCPM5-1B'),
+                Model('OpenBMB/MiniCPM5-1B-Base', 'openbmb/MiniCPM5-1B-Base'),
+                Model('OpenBMB/MiniCPM5-1B-SFT', 'openbmb/MiniCPM5-1B-SFT'),
+            ],
+                       TemplateType.minicpm5,
+                       requires=['transformers>=5.6']),
+            ModelGroup([
+                Model('LLM-Research/Reflection-Llama-3.1-70B', 'mattshumer/Reflection-Llama-3.1-70B'),
+            ],
+                       TemplateType.reflection,
+                       requires=['transformers>=4.43']),
+        ],
+        LlamaLoader,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))
+
+
+class Llama3_2VisionLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import MllamaForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or MllamaForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llama3_2_vision,
+        [
+            ModelGroup([
+                Model('LLM-Research/Llama-3.2-11B-Vision-Instruct', 'meta-llama/Llama-3.2-11B-Vision-Instruct'),
+                Model('LLM-Research/Llama-3.2-90B-Vision-Instruct', 'meta-llama/Llama-3.2-90B-Vision-Instruct'),
+                Model('LLM-Research/Llama-3.2-11B-Vision', 'meta-llama/Llama-3.2-11B-Vision'),
+                Model('LLM-Research/Llama-3.2-90B-Vision', 'meta-llama/Llama-3.2-90B-Vision'),
+            ])
+        ],
+        Llama3_2VisionLoader,
+        template=TemplateType.llama3_2_vision,
+        requires=['transformers>=4.45'],
+        architectures=['MllamaForConditionalGeneration'],
+        model_arch=ModelArch.llama3_2_vision,
+        tags=['vision'],
+    ))
+
+
+class Llama4Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import Llama4ForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Llama4ForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llama4,
+        [
+            ModelGroup([
+                Model('LLM-Research/Llama-4-Scout-17B-16E', 'meta-llama/Llama-4-Scout-17B-16E'),
+                Model('LLM-Research/Llama-4-Maverick-17B-128E', 'meta-llama/Llama-4-Maverick-17B-128E'),
+                Model('LLM-Research/Llama-4-Scout-17B-16E-Instruct', 'meta-llama/Llama-4-Scout-17B-16E-Instruct'),
+                Model('LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8',
+                      'meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8'),
+                Model('LLM-Research/Llama-4-Maverick-17B-128E-Instruct',
+                      'meta-llama/Llama-4-Maverick-17B-128E-Instruct'),
+            ])
+        ],
+        Llama4Loader,
+        template=TemplateType.llama4,
+        requires=['transformers>=4.51'],
+        model_arch=ModelArch.llama4,
+        architectures=['Llama4ForConditionalGeneration'],
+        tags=['vision'],
+    ))
+
+
+class Llama3OmniLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/ictnlp/LLaMA-Omni')
+        sys.path.append(self.local_repo_path)
+        import whisper
+        from omni_speech.model import OmniSpeech2SLlamaForCausalLM, OmniSpeechLlamaForCausalLM
+        config.speech_encoder = os.path.join(model_dir, 'large-v3.pt')
+        if not os.path.exists(config.speech_encoder):
+            whisper.load_model('large-v3', download_root=model_dir)
+        self.auto_model_cls = self.auto_model_cls or OmniSpeech2SLlamaForCausalLM
+        for key in ['forward', 'generate']:
+            try:
+                delattr(OmniSpeech2SLlamaForCausalLM, key)
+                delattr(OmniSpeechLlamaForCausalLM, key)
+            except AttributeError:
+                pass
+        # not support device_map='auto'
+        device_map = model_kwargs['device_map']
+        model_kwargs['device_map'] = None
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.to(get_device() if device_map == 'auto' else device_map)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llama3_1_omni,
+        [ModelGroup([
+            Model('ICTNLP/Llama-3.1-8B-Omni', 'ICTNLP/Llama-3.1-8B-Omni'),
+        ], )],
+        Llama3OmniLoader,
+        template=TemplateType.llama3_1_omni,
+        architectures=['OmniSpeech2SLlamaForCausalLM'],
+        model_arch=ModelArch.llama3_1_omni,
+        requires=['openai-whisper'],
+        tags=['audio'],
+    ))

swift/model/models/llava.py

@@ -0,0 +1,455 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import sys
+from functools import wraps
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.template import TemplateType
+from swift.utils import git_clone_github, safe_snapshot_download
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_get_input_embeddings
+from ..register import ModelLoader, register_model
+
+
+class LlavaLlamaHfLoader(ModelLoader):
+
+    def get_config(self, model_dir: str):
+        from transformers import LlavaConfig
+        self.auto_config_cls = LlavaConfig
+        return super().get_config(model_dir)
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import LlavaForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_llama3_hf,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/llava-llama-3-8b-v1_1-transformers', 'xtuner/llava-llama-3-8b-v1_1-transformers'),
+            ]),
+        ],
+        LlavaLlamaHfLoader,
+        template=TemplateType.llava_llama3_hf,
+        architectures=['LlavaForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.36'],
+        tags=['vision'],
+    ))
+
+
+def _patch_llava(model):
+    if hasattr(model, '__old_generate'):
+        return
+    generate = model.generate
+    model.__old_generate = generate
+
+    @wraps(generate)
+    def _new_generate(inputs=None, *args, **kwargs):
+        input_ids = kwargs.pop('input_ids', None)
+        if inputs is None and input_ids is not None:
+            inputs = input_ids
+        return generate(inputs, *args, **kwargs)
+
+    model.generate = _new_generate
+
+
+class LlavahfLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import LlavaForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava1_5_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llava-1.5-7b-hf', 'llava-hf/llava-1.5-7b-hf'),
+                Model('llava-hf/llava-1.5-13b-hf', 'llava-hf/llava-1.5-13b-hf'),
+            ]),
+        ],
+        LlavahfLoader,
+        template=TemplateType.llava1_5_hf,
+        architectures=['LlavaForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.36'],
+        tags=['vision'],
+    ))
+
+
+class LlavaOnevisionHfLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import LlavaOnevisionForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or LlavaOnevisionForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_onevision_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llava-onevision-qwen2-0.5b-ov-hf', 'llava-hf/llava-onevision-qwen2-0.5b-ov-hf'),
+                Model('llava-hf/llava-onevision-qwen2-7b-ov-hf', 'llava-hf/llava-onevision-qwen2-7b-ov-hf'),
+                Model('llava-hf/llava-onevision-qwen2-72b-ov-hf', 'llava-hf/llava-onevision-qwen2-72b-ov-hf'),
+            ], ),
+        ],
+        LlavaOnevisionHfLoader,
+        template=TemplateType.llava_onevision_hf,
+        architectures=['LlavaOnevisionForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.45'],
+        tags=['vision', 'video'],
+    ))
+
+
+class LlavaNextHfLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import LlavaNextForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or LlavaNextForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_next_qwen_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llava-next-72b-hf', 'llava-hf/llava-next-72b-hf'),
+                Model('llava-hf/llava-next-110b-hf', 'llava-hf/llava-next-110b-hf'),
+            ], ),
+        ],
+        LlavaNextHfLoader,
+        template=TemplateType.llava_next_qwen_hf,
+        architectures=['LlavaNextForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.39'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llama3_llava_next_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llama3-llava-next-8b-hf', 'llava-hf/llama3-llava-next-8b-hf'),
+            ], ),
+        ],
+        LlavaNextHfLoader,
+        template=TemplateType.llama3_llava_next_hf,
+        architectures=['LlavaNextForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.39'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava1_6_vicuna_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llava-v1.6-vicuna-7b-hf', 'llava-hf/llava-v1.6-vicuna-7b-hf'),
+                Model('llava-hf/llava-v1.6-vicuna-13b-hf', 'llava-hf/llava-v1.6-vicuna-13b-hf'),
+            ], ),
+        ],
+        LlavaNextHfLoader,
+        template=TemplateType.llava1_6_vicuna_hf,
+        architectures=['LlavaNextForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.39'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava1_6_mistral_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llava-v1.6-mistral-7b-hf', 'llava-hf/llava-v1.6-mistral-7b-hf'),
+            ], ),
+        ],
+        LlavaNextHfLoader,
+        template=TemplateType.llava1_6_mistral_hf,
+        architectures=['LlavaNextForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.39'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_llama3_1_hf,
+        [
+            ModelGroup([
+                Model('swift/llava-llama3.1-8b'),
+            ], ),
+        ],
+        LlavaNextHfLoader,
+        template=TemplateType.llava_llama3_1_hf,
+        architectures=['LlavaNextForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.41'],
+        tags=['vision'],
+    ))
+
+
+class LlavaNextYiHfLoader(LlavaNextHfLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        config = super().get_config(model_dir)
+        config.image_token_index = 64003
+        return config
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava1_6_yi_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/llava-v1.6-34b-hf', 'llava-hf/llava-v1.6-34b-hf'),
+            ], ),
+        ],
+        LlavaNextHfLoader,
+        template=TemplateType.llava1_6_yi_hf,
+        architectures=['LlavaNextForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.39'],
+        tags=['vision'],
+    ))
+
+
+class LlavaNextVideoHfLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import LlavaNextVideoForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or LlavaNextVideoForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_next_video_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/LLaVA-NeXT-Video-7B-DPO-hf', 'llava-hf/LLaVA-NeXT-Video-7B-DPO-hf'),
+                Model('llava-hf/LLaVA-NeXT-Video-7B-32K-hf', 'llava-hf/LLaVA-NeXT-Video-7B-32K-hf'),
+                Model('llava-hf/LLaVA-NeXT-Video-7B-hf', 'llava-hf/LLaVA-NeXT-Video-7B-hf'),
+            ], ),
+        ],
+        LlavaNextVideoHfLoader,
+        template=TemplateType.llava_next_video_hf,
+        architectures=['LlavaNextVideoForConditionalGeneration'],
+        model_arch=ModelArch.llava_next_video_hf,
+        requires=['transformers>=4.42', 'av'],
+        tags=['video'],
+    ))
+
+
+class LlavaNextVideoYiHfLoader(LlavaNextVideoHfLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        config = super().get_config(model_dir)
+        config.video_token_index = 64003
+        config.image_token_index = 64004
+        return config
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_next_video_yi_hf,
+        [
+            ModelGroup([
+                Model('llava-hf/LLaVA-NeXT-Video-34B-hf', 'llava-hf/LLaVA-NeXT-Video-34B-hf'),
+            ], ),
+        ],
+        LlavaNextVideoYiHfLoader,
+        template=TemplateType.llava_next_video_hf,
+        architectures=['LlavaNextVideoForConditionalGeneration'],
+        model_arch=ModelArch.llava_next_video_hf,
+        requires=['transformers>=4.42', 'av'],
+        tags=['video'],
+    ))
+
+
+class LlavaLoader(ModelLoader):
+    llm_model_type = None
+
+    def get_config(self, model_dir: str):
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            if 'next' in self.llm_model_type:
+                repo_path = 'https://github.com/LLaVA-VL/LLaVA-NeXT'
+            else:
+                repo_path = 'https://github.com/haotian-liu/LLaVA'
+            local_repo_path = git_clone_github(repo_path)
+        sys.path.append(local_repo_path)
+        if self.llm_model_type == 'mistral':
+            from llava.model import LlavaMistralConfig
+            self.auto_config_cls = LlavaMistralConfig
+        elif 'llama' in self.llm_model_type:  # llama
+            from llava.model import LlavaConfig
+            self.auto_config_cls = LlavaConfig
+        config = super().get_config(model_dir)
+        if not hasattr(config, 'max_sequence_length'):
+            config.max_sequence_length = 2048
+        return config
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        if self.llm_model_type == 'mistral':
+            from llava.model import LlavaMistralForCausalLM
+            auto_model_cls = LlavaMistralForCausalLM
+        elif 'llama' in self.llm_model_type:  # llama
+            from llava.model import LlavaLlamaForCausalLM
+            if not hasattr(LlavaLlamaForCausalLM, '__old_forward'):  # Avoid double patching
+                forward = LlavaLlamaForCausalLM.forward
+                LlavaLlamaForCausalLM.__old_forward = forward
+
+                @wraps(forward)
+                def _new_forward(*args, **kwargs):
+                    kwargs.pop('cache_position', None)
+                    return forward(*args, **kwargs)
+
+                LlavaLlamaForCausalLM.forward = _new_forward
+            auto_model_cls = LlavaLlamaForCausalLM
+        else:  # qwen
+            from llava.model import LlavaQwenForCausalLM
+            auto_model_cls = LlavaQwenForCausalLM
+
+        config.mm_vision_tower = safe_snapshot_download('AI-ModelScope/clip-vit-large-patch14-336', check_local=True)
+        self.auto_model_cls = self.auto_model_cls or auto_model_cls
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        vision_tower = model.get_vision_tower()
+        device_map = str(model_kwargs.get('device_map', str(model.device)))
+        if not vision_tower.is_loaded:
+            vision_tower.load_model(device_map=device_map)
+        _patch_llava(model)
+        model.resize_token_embeddings(len(processor))
+        processor.image_processor = vision_tower.image_processor
+        return model
+
+
+class Llama3LlavaNextLoader(LlavaLoader):
+    llm_model_type = 'next_llama'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llama3_llava_next,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/llama3-llava-next-8b', 'lmms-lab/llama3-llava-next-8b'),
+            ], ),
+        ],
+        Llama3LlavaNextLoader,
+        template=TemplateType.llama3_llava_next,
+        architectures=['LlavaLlamaForCausalLM'],
+        model_arch=ModelArch.llava_llama,
+        requires=['transformers>=4.42', 'av'],
+        tags=['vision'],
+    ))
+
+
+class LlavaMistralLoader(LlavaLoader):
+    llm_model_type = 'next_llama'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava1_6_mistral,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/llava-v1.6-mistral-7b', 'liuhaotian/llava-v1.6-mistral-7b'),
+            ], ),
+        ],
+        LlavaMistralLoader,
+        template=TemplateType.llava1_6_mistral,
+        requires=['transformers>=4.34'],
+        architectures=['LlavaMistralForCausalLM'],
+        model_arch=ModelArch.llava_mistral,
+        tags=['vision'],
+    ))
+
+
+class LlavaLlamaLoader(LlavaLoader):
+    llm_model_type = 'llama'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava1_6_yi, [
+            ModelGroup([
+                Model('AI-ModelScope/llava-v1.6-34b', 'liuhaotian/llava-v1.6-34b'),
+            ], ),
+        ],
+        LlavaLlamaLoader,
+        template=TemplateType.llava1_6_yi,
+        requires=['transformers>=4.34'],
+        architectures=['LlavaLlamaForCausalLM'],
+        tags=['vision'],
+        model_arch=None))
+
+
+class LlavaNextQwenLoader(LlavaLoader):
+    llm_model_type = 'next_qwen'
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_next_qwen, [
+            ModelGroup([
+                Model('AI-ModelScope/llava-next-72b', 'lmms-lab/llava-next-72b'),
+                Model('AI-ModelScope/llava-next-110b', 'lmms-lab/llava-next-110b'),
+            ], ),
+        ],
+        LlavaNextQwenLoader,
+        template=TemplateType.llava_next_qwen,
+        architectures=['LlavaQwenForCausalLM'],
+        requires=['transformers>=4.42', 'av'],
+        tags=['vision'],
+        model_arch=None))
+
+
+class LlavaOnevisionLoader(ModelLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        config = super().get_config(model_dir)
+        config.vision_start_token_id = 151652
+        return config
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model_cls = get_class_from_dynamic_module(
+            'modeling_llavaonevision1_5.LLaVAOneVision1_5_ForConditionalGeneration', model_dir)
+        model_cls._no_split_modules = ['LLaVAOneVision1_5_DecoderLayer', 'RiceBlock']
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_get_input_embeddings(model.visual, 'patch_embed')
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.llava_onevision1_5,
+        [
+            ModelGroup([
+                Model('lmms-lab/LLaVA-OneVision-1.5-4B-Instruct', 'lmms-lab/LLaVA-OneVision-1.5-4B-Instruct'),
+                Model('lmms-lab/LLaVA-OneVision-1.5-8B-Instruct', 'lmms-lab/LLaVA-OneVision-1.5-8B-Instruct'),
+                Model('lmms-lab/LLaVA-OneVision-1.5-4B-Base', 'lmms-lab/LLaVA-OneVision-1.5-4B-Base'),
+                Model('lmms-lab/LLaVA-OneVision-1.5-8B-Base', 'lmms-lab/LLaVA-OneVision-1.5-8B-Base'),
+            ], ),
+        ],
+        LlavaOnevisionLoader,
+        template=TemplateType.llava_onevision1_5,
+        architectures=['LLaVAOneVision1_5_ForConditionalGeneration'],
+        model_arch=ModelArch.llava_onevision1_5,
+        requires=['transformers>=4.53.0', 'qwen_vl_utils'],
+        tags=['vision'],
+    ))

swift/model/models/llm.py

@@ -0,0 +1,441 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import AutoTokenizer, PretrainedConfig
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_logger, safe_snapshot_download
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, SentenceTransformersLoader, register_model
+
+logger = get_logger()
+
+
+class GrokLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer_dir = safe_snapshot_download('AI-ModelScope/grok-1-tokenizer', download_model=False, check_local=True)
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, trust_remote_code=True)
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.grok, [
+            ModelGroup([
+                Model('colossalai/grok-1-pytorch', 'hpcai-tech/grok-1'),
+            ]),
+        ],
+        GrokLoader,
+        template=TemplateType.default,
+        architectures=['Grok1ModelForCausalLM'],
+        model_arch=ModelArch.llama))
+
+
+class PolyLMLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False, legacy=True)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.polylm,
+        [
+            ModelGroup(
+                [
+                    # base
+                    Model('damo/nlp_polylm_13b_text_generation', 'DAMO-NLP-MT/polylm-13b'),
+                ], ),
+        ],
+        PolyLMLoader,
+        template=TemplateType.default,
+        architectures=['GPT2LMHeadModel'],
+        model_arch=ModelArch.qwen))
+
+
+class YuanLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_dir, add_eos_token=False, add_bos_token=False, eos_token='<eod>', legacy=True)
+        addi_tokens = [
+            '<sep>', '<pad>', '<mask>', '<predict>', '<FIM_SUFFIX>', '<FIM_PREFIX>', '<FIM_MIDDLE>', '<commit_before>',
+            '<commit_msg>', '<commit_after>', '<jupyter_start>', '<jupyter_text>', '<jupyter_code>', '<jupyter_output>',
+            '<empty_output>'
+        ]
+        tokenizer.add_tokens(addi_tokens, special_tokens=True)
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.yuan2,
+        [
+            ModelGroup([
+                Model('IEITYuan/Yuan2.0-2B-hf', 'IEITYuan/Yuan2-2B-hf'),
+                Model('IEITYuan/Yuan2.0-51B-hf', 'IEITYuan/Yuan2-51B-hf'),
+                Model('IEITYuan/Yuan2.0-102B-hf', 'IEITYuan/Yuan2-102B-hf'),
+                Model('IEITYuan/Yuan2-2B-Janus-hf', 'IEITYuan/Yuan2-2B-Janus-hf'),
+            ]),
+            ModelGroup([
+                Model('IEITYuan/Yuan2-M32-hf', 'IEITYuan/Yuan2-M32-hf'),
+            ]),
+        ],
+        YuanLoader,
+        template=TemplateType.yuan,
+        model_arch=ModelArch.llama,
+        architectures=['YuanForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.orion,
+        [
+            ModelGroup([
+                Model('OrionStarAI/Orion-14B-Chat', 'OrionStarAI/Orion-14B-Chat'),
+                Model('OrionStarAI/Orion-14B-Base', 'OrionStarAI/Orion-14B-Base'),
+            ]),
+        ],
+        template=TemplateType.orion,
+        model_arch=ModelArch.llama,
+        architectures=['OrionForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.dbrx, [
+            ModelGroup([
+                Model('AI-ModelScope/dbrx-base', 'databricks/dbrx-base'),
+                Model('AI-ModelScope/dbrx-instruct', 'databricks/dbrx-instruct'),
+            ]),
+        ],
+        template=TemplateType.dbrx,
+        model_arch=ModelArch.dbrx,
+        architectures=['DbrxForCausalLM'],
+        requires=['transformers>=4.36']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.bluelm,
+        [
+            ModelGroup([
+                Model('vivo-ai/BlueLM-7B-Chat-32K', 'vivo-ai/BlueLM-7B-Chat-32K'),
+                Model('vivo-ai/BlueLM-7B-Chat', 'vivo-ai/BlueLM-7B-Chat'),
+                Model('vivo-ai/BlueLM-7B-Base-32K', 'vivo-ai/BlueLM-7B-Base-32K'),
+                Model('vivo-ai/BlueLM-7B-Base', 'vivo-ai/BlueLM-7B-Base'),
+            ]),
+        ],
+        template=TemplateType.bluelm,
+        model_arch=ModelArch.llama,
+        architectures=['BlueLMForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.seggpt,
+        [
+            ModelGroup([
+                Model('damo/nlp_seqgpt-560m', 'DAMO-NLP/SeqGPT-560M'),
+            ]),
+        ],
+        template=TemplateType.default,
+        model_arch=None,
+        architectures=['BloomForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.xverse,
+        [
+            ModelGroup([
+                Model('xverse/XVERSE-7B-Chat', 'xverse/XVERSE-7B-Chat'),
+                Model('xverse/XVERSE-7B', 'xverse/XVERSE-7B'),
+                Model('xverse/XVERSE-13B', 'xverse/XVERSE-13B'),
+                Model('xverse/XVERSE-13B-Chat', 'xverse/XVERSE-13B-Chat'),
+                Model('xverse/XVERSE-65B', 'xverse/XVERSE-65B'),
+                Model('xverse/XVERSE-65B-2', 'xverse/XVERSE-65B-2'),
+                Model('xverse/XVERSE-65B-Chat', 'xverse/XVERSE-65B-Chat'),
+                Model('xverse/XVERSE-13B-256K', 'xverse/XVERSE-13B-256K', ms_revision='v1.0.0'),
+            ]),
+        ],
+        template=TemplateType.xverse,
+        model_arch=ModelArch.llama,
+        architectures=['XverseForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.xverse_moe,
+        [
+            ModelGroup([
+                Model('xverse/XVERSE-MoE-A4.2B', 'xverse/XVERSE-MoE-A4.2B'),
+            ]),
+        ],
+        template=TemplateType.xverse,
+        model_arch=ModelArch.llama,
+        architectures=['XverseForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.c4ai,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/c4ai-command-r-v01', 'CohereForAI/c4ai-command-r-v01'),
+                Model('AI-ModelScope/c4ai-command-r-plus', 'CohereForAI/c4ai-command-r-plus'),
+            ]),
+        ],
+        template=TemplateType.c4ai,
+        model_arch=ModelArch.llama,
+        architectures=['CohereForCausalLM'],
+        requires=['transformers>=4.39'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.aya, [
+            ModelGroup([
+                Model('AI-ModelScope/aya-expanse-8b', 'CohereForAI/aya-expanse-8b'),
+                Model('AI-ModelScope/aya-expanse-32b', 'CohereForAI/aya-expanse-32b'),
+            ]),
+        ],
+        template=TemplateType.aya,
+        model_arch=ModelArch.llama,
+        architectures=['CohereForCausalLM'],
+        requires=['transformers>=4.44.0']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.ling,
+        [
+            ModelGroup([
+                Model('inclusionAI/Ling-lite', 'inclusionAI/Ling-lite'),
+                Model('inclusionAI/Ling-plus', 'inclusionAI/Ling-plus'),
+                Model('inclusionAI/Ling-lite-base', 'inclusionAI/Ling-lite-base'),
+                Model('inclusionAI/Ling-plus-base', 'inclusionAI/Ling-plus-base'),
+            ]),
+        ],
+        template=TemplateType.ling,
+        architectures=['BailingMoeForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_gte, [
+            ModelGroup([
+                Model('iic/gte_Qwen2-1.5B-instruct', 'Alibaba-NLP/gte-Qwen2-1.5B-instruct'),
+                Model('iic/gte_Qwen2-7B-instruct', 'Alibaba-NLP/gte-Qwen2-7B-instruct'),
+            ]),
+        ],
+        SentenceTransformersLoader,
+        template=TemplateType.dummy,
+        architectures=['Qwen2ForCausalLM']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mimo, [
+            ModelGroup([
+                Model('XiaomiMiMo/MiMo-7B-Base', 'XiaomiMiMo/MiMo-7B-Base'),
+                Model('XiaomiMiMo/MiMo-7B-SFT', 'XiaomiMiMo/MiMo-7B-SFT'),
+                Model('XiaomiMiMo/MiMo-7B-RL-Zero', 'XiaomiMiMo/MiMo-7B-RL-Zero'),
+                Model('XiaomiMiMo/MiMo-7B-RL', 'XiaomiMiMo/MiMo-7B-RL'),
+            ], TemplateType.qwen),
+            ModelGroup([
+                Model('XiaomiMiMo/MiMo-7B-RL-0530', 'XiaomiMiMo/MiMo-7B-RL-0530'),
+            ], TemplateType.mimo_rl),
+        ],
+        model_arch=ModelArch.llama,
+        architectures=['MiMoForCausalLM'],
+        requires=['transformers>=4.37']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.dots1,
+        [
+            ModelGroup([
+                Model('rednote-hilab/dots.llm1.base', 'rednote-hilab/dots.llm1.base'),
+                Model('rednote-hilab/dots.llm1.inst', 'rednote-hilab/dots.llm1.inst'),
+            ])
+        ],
+        template=TemplateType.dots1,
+        architectures=['Dots1ForCausalLM'],
+        requires=['transformers>=4.53'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.hunyuan,
+        [ModelGroup([
+            Model('Tencent-Hunyuan/Hunyuan-A13B-Instruct', 'tencent/Hunyuan-A13B-Instruct'),
+        ])],
+        template=TemplateType.hunyuan_moe,
+        architectures=['HunYuanMoEV1ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.hunyuan_v1_dense,
+        [
+            ModelGroup([
+                Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct', 'tencent/Hunyuan-0.5B-Instruct'),
+                Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct', 'tencent/Hunyuan-1.8B-Instruct'),
+                Model('Tencent-Hunyuan/Hunyuan-4B-Instruct', 'tencent/Hunyuan-4B-Instruct'),
+                Model('Tencent-Hunyuan/Hunyuan-7B-Instruct', 'tencent/Hunyuan-7B-Instruct'),
+                # pretrain
+                Model('Tencent-Hunyuan/Hunyuan-0.5B-Pretrain', 'tencent/Hunyuan-0.5B-Pretrain'),
+                Model('Tencent-Hunyuan/Hunyuan-1.8B-Pretrain', 'tencent/Hunyuan-1.8B-Pretrain'),
+                Model('Tencent-Hunyuan/Hunyuan-4B-Pretrain', 'tencent/Hunyuan-4B-Pretrain'),
+                Model('Tencent-Hunyuan/Hunyuan-7B-Pretrain', 'tencent/Hunyuan-7B-Pretrain'),
+                # fp8
+                Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct-FP8', 'tencent/Hunyuan-0.5B-Instruct-FP8'),
+                Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct-FP8', 'tencent/Hunyuan-1.8B-Instruct-FP8'),
+                Model('Tencent-Hunyuan/Hunyuan-4B-Instruct-FP8', 'tencent/Hunyuan-4B-Instruct-FP8'),
+                Model('Tencent-Hunyuan/Hunyuan-7B-Instruct-FP8', 'tencent/Hunyuan-7B-Instruct-FP8'),
+                # awq
+                Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct-AWQ-Int4', 'tencent/Hunyuan-0.5B-Instruct-AWQ-Int4'),
+                Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct-AWQ-Int4', 'tencent/Hunyuan-1.8B-Instruct-AWQ-Int4'),
+                Model('Tencent-Hunyuan/Hunyuan-4B-Instruct-AWQ-Int4', 'tencent/Hunyuan-4B-Instruct-AWQ-Int4'),
+                Model('Tencent-Hunyuan/Hunyuan-7B-Instruct-AWQ-Int4', 'tencent/Hunyuan-7B-Instruct-AWQ-Int4'),
+                # gptq
+                Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-0.5B-Instruct-GPTQ-Int4'),
+                Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-1.8B-Instruct-GPTQ-Int4'),
+                Model('Tencent-Hunyuan/Hunyuan-4B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-4B-Instruct-GPTQ-Int4'),
+                Model('Tencent-Hunyuan/Hunyuan-7B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-7B-Instruct-GPTQ-Int4'),
+            ])
+        ],
+        template=TemplateType.hunyuan,
+        requires=['transformers>=4.55.0.dev0'],
+        architectures=['HunYuanDenseV1ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.hy_v3,
+        [
+            ModelGroup([
+                Model('Tencent-Hunyuan/Hy3-preview', 'tencent/Hy3-preview'),
+                Model('Tencent-Hunyuan/Hy3-preview-Base', 'tencent/Hy3-preview-Base'),
+            ],
+                       template=TemplateType.hy_v3_preview),
+            ModelGroup([
+                Model('Tencent-Hunyuan/Hy3', 'tencent/Hy3'),
+                Model('Tencent-Hunyuan/Hy3-FP8', 'tencent/Hy3-FP8'),
+            ],
+                       template=TemplateType.hy_v3),
+        ],
+        requires=['transformers>=5.6.0'],
+        architectures=['HYV3ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.gpt_oss, [
+            ModelGroup([
+                Model('openai-mirror/gpt-oss-20b', 'openai/gpt-oss-20b'),
+                Model('openai-mirror/gpt-oss-120b', 'openai/gpt-oss-120b'),
+            ])
+        ],
+        template=TemplateType.gpt_oss,
+        ignore_patterns=['metal/', 'original/'],
+        architectures=['GptOssForCausalLM'],
+        requires=['transformers>=4.55']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.longchat,
+        [
+            ModelGroup([
+                Model('meituan-longcat/LongCat-Flash-Chat', 'meituan-longcat/LongCat-Flash-Chat'),
+                Model('meituan-longcat/LongCat-Flash-Chat-FP8', 'meituan-longcat/LongCat-Flash-Chat-FP8'),
+            ])
+        ],
+        template=TemplateType.longchat,
+        architectures=['LongcatFlashForCausalLM'],
+        requires=['transformers>=4.54,<4.56'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.bailing_moe,
+        [
+            ModelGroup([
+                Model('inclusionAI/Ling-mini-2.0', 'inclusionAI/Ling-mini-2.0'),
+                Model('inclusionAI/Ling-mini-base-2.0', 'inclusionAI/Ling-mini-base-2.0'),
+                Model('inclusionAI/Ling-1T', 'inclusionAI/Ling-1T'),
+            ],
+                       template=TemplateType.ling2),
+            ModelGroup([
+                Model('inclusionAI/Ring-mini-2.0', 'inclusionAI/Ring-mini-2.0'),
+            ], template=TemplateType.ring2)
+        ],
+        architectures=['BailingMoeV2ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.bailing_hybrid,
+        [
+            ModelGroup([
+                Model('inclusionAI/Ling-2.5-1T', 'inclusionAI/Ling-2.5-1T'),
+                Model('inclusionAI/Ling-2.6-1T', 'inclusionAI/Ling-2.6-1T'),
+                Model('inclusionAI/Ling-2.6-flash', 'inclusionAI/Ling-2.6-flash'),
+            ],
+                       template=TemplateType.ling2),
+            ModelGroup([
+                Model('inclusionAI/Ring-2.5-1T', 'inclusionAI/Ring-2.5-1T'),
+                Model('inclusionAI/Ring-2.6-1T', 'inclusionAI/Ring-2.6-1T'),
+            ],
+                       template=TemplateType.ring2_5),
+        ],
+        architectures=['BailingMoeV2_5ForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.iquestcoder,
+        [
+            ModelGroup([
+                Model('IQuestLab/IQuest-Coder-V1-40B-Base-Stage1', 'IQuestLab/IQuest-Coder-V1-40B-Base-Stage1'),
+                Model('IQuestLab/IQuest-Coder-V1-40B-Base', 'IQuestLab/IQuest-Coder-V1-40B-Base'),
+                Model('IQuestLab/IQuest-Coder-V1-40B-Instruct', 'IQuestLab/IQuest-Coder-V1-40B-Instruct'),
+            ])
+        ],
+        template=TemplateType.iquestcoder,
+        requires=['transformers==4.52.4'],
+        architectures=['IQuestCoderForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.youtu_llm,
+        [
+            ModelGroup([
+                Model('Tencent-YouTu-Research/Youtu-LLM-2B', 'tencent/Youtu-LLM-2B'),
+                Model('Tencent-YouTu-Research/Youtu-LLM-2B-Base', 'tencent/Youtu-LLM-2B-Base'),
+            ])
+        ],
+        template=TemplateType.youtu_llm,
+        architectures=['YoutuForCausalLM'],
+        requires=['transformers>=4.56'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.olmoe,
+        [
+            ModelGroup([
+                Model('allenai/OLMoE-1B-7B-0125', 'allenai/OLMoE-1B-7B-0125'),
+                Model('allenai/OLMoE-1B-7B-0125-Instruct', 'allenai/OLMoE-1B-7B-0125-Instruct'),
+            ],
+                       template=TemplateType.olmoe),
+            ModelGroup([
+                Model('allenai/OLMoE-1B-7B-0924', 'allenai/OLMoE-1B-7B-0924'),
+                Model('allenai/OLMoE-1B-7B-0924-Instruct', 'allenai/OLMoE-1B-7B-0924-Instruct'),
+                Model('allenai/OLMoE-1B-7B-0924-SFT', 'allenai/OLMoE-1B-7B-0924-SFT'),
+            ],
+                       template=TemplateType.olmoe_0924)
+        ],
+        architectures=['OlmoeForCausalLM'],
+    ))

swift/model/models/mamba.py

@@ -0,0 +1,40 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import PreTrainedModel
+
+from swift.template import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+
+class MambaLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        logger.info(
+            '[IMPORTANT] Remember installing causal-conv1d>=1.2.0 and mamba-ssm, or you training and inference will'
+            'be really slow!')
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.mamba,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/mamba-130m-hf', 'state-spaces/mamba-130m-hf'),
+                Model('AI-ModelScope/mamba-370m-hf', 'state-spaces/mamba-370m-hf'),
+                Model('AI-ModelScope/mamba-390m-hf', 'state-spaces/mamba-390m-hf'),
+                Model('AI-ModelScope/mamba-790m-hf', 'state-spaces/mamba-790m-hf'),
+                Model('AI-ModelScope/mamba-1.4b-hf', 'state-spaces/mamba-1.4b-hf'),
+                Model('AI-ModelScope/mamba-2.8b-hf', 'state-spaces/mamba-2.8b-hf'),
+            ])
+        ],
+        MambaLoader,
+        template=TemplateType.default,
+        architectures=['MambaForCausalLM'],
+        model_arch=None,
+        requires=['transformers>=4.39.0'],
+    ))

swift/model/models/microsoft.py

@@ -0,0 +1,210 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import PretrainedConfig, PreTrainedModel
+from types import MethodType
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_device, get_env_args
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_ignore_check_imports, patch_output_clone
+from ..register import ModelLoader, register_model
+from ..utils import use_submodel_func
+
+
+class Phi3VisionLoader(ModelLoader):
+    num_crops = 4
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        processor_kwargs = {'num_crops': get_env_args('num_crops', int, self.num_crops)}
+        from transformers import AutoProcessor
+        processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True, **processor_kwargs)
+        return processor
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.model.vision_embed_tokens.wte)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.phi3_vision,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3-vision-128k-instruct', 'microsoft/Phi-3-vision-128k-instruct'),
+                Model('LLM-Research/Phi-3.5-vision-instruct', 'microsoft/Phi-3.5-vision-instruct'),
+            ])
+        ],
+        Phi3VisionLoader,
+        template=TemplateType.phi3_vision,
+        architectures=['Phi3VForCausalLM'],
+        model_arch=ModelArch.phi3_vision,
+        requires=['transformers>=4.36'],
+        tags=['vision'],
+    ))
+
+
+class Phi4MultimodalLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        processor = super().get_processor(model_dir, config)
+        processor.audio_processor.audio_compression_rate = processor.audio_processor.compression_rate
+        processor.audio_processor.audio_downsample_rate = processor.audio_processor.qformer_compression_rate
+        processor.audio_processor.audio_feat_stride = processor.audio_processor.feat_stride
+        del processor.audio_processor.feature_size
+        del processor.audio_processor.sampling_rate
+        del processor.audio_processor.padding_value
+        del processor.__class__.chat_template
+        processor.chat_template = None
+        return processor
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        model.set_lora_adapter(['vision', 'speech'])
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.phi4_multimodal,
+        [ModelGroup([
+            Model('LLM-Research/Phi-4-multimodal-instruct', 'microsoft/Phi-4-multimodal-instruct'),
+        ])],
+        Phi4MultimodalLoader,
+        template=TemplateType.phi4_multimodal,
+        architectures=['Phi4MMForCausalLM'],
+        model_arch=ModelArch.phi4_multimodal,
+        requires=['transformers>=4.36,<4.49', 'backoff', 'soundfile'],
+        tags=['vision', 'audio'],
+    ))
+
+
+class FlorenceLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        config.vision_config.model_type = 'davit'  # fix merge-lora
+        if model_kwargs['device_map'] == 'auto':
+            model_kwargs['device_map'] = get_device()
+        with patch_ignore_check_imports():
+            model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.vision_tower.enable_checkpoint = True
+        use_submodel_func(model, 'language_model', ['generate', 'forward'])
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.florence,
+        [
+            # llama2
+            ModelGroup([
+                Model('AI-ModelScope/Florence-2-base-ft', 'microsoft/Florence-2-base-ft'),
+                Model('AI-ModelScope/Florence-2-base', 'microsoft/Florence-2-base'),
+                Model('AI-ModelScope/Florence-2-large', 'microsoft/Florence-2-large'),
+                Model('AI-ModelScope/Florence-2-large-ft', 'microsoft/Florence-2-large-ft'),
+            ]),
+        ],
+        FlorenceLoader,
+        template=TemplateType.florence,
+        architectures=['Florence2ForConditionalGeneration'],
+        model_arch=ModelArch.florence,
+        tags=['vision'],
+    ))
+
+
+class Phi3SmallLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+
+        def rotary_emb(self, query_states, key_states, **kwargs):
+            q_type = query_states.dtype
+            k_type = key_states.dtype
+            query_states, key_states = self.rotory_emb_origin(query_states, key_states, **kwargs)
+            query_states = query_states.to(q_type)
+            key_states = key_states.to(k_type)
+            return query_states, key_states
+
+        for i in range(32):  # TODO: 32
+            re = model.model.layers[i].self_attn.rotary_emb
+            re.rotory_emb_origin = re.forward
+            re.forward = MethodType(rotary_emb, re)
+        return model
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi3_small,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3-small-8k-instruct', 'microsoft/Phi-3-small-8k-instruct'),
+                Model('LLM-Research/Phi-3-small-128k-instruct', 'microsoft/Phi-3-small-128k-instruct'),
+            ]),
+        ],
+        Phi3SmallLoader,
+        template=TemplateType.phi3,
+        architectures=['Phi3SmallForCausalLM'],
+        model_arch=ModelArch.phi3_small,
+        requires=['transformers>=4.36'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi2,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/phi-2', 'microsoft/phi-2'),
+            ]),
+        ],
+        template=TemplateType.default,
+        architectures=['PhiForCausalLM'],
+        model_arch=ModelArch.phi2,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi3,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3-mini-4k-instruct', 'microsoft/Phi-3-mini-4k-instruct'),
+                Model('LLM-Research/Phi-3-mini-128k-instruct', 'microsoft/Phi-3-mini-128k-instruct'),
+                Model('LLM-Research/Phi-3-medium-4k-instruct', 'microsoft/Phi-3-medium-4k-instruct'),
+                Model('LLM-Research/Phi-3-medium-128k-instruct', 'microsoft/Phi-3-medium-128k-instruct'),
+                Model('LLM-Research/Phi-3.5-mini-instruct', 'microsoft/Phi-3.5-mini-instruct'),
+            ]),
+            ModelGroup([Model('LLM-Research/Phi-4-mini-instruct', 'microsoft/Phi-4-mini-instruct')])
+        ],
+        template=TemplateType.phi3,
+        architectures=['Phi3ForCausalLM'],
+        requires=['transformers>=4.36'],
+        model_arch=ModelArch.phi3,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi4,
+        [
+            ModelGroup([
+                Model('LLM-Research/phi-4', 'microsoft/phi-4'),
+            ]),
+        ],
+        template=TemplateType.phi4,
+        architectures=['Phi3ForCausalLM'],
+        requires=['transformers>=4.36'],
+        model_arch=ModelArch.phi3,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi3_moe,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3.5-MoE-instruct', 'microsoft/Phi-3.5-MoE-instruct'),
+            ]),
+        ],
+        template=TemplateType.phi3,
+        architectures=['PhiMoEForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))

swift/model/models/minicpm.py

@@ -0,0 +1,268 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from transformers import PreTrainedModel
+from transformers.utils import strtobool
+from types import MethodType
+
+from swift.template import TemplateType
+from swift.utils import get_env_args
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_device_map, patch_fixed_device, patch_output_clone
+from ..register import ModelLoader, register_model
+from ..utils import use_submodel_func
+from .deepseek import DeepseekLoader
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm_moe,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-MoE-8x2B', 'openbmb/MiniCPM-MoE-8x2B'),
+            ]),
+        ],
+        DeepseekLoader,
+        template=TemplateType.minicpm,
+        architectures=['MiniCPMForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.36'],
+    ))
+
+
+def _patch_minicpmv_device_map(model) -> None:
+    if not hasattr(model, 'hf_device_map') or len(model.hf_device_map.values()) == 1:
+        return
+
+    device = list(model.hf_device_map.values())[0]
+    if hasattr(model, 'get_vision_embedding') and not hasattr(model, '_old_get_vision_embedding'):
+        # minicpm-v-v2-chat; avoid double patching
+        _old_get_vision_embedding = model.__class__.get_vision_embedding
+
+        def _get_vision_embedding(self, pixel_values):
+            output = _old_get_vision_embedding(self, pixel_values)
+            if len(pixel_values) == 0:
+                return output
+            if isinstance(output, list):
+                return [x.to(device=device) if isinstance(x, torch.Tensor) else x for x in output]
+            else:
+                return output.to(device=device)
+
+        model.__class__._old_get_vision_embedding = _old_get_vision_embedding
+        model.__class__.get_vision_embedding = _get_vision_embedding
+
+    if hasattr(model, 'resampler'):  # minicpm-v-v2_5-chat
+        patch_fixed_device(model.resampler, device)
+
+
+class MiniCPMVLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.resampler.to(self.torch_dtype)  # fix float32
+        _patch_minicpmv_device_map(model)
+        func_list = ['generate', 'get_input_embeddings', 'forward']
+        use_submodel_func(model, 'llm', func_list)
+        if hasattr(model, 'get_slice_image_placeholder'):
+            processor.get_slice_image_placeholder = MethodType(model.get_slice_image_placeholder, processor)
+            processor.transform = MethodType(model.transform, processor)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V', 'openbmb/MiniCPM-V'),
+                Model('OpenBMB/MiniCPM-V-2', 'openbmb/MiniCPM-V-2'),
+            ], ),
+        ],
+        MiniCPMVLoader,
+        template=TemplateType.minicpmv,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers<4.42'],
+        tags=['vision'],
+    ))
+
+
+class MiniCPMV2Loader(MiniCPMVLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        with patch_device_map():
+            model = super().get_model(model_dir, *args, **kwargs)
+        embedding = model.get_input_embeddings()
+        patch_output_clone(embedding)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv2_5,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-Llama3-V-2_5', 'openbmb/MiniCPM-Llama3-V-2_5'),
+            ], ),
+        ],
+        MiniCPMV2Loader,
+        template=TemplateType.minicpmv2_5,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv2_6,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V-2_6', 'openbmb/MiniCPM-V-2_6'),
+            ], ),
+        ],
+        MiniCPMV2Loader,
+        template=TemplateType.minicpmv2_6,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36', 'decord'],
+        tags=['vision', 'video'],
+    ))
+
+
+class MiniCPMO2Loader(MiniCPMV2Loader):
+
+    def get_model(self, model_dir: str, config, *args, **kwargs) -> PreTrainedModel:
+        config.init_tts = strtobool(get_env_args('init_tts', str, 'false'))
+        config.init_audio = strtobool(get_env_args('init_audio', str, 'true'))
+        return super().get_model(model_dir, config, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmo,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-o-2_6', 'openbmb/MiniCPM-o-2_6'),
+            ], template=TemplateType.minicpmo),
+            ModelGroup(
+                [
+                    Model('OpenBMB/MiniCPM-o-4_5', 'openbmb/MiniCPM-o-4_5'),
+                ],
+                template=TemplateType.minicpmo4_5,
+                requires=['timm', 'transformers==4.51.3', 'decord', 'soundfile', 'minicpmo-utils==1.0.6'],
+            ),
+        ],
+        MiniCPMO2Loader,
+        architectures=['MiniCPMO'],
+        model_arch=ModelArch.minicpmo,
+        requires=['timm', 'transformers>=4.36', 'decord', 'soundfile'],
+        tags=['vision', 'video', 'omni', 'audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv4,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V-4', 'openbmb/MiniCPM-V-4'),
+            ], ),
+        ],
+        MiniCPMV2Loader,
+        template=TemplateType.minicpmv4,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36', 'decord'],
+        tags=['vision', 'video'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv4_5,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V-4_5', 'openbmb/MiniCPM-V-4_5'),
+            ], ),
+        ],
+        MiniCPMV2Loader,
+        template=TemplateType.minicpmv4_5,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36', 'decord'],
+        tags=['vision', 'video'],
+    ))
+
+
+class MiniCPMV4_6Loader(ModelLoader):
+
+    def get_model(self, *args, **kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForImageTextToText
+        self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
+        from .qwen import _patch_qwen3_5_linear_attention_sequence_parallel
+        _patch_qwen3_5_linear_attention_sequence_parallel()
+        return super().get_model(*args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv4_6,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V-4.6', 'openbmb/MiniCPM-V-4.6'),
+            ], ),
+        ],
+        MiniCPMV4_6Loader,
+        template=TemplateType.minicpmv4_6,
+        architectures=['MiniCPMV4_6ForConditionalGeneration'],
+        model_arch=ModelArch.minicpmv4_6,
+        requires=['transformers>=5.7.0'],
+        tags=['vision', 'video'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-2B-sft-fp32', 'openbmb/MiniCPM-2B-sft-fp32'),
+                Model('OpenBMB/MiniCPM-2B-dpo-fp32', 'openbmb/MiniCPM-2B-dpo-fp32'),
+                Model('OpenBMB/MiniCPM-1B-sft-bf16', 'openbmb/MiniCPM-1B-sft-bf16'),
+            ], ),
+        ],
+        template=TemplateType.minicpm,
+        architectures=['MiniCPMForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.36.0'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm_chatml,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-2B-128k', 'openbmb/MiniCPM-2B-128k'),
+            ]),
+            ModelGroup([
+                Model('OpenBMB/MiniCPM4-0.5B', 'openbmb/MiniCPM4-0.5B'),
+                Model('OpenBMB/MiniCPM4-8B', 'openbmb/MiniCPM4-8B'),
+            ]),
+        ],
+        template=TemplateType.chatml,
+        architectures=['MiniCPMForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.36'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm3,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM3-4B', 'openbmb/MiniCPM3-4B'),
+            ]),
+        ],
+        template=TemplateType.chatml,
+        architectures=['MiniCPM3ForCausalLM'],
+        model_arch=ModelArch.deepseek_v2,
+        requires=['transformers>=4.36'],
+    ))

swift/model/models/minimax.py

@@ -0,0 +1,193 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import json
+import os
+from transformers import AutoProcessor, PretrainedConfig, PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_device, get_device_count, get_dist_setting, get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_ignore_check_imports
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+
+class MiniMaxVLLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        logger.warn('NOTE: minimax-vl-01 model does not support training.')
+        n_gpu = get_device_count()
+        _, local_rank, _, local_world_size = get_dist_setting()
+        device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
+        if 'quantization_config' in model_kwargs:
+            quantization_config = model_kwargs['quantization_config']
+            from transformers import QuantoConfig
+            if isinstance(quantization_config, QuantoConfig):
+                quantization_config.modules_to_not_convert = (
+                    [
+                        'vision_tower',
+                        'image_newline',
+                        'multi_modal_projector',
+                        'lm_head',
+                        'embed_tokens',
+                    ] + [f'model.layers.{i}.coefficient' for i in range(config.text_config.num_hidden_layers)]
+                    + [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.text_config.num_hidden_layers)])
+
+        if len(device_ids) > 1:
+            model_safetensors_index_path = os.path.join(model_dir, 'model.safetensors.index.json')
+            with open(model_safetensors_index_path, 'r') as f:
+                model_safetensors_index = json.load(f)
+            weight_map = model_safetensors_index['weight_map']
+            vision_map = {}
+            for key, value in weight_map.items():
+                if 'vision_tower' in key or 'image_newline' in key or 'multi_modal_projector' in key:
+                    new_key = key.replace('.weight', '').replace('.bias', '')
+                    if new_key not in vision_map:
+                        vision_map[new_key] = value
+
+            device_map = {
+                'language_model.model.embed_tokens': get_device(device_ids[0]),
+                'language_model.model.norm': get_device(device_ids[len(device_ids) - 1]),
+                'language_model.lm_head': get_device(device_ids[len(device_ids) - 1])
+            }
+            for key, value in vision_map.items():
+                device_map[key] = get_device(device_ids[0])
+            device_map['vision_tower.vision_model.post_layernorm'] = get_device(device_ids[0])
+            layers_per_device = config.text_config.num_hidden_layers // len(device_ids)
+            for i in range(len(device_ids)):
+                for j in range(layers_per_device):
+                    device_map[f'language_model.model.layers.{i * layers_per_device + j}'] = get_device(device_ids[i])
+            model_kwargs['device_map'] = device_map
+        with patch_ignore_check_imports():
+            return super().get_model(model_dir, config, processor, model_kwargs)
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        MiniMaxVL01ProcessorKwargs = get_class_from_dynamic_module(
+            'processing_minimax_vl_01.MiniMaxVL01ProcessorKwargs', model_dir)
+        get_hw_multiple_of = get_class_from_dynamic_module('processing_minimax_vl_01.get_hw_multiple_of', model_dir)
+        get_num_token = get_class_from_dynamic_module('processing_minimax_vl_01.get_num_token', model_dir)
+
+        processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+        processor.MiniMaxVL01ProcessorKwargs = MiniMaxVL01ProcessorKwargs
+        processor.get_hw_multiple_of = get_hw_multiple_of
+        processor.get_num_token = get_num_token
+        return processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minimax_vl, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-VL-01', 'MiniMaxAI/MiniMax-VL-01'),
+            ]),
+        ],
+        MiniMaxVLLoader,
+        template=TemplateType.minimax_vl,
+        architectures=['MiniMaxVL01ForConditionalGeneration'],
+        tags=['vision']))
+
+
+class MinimaxTextLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        logger.warn('NOTE: minimax-text-01 model does not support training.')
+        n_gpu = get_device_count()
+        _, local_rank, _, local_world_size = get_dist_setting()
+        device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
+        if 'quantization_config' in model_kwargs:
+            quantization_config = model_kwargs['quantization_config']
+            from transformers import QuantoConfig
+            if isinstance(quantization_config, QuantoConfig):
+                quantization_config.modules_to_not_convert = (
+                    [
+                        'lm_head',
+                        'embed_tokens',
+                    ] + [f'model.layers.{i}.coefficient' for i in range(config.num_hidden_layers)]
+                    + [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.num_hidden_layers)])
+
+        if len(device_ids) > 1:
+            layers_per_device = config.num_hidden_layers // len(device_ids)
+            # set device map
+            device_map = {
+                'model.embed_tokens': get_device(0),
+                'model.norm': get_device(len(device_ids) - 1),
+                'lm_head': get_device(len(device_ids) - 1)
+            }
+            for i in range(len(device_ids)):
+                for j in range(layers_per_device):
+                    device_map[f'model.layers.{i * layers_per_device + j}'] = get_device(i)
+            model_kwargs['device_map'] = device_map
+        with patch_ignore_check_imports():
+            return super().get_model(model_dir, config, processor, model_kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.minimax, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-Text-01', 'MiniMaxAI/MiniMax-Text-01'),
+            ]),
+        ],
+        MinimaxTextLoader,
+        template=TemplateType.minimax,
+        architectures=['MiniMaxText01ForCausalLM']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minimax_m1, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-M1-40k', 'MiniMaxAI/MiniMax-M1-40k'),
+                Model('MiniMax/MiniMax-M1-80k', 'MiniMaxAI/MiniMax-M1-80k'),
+            ]),
+        ],
+        MinimaxTextLoader,
+        template=TemplateType.minimax_m1,
+        architectures=['MiniMaxM1ForCausalLM']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minimax_m2, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-M2', 'MiniMaxAI/MiniMax-M2'),
+            ], TemplateType.minimax_m2),
+            ModelGroup([
+                Model('MiniMax/MiniMax-M2.1', 'MiniMaxAI/MiniMax-M2.1'),
+            ], TemplateType.minimax_m2_1),
+            ModelGroup([
+                Model('MiniMax/MiniMax-M2.5', 'MiniMaxAI/MiniMax-M2.5'),
+            ], TemplateType.minimax_m2_5),
+            ModelGroup([
+                Model('MiniMax/MiniMax-M2.7', 'MiniMaxAI/MiniMax-M2.7'),
+            ], TemplateType.minimax_m2_7),
+        ],
+        requires=['transformers==4.57.1'],
+        architectures=['MiniMaxM2ForCausalLM']))
+
+
+class MinimaxM3VLLoader(ModelLoader):
+    default_trust_remote_code = False
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        return AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForImageTextToText
+        self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
+        return super().get_model(model_dir, config, processor, model_kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minimax_m3_vl, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-M3', 'MiniMaxAI/MiniMax-M3'),
+            ]),
+        ],
+        MinimaxM3VLLoader,
+        template=TemplateType.minimax_m3_vl,
+        model_arch=ModelArch.minimax_m3_vl,
+        architectures=['MiniMaxM3SparseForConditionalGeneration'],
+        tags=['vision', 'video']))

swift/model/models/mistral.py

@@ -0,0 +1,210 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import AutoProcessor, AutoTokenizer, PretrainedConfig, PreTrainedModel
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, safe_snapshot_download
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+register_model(
+    ModelMeta(
+        LLMModelType.mistral,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Mistral-7B-Instruct-v0.1', 'mistralai/Mistral-7B-Instruct-v0.1'),
+                Model('AI-ModelScope/Mistral-7B-Instruct-v0.2', 'mistralai/Mistral-7B-Instruct-v0.2'),
+                Model('LLM-Research/Mistral-7B-Instruct-v0.3', 'mistralai/Mistral-7B-Instruct-v0.3'),
+                Model('AI-ModelScope/Mistral-7B-v0.1', 'mistralai/Mistral-7B-v0.1'),
+                Model('AI-ModelScope/Mistral-7B-v0.2-hf', 'alpindale/Mistral-7B-v0.2-hf'),
+            ]),
+            ModelGroup([
+                Model('swift/Codestral-22B-v0.1', 'mistralai/Codestral-22B-v0.1'),
+            ]),
+        ],
+        template=TemplateType.llama,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.34'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mixtral, [
+            ModelGroup([
+                Model('AI-ModelScope/Mixtral-8x7B-Instruct-v0.1', 'mistralai/Mixtral-8x7B-Instruct-v0.1'),
+                Model('AI-ModelScope/Mixtral-8x7B-v0.1', 'mistralai/Mixtral-8x7B-v0.1'),
+                Model('AI-ModelScope/Mixtral-8x22B-v0.1', 'mistral-community/Mixtral-8x22B-v0.1'),
+            ],
+                       requires=['transformers>=4.36']),
+            ModelGroup([
+                Model('AI-ModelScope/Mixtral-8x7b-AQLM-2Bit-1x16-hf', 'ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf'),
+            ],
+                       requires=['transformers>=4.38', 'aqlm', 'torch>=2.2.0']),
+        ],
+        template=TemplateType.llama,
+        architectures=['MixtralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mistral_nemo, [
+            ModelGroup([
+                Model('AI-ModelScope/Mistral-Small-Instruct-2409', 'mistralai/Mistral-Small-Instruct-2409'),
+                Model('LLM-Research/Mistral-Large-Instruct-2407', 'mistralai/Mistral-Large-Instruct-2407'),
+                Model('AI-ModelScope/Mistral-Nemo-Base-2407', 'mistralai/Mistral-Nemo-Base-2407'),
+                Model('AI-ModelScope/Mistral-Nemo-Instruct-2407', 'mistralai/Mistral-Nemo-Instruct-2407'),
+            ],
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('AI-ModelScope/Ministral-8B-Instruct-2410', 'mistralai/Ministral-8B-Instruct-2410'),
+            ],
+                       requires=['transformers>=4.46']),
+        ],
+        template=TemplateType.mistral_nemo,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mistral_2501, [
+            ModelGroup([
+                Model('mistralai/Mistral-Small-24B-Base-2501', 'mistralai/Mistral-Small-24B-Base-2501'),
+                Model('mistralai/Mistral-Small-24B-Instruct-2501', 'mistralai/Mistral-Small-24B-Instruct-2501'),
+            ]),
+        ],
+        template=TemplateType.mistral_2501,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.zephyr,
+        [
+            ModelGroup([
+                Model('modelscope/zephyr-7b-beta', 'HuggingFaceH4/zephyr-7b-beta'),
+            ]),
+        ],
+        template=TemplateType.zephyr,
+        model_arch=ModelArch.llama,
+        architectures=['MistralForCausalLM'],
+        requires=['transformers>=4.34'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.wizardlm2_moe,
+        [ModelGroup([
+            Model('AI-ModelScope/WizardLM-2-8x22B', 'alpindale/WizardLM-2-8x22B'),
+        ])],
+        template=TemplateType.wizardlm2_moe,
+        architectures=['MixtralForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.wizardlm2,
+        [ModelGroup([
+            Model('AI-ModelScope/WizardLM-2-7B-AWQ', 'MaziyarPanahi/WizardLM-2-7B-AWQ'),
+        ])],
+        template=TemplateType.wizardlm2,
+        architectures=['MistralForCausalLM'],
+        requires=['transformers>=4.34'],
+    ))
+
+
+class DevstralLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        # src: sglang did the same (https://github.com/sgl-project/sglang/pull/6547)
+        tokenizer_dir = safe_snapshot_download('mistralai/Mistral-Small-3.1-24B-Instruct-2503', download_model=False)
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.devstral, [
+            ModelGroup([
+                Model('mistralai/Devstral-Small-2505', 'mistralai/Devstral-Small-2505'),
+            ],
+                       requires=['transformers>=4.43', 'mistral-common>=1.5.5'])
+        ],
+        DevstralLoader,
+        template=TemplateType.devstral,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+
+class Mistral3Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import Mistral3ForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or Mistral3ForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mistral3,
+        [
+            ModelGroup([
+                Model('mistralai/Mistral-Small-3.1-24B-Base-2503', 'mistralai/Mistral-Small-3.1-24B-Base-2503'),
+                Model('mistralai/Mistral-Small-3.1-24B-Instruct-2503', 'mistralai/Mistral-Small-3.1-24B-Instruct-2503'),
+            ],
+                       requires=['transformers>=4.49']),
+            ModelGroup([
+                Model('mistralai/Ministral-3-3B-Base-2512', 'mistralai/Ministral-3-3B-Base-2512'),
+                Model('mistralai/Ministral-3-3B-Instruct-2512', 'mistralai/Ministral-3-3B-Instruct-2512'),
+                Model('mistralai/Ministral-3-3B-Instruct-2512-BF16', 'mistralai/Ministral-3-3B-Instruct-2512-BF16'),
+                Model('mistralai/Ministral-3-8B-Base-2512', 'mistralai/Ministral-3-8B-Base-2512'),
+                Model('mistralai/Ministral-3-8B-Instruct-2512', 'mistralai/Ministral-3-8B-Instruct-2512'),
+                Model('mistralai/Ministral-3-8B-Instruct-2512-BF16', 'mistralai/Ministral-3-8B-Instruct-2512-BF16'),
+                Model('mistralai/Ministral-3-14B-Base-2512', 'mistralai/Ministral-3-14B-Base-2512'),
+                Model('mistralai/Ministral-3-14B-Instruct-2512', 'mistralai/Ministral-3-14B-Instruct-2512'),
+                Model('mistralai/Ministral-3-14B-Instruct-2512-BF16', 'mistralai/Ministral-3-14B-Instruct-2512-BF16'),
+            ],
+                       TemplateType.mistral_2512,
+                       requires=['transformers>=5.0.0.dev0', 'mistral-common>=1.8.6']),
+            ModelGroup([
+                Model('mistralai/Ministral-3-3B-Reasoning-2512', 'mistralai/Ministral-3-3B-Reasoning-2512'),
+                Model('mistralai/Ministral-3-8B-Reasoning-2512', 'mistralai/Ministral-3-8B-Reasoning-2512'),
+                Model('mistralai/Ministral-3-14B-Reasoning-2512', 'mistralai/Ministral-3-14B-Reasoning-2512'),
+            ],
+                       TemplateType.mistral_2512_thinking,
+                       requires=['transformers>=5.0.0.dev0', 'mistral-common>=1.8.6']),
+        ],
+        Mistral3Loader,
+        template=TemplateType.mistral_2503,
+        model_arch=ModelArch.llava_hf,
+        architectures=['Mistral3ForConditionalGeneration'],
+        tags=['vision'],
+        ignore_patterns=[],
+    ))
+
+
+class Mistral3_2506Loader(Mistral3Loader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer_dir = safe_snapshot_download('mistralai/Mistral-Small-3.1-24B-Instruct-2503', download_model=False)
+        processor = AutoProcessor.from_pretrained(tokenizer_dir)
+        return processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mistral3_2506,
+        [
+            ModelGroup([
+                Model('mistralai/Mistral-Small-3.2-24B-Instruct-2506', 'mistralai/Mistral-Small-3.2-24B-Instruct-2506'),
+            ]),
+        ],
+        Mistral3_2506Loader,
+        template=TemplateType.mistral_2506,
+        architectures=['Mistral3ForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.49'],
+    ))

swift/model/models/mllm.py

@@ -0,0 +1,390 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from types import MethodType
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_logger
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_output_clone
+from ..register import ModelLoader, register_model
+from ..utils import use_submodel_func
+from .qwen import Qwen2VLLoader, patch_qwen_vl_utils
+
+logger = get_logger()
+
+
+class Idefics3Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForVision2Seq
+        self.auto_model_cls = self.auto_model_cls or AutoModelForVision2Seq
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.idefics3,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Idefics3-8B-Llama3', 'HuggingFaceM4/Idefics3-8B-Llama3'),
+            ]),
+        ],
+        Idefics3Loader,
+        template=TemplateType.idefics3,
+        model_arch=ModelArch.idefics3,
+        architectures=['Idefics3ForConditionalGeneration'],
+        tags=['vision'],
+        requires=['transformers>=4.45'],
+    ))
+
+
+class PixtralLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import LlavaForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.pixtral,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/pixtral-12b', 'mistral-community/pixtral-12b'),
+            ]),
+        ],
+        PixtralLoader,
+        template=TemplateType.pixtral,
+        model_arch=ModelArch.llava_hf,
+        architectures=['LlavaForConditionalGeneration'],
+        requires=['transformers>=4.45'],
+        tags=['vision'],
+    ))
+
+
+class MolMoeLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+
+        # fix bug for molmoe-1b
+        def to_dict(self, *args, **kwargs):
+            res = self._to_dict(*args, **kwargs)
+            res['vision_backbone'] = self.vision_backbone.__dict__
+            res.pop('to_dict')
+            res.pop('_to_dict')
+            return res
+
+        model.config._to_dict = model.config.to_dict
+        model.config.to_dict = MethodType(to_dict, model.config)
+        patch_output_clone(model.model.transformer.wte)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.molmoe,
+        [
+            ModelGroup([
+                Model('LLM-Research/MolmoE-1B-0924', 'allenai/MolmoE-1B-0924'),
+            ]),
+        ],
+        MolMoeLoader,
+        template=TemplateType.molmo,
+        model_arch=ModelArch.molmo,
+        torch_dtype=torch.float32,
+        architectures=['OLMoForCausalLM'],
+        tags=['vision'],
+        requires=['transformers>=4.45'],
+    ))
+
+
+class MolmoLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model_cls = get_class_from_dynamic_module('modeling_molmo.MolmoForCausalLM', model_dir)
+        model_cls._no_split_modules = ['MolmoSequentialBlock']
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.model.transformer.wte)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.molmo,
+        [
+            ModelGroup([
+                Model('LLM-Research/Molmo-7B-O-0924', 'allenai/Molmo-7B-O-0924'),
+                Model('LLM-Research/Molmo-7B-D-0924', 'allenai/Molmo-7B-D-0924'),
+                Model('LLM-Research/Molmo-72B-0924', 'allenai/Molmo-72B-0924'),
+            ]),
+        ],
+        MolmoLoader,
+        template=TemplateType.molmo,
+        model_arch=ModelArch.molmo,
+        architectures=['MolmoForCausalLM'],
+        tags=['vision'],
+        requires=['transformers>=4.45'],
+    ))
+
+
+class Molmo2Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import AutoModelForImageTextToText
+        model_cls = get_class_from_dynamic_module('modeling_molmo2.Molmo2ForConditionalGeneration', model_dir)
+        no_split_modules = getattr(model_cls, '_no_split_modules', []) or []
+        if 'MolmoSequentialBlock' not in no_split_modules:
+            model_cls._no_split_modules = no_split_modules + ['MolmoSequentialBlock']
+        self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.model.transformer.wte)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.molmo2,
+        [
+            ModelGroup([
+                Model('allenai/Molmo2-4B', 'allenai/Molmo2-4B'),
+                Model('allenai/Molmo2-8B', 'allenai/Molmo2-8B'),
+                Model('allenai/Molmo2-O-7B', 'allenai/Molmo2-O-7B'),
+            ]),
+        ],
+        Molmo2Loader,
+        template=TemplateType.molmo2,
+        model_arch=ModelArch.molmo,
+        architectures=['Molmo2ForConditionalGeneration'],
+        tags=['vision', 'video'],
+        requires=['transformers>=4.57.1,<5', 'decord'],
+    ))
+
+
+class MegrezOmniLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model_cls = get_class_from_dynamic_module('modeling_megrezo.MegrezO', model_dir)
+        model_cls._no_split_modules = ['ResidualAttentionBlock', 'LlamaDecoderLayer']
+        model_cls = get_class_from_dynamic_module('modeling_megrezo.SiglipVisionTransformer', model_dir)
+        model_cls._no_split_modules = ['SiglipEncoderLayer']
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.llm.model.embed_tokens)
+        use_submodel_func(model, 'llm')
+        return model
+
+    def _get_model_processor(self, model_dir, config):
+        model, processor = super().get_processor(model_dir, config)
+        if model:
+            processor = model._get_or_init_processor()
+        return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.megrez_omni,
+        [
+            ModelGroup([
+                Model('InfiniAI/Megrez-3B-Omni', 'Infinigence/Megrez-3B-Omni'),
+            ]),
+        ],
+        MegrezOmniLoader,
+        template=TemplateType.megrez_omni,
+        model_arch=ModelArch.megrez_omni,
+        architectures=['MegrezO'],
+        tags=['vision', 'audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_gme, [
+            ModelGroup([
+                Model('iic/gme-Qwen2-VL-2B-Instruct', 'Alibaba-NLP/gme-Qwen2-VL-2B-Instruct'),
+                Model('iic/gme-Qwen2-VL-7B-Instruct', 'Alibaba-NLP/gme-Qwen2-VL-7B-Instruct'),
+            ]),
+        ],
+        Qwen2VLLoader,
+        template=TemplateType.qwen2_gme,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2VLForConditionalGeneration'],
+        tags=['vision']))
+
+
+class JinaRerankerM0Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        # Use AutoModel to respect the model repo's dynamic class mapping
+        # and load the custom Jina reranker head via trust_remote_code.
+        from transformers import AutoModel
+        from transformers.modeling_outputs import SequenceClassifierOutputWithPast
+        self.auto_model_cls = self.auto_model_cls or AutoModel
+        model = super().get_model(model_dir, *args, **kwargs)
+        # Patch forward to return a sequence-classification-style output with `.logits`
+        # Use the model's own head (already present in jina-reranker-m0), just wrap outputs.
+
+        if not hasattr(model, '_forward_origin'):
+            model._forward_origin = model.forward
+            model.logit_bias = 2.65
+
+            def forward(self,
+                        input_ids=None,
+                        attention_mask=None,
+                        position_ids=None,
+                        inputs_embeds=None,
+                        pixel_values=None,
+                        image_grid_thw=None,
+                        video_grid_thw=None,
+                        output_attentions=None,
+                        output_hidden_states=None,
+                        return_dict=None,
+                        **kwargs):
+                # Remove labels to avoid upstream asserts in ranking models
+                kwargs.pop('labels', None)
+                if return_dict is None:
+                    return_dict = True
+
+                out = self._forward_origin(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    inputs_embeds=inputs_embeds,
+                    pixel_values=pixel_values,
+                    image_grid_thw=image_grid_thw,
+                    video_grid_thw=video_grid_thw,
+                    output_attentions=output_attentions,
+                    output_hidden_states=output_hidden_states,
+                    return_dict=return_dict,
+                    **kwargs)
+
+                logits = out.unsqueeze(-1) - self.logit_bias
+
+                if not return_dict:
+                    return (logits, )
+
+                return SequenceClassifierOutputWithPast(logits=logits)
+
+            model.forward = MethodType(forward, model)
+
+            def padding_free_fn(self, output, kwargs, padding_side):
+                return_dict = kwargs.get('return_dict', None)
+
+                output.logits = output['last_hidden_state'][:, -1]
+                logits = self.score(output.logits)
+                logits = logits - self.logit_bias
+
+                if not return_dict:
+                    return (logits, )
+
+                return SequenceClassifierOutputWithPast(logits=logits)
+
+            model.padding_free_fn = MethodType(padding_free_fn, model)
+            return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.jina_reranker_m0,
+        [ModelGroup([Model('JinaAI/jina-reranker-m0', 'JinaAI/jina-reranker-m0')])],
+        JinaRerankerM0Loader,
+        template=TemplateType.jina_reranker_m0,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['JinaRerankerM0ForConditionalGeneration'],
+        task_type='reranker',
+        tags=['reranker', 'vision'],
+    ))
+
+
+class KeyeVLLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        processor = super().get_processor(model_dir, config)
+        from keye_vl_utils import vision_process
+        global_vars = patch_qwen_vl_utils(vision_process)
+        processor.global_vars = global_vars
+        return processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.keye_vl,
+        [
+            ModelGroup([
+                Model('Kwai-Keye/Keye-VL-8B-Preview', 'Kwai-Keye/Keye-VL-8B-Preview'),
+            ]),
+        ],
+        KeyeVLLoader,
+        template=TemplateType.keye_vl,
+        model_arch=ModelArch.keye_vl,
+        architectures=['KeyeForConditionalGeneration'],
+        tags=['vision'],
+        requires=['keye_vl_utils'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.keye_vl_1_5,
+        [
+            ModelGroup([
+                Model('Kwai-Keye/Keye-VL-1_5-8B', 'Kwai-Keye/Keye-VL-1_5-8B'),
+            ]),
+        ],
+        KeyeVLLoader,
+        template=TemplateType.keye_vl_1_5,
+        model_arch=ModelArch.keye_vl,
+        architectures=['KeyeVL1_5ForConditionalGeneration'],
+        tags=['vision'],
+        requires=['keye_vl_utils>=1.5.2', 'transformers==4.52.4'],
+    ))
+
+
+class DotsOCRLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model_cls = get_class_from_dynamic_module('modeling_dots_vision.DotsVisionTransformer', model_dir)
+        model_cls._no_split_modules = ['DotsVisionBlock']
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.dots_ocr,
+        [ModelGroup([
+            Model('rednote-hilab/dots.ocr', 'rednote-hilab/dots.ocr'),
+        ])],
+        DotsOCRLoader,
+        template=TemplateType.dots_ocr,
+        model_arch=ModelArch.dots_ocr,
+        architectures=['DotsOCRForCausalLM'],
+        requires=['transformers>=4.51.0'],
+    ))
+
+
+class Sail2VLLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        use_submodel_func(model, 'language_model')
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.sail_vl2, [
+            ModelGroup([
+                Model('BytedanceDouyinContent/SAIL-VL2-2B', 'BytedanceDouyinContent/SAIL-VL2-2B'),
+                Model('BytedanceDouyinContent/SAIL-VL2-2B-Thinking', 'BytedanceDouyinContent/SAIL-VL2-2B-Thinking'),
+                Model('BytedanceDouyinContent/SAIL-VL2-8B', 'BytedanceDouyinContent/SAIL-VL2-8B'),
+                Model('BytedanceDouyinContent/SAIL-VL2-8B-Thinking', 'BytedanceDouyinContent/SAIL-VL2-8B-Thinking'),
+            ])
+        ],
+        Sail2VLLoader,
+        template=TemplateType.sail_vl2,
+        model_arch=ModelArch.internvl,
+        architectures=['SAILVLModel'],
+        requires=['transformers<=4.51.3'],
+        tags=['vision']))

swift/model/models/moonshot.py

@@ -0,0 +1,57 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.template import TemplateType
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_get_input_embeddings
+from ..register import ModelLoader, register_model
+
+
+class KimiVLLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        KimiVLPreTrainedModel = get_class_from_dynamic_module('modeling_kimi_vl.KimiVLPreTrainedModel', model_dir)
+        try:
+            del KimiVLPreTrainedModel._supports_sdpa
+        except AttributeError:
+            pass
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_get_input_embeddings(model.vision_tower, 'patch_embed')
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.kimi_vl,
+        [
+            ModelGroup([
+                Model('moonshotai/Kimi-VL-A3B-Instruct', 'moonshotai/Kimi-VL-A3B-Instruct'),
+                Model('moonshotai/Kimi-VL-A3B-Thinking', 'moonshotai/Kimi-VL-A3B-Thinking'),
+                Model('moonshotai/Kimi-VL-A3B-Thinking-2506', 'moonshotai/Kimi-VL-A3B-Thinking-2506'),
+            ])
+        ],
+        KimiVLLoader,
+        template=TemplateType.kimi_vl,
+        model_arch=ModelArch.llava_hf_legacy,
+        architectures=['KimiVLForConditionalGeneration'],
+        requires=['transformers<4.49'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.kimi_k25,
+        [
+            ModelGroup([
+                Model('moonshotai/Kimi-K2.5', 'moonshotai/Kimi-K2.5'),
+                Model('moonshotai/Kimi-K2.6', 'moonshotai/Kimi-K2.6'),
+                Model('moonshotai/Kimi-K2.7-Code', 'moonshotai/Kimi-K2.7-Code'),
+            ])
+        ],
+        template=TemplateType.kimi_k25,
+        model_arch=ModelArch.kimi_k25,
+        architectures=['KimiK25ForConditionalGeneration'],
+        requires=['transformers>=4.57.1,<5.0.0'],
+    ))

swift/model/models/mplug.py

@@ -0,0 +1,163 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+import sys
+from collections import OrderedDict
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_logger, git_clone_github
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+from ..utils import use_submodel_func
+from .qwen import QwenLoader
+
+logger = get_logger()
+
+
+class MplugOwl2Loader(ModelLoader):
+
+    def _get_model(self, model_dir: str, vocab_size, *args, **kwargs) -> PreTrainedModel:
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/X-PLUG/mPLUG-Owl')
+        local_repo_path = os.path.join(local_repo_path, 'mPLUG-Owl2')
+        sys.path.append(local_repo_path)
+        # register
+        # https://github.com/X-PLUG/mPLUG-Owl/blob/main/mPLUG-Owl2/mplug_owl2/model/modeling_mplug_owl2.py#L447
+        from mplug_owl2 import MPLUGOwl2LlamaForCausalLM
+        if vocab_size is not None:
+            config.vocab_size = vocab_size
+        model = super().get_model(model_dir, *args, **kwargs)
+        logger.info('Please ignore the unimported warning.')
+        return model
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        return self._get_model(model_dir, None, *args, **kwargs)
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        from transformers.models.clip.image_processing_clip import CLIPImageProcessor
+        processor = CLIPImageProcessor.from_pretrained(model_dir)
+        return processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl2, [ModelGroup([
+            Model('iic/mPLUG-Owl2', 'MAGAer13/mplug-owl2-llama2-7b'),
+        ])],
+        MplugOwl2Loader,
+        template=TemplateType.mplug_owl2,
+        model_arch=ModelArch.mplug_owl2,
+        requires=['transformers<4.35', 'icecream'],
+        tags=['vision']), )
+
+
+class MplugOwl2_1Loader(QwenLoader, MplugOwl2Loader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        return self._get_model(model_dir, 151851, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl2_1, [ModelGroup([
+            Model('iic/mPLUG-Owl2.1', 'Mizukiluke/mplug_owl_2_1'),
+        ])],
+        MplugOwl2_1Loader,
+        template=TemplateType.mplug_owl2,
+        model_arch=ModelArch.mplug_owl2_1,
+        requires=['transformers<4.35', 'icecream'],
+        tags=['vision']))
+
+
+class MplugOwl3Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        get_class_from_dynamic_module('configuration_hyper_qwen2.HyperQwen2Config', model_dir)
+        model_cls = get_class_from_dynamic_module('modeling_mplugowl3.mPLUGOwl3Model', model_dir)
+        model_cls._no_split_modules = ['SiglipEncoderLayer']
+        model = super().get_model(model_dir, *args, **kwargs)
+        func_list = ['generate', 'forward']
+        use_submodel_func(model, 'language_model', func_list)
+
+        all_hooks = OrderedDict()
+        hooks_with_kwargs = OrderedDict()
+
+        def append_hooks(sub_module, inc_id=0):
+            for id, hook in sub_module._forward_hooks.items():
+                all_hooks[inc_id] = hook
+                if id in sub_module._forward_hooks_with_kwargs:
+                    hooks_with_kwargs[inc_id] = sub_module._forward_hooks_with_kwargs[id]
+                inc_id += 1
+            return inc_id
+
+        inc_id = append_hooks(model.language_model)
+        append_hooks(model, inc_id)
+        model._forward_hooks = all_hooks
+        model._forward_hooks_with_kwargs = hooks_with_kwargs
+        return model
+
+    def _get_model_processor(self, model_dir, config):
+        model, tokenizer = super()._get_model_processor(model_dir, config)
+        if model:
+            tokenizer = model.init_processor(tokenizer)
+        return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl3, [
+            ModelGroup([
+                Model('iic/mPLUG-Owl3-1B-241014', 'mPLUG/mPLUG-Owl3-1B-241014'),
+                Model('iic/mPLUG-Owl3-2B-241014', 'mPLUG/mPLUG-Owl3-2B-241014'),
+                Model('iic/mPLUG-Owl3-7B-240728', 'mPLUG/mPLUG-Owl3-7B-240728'),
+            ]),
+        ],
+        MplugOwl3Loader,
+        template=TemplateType.mplug_owl3,
+        architectures=['mPLUGOwl3Model'],
+        model_arch=ModelArch.mplug_owl3,
+        requires=['transformers>=4.36', 'icecream', 'decord'],
+        tags=['vision', 'video']))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl3_241101, [
+            ModelGroup([
+                Model('iic/mPLUG-Owl3-7B-241101', 'mPLUG/mPLUG-Owl3-7B-241101'),
+            ]),
+        ],
+        MplugOwl3Loader,
+        template=TemplateType.mplug_owl3_241101,
+        architectures=['mPLUGOwl3Model'],
+        model_arch=ModelArch.mplug_owl3,
+        requires=['transformers>=4.36', 'icecream'],
+        tags=['vision', 'video']))
+
+
+class DocOwl2Loader(ModelLoader):
+
+    def _get_model_processor(self, model_dir, config):
+        model, tokenizer = super()._get_model_processor(model_dir, config)
+        if model:
+            tokenizer = model.init_processor(tokenizer, basic_image_size=504, crop_anchors='grid_12')
+        return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.doc_owl2, [
+            ModelGroup([
+                Model('iic/DocOwl2', 'mPLUG/DocOwl2'),
+            ]),
+        ],
+        DocOwl2Loader,
+        template=TemplateType.doc_owl2,
+        architectures=['mPLUGDocOwl2'],
+        model_arch=ModelArch.doc_owl2,
+        requires=['transformers>=4.36', 'icecream'],
+        tags=['vision']))

swift/model/models/openbuddy.py

@@ -0,0 +1,74 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.template import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import register_model
+
+logger = get_logger()
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_llama,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama-65b-v8-bf16', 'OpenBuddy/openbuddy-llama-65b-v8-bf16'),
+            ], TemplateType.openbuddy),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama2-13b-v8.1-fp16', 'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16'),
+                Model('OpenBuddy/openbuddy-llama2-70b-v10.1-bf16', 'OpenBuddy/openbuddy-llama2-70b-v10.1-bf16'),
+            ], TemplateType.openbuddy),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-deepseek-67b-v15.2', 'OpenBuddy/openbuddy-deepseek-67b-v15.2'),
+            ], TemplateType.openbuddy),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama3-8b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-8b-v21.1-8k'),
+                Model('OpenBuddy/openbuddy-llama3-70b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-70b-v21.1-8k'),
+                Model('OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k', 'OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k'),
+            ], TemplateType.openbuddy2),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k', 'OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k'),
+                Model('OpenBuddy/openbuddy-nemotron-70b-v23.2-131k', 'OpenBuddy/openbuddy-nemotron-70b-v23.2-131k'),
+            ],
+                       TemplateType.openbuddy2,
+                       requires=['transformers>=4.43']),
+            ModelGroup(
+                [Model('OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k', 'OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k')],
+                TemplateType.openbuddy2,
+                requires=['transformers>=4.45']),
+        ],
+        model_arch=ModelArch.llama,
+        mcore_model_type='gpt',
+        architectures=['LlamaForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_mistral,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-mistral-7b-v17.1-32k', 'OpenBuddy/openbuddy-mistral-7b-v17.1-32k'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-zephyr-7b-v14.1', 'OpenBuddy/openbuddy-zephyr-7b-v14.1'),
+            ]),
+        ],
+        template=TemplateType.openbuddy,
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.34'],
+        architectures=['MistralForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_mixtral,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k', 'OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k'),
+            ], ),
+        ],
+        template=TemplateType.openbuddy,
+        architectures=['MixtralForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))

swift/model/models/seed.py

@@ -0,0 +1,21 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.template import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import register_model
+
+logger = get_logger()
+
+register_model(
+    ModelMeta(
+        LLMModelType.seed_oss, [
+            ModelGroup([
+                Model('ByteDance-Seed/Seed-OSS-36B-Instruct', 'ByteDance-Seed/Seed-OSS-36B-Instruct'),
+                Model('ByteDance-Seed/Seed-OSS-36B-Base', 'ByteDance-Seed/Seed-OSS-36B-Base'),
+                Model('ByteDance-Seed/Seed-OSS-36B-Base-woSyn', 'ByteDance-Seed/Seed-OSS-36B-Base-woSyn'),
+            ])
+        ],
+        template=TemplateType.seed_oss,
+        architectures=['SeedOssForCausalLM'],
+        requires=['transformers>=4.56']))

swift/model/models/skywork.py

@@ -0,0 +1,70 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+from transformers import PretrainedConfig
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor
+from ..constant import LLMModelType, RMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+
+class SkyworkLoader(ModelLoader):
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        tokenizer = super().get_processor(model_dir, config)
+        tokenizer.add_tokens('[USER]')
+        tokenizer.add_tokens('[BOT]')
+        tokenizer.add_tokens('[SEP]')
+        return tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.skywork,
+        [
+            ModelGroup([
+                Model('skywork/Skywork-13B-base', 'skywork/Skywork-13B-base'),
+                Model('skywork/Skywork-13B-chat'),
+            ]),
+        ],
+        template=TemplateType.skywork,
+        architectures=['SkyworkForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.llama3_2_reward,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B', 'Skywork/Skywork-Reward-Llama-3.1-8B'),
+                Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2', 'Skywork/Skywork-Reward-Llama-3.1-8B-v0.2'),
+            ]),
+            ModelGroup([
+                Model('AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft', 'Ray2333/GRM_Llama3.1_8B_rewardmodel-ft'),
+                Model('AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft', 'Ray2333/GRM-llama3.2-3B-rewardmodel-ft'),
+            ])
+        ],
+        template=TemplateType.llama3_2,
+        requires=['transformers>=4.43'],
+        architectures=['LlamaForSequenceClassification'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.gemma_reward,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B', 'Skywork/Skywork-Reward-Gemma-2-27B'),
+                Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2', 'Skywork/Skywork-Reward-Gemma-2-27B-v0.2'),
+            ]),
+        ],
+        template=TemplateType.gemma,
+        requires=['transformers>=4.42'],
+        architectures=['Gemma2ForSequenceClassification'],
+        model_arch=ModelArch.llama,
+    ))

swift/model/models/stepfun.py

@@ -0,0 +1,167 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import sys
+from functools import wraps
+from transformers import AutoModel, PretrainedConfig, PreTrainedModel
+
+from swift.template import TemplateType
+from swift.utils import Processor, git_clone_github, safe_snapshot_download
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..patcher import patch_output_clone
+from ..register import ModelLoader, register_model
+
+
+class GotOCR2Loader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        self.auto_model_cls = AutoModel
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.got_ocr2, [
+            ModelGroup([
+                Model('stepfun-ai/GOT-OCR2_0', 'stepfun-ai/GOT-OCR2_0'),
+            ]),
+        ],
+        GotOCR2Loader,
+        template=TemplateType.got_ocr2,
+        model_arch=ModelArch.got_ocr2,
+        architectures=['GOTQwenForCausalLM'],
+        tags=['vision']))
+
+
+class GotOCR2HfLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers.models.got_ocr2 import GotOcr2ForConditionalGeneration
+        GotOcr2ForConditionalGeneration._no_split_modules = ['GotOcr2VisionLayer']
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.got_ocr2_hf, [
+            ModelGroup([
+                Model('stepfun-ai/GOT-OCR-2.0-hf', 'stepfun-ai/GOT-OCR-2.0-hf'),
+            ]),
+        ],
+        GotOCR2HfLoader,
+        template=TemplateType.got_ocr2_hf,
+        model_arch=ModelArch.llava_hf,
+        architectures=['GotOcr2ForConditionalGeneration'],
+        tags=['vision']))
+
+
+class StepAudioLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/stepfun-ai/Step-Audio.git')
+        sys.path.append(local_repo_path)
+        from tokenizer import StepAudioTokenizer
+        encoder_path = safe_snapshot_download('stepfun-ai/Step-Audio-Tokenizer', check_local=True)
+        model = super().get_model(model_dir, *args, **kwargs)
+        model.encoder = StepAudioTokenizer(encoder_path)
+        # from tts import StepAudioTTS
+        # if not os.path.exists('speakers'):
+        #     shutil.copytree(os.path.join(local_repo_path, 'speakers'), 'speakers')
+        # decoder_path = safe_snapshot_download('stepfun-ai/Step-Audio-TTS-3B', check_local=True)
+        # model.decoder = StepAudioTTS(decoder_path, model.encoder)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.step_audio, [
+            ModelGroup([
+                Model('stepfun-ai/Step-Audio-Chat', 'stepfun-ai/Step-Audio-Chat'),
+            ]),
+        ],
+        StepAudioLoader,
+        template=TemplateType.step_audio,
+        architectures=['Step1ForCausalLM'],
+        requires=['funasr', 'sox', 'conformer', 'openai-whisper', 'librosa'],
+        tags=['audio']))
+
+
+def _patch_step_audio2_mini(model):
+    if hasattr(model.__class__, 'origin_forward'):
+        return
+
+    model.__class__.origin_forward = model.__class__.forward
+
+    @wraps(model.__class__.origin_forward)
+    def _forward(self, *args, **kwargs):
+        labels = kwargs.get('labels')
+        output = self.origin_forward(*args, **kwargs)
+        if labels is not None and output.loss is None:
+            output['loss'] = self.loss_function(
+                logits=output.logits, labels=labels, vocab_size=self.config.get_text_config().vocab_size)
+        return output
+
+    model.__class__.forward = _forward
+
+
+class StepAudio2MiniLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, *args, **kwargs)
+        patch_output_clone(model.model.embed_tokens)
+        _patch_step_audio2_mini(model)
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.step_audio2_mini,
+        [ModelGroup([
+            Model('stepfun-ai/Step-Audio-2-mini', 'stepfun-ai/Step-Audio-2-mini'),
+        ])],
+        StepAudio2MiniLoader,
+        template=TemplateType.step_audio2_mini,
+        model_arch=ModelArch.step_audio2_mini,
+        architectures=['StepAudio2ForCausalLM'],
+        requires=['transformers==4.53.3', 'torchaudio', 'librosa'],
+        tags=['audio'],
+    ))
+
+
+class Step3VLLoader(ModelLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        config = super().get_config(model_dir)
+        config.vocab_size = config.text_config.vocab_size
+        return config
+
+    def get_model(self, model_dir: str, config: PretrainedConfig, processor: Processor,
+                  model_kwargs) -> PreTrainedModel:
+        key_mapping = {
+            '^vision_model': 'model.vision_model',
+            r'^model(?!\.(language_model|vision_model))': 'model.language_model',
+            'vit_large_projector': 'model.vit_large_projector',
+        }
+        model_kwargs = model_kwargs.copy()
+        model_kwargs['key_mapping'] = key_mapping
+        return super().get_model(model_dir, config, processor, model_kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.step3_vl,
+        [
+            ModelGroup([
+                Model('stepfun-ai/Step3-VL-10B-Base', 'stepfun-ai/Step3-VL-10B-Base'),
+                Model('stepfun-ai/Step3-VL-10B', 'stepfun-ai/Step3-VL-10B'),
+            ])
+        ],
+        Step3VLLoader,
+        template=TemplateType.step3_vl,
+        model_arch=ModelArch.step3_vl,
+        architectures=['StepVLForConditionalGeneration'],
+        requires=['transformers>=4.57.0'],
+        tags=['vision'],
+    ))

swift/model/models/telechat.py

@@ -0,0 +1,60 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+from transformers import GenerationConfig, PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
+
+from swift.template import TemplateType
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+
+class TeleChatLoader(ModelLoader):
+
+    def get_model(self, model_dir: str, config, processor, **kwargs) -> PreTrainedModel:
+        model = super().get_model(model_dir, config, processor, **kwargs)
+        generation_config = GenerationConfig.from_pretrained(model_dir)
+        for k in ['bos_token_id', 'eos_token_id', 'pad_token_id', 'user_token_id', 'bot_token_id']:
+            setattr(processor, k, getattr(generation_config, k))
+        return model
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.telechat,
+        [
+            ModelGroup([
+                Model('TeleAI/TeleChat-7B', 'Tele-AI/telechat-7B'),
+                Model('TeleAI/TeleChat-12B', 'Tele-AI/TeleChat-12B'),
+                Model('TeleAI/TeleChat-12B-v2', 'Tele-AI/TeleChat-12B-v2'),
+                Model('TeleAI/TeleChat-52B', 'TeleAI/TeleChat-52B'),
+            ]),
+            ModelGroup([
+                Model('swift/TeleChat-12B-V2-GPTQ-Int4'),
+            ]),
+            ModelGroup([
+                Model('TeleAI/TeleChat2-35B', 'Tele-AI/TeleChat2-35B'),
+                Model('TeleAI/TeleChat2-115B', 'Tele-AI/TeleChat2-115B'),
+            ]),
+        ],
+        TeleChatLoader,
+        template=TemplateType.telechat,
+        model_arch=ModelArch.telechat,
+        architectures=['TelechatForCausalLM', 'TeleChatForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.telechat2,
+        [
+            ModelGroup([
+                Model('TeleAI/TeleChat2-3B', 'Tele-AI/TeleChat2-3B'),
+                Model('TeleAI/TeleChat2-7B-32K', 'Tele-AI/TeleChat2-7B-32K'),
+                Model('TeleAI/TeleChat2-35B-32K', 'Tele-AI/TeleChat2-35B-32K'),
+                Model('TeleAI/TeleChat2-35B-Nov', 'Tele-AI/TeleChat2-35B-Nov'),
+            ]),
+        ],
+        template=TemplateType.telechat2,
+        model_arch=ModelArch.telechat,
+        architectures=['TeleChat2ForCausalLM'],
+    ))

swift/model/models/tencent.py

@@ -0,0 +1,37 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+
+from transformers import PreTrainedModel
+
+from swift.template import TemplateType
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+
+class HunyuanVLLoader(ModelLoader):
+
+    def get_config(self, model_dir: str):
+        self.attn_impl = self.attn_impl or 'eager'
+        return super().get_config(model_dir)
+
+    def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
+        from transformers import HunYuanVLForConditionalGeneration
+        self.auto_model_cls = self.auto_model_cls or HunYuanVLForConditionalGeneration
+        return super().get_model(model_dir, *args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.hunyuan_ocr,
+        [
+            ModelGroup([
+                Model('Tencent-Hunyuan/HunyuanOCR', 'tencent/HunyuanOCR'),
+            ]),
+        ],
+        HunyuanVLLoader,
+        template=TemplateType.hunyuan_ocr,
+        architectures=['HunYuanVLForConditionalGeneration'],
+        model_arch=ModelArch.hunyuan_vl,
+        requires=['transformers>=4.49.0'],
+    ))

swift/model/models/valley.py

@@ -0,0 +1,80 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import sys
+from functools import wraps
+from transformers import PreTrainedModel
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import git_clone_github, safe_snapshot_download
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+
+class ValleyLoader(ModelLoader):
+
+    def get_config(self, model_dir: str):
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            repo_path = 'https://github.com/bytedance/Valley.git'
+            local_repo_path = git_clone_github(repo_path)
+        sys.path.append(local_repo_path)
+        from valley_eagle.model.language_model.valley_qwen2 import ValleyConfig
+        self.auto_config_cls = ValleyConfig
+        return super().get_config(model_dir)
+
+    def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
+        from transformers.modeling_outputs import CausalLMOutputWithPast
+        from valley_eagle.model.language_model.valley_qwen2 import ValleyQwen2ForCausalLM
+        config.mm_vision_tower = safe_snapshot_download('AI-ModelScope/siglip-so400m-patch14-384', check_local=True)
+        config.eagle_vision_tower = safe_snapshot_download('Qwen/Qwen2-VL-7B-Instruct', check_local=True)
+        auto_model_cls = ValleyQwen2ForCausalLM
+
+        if not hasattr(ValleyQwen2ForCausalLM, '_origin_forward'):
+            forward = ValleyQwen2ForCausalLM.forward
+            ValleyQwen2ForCausalLM._origin_forward = forward
+
+            @wraps(forward)
+            def new_forward(*args, **kwargs):
+                import torch
+                outputs = forward(*args, **kwargs)
+                loss = outputs.loss
+                if loss is not None and loss.shape[-1] > 0:
+                    loss = torch.mean(loss, dim=-1)
+                return CausalLMOutputWithPast(
+                    loss=loss,
+                    logits=outputs.logits,
+                    past_key_values=outputs.past_key_values,
+                    hidden_states=outputs.hidden_states,
+                    attentions=outputs.attentions,
+                )
+
+            ValleyQwen2ForCausalLM.forward = new_forward
+        self.auto_model_cls = auto_model_cls
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        model.generation_config.repetition_penalty = 1.0  # Otherwise, Error. Same for original code.
+
+        from transformers import AutoProcessor, SiglipImageProcessor
+        processor.image_processor = SiglipImageProcessor.from_pretrained(model.config.mm_vision_tower)
+        processor.qwen2vl_processor = AutoProcessor.from_pretrained(
+            model.config.eagle_vision_tower, max_pixels=1280 * 28 * 28)
+        processor.image_processor.crop_size = processor.image_processor.size['height']
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.valley,
+        [
+            ModelGroup([
+                Model('bytedance-research/Valley-Eagle-7B'),
+            ], ),
+        ],
+        ValleyLoader,
+        template=TemplateType.valley,
+        architectures=['ValleyQwen2ForCausalLM'],
+        model_arch=ModelArch.valley,
+        requires=['transformers>=4.42', 'av'],
+        tags=['vision'],
+    ))

swift/model/models/yi.py

@@ -0,0 +1,123 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+import sys
+from transformers import AutoTokenizer, PretrainedConfig, PreTrainedModel
+from typing import Any, Dict
+
+from swift.template import TemplateType
+from swift.utils import Processor, get_logger, git_clone_github
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..model_meta import Model, ModelGroup, ModelMeta
+from ..register import ModelLoader, register_model
+
+logger = get_logger()
+
+
+class YiVLLoader(ModelLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        local_repo_path = self.local_repo_path
+        if not local_repo_path:
+            local_repo_path = git_clone_github('https://github.com/01-ai/Yi')
+        sys.path.append(os.path.join(local_repo_path, 'VL'))
+        from llava.model import LlavaConfig
+        config = LlavaConfig.from_pretrained(model_dir)
+        mm_vision_tower = config.mm_vision_tower
+        config.mm_vision_tower = os.path.join(model_dir, *mm_vision_tower.rsplit('/', maxsplit=2)[-2:])
+        config.attention_dropout = 0.
+        if not hasattr(config, 'max_sequence_length'):
+            config.max_sequence_length = 2048
+        return config
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False)
+
+    def get_model(self, model_dir: str, config, processor, **kwargs) -> PreTrainedModel:
+        from llava.model import LlavaLlamaForCausalLM
+        from llava.model.constants import key_info
+        key_info['model_path'] = model_dir
+        self.auto_model_cls = self.auto_model_cls or LlavaLlamaForCausalLM
+        model = super().get_model(model_dir, config, processor, **kwargs)
+        vision_tower = model.get_vision_tower()
+        vision_tower.load_model()
+        vision_tower.to(device=model.device, dtype=config.torch_dtype)
+
+        logger.info('Please ignore the above warning.')
+        logger.info('Loading the parameters of vision_tower...')
+        model.resize_token_embeddings(len(processor))
+        processor.image_processor = vision_tower.image_processor
+        return model
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.yi_vl,
+        [
+            ModelGroup([
+                Model('01ai/Yi-VL-6B', '01-ai/Yi-VL-6B'),
+                Model('01ai/Yi-VL-34B', '01-ai/Yi-VL-34B'),
+            ], ),
+        ],
+        YiVLLoader,
+        template=TemplateType.yi_vl,
+        model_arch=ModelArch.llava_llama,
+        architectures=['LlavaLlamaForCausalLM'],
+        requires=['transformers>=4.34'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.yi,
+        [  # yi
+            ModelGroup([
+                Model('01ai/Yi-6B', '01-ai/Yi-6B'),
+                Model('01ai/Yi-6B-200K', '01-ai/Yi-6B-200K'),
+                Model('01ai/Yi-6B-Chat', '01-ai/Yi-6B-Chat'),
+                Model('01ai/Yi-6B-Chat-4bits', '01-ai/Yi-6B-Chat-4bits'),
+                Model('01ai/Yi-6B-Chat-8bits', '01-ai/Yi-6B-Chat-8bits'),
+                Model('01ai/Yi-9B', '01-ai/Yi-9B'),
+                Model('01ai/Yi-9B-200K', '01-ai/Yi-9B-200K'),
+                Model('01ai/Yi-34B', '01-ai/Yi-34B'),
+                Model('01ai/Yi-34B-200K', '01-ai/Yi-34B-200K'),
+                Model('01ai/Yi-34B-Chat', '01-ai/Yi-34B-Chat'),
+                Model('01ai/Yi-34B-Chat-4bits', '01-ai/Yi-34B-Chat-4bits'),
+                Model('01ai/Yi-34B-Chat-8bits', '01-ai/Yi-34B-Chat-8bits'),
+            ], TemplateType.chatml),
+            # yi1.5
+            ModelGroup([
+                Model('01ai/Yi-1.5-6B', '01-ai/Yi-1.5-6B'),
+                Model('01ai/Yi-1.5-6B-Chat', '01-ai/Yi-1.5-6B-Chat'),
+                Model('01ai/Yi-1.5-9B', '01-ai/Yi-1.5-9B'),
+                Model('01ai/Yi-1.5-9B-Chat', '01-ai/Yi-1.5-9B-Chat'),
+                Model('01ai/Yi-1.5-9B-Chat-16K', '01-ai/Yi-1.5-9B-Chat-16K'),
+                Model('01ai/Yi-1.5-34B', '01-ai/Yi-1.5-34B'),
+                Model('01ai/Yi-1.5-34B-Chat', '01-ai/Yi-1.5-34B-Chat'),
+                Model('01ai/Yi-1.5-34B-Chat-16K', '01-ai/Yi-1.5-34B-Chat-16K'),
+            ], TemplateType.chatml),
+            # yi1.5-quant
+            ModelGroup([
+                Model('AI-ModelScope/Yi-1.5-6B-Chat-GPTQ', 'modelscope/Yi-1.5-6B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-6B-Chat-AWQ', 'modelscope/Yi-1.5-6B-Chat-AWQ'),
+                Model('AI-ModelScope/Yi-1.5-9B-Chat-GPTQ', 'modelscope/Yi-1.5-9B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-9B-Chat-AWQ', 'modelscope/Yi-1.5-9B-Chat-AWQ'),
+                Model('AI-ModelScope/Yi-1.5-34B-Chat-GPTQ', 'modelscope/Yi-1.5-34B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-34B-Chat-AWQ', 'modelscope/Yi-1.5-34B-Chat-AWQ'),
+            ], TemplateType.chatml),
+            ModelGroup([
+                Model('01ai/Yi-Coder-1.5B', '01-ai/Yi-Coder-1.5B'),
+                Model('01ai/Yi-Coder-9B', '01-ai/Yi-Coder-9B'),
+                Model('01ai/Yi-Coder-1.5B-Chat', '01-ai/Yi-Coder-1.5B-Chat'),
+                Model('01ai/Yi-Coder-9B-Chat', '01-ai/Yi-Coder-9B-Chat'),
+            ],
+                       TemplateType.yi_coder,
+                       tags=['coding']),
+            ModelGroup([
+                Model('SUSTC/SUS-Chat-34B', 'SUSTech/SUS-Chat-34B'),
+            ], TemplateType.sus),
+        ],
+        architectures=['LlamaForCausalLM'],
+        mcore_model_type='gpt',
+        model_arch=ModelArch.llama,
+    ))