This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,14 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers.utils import is_torch_npu_available
|
||||
|
||||
from . import models
|
||||
from .constant import LLMModelType, MLLMModelType, ModelType
|
||||
from .model_arch import MODEL_ARCH_MAPPING, ModelArch, ModelKeys, MultiModelKeys, get_model_arch, register_model_arch
|
||||
from .model_meta import Model, ModelGroup, ModelInfo, ModelMeta, get_matched_model_meta, get_model_name
|
||||
from .patcher import get_lm_head_model, patch_module_forward
|
||||
from .register import (MODEL_MAPPING, ModelLoader, fix_do_sample_warning, get_default_device_map, get_model_info_meta,
|
||||
get_model_list, get_model_processor, get_processor, load_by_unsloth, register_model)
|
||||
from .utils import get_ckpt_dir, get_default_torch_dtype, get_llm_model, save_checkpoint
|
||||
|
||||
if is_torch_npu_available():
|
||||
from . import npu_patcher
|
||||
@@ -0,0 +1,360 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025, HUAWEI CORPORATION. All rights reserved.
|
||||
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
||||
|
||||
import torch
|
||||
import warnings
|
||||
from mindspeed.ops.triton.chunk_delta_h import chunk_gated_delta_rule_bwd_dhu, chunk_gated_delta_rule_fwd_h
|
||||
from mindspeed.ops.triton.chunk_o import chunk_bwd_dqkwg, chunk_bwd_dv_local, chunk_fwd_o
|
||||
from mindspeed.ops.triton.chunk_scaled_dot_kkt import chunk_scaled_dot_kkt_fwd
|
||||
from mindspeed.ops.triton.cumsum import chunk_local_cumsum
|
||||
from mindspeed.ops.triton.solve_tril import solve_tril
|
||||
from mindspeed.ops.triton.utils import autocast_custom_bwd, autocast_custom_fwd, input_guard
|
||||
from mindspeed.ops.triton.wy_fast import prepare_wy_repr_bwd, recompute_w_u_fwd
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def _torch_l2norm_fwd(
|
||||
x: torch.Tensor,
|
||||
eps: float = 1e-6,
|
||||
output_dtype: Optional[torch.dtype] = None,
|
||||
):
|
||||
x_shape_og = x.shape
|
||||
x = x.view(-1, x.shape[-1])
|
||||
x_float = x.float()
|
||||
rstd = torch.rsqrt(torch.sum(x_float * x_float, dim=-1) + eps)
|
||||
y = x_float * rstd.unsqueeze(-1)
|
||||
y = y.to(output_dtype if output_dtype is not None else x.dtype)
|
||||
return y.view(x_shape_og), rstd.view(x_shape_og[:-1])
|
||||
|
||||
|
||||
def _torch_l2norm_bwd(
|
||||
y: torch.Tensor,
|
||||
rstd: torch.Tensor,
|
||||
dy: torch.Tensor,
|
||||
eps: float = 1e-6,
|
||||
):
|
||||
y_shape_og = y.shape
|
||||
y = y.view(-1, y.shape[-1])
|
||||
dy = dy.view(-1, dy.shape[-1])
|
||||
y_float = y.float()
|
||||
dy_float = dy.float()
|
||||
rstd = rstd.view(-1).float()
|
||||
dx = dy_float * rstd.unsqueeze(-1)
|
||||
dx = dx - torch.sum(dy_float * y_float, dim=-1, keepdim=True) * y_float * rstd.unsqueeze(-1)
|
||||
return dx.to(y.dtype).view(y_shape_og)
|
||||
|
||||
|
||||
def chunk_gated_delta_rule_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: Optional[torch.LongTensor] = None,
|
||||
chunk_size: int = 64,
|
||||
):
|
||||
g = chunk_local_cumsum(g, chunk_size=chunk_size, cu_seqlens=cu_seqlens, head_first=False)
|
||||
# obtain WY representation. u is actually the new v.
|
||||
A = chunk_scaled_dot_kkt_fwd(
|
||||
k=k, g=g, beta=beta, cu_seqlens=cu_seqlens, chunk_size=chunk_size, output_dtype=torch.float32)
|
||||
A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype)
|
||||
w, u = recompute_w_u_fwd(
|
||||
k=k,
|
||||
v=v,
|
||||
beta=beta,
|
||||
A=A,
|
||||
g=g,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
h, v_new, final_state = chunk_gated_delta_rule_fwd_h(
|
||||
k=k,
|
||||
w=w,
|
||||
u=u,
|
||||
g=g,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
chunk_size=chunk_size,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
o = chunk_fwd_o(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v_new,
|
||||
h=h,
|
||||
g=g,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
return g, o, A, final_state
|
||||
|
||||
|
||||
def chunk_gated_delta_rule_bwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
do: torch.Tensor,
|
||||
dht: torch.Tensor,
|
||||
cu_seqlens: Optional[torch.LongTensor] = None,
|
||||
chunk_size: int = 64,
|
||||
):
|
||||
w, u = recompute_w_u_fwd(
|
||||
k=k,
|
||||
v=v,
|
||||
beta=beta,
|
||||
A=A,
|
||||
g=g,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
h, v_new, _ = chunk_gated_delta_rule_fwd_h(
|
||||
k=k,
|
||||
w=w,
|
||||
u=u,
|
||||
g=g,
|
||||
initial_state=initial_state,
|
||||
output_final_state=False,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
dv = chunk_bwd_dv_local(
|
||||
q=q,
|
||||
k=k,
|
||||
g=g,
|
||||
do=do,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
dh, dh0, dv = chunk_gated_delta_rule_bwd_dhu(
|
||||
q=q,
|
||||
k=k,
|
||||
w=w,
|
||||
g=g,
|
||||
h0=initial_state,
|
||||
dht=dht,
|
||||
do=do,
|
||||
dv=dv,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
dq, dk, dw, dg = chunk_bwd_dqkwg(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v_new,
|
||||
w=w,
|
||||
g=g,
|
||||
h=h,
|
||||
dv=dv,
|
||||
do=do,
|
||||
dh=dh,
|
||||
chunk_size=chunk_size,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
dk2, dv, db, dg2 = prepare_wy_repr_bwd(
|
||||
k=k, v=v, beta=beta, g=g, A=A, dw=dw, du=dv, cu_seqlens=cu_seqlens, chunk_size=chunk_size)
|
||||
dk.add_(dk2)
|
||||
dg.add_(dg2)
|
||||
if dg.dtype != torch.float32:
|
||||
raise ValueError(f'dg current type is {dg.dtype} , should be float32')
|
||||
dg = chunk_local_cumsum(dg, chunk_size=chunk_size, reverse=True, cu_seqlens=cu_seqlens, head_first=False)
|
||||
return dq, dk, dv, db, dg, dh0
|
||||
|
||||
|
||||
class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
|
||||
|
||||
@staticmethod
|
||||
@input_guard
|
||||
@autocast_custom_fwd
|
||||
def forward(
|
||||
ctx,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: Optional[torch.LongTensor] = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
chunk_size: int = 64,
|
||||
):
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q, q_rstd = _torch_l2norm_fwd(q)
|
||||
k, k_rstd = _torch_l2norm_fwd(k)
|
||||
else:
|
||||
q_rstd, k_rstd = None, None
|
||||
|
||||
g, o, A, final_state = chunk_gated_delta_rule_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=chunk_size)
|
||||
ctx.save_for_backward(q, q_rstd, k, k_rstd, v, g, beta, A, initial_state, cu_seqlens)
|
||||
ctx.scale = scale
|
||||
ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel
|
||||
ctx.chunk_size = chunk_size
|
||||
return o.to(q.dtype), final_state
|
||||
|
||||
@staticmethod
|
||||
@input_guard
|
||||
@autocast_custom_bwd
|
||||
def backward(ctx, do: torch.Tensor, dht: torch.Tensor):
|
||||
q, q_rstd, k, k_rstd, v, g, beta, A, initial_state, cu_seqlens = ctx.saved_tensors
|
||||
dq, dk, dv, db, dg, dh0 = chunk_gated_delta_rule_bwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
A=A,
|
||||
scale=ctx.scale,
|
||||
initial_state=initial_state,
|
||||
do=do,
|
||||
dht=dht,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=ctx.chunk_size,
|
||||
)
|
||||
if ctx.use_qk_l2norm_in_kernel:
|
||||
dq = _torch_l2norm_bwd(q, q_rstd, dq)
|
||||
dk = _torch_l2norm_bwd(k, k_rstd, dk)
|
||||
return dq.to(q), dk.to(k), dv.to(v), dg.to(g), db.to(beta), None, dh0, None, None, None, None
|
||||
|
||||
|
||||
@torch.compiler.disable
|
||||
def chunk_gated_delta_rule(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float = None,
|
||||
initial_state: torch.Tensor = None,
|
||||
output_final_state: bool = False,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: Optional[torch.LongTensor] = None,
|
||||
chunk_size: int = 64,
|
||||
head_first: bool = False,
|
||||
):
|
||||
r"""
|
||||
Args:
|
||||
q (torch.Tensor):
|
||||
queries of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
keys of shape `[B, T, H, K]`.
|
||||
v (torch.Tensor):
|
||||
values of shape `[B, T, H, V]`.
|
||||
g (torch.Tensor):
|
||||
(forget) gating tensor (in log space!) of shape `[B, T, H]`.
|
||||
beta (torch.Tensor):
|
||||
betas of shape `[B, T, H]`.
|
||||
scale (Optional[float]):
|
||||
Scale factor for the RetNet attention scores.
|
||||
If not provided, it will default to `1 / sqrt(K)`. Default: `None`.
|
||||
initial_state (Optional[torch.Tensor]):
|
||||
Initial state of shape `[N, H, K, V]` for `N` input sequences.
|
||||
For equal-length input sequences, `N` equals the batch size `B`.
|
||||
Default: `None`.
|
||||
output_final_state (Optional[bool]):
|
||||
Whether to output the final state of shape `[N, H, K, V]`. Default: `False`.
|
||||
use_qk_l2norm_in_kernel (bool):
|
||||
Whether to apply L2norm to the q/k tensor internally. Default: `False`.
|
||||
cu_seqlens (torch.LongTensor):
|
||||
Cumulative sequence lengths of shape `[N+1]` used for variable-length training,
|
||||
consistent with the FlashAttention API.
|
||||
head_first (Optional[bool]):
|
||||
Whether the inputs are in the head-first format. Default: `False`.
|
||||
This argument has been deprecated.
|
||||
|
||||
Returns:
|
||||
o (torch.Tensor):
|
||||
Outputs of shape `[B, T, H, V]`.
|
||||
final_state (torch.Tensor):
|
||||
Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`.
|
||||
|
||||
Examples::
|
||||
>>> import torch
|
||||
>>> import torch.nn.functional as F
|
||||
>>> from einops import rearrange
|
||||
>>> from fla.ops.gated_delta_rule import chunk_gated_delta_rule
|
||||
# inputs with equal lengths
|
||||
>>> B, T, H, K, V = 4, 2048, 4, 512, 512
|
||||
>>> q = torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda')
|
||||
>>> k = F.normalize(torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda'), p=2, dim=-1)
|
||||
>>> v = torch.randn(B, T, H, V, dtype=torch.bfloat16, device='cuda')
|
||||
>>> beta = torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda').sigmoid()
|
||||
>>> g = F.logsigmoid(torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda'))
|
||||
>>> h0 = torch.randn(B, H, K, V, dtype=torch.bfloat16, device='cuda')
|
||||
>>> o, ht = chunk_gated_delta_rule(
|
||||
q, k, v, g, beta,
|
||||
initial_state=h0,
|
||||
output_final_state=True
|
||||
)
|
||||
# for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required
|
||||
>>> q, k, v, beta, g = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, beta, g))
|
||||
# for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected
|
||||
>>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.long)
|
||||
>>> o, ht = chunk_gated_delta_rule(
|
||||
q, k, v, g, beta,
|
||||
initial_state=h0,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens
|
||||
)
|
||||
"""
|
||||
if q.dtype != k.dtype or k.dtype != v.dtype:
|
||||
raise ValueError(
|
||||
f'q current type is {q.dtype}, k current type is {k.dtype}, v current type is {v.dtype}, should be equal')
|
||||
if q.dtype == torch.float32:
|
||||
raise ValueError('ChunkGatedDeltaRuleFunction does not support float32. Please use bfloat16.')
|
||||
if len(beta.shape) != 3:
|
||||
raise ValueError(f'beta current shape len is {len(beta.shape)}, beta must be of shape [B, T, H] '
|
||||
f'if head_first=False, or [B, H, T] otherwise.')
|
||||
if head_first:
|
||||
warnings.warn('head_first is deprecated and will be removed in a future version. '
|
||||
'Please use head_first=False for now instead.')
|
||||
if not head_first and q.shape[1] < q.shape[2]:
|
||||
warnings.warn(
|
||||
f'Input tensor shape suggests format mismatch: seq_len ({q.shape[1]}) < num_heads ({q.shape[2]}). '
|
||||
'This may indicate the inputs were passed in head-first format [B, H, T, ...] '
|
||||
'when head_first=False was specified. '
|
||||
'Please verify your input tensor format matches the expected shape [B, T, H, ...].')
|
||||
if cu_seqlens is not None:
|
||||
if q.shape[0] != 1:
|
||||
raise ValueError(f'The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`.'
|
||||
f'Please flatten variable-length inputs before processing.')
|
||||
if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1:
|
||||
raise ValueError(f'The number of initial states is expected to be equal to the number of input sequences, '
|
||||
f'i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}.')
|
||||
if scale is None:
|
||||
scale = k.shape[-1]**-0.5
|
||||
o, final_state = ChunkGatedDeltaRuleFunction.apply(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
scale,
|
||||
initial_state,
|
||||
output_final_state,
|
||||
cu_seqlens,
|
||||
use_qk_l2norm_in_kernel,
|
||||
chunk_size,
|
||||
)
|
||||
return o, final_state
|
||||
@@ -0,0 +1,287 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from itertools import chain
|
||||
from typing import List
|
||||
|
||||
|
||||
class LLMModelType:
|
||||
qwen = 'qwen'
|
||||
qwen2 = 'qwen2'
|
||||
qwen2_moe = 'qwen2_moe'
|
||||
qwen3 = 'qwen3'
|
||||
qwen3_moe = 'qwen3_moe'
|
||||
qwen3_next = 'qwen3_next'
|
||||
qwen3_emb = 'qwen3_emb'
|
||||
qwen3_reranker = 'qwen3_reranker'
|
||||
|
||||
qwen2_gte = 'qwen2_gte'
|
||||
|
||||
codefuse_qwen = 'codefuse_qwen'
|
||||
modelscope_agent = 'modelscope_agent'
|
||||
|
||||
llama = 'llama'
|
||||
yi = 'yi'
|
||||
gpt_oss = 'gpt_oss'
|
||||
seed_oss = 'seed_oss'
|
||||
|
||||
codefuse_codellama = 'codefuse_codellama'
|
||||
|
||||
chatglm2 = 'chatglm2'
|
||||
chatglm3 = 'chatglm3'
|
||||
chatglm4 = 'chatglm4'
|
||||
glm4 = 'glm4'
|
||||
glm4_moe = 'glm4_moe'
|
||||
glm4_moe_lite = 'glm4_moe_lite'
|
||||
glm_moe_dsa = 'glm_moe_dsa'
|
||||
|
||||
glm_edge = 'glm_edge'
|
||||
codefuse_codegeex2 = 'codefuse_codegeex2'
|
||||
codegeex4 = 'codegeex4'
|
||||
|
||||
internlm = 'internlm'
|
||||
internlm2 = 'internlm2'
|
||||
internlm3 = 'internlm3'
|
||||
|
||||
deepseek = 'deepseek'
|
||||
deepseek_v2 = 'deepseek_v2'
|
||||
deepseek_v3 = 'deepseek_v3'
|
||||
deepseek_v32 = 'deepseek_v32'
|
||||
deepseek_v4 = 'deepseek_v4'
|
||||
|
||||
openbuddy_llama = 'openbuddy_llama'
|
||||
openbuddy_mistral = 'openbuddy_mistral'
|
||||
openbuddy_mixtral = 'openbuddy_mixtral'
|
||||
|
||||
baichuan = 'baichuan'
|
||||
baichuan2 = 'baichuan2'
|
||||
baichuan_m1 = 'baichuan_m1'
|
||||
|
||||
minicpm = 'minicpm'
|
||||
minicpm_chatml = 'minicpm_chatml'
|
||||
minicpm3 = 'minicpm3'
|
||||
minicpm_moe = 'minicpm_moe'
|
||||
|
||||
telechat = 'telechat'
|
||||
telechat2 = 'telechat2'
|
||||
|
||||
mistral = 'mistral'
|
||||
devstral = 'devstral'
|
||||
zephyr = 'zephyr'
|
||||
mixtral = 'mixtral'
|
||||
mistral_nemo = 'mistral_nemo'
|
||||
mistral_2501 = 'mistral_2501'
|
||||
wizardlm2 = 'wizardlm2'
|
||||
wizardlm2_moe = 'wizardlm2_moe'
|
||||
|
||||
phi2 = 'phi2'
|
||||
phi3_small = 'phi3_small'
|
||||
phi3 = 'phi3'
|
||||
phi3_moe = 'phi3_moe'
|
||||
phi4 = 'phi4'
|
||||
|
||||
minimax = 'minimax'
|
||||
minimax_m1 = 'minimax_m1'
|
||||
minimax_m2 = 'minimax_m2'
|
||||
|
||||
gemma = 'gemma'
|
||||
gemma2 = 'gemma2'
|
||||
gemma3_text = 'gemma3_text'
|
||||
|
||||
skywork = 'skywork'
|
||||
|
||||
ling = 'ling'
|
||||
bailing_moe = 'bailing_moe'
|
||||
bailing_hybrid = 'bailing_hybrid'
|
||||
yuan2 = 'yuan2'
|
||||
orion = 'orion'
|
||||
xverse = 'xverse'
|
||||
xverse_moe = 'xverse_moe'
|
||||
seggpt = 'seggpt'
|
||||
bluelm = 'bluelm'
|
||||
c4ai = 'c4ai'
|
||||
dbrx = 'dbrx'
|
||||
grok = 'grok'
|
||||
mamba = 'mamba'
|
||||
polylm = 'polylm'
|
||||
aya = 'aya'
|
||||
mimo = 'mimo'
|
||||
dots1 = 'dots1'
|
||||
hunyuan = 'hunyuan'
|
||||
hunyuan_v1_dense = 'hunyuan_v1_dense'
|
||||
hy_v3 = 'hy_v3'
|
||||
ernie4_5 = 'ernie4_5'
|
||||
ernie4_5_moe = 'ernie4_5_moe'
|
||||
gemma_emb = 'gemma_emb'
|
||||
longchat = 'longchat'
|
||||
iquestcoder = 'iquestcoder'
|
||||
youtu_llm = 'youtu_llm'
|
||||
|
||||
modern_bert_gte_reranker = 'modern_bert_gte_reranker'
|
||||
bge_reranker = 'bge_reranker'
|
||||
|
||||
olmoe = 'olmoe'
|
||||
|
||||
|
||||
class BertModelType:
|
||||
modern_bert = 'modern_bert'
|
||||
modern_bert_gte = 'modern_bert_gte'
|
||||
bert = 'bert'
|
||||
|
||||
|
||||
class RMModelType:
|
||||
internlm2_reward = 'internlm2_reward'
|
||||
qwen2_reward = 'qwen2_reward'
|
||||
qwen2_5_prm = 'qwen2_5_prm'
|
||||
llama3_2_reward = 'llama3_2_reward'
|
||||
gemma_reward = 'gemma_reward'
|
||||
|
||||
|
||||
class MLLMModelType:
|
||||
qwen_vl = 'qwen_vl'
|
||||
qwen_audio = 'qwen_audio'
|
||||
qwen2_vl = 'qwen2_vl'
|
||||
qwen2_5_vl = 'qwen2_5_vl'
|
||||
qwen2_5_omni = 'qwen2_5_omni'
|
||||
qwen3_omni_moe = 'qwen3_omni_moe'
|
||||
qwen2_audio = 'qwen2_audio'
|
||||
qwen3_asr = 'qwen3_asr'
|
||||
qwen3_tts = 'qwen3_tts'
|
||||
qwen3_vl = 'qwen3_vl'
|
||||
qwen3_vl_moe = 'qwen3_vl_moe'
|
||||
qwen3_vl_emb = 'qwen3_vl_emb'
|
||||
qwen3_vl_reranker = 'qwen3_vl_reranker'
|
||||
qwen3_5 = 'qwen3_5'
|
||||
qwen3_5_moe = 'qwen3_5_moe'
|
||||
|
||||
qwen2_gme = 'qwen2_gme'
|
||||
ovis1_6 = 'ovis1_6'
|
||||
ovis2 = 'ovis2'
|
||||
ovis2_5 = 'ovis2_5'
|
||||
midashenglm = 'midashenglm'
|
||||
|
||||
chatglm4v = 'chatglm4v'
|
||||
glm4v = 'glm4v'
|
||||
glm4v_moe = 'glm4v_moe'
|
||||
glm_edge_v = 'glm_edge_v'
|
||||
glm_ocr = 'glm_ocr'
|
||||
cogvlm = 'cogvlm'
|
||||
cogagent_vqa = 'cogagent_vqa'
|
||||
cogagent_chat = 'cogagent_chat'
|
||||
cogvlm2 = 'cogvlm2'
|
||||
cogvlm2_video = 'cogvlm2_video'
|
||||
|
||||
internvl_chat = 'internvl_chat'
|
||||
internvl = 'internvl'
|
||||
interns1 = 'interns1'
|
||||
xcomposer2 = 'xcomposer2'
|
||||
xcomposer2_4khd = 'xcomposer2_4khd'
|
||||
xcomposer2_5 = 'xcomposer2_5'
|
||||
xcomposer2_5_ol_audio = 'xcomposer2_5_ol_audio'
|
||||
|
||||
llama3_2_vision = 'llama3_2_vision'
|
||||
llama4 = 'llama4'
|
||||
llama3_1_omni = 'llama3_1_omni'
|
||||
|
||||
llava1_5_hf = 'llava1_5_hf'
|
||||
llava1_6_mistral_hf = 'llava1_6_mistral_hf'
|
||||
llava1_6_vicuna_hf = 'llava1_6_vicuna_hf'
|
||||
llava1_6_yi_hf = 'llava1_6_yi_hf'
|
||||
llama3_llava_next_hf = 'llama3_llava_next_hf'
|
||||
llava_next_qwen_hf = 'llava_next_qwen_hf'
|
||||
llava_next_video_hf = 'llava_next_video_hf'
|
||||
llava_next_video_yi_hf = 'llava_next_video_yi_hf'
|
||||
llava_onevision_hf = 'llava_onevision_hf'
|
||||
yi_vl = 'yi_vl'
|
||||
ernie_vl = 'ernie_vl'
|
||||
|
||||
llava_llama3_1_hf = 'llava_llama3_1_hf' # DaozeZhang
|
||||
llava_llama3_hf = 'llava_llama3_hf' # xtuner
|
||||
|
||||
llava1_6_mistral = 'llava1_6_mistral'
|
||||
llava1_6_yi = 'llava1_6_yi'
|
||||
llava_next_qwen = 'llava_next_qwen'
|
||||
llama3_llava_next = 'llama3_llava_next'
|
||||
llava_onevision1_5 = 'llava_onevision1_5'
|
||||
|
||||
deepseek_vl = 'deepseek_vl'
|
||||
deepseek_vl2 = 'deepseek_vl2'
|
||||
deepseek_janus = 'deepseek_janus'
|
||||
deepseek_janus_pro = 'deepseek_janus_pro'
|
||||
deepseek_ocr = 'deepseek_ocr'
|
||||
deepseek_ocr2 = 'deepseek_ocr2'
|
||||
unlimited_ocr = 'unlimited-ocr'
|
||||
|
||||
minicpmv = 'minicpmv'
|
||||
minicpmv2_5 = 'minicpmv2_5'
|
||||
minicpmv2_6 = 'minicpmv2_6'
|
||||
minicpmv4 = 'minicpmv4'
|
||||
minicpmv4_5 = 'minicpmv4_5'
|
||||
minicpmv4_6 = 'minicpmv4_6'
|
||||
minicpmo = 'minicpmo'
|
||||
|
||||
minimax_vl = 'minimax_vl'
|
||||
minimax_m3_vl = 'minimax_m3_vl'
|
||||
|
||||
mplug_owl2 = 'mplug_owl2'
|
||||
mplug_owl2_1 = 'mplug_owl2_1'
|
||||
mplug_owl3 = 'mplug_owl3'
|
||||
mplug_owl3_241101 = 'mplug_owl3_241101'
|
||||
doc_owl2 = 'doc_owl2'
|
||||
|
||||
emu3_gen = 'emu3_gen'
|
||||
emu3_chat = 'emu3_chat'
|
||||
got_ocr2 = 'got_ocr2'
|
||||
got_ocr2_hf = 'got_ocr2_hf'
|
||||
step_audio = 'step_audio'
|
||||
step_audio2_mini = 'step_audio2_mini'
|
||||
kimi_vl = 'kimi_vl'
|
||||
kimi_k25 = 'kimi_k25'
|
||||
keye_vl = 'keye_vl'
|
||||
keye_vl_1_5 = 'keye_vl_1_5'
|
||||
dots_ocr = 'dots_ocr'
|
||||
sail_vl2 = 'sail_vl2'
|
||||
|
||||
phi3_vision = 'phi3_vision'
|
||||
phi4_multimodal = 'phi4_multimodal'
|
||||
florence = 'florence'
|
||||
idefics3 = 'idefics3'
|
||||
paligemma = 'paligemma'
|
||||
molmo = 'molmo'
|
||||
molmo2 = 'molmo2'
|
||||
molmoe = 'molmoe'
|
||||
pixtral = 'pixtral'
|
||||
megrez_omni = 'megrez_omni'
|
||||
valley = 'valley'
|
||||
gemma3_vision = 'gemma3_vision'
|
||||
gemma3n = 'gemma3n'
|
||||
gemma4 = 'gemma4'
|
||||
gemma4_unified = 'gemma4_unified'
|
||||
diffusion_gemma = 'diffusion_gemma'
|
||||
mistral3 = 'mistral3'
|
||||
mistral3_2506 = 'mistral3_2506'
|
||||
paddle_ocr = 'paddle_ocr'
|
||||
paddleocr_vl = 'paddleocr_vl'
|
||||
hunyuan_ocr = 'hunyuan_ocr'
|
||||
step3_vl = 'step3_vl'
|
||||
|
||||
jina_reranker_m0 = 'jina_reranker_m0'
|
||||
|
||||
|
||||
class ModelType(LLMModelType, MLLMModelType, BertModelType, RMModelType):
|
||||
|
||||
@classmethod
|
||||
def get_model_name_list(cls) -> List[str]:
|
||||
|
||||
def _get_model_name_list(cls):
|
||||
res = []
|
||||
for k in cls.__dict__:
|
||||
if k.startswith('__'):
|
||||
continue
|
||||
value = getattr(cls, k)
|
||||
if isinstance(value, str):
|
||||
res.append(value)
|
||||
return res
|
||||
|
||||
return list(
|
||||
chain.from_iterable(
|
||||
_get_model_name_list(model_type_cls)
|
||||
for model_type_cls in [LLMModelType, MLLMModelType, BertModelType, RMModelType]))
|
||||
@@ -0,0 +1,844 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import transformers
|
||||
from dataclasses import dataclass, field
|
||||
from packaging import version
|
||||
from typing import List, Optional, Union
|
||||
|
||||
transformers_ge_4_52 = version.parse(transformers.__version__) >= version.parse('4.52')
|
||||
|
||||
|
||||
class LLMModelArch:
|
||||
qwen = 'qwen'
|
||||
llama = 'llama'
|
||||
internlm2 = 'internlm2'
|
||||
chatglm = 'chatglm'
|
||||
deepseek_v2 = 'deepseek_v2'
|
||||
baichuan = 'baichuan'
|
||||
|
||||
yuan = 'yuan'
|
||||
codefuse = 'codefuse'
|
||||
phi2 = 'phi2'
|
||||
phi3 = 'phi3'
|
||||
phi3_small = 'phi3_small'
|
||||
telechat = 'telechat'
|
||||
dbrx = 'dbrx'
|
||||
|
||||
|
||||
class MLLMModelArch:
|
||||
qwen_vl = 'qwen_vl'
|
||||
qwen_audio = 'qwen_audio'
|
||||
qwen2_vl = 'qwen2_vl'
|
||||
qwen2_audio = 'qwen2_audio'
|
||||
qwen2_5_omni = 'qwen2_5_omni'
|
||||
qwen3_vl = 'qwen3_vl'
|
||||
qwen3_omni = 'qwen3_omni'
|
||||
qwen3_asr = 'qwen3_asr'
|
||||
qwen3_tts = 'qwen3_tts'
|
||||
|
||||
cogvlm = 'cogvlm'
|
||||
chatglm4v = 'chatglm4v'
|
||||
glm4v = 'glm4v'
|
||||
glm_edge_v = 'glm_edge_v'
|
||||
|
||||
llama3_1_omni = 'llama3_1_omni'
|
||||
llama3_2_vision = 'llama3_2_vision'
|
||||
llama4 = 'llama4'
|
||||
|
||||
llava_hf = 'llava_hf'
|
||||
llava_hf_legacy = 'llava_hf_legacy' # transformers<4.52
|
||||
llava_next_video_hf = 'llava_next_video_hf'
|
||||
llava_onevision1_5 = 'llava_onevision1_5'
|
||||
|
||||
llava_llama = 'llava_llama'
|
||||
llava_mistral = 'llava_mistral'
|
||||
|
||||
xcomposer = 'xcomposer'
|
||||
internvl = 'internvl'
|
||||
interns1 = 'interns1'
|
||||
minicpmv = 'minicpmv'
|
||||
minicpmo = 'minicpmo'
|
||||
minicpmv4_6 = 'minicpmv4_6'
|
||||
deepseek_vl = 'deepseek_vl'
|
||||
deepseek_vl2 = 'deepseek_vl2'
|
||||
deepseek_janus = 'deepseek_janus'
|
||||
deepseek_ocr = 'deepseek_ocr'
|
||||
deepseek_ocr2 = 'deepseek_ocr2'
|
||||
unlimited_ocr = 'unlimited-ocr'
|
||||
kimi_k25 = 'kimi_k25'
|
||||
|
||||
mplug_owl2 = 'mplug_owl2'
|
||||
mplug_owl2_1 = 'mplug_owl2_1'
|
||||
mplug_owl3 = 'mplug_owl3'
|
||||
doc_owl2 = 'doc_owl2'
|
||||
|
||||
phi3_vision = 'phi3_vision'
|
||||
phi4_multimodal = 'phi4_multimodal'
|
||||
florence = 'florence'
|
||||
idefics3 = 'idefics3'
|
||||
|
||||
got_ocr2 = 'got_ocr2'
|
||||
dots_ocr = 'dots_ocr'
|
||||
ernie_vl = 'ernie_vl'
|
||||
|
||||
ovis = 'ovis'
|
||||
ovis2_5 = 'ovis2_5'
|
||||
molmo = 'molmo'
|
||||
emu3_chat = 'emu3_chat'
|
||||
megrez_omni = 'megrez_omni'
|
||||
valley = 'valley'
|
||||
gemma3n = 'gemma3n'
|
||||
gemma4_unified = 'gemma4_unified'
|
||||
diffusion_gemma = 'diffusion_gemma'
|
||||
keye_vl = 'keye_vl'
|
||||
|
||||
midashenglm = 'midashenglm'
|
||||
step_audio2_mini = 'step_audio2_mini'
|
||||
hunyuan_vl = 'hunyuan_vl'
|
||||
step3_vl = 'step3_vl'
|
||||
paddleocr_vl = 'paddleocr_vl'
|
||||
minimax_m3_vl = 'minimax_m3_vl'
|
||||
|
||||
|
||||
class ModelArch(LLMModelArch, MLLMModelArch):
|
||||
# Multimodal models typically require specifying model_arch,
|
||||
# while text-only models usually do not need to specify model_arch.
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelKeys:
|
||||
"""Used to support training of tuners such as llama-pro"""
|
||||
arch_name: str = None
|
||||
|
||||
embedding: str = None
|
||||
module_list: str = None
|
||||
lm_head: str = None
|
||||
|
||||
q_proj: str = None
|
||||
k_proj: str = None
|
||||
v_proj: str = None
|
||||
o_proj: str = None
|
||||
attention: str = None
|
||||
|
||||
mlp: str = None
|
||||
down_proj: str = None
|
||||
|
||||
qkv_proj: str = None
|
||||
qk_proj: str = None
|
||||
qa_proj: str = None
|
||||
qb_proj: str = None
|
||||
kv_proj: str = None
|
||||
kva_proj: str = None
|
||||
kvb_proj: str = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class MultiModelKeys(ModelKeys):
|
||||
"""Used to support freeze_vit/freeze_aligner/freeze_llm"""
|
||||
language_model: Union[str, List[str]] = field(default_factory=list)
|
||||
aligner: Union[str, List[str]] = field(default_factory=list)
|
||||
vision_tower: Union[str, List[str]] = field(default_factory=list)
|
||||
generator: Union[str, List[str]] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
for key in ['language_model', 'aligner', 'vision_tower', 'generator']:
|
||||
v = getattr(self, key)
|
||||
if isinstance(v, str):
|
||||
setattr(self, key, [v])
|
||||
if v is None:
|
||||
setattr(self, key, [])
|
||||
|
||||
|
||||
MODEL_ARCH_MAPPING = {}
|
||||
|
||||
|
||||
def register_model_arch(model_arch: ModelKeys, *, exist_ok: bool = False) -> None:
|
||||
"""
|
||||
model_type: The unique ID for the model type. Models with the same model_type share
|
||||
the same architectures, template, get_function, etc.
|
||||
"""
|
||||
arch_name = model_arch.arch_name
|
||||
if not exist_ok and arch_name in MODEL_ARCH_MAPPING:
|
||||
raise ValueError(f'The `{arch_name}` has already been registered in the MODEL_ARCH_MAPPING.')
|
||||
|
||||
MODEL_ARCH_MAPPING[arch_name] = model_arch
|
||||
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.llama,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.down_proj',
|
||||
attention='model.layers.{}.self_attn',
|
||||
o_proj='model.layers.{}.self_attn.o_proj',
|
||||
q_proj='model.layers.{}.self_attn.q_proj',
|
||||
k_proj='model.layers.{}.self_attn.k_proj',
|
||||
v_proj='model.layers.{}.self_attn.v_proj',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.internlm2,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.feed_forward',
|
||||
down_proj='model.layers.{}.feed_forward.w2',
|
||||
attention='model.layers.{}.attention',
|
||||
o_proj='model.layers.{}.attention.wo',
|
||||
qkv_proj='model.layers.{}.attention.wqkv',
|
||||
embedding='model.tok_embeddings',
|
||||
lm_head='output',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.chatglm,
|
||||
module_list='transformer.encoder.layers',
|
||||
mlp='transformer.encoder.layers.{}.mlp',
|
||||
down_proj='transformer.encoder.layers.{}.mlp.dense_4h_to_h',
|
||||
attention='transformer.encoder.layers.{}.self_attention',
|
||||
o_proj='transformer.encoder.layers.{}.self_attention.dense',
|
||||
qkv_proj='transformer.encoder.layers.{}.self_attention.query_key_value',
|
||||
embedding='transformer.embedding',
|
||||
lm_head='transformer.output_layer'))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.telechat,
|
||||
module_list='transformer.h',
|
||||
mlp='transformer.h.{}.mlp',
|
||||
down_proj='transformer.h.{}.mlp.down_proj',
|
||||
attention='transformer.h.{}.self_attention',
|
||||
o_proj='transformer.h.{}.self_attention.dense',
|
||||
q_proj='transformer.h.{}.self_attention.query',
|
||||
kv_proj='transformer.h.{}.self_attention.key_value',
|
||||
embedding='transformer.word_embeddings',
|
||||
lm_head='lm_head'))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.baichuan,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.down_proj',
|
||||
attention='model.layers.{}.self_attn',
|
||||
qkv_proj='model.layers.{}.self_attn.W_pack',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.yuan,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.down_proj',
|
||||
attention='model.layers.{}.self_attn',
|
||||
qk_proj='model.layers.{}.self_attn.qk_proj',
|
||||
o_proj='model.layers.{}.self_attn.o_proj',
|
||||
q_proj='model.layers.{}.self_attn.q_proj',
|
||||
k_proj='model.layers.{}.self_attn.k_proj',
|
||||
v_proj='model.layers.{}.self_attn.v_proj',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.codefuse,
|
||||
module_list='gpt_neox.layers',
|
||||
mlp='gpt_neox.layers.{}.mlp',
|
||||
down_proj='gpt_neox.layers.{}.mlp.dense_4h_to_h',
|
||||
attention='gpt_neox.layers.{}.attention',
|
||||
o_proj='gpt_neox.layers.{}.attention.dense',
|
||||
qkv_proj='gpt_neox.layers.{}.attention.query_key_value',
|
||||
embedding='gpt_neox.embed_in',
|
||||
lm_head='gpt_neox.embed_out',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.phi2,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.fc2',
|
||||
attention='model.layers.{}.self_attn',
|
||||
o_proj='model.layers.{}.self_attn.dense',
|
||||
q_proj='model.layers.{}.self_attn.q_proj',
|
||||
k_proj='model.layers.{}.self_attn.k_proj',
|
||||
v_proj='model.layers.{}.self_attn.v_proj',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.qwen,
|
||||
module_list='transformer.h',
|
||||
mlp='transformer.h.{}.mlp',
|
||||
down_proj='transformer.h.{}.mlp.c_proj',
|
||||
attention='transformer.h.{}.attn',
|
||||
o_proj='transformer.h.{}.attn.c_proj',
|
||||
qkv_proj='transformer.h.{}.attn.c_attn',
|
||||
embedding='transformer.wte',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.dbrx,
|
||||
module_list='transformer.blocks',
|
||||
mlp='transformer.blocks.{}.ffn',
|
||||
attention='transformer.blocks.{}.norm_attn_norm.attn',
|
||||
o_proj='transformer.blocks.{}.norm_attn_norm.attn.out_proj',
|
||||
qkv_proj='transformer.blocks.{}.norm_attn_norm.attn.Wqkv',
|
||||
embedding='transformer.wte',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.phi3,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.down_proj',
|
||||
attention='model.layers.{}.self_attn',
|
||||
o_proj='model.layers.{}.self_attn.o_proj',
|
||||
qkv_proj='model.layers.{}.self_attn.qkv_proj',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.phi3_small,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.down_proj',
|
||||
attention='model.layers.{}.self_attn',
|
||||
o_proj='model.layers.{}.self_attn.dense',
|
||||
qkv_proj='model.layers.{}.self_attn.query_key_value',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
ModelKeys(
|
||||
LLMModelArch.deepseek_v2,
|
||||
module_list='model.layers',
|
||||
mlp='model.layers.{}.mlp',
|
||||
down_proj='model.layers.{}.mlp.down_proj',
|
||||
attention='model.layers.{}.self_attn',
|
||||
o_proj='model.layers.{}.self_attn.o_proj',
|
||||
qa_proj='model.layers.{}.self_attn.q_a_proj',
|
||||
qb_proj='model.layers.{}.self_attn.q_b_proj',
|
||||
kva_proj='model.layers.{}.self_attn.kv_a_proj_with_mqa',
|
||||
kvb_proj='model.layers.{}.self_attn.kv_b_proj',
|
||||
embedding='model.embed_tokens',
|
||||
lm_head='lm_head',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_hf_legacy,
|
||||
language_model='language_model',
|
||||
aligner='multi_modal_projector',
|
||||
vision_tower='vision_tower',
|
||||
))
|
||||
|
||||
if transformers_ge_4_52:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_hf,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.multi_modal_projector',
|
||||
vision_tower='model.vision_tower',
|
||||
))
|
||||
else:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_hf,
|
||||
language_model='language_model',
|
||||
aligner='multi_modal_projector',
|
||||
vision_tower='vision_tower',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_mistral,
|
||||
language_model='model.layers',
|
||||
aligner='model.mm_projector',
|
||||
vision_tower='model.vision_tower',
|
||||
))
|
||||
|
||||
if transformers_ge_4_52:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_next_video_hf,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner=['model.multi_modal_projector'],
|
||||
vision_tower='model.vision_tower'))
|
||||
else:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_next_video_hf,
|
||||
language_model='language_model',
|
||||
aligner=['multi_modal_projector'],
|
||||
vision_tower='vision_tower'))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_llama,
|
||||
language_model='model.layers',
|
||||
aligner='model.mm_projector',
|
||||
vision_tower='model.vision_tower',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.kimi_k25,
|
||||
language_model='language_model',
|
||||
aligner='mm_projector',
|
||||
vision_tower='vision_tower',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.xcomposer,
|
||||
language_model='model',
|
||||
aligner='vision_proj',
|
||||
vision_tower='vit',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.internvl,
|
||||
language_model='language_model',
|
||||
aligner='mlp1',
|
||||
vision_tower='vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.interns1,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.multi_modal_projector',
|
||||
vision_tower='model.vision_tower',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.mplug_owl3,
|
||||
language_model='language_model',
|
||||
aligner='vision2text_model',
|
||||
vision_tower='vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.doc_owl2,
|
||||
language_model='model.layers',
|
||||
aligner=['model.vision2text', 'model.hr_compressor'],
|
||||
vision_tower='model.vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.deepseek_vl,
|
||||
language_model='language_model',
|
||||
aligner='aligner',
|
||||
vision_tower='vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.deepseek_janus,
|
||||
language_model='language_model',
|
||||
vision_tower='vision_model',
|
||||
aligner='aligner',
|
||||
generator=['gen_vision_model', 'gen_aligner', 'gen_head', 'gen_embed']))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.deepseek_ocr,
|
||||
language_model='model.layers',
|
||||
vision_tower=['model.sam_model', 'model.vision_model'],
|
||||
aligner='model.projector',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.deepseek_ocr2,
|
||||
language_model='model.layers',
|
||||
vision_tower=['model.sam_model', 'model.qwen2_model'],
|
||||
aligner='model.projector',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.unlimited_ocr,
|
||||
language_model=['model.model.embed_tokens', 'model.model.layers', 'model.model.norm', 'model.lm_head'],
|
||||
vision_tower=['model.model.vision_model', 'model.model.sam_model'],
|
||||
aligner=['model.model.projector'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.deepseek_vl2,
|
||||
language_model='language',
|
||||
vision_tower='vision',
|
||||
aligner='projector',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.minicpmv,
|
||||
language_model='llm',
|
||||
aligner='resampler',
|
||||
vision_tower='vpm',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.minicpmv4_6,
|
||||
language_model='model.language_model',
|
||||
aligner='model.merger',
|
||||
vision_tower='model.vision_tower',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.minicpmo,
|
||||
language_model='llm',
|
||||
aligner='resampler',
|
||||
vision_tower=['vpm', 'apm'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.phi3_vision,
|
||||
language_model='model.layers',
|
||||
aligner='model.vision_embed_tokens.img_projection',
|
||||
vision_tower='model.vision_embed_tokens.img_processor',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.phi4_multimodal,
|
||||
language_model='model.layers',
|
||||
aligner=[
|
||||
'model.embed_tokens_extend.image_embed.img_projection',
|
||||
'model.embed_tokens_extend.audio_embed.audio_projection'
|
||||
],
|
||||
vision_tower=[
|
||||
'model.embed_tokens_extend.image_embed.img_processor', 'model.embed_tokens_extend.audio_embed.encoder'
|
||||
],
|
||||
))
|
||||
|
||||
register_model_arch(MultiModelKeys(
|
||||
MLLMModelArch.cogvlm,
|
||||
language_model='model.layers',
|
||||
vision_tower='model.vision',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.florence,
|
||||
language_model='language_model',
|
||||
vision_tower='vision_tower',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen_vl,
|
||||
language_model='transformer.h',
|
||||
vision_tower='transformer.visual',
|
||||
))
|
||||
# TODO: check lm_head, ALL
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen_audio,
|
||||
language_model='transformer.h',
|
||||
vision_tower='transformer.audio',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen2_audio,
|
||||
language_model='language_model',
|
||||
aligner='multi_modal_projector',
|
||||
vision_tower='audio_tower',
|
||||
))
|
||||
|
||||
if transformers_ge_4_52:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen2_vl,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.visual.merger',
|
||||
vision_tower='model.visual',
|
||||
))
|
||||
else:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen2_vl,
|
||||
language_model=['model', 'lm_head'],
|
||||
aligner='visual.merger',
|
||||
vision_tower='visual',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen3_vl,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner=['model.visual.merger', 'model.visual.deepstack_merger_list'],
|
||||
vision_tower='model.visual',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen2_5_omni,
|
||||
language_model=['thinker.model', 'thinker.lm_head'],
|
||||
vision_tower=['thinker.audio_tower', 'thinker.visual'],
|
||||
aligner=['thinker.audio_tower.proj', 'thinker.visual.merger'],
|
||||
generator=['talker', 'token2wav'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen3_omni,
|
||||
language_model=['thinker.model', 'thinker.lm_head'],
|
||||
vision_tower=['thinker.audio_tower', 'thinker.visual'],
|
||||
aligner=[
|
||||
'thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger',
|
||||
'thinker.visual.merger_list'
|
||||
],
|
||||
generator=['talker', 'code2wav'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen3_asr,
|
||||
language_model=['thinker.model', 'thinker.lm_head'],
|
||||
vision_tower='thinker.audio_tower',
|
||||
aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.qwen3_tts,
|
||||
language_model='talker',
|
||||
generator='speaker_encoder', # no grad
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.midashenglm,
|
||||
language_model='decoder',
|
||||
aligner=['audio_projector'],
|
||||
vision_tower=['audio_encoder'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.step_audio2_mini,
|
||||
language_model=['model', 'lm_head'],
|
||||
aligner=['adapter'],
|
||||
vision_tower=['encoder'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.chatglm4v,
|
||||
language_model='transformer.encoder',
|
||||
vision_tower='transformer.vision',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.glm4v,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.visual.merger',
|
||||
vision_tower='model.visual',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.idefics3,
|
||||
language_model='model.text_model',
|
||||
aligner='model.connector',
|
||||
vision_tower='model.vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llama3_1_omni,
|
||||
language_model='model.layers',
|
||||
aligner='model.speech_projector',
|
||||
vision_tower='model.speech_encoder',
|
||||
generator='speech_generator',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.got_ocr2,
|
||||
language_model='model.layers',
|
||||
aligner='model.mm_projector_vary',
|
||||
vision_tower='model.vision_tower_high',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.ernie_vl,
|
||||
language_model=['model', 'lm_head'],
|
||||
aligner='model.resampler_model',
|
||||
vision_tower='vision_model',
|
||||
))
|
||||
|
||||
if transformers_ge_4_52:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llama3_2_vision,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.multi_modal_projector',
|
||||
vision_tower='model.vision_model',
|
||||
))
|
||||
else:
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llama3_2_vision,
|
||||
language_model='language_model',
|
||||
aligner='multi_modal_projector',
|
||||
vision_tower='vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llama4,
|
||||
language_model='language_model',
|
||||
aligner='multi_modal_projector',
|
||||
vision_tower='vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.ovis,
|
||||
language_model='llm',
|
||||
vision_tower=['visual_tokenizer', 'vte'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.ovis2_5,
|
||||
language_model='llm',
|
||||
aligner='visual_tokenizer.head',
|
||||
vision_tower=['visual_tokenizer.vit', 'vte'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.molmo,
|
||||
language_model='model.transformer',
|
||||
vision_tower='model.vision_backbone',
|
||||
aligner='model.vision_backbone.image_projector'))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.megrez_omni,
|
||||
language_model='llm',
|
||||
vision_tower=['vision', 'audio'],
|
||||
))
|
||||
|
||||
register_model_arch(MultiModelKeys(MLLMModelArch.emu3_chat, language_model='model'))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(MLLMModelArch.glm_edge_v, language_model='model.layers', vision_tower='model.vision'))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.valley,
|
||||
language_model='model',
|
||||
vision_tower=['model.vision_tower', 'model.qwen2vl_vision_tower'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.gemma3n,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner=['model.embed_vision', 'model.embed_audio'],
|
||||
vision_tower=['model.vision_tower', 'model.audio_tower'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.gemma4_unified,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner=['model.embed_vision', 'model.embed_audio'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.diffusion_gemma,
|
||||
language_model=['model.encoder.language_model', 'model.decoder', 'lm_head'],
|
||||
vision_tower=['model.encoder.vision_tower'],
|
||||
aligner=['model.encoder.embed_vision'],
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.keye_vl,
|
||||
language_model=['model', 'lm_head'],
|
||||
aligner='mlp_AR',
|
||||
vision_tower='visual',
|
||||
))
|
||||
|
||||
register_model_arch(MultiModelKeys(
|
||||
MLLMModelArch.dots_ocr,
|
||||
language_model='model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.llava_onevision1_5,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.visual.merger',
|
||||
vision_tower='model.visual',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.hunyuan_vl,
|
||||
language_model='model',
|
||||
aligner='vit.perceive',
|
||||
vision_tower='vit',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.step3_vl,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.vit_large_projector',
|
||||
vision_tower='model.vision_model',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.paddleocr_vl,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.projector',
|
||||
vision_tower='model.visual',
|
||||
))
|
||||
|
||||
register_model_arch(
|
||||
MultiModelKeys(
|
||||
MLLMModelArch.minimax_m3_vl,
|
||||
language_model=['model.language_model', 'lm_head'],
|
||||
aligner='model.multi_modal_projector',
|
||||
vision_tower='model.vision_tower',
|
||||
))
|
||||
|
||||
|
||||
def get_model_arch(arch_name: Optional[str]) -> Optional[MultiModelKeys]:
|
||||
return MODEL_ARCH_MAPPING.get(arch_name)
|
||||
@@ -0,0 +1,325 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import torch
|
||||
from abc import ABC, abstractmethod
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from transformers import AutoConfig, PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
|
||||
from transformers.utils.versions import require_version
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Type
|
||||
|
||||
from swift.utils import HfConfigFactory, get_logger, safe_snapshot_download
|
||||
from .utils import get_default_torch_dtype
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
ms_model_id: Optional[str] = None
|
||||
hf_model_id: Optional[str] = None
|
||||
model_path: Optional[str] = None
|
||||
|
||||
ms_revision: Optional[str] = None
|
||||
hf_revision: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelGroup:
|
||||
models: List[Model]
|
||||
|
||||
# Higher priority. If set to None, the attributes of the ModelMeta will be used.
|
||||
template: Optional[str] = None
|
||||
ignore_patterns: Optional[List[str]] = None
|
||||
requires: Optional[List[str]] = None
|
||||
tags: List[str] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
assert not isinstance(self.template, (list, tuple)) # check ms-swift4.0
|
||||
assert isinstance(self.models, (tuple, list)), f'self.models: {self.models}'
|
||||
|
||||
|
||||
class BaseModelLoader(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def __init__(self, model_info, model_meta, *args, **kwargs):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load(self) -> Tuple[Optional[PreTrainedModel], PreTrainedTokenizerBase]:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelMeta:
|
||||
model_type: Optional[str]
|
||||
# Used to list the model_ids from modelscope/huggingface,
|
||||
# which participate in the automatic inference of the model_type.
|
||||
model_groups: List[ModelGroup]
|
||||
loader: Optional[Type[BaseModelLoader]] = None
|
||||
|
||||
template: Optional[str] = None
|
||||
model_arch: Optional[str] = None
|
||||
mcore_model_type: Optional[str] = None
|
||||
architectures: List[str] = field(default_factory=list)
|
||||
# Additional files that need to be saved for full parameter training/merge-lora.
|
||||
additional_saved_files: List[str] = field(default_factory=list)
|
||||
torch_dtype: Optional[torch.dtype] = None
|
||||
|
||||
is_multimodal: bool = False
|
||||
is_reward: bool = False
|
||||
task_type: Optional[str] = None
|
||||
|
||||
# File patterns to ignore when downloading the model.
|
||||
ignore_patterns: Optional[List[str]] = None
|
||||
# Usually specifies the version limits of transformers.
|
||||
requires: List[str] = field(default_factory=list)
|
||||
tags: List[str] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
from .constant import MLLMModelType, RMModelType
|
||||
from .register import ModelLoader
|
||||
assert not isinstance(self.loader, str) # check ms-swift4.0
|
||||
if self.loader is None:
|
||||
self.loader = ModelLoader
|
||||
if not isinstance(self.model_groups, (list, tuple)):
|
||||
self.model_groups = [self.model_groups]
|
||||
self.candidate_templates = list(
|
||||
dict.fromkeys(t for t in [self.template] + [mg.template for mg in self.model_groups] if t is not None))
|
||||
if self.model_type in MLLMModelType.__dict__:
|
||||
self.is_multimodal = True
|
||||
if self.model_type in RMModelType.__dict__:
|
||||
self.is_reward = True
|
||||
|
||||
def get_matched_model_group(self, model_name: str) -> Optional[ModelGroup]:
|
||||
for model_group in self.model_groups:
|
||||
for model in model_group.models:
|
||||
for key in ['ms_model_id', 'hf_model_id', 'model_path']:
|
||||
value = getattr(model, key)
|
||||
|
||||
if isinstance(value, str) and model_name == value.rsplit('/', 1)[-1].lower():
|
||||
return model_group
|
||||
|
||||
def check_requires(self, model_info=None):
|
||||
extra_requires = []
|
||||
if model_info and model_info.quant_method:
|
||||
mapping = {'bnb': ['bitsandbytes'], 'awq': ['autoawq'], 'gptq': ['auto_gptq'], 'aqlm': ['aqlm']}
|
||||
extra_requires += mapping.get(model_info.quant_method, [])
|
||||
requires = []
|
||||
for require in self.requires + extra_requires:
|
||||
try:
|
||||
require_version(require)
|
||||
except ImportError:
|
||||
requires.append(f'"{require}"')
|
||||
if requires:
|
||||
requires = ' '.join(requires)
|
||||
logger.warning(f'Please install the package: `pip install {requires} -U`.')
|
||||
|
||||
|
||||
MODEL_MAPPING: Dict[str, ModelMeta] = {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelInfo:
|
||||
model_type: str
|
||||
model_dir: str
|
||||
torch_dtype: torch.dtype
|
||||
max_model_len: int
|
||||
quant_method: Literal['gptq', 'awq', 'bnb', 'aqlm', 'hqq', None]
|
||||
quant_bits: int
|
||||
|
||||
# extra
|
||||
rope_scaling: Optional[Dict[str, Any]] = None
|
||||
is_moe_model: bool = False
|
||||
is_multimodal: bool = False
|
||||
config: Optional[PretrainedConfig] = None
|
||||
task_type: Optional[str] = None
|
||||
num_labels: Optional[int] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.model_name = get_model_name(self.model_dir)
|
||||
|
||||
|
||||
def get_model_name(model_id_or_path: str) -> Optional[str]:
|
||||
assert isinstance(model_id_or_path, str), f'model_id_or_path: {model_id_or_path}'
|
||||
# compat hf hub
|
||||
model_id_or_path = model_id_or_path.rstrip('/')
|
||||
match_ = re.search('/models--.+?--(.+?)/snapshots/', model_id_or_path)
|
||||
if match_ is not None:
|
||||
return match_.group(1)
|
||||
|
||||
model_name = model_id_or_path.rsplit('/', 1)[-1]
|
||||
if platform.system().lower() == 'windows':
|
||||
model_name = model_name.rsplit('\\', 1)[-1]
|
||||
# compat modelscope snapshot_download
|
||||
model_name = model_name.replace('___', '.')
|
||||
return model_name
|
||||
|
||||
|
||||
def get_matched_model_meta(model_id_or_path: str) -> Optional[ModelMeta]:
|
||||
model_name = get_model_name(model_id_or_path).lower()
|
||||
for model_type, model_meta in MODEL_MAPPING.items():
|
||||
model_group = ModelMeta.get_matched_model_group(model_meta, model_name)
|
||||
if model_group is not None:
|
||||
model_meta = deepcopy(model_meta)
|
||||
for k, v in asdict(model_group).items():
|
||||
if v is not None and k in model_meta.__dict__:
|
||||
setattr(model_meta, k, v)
|
||||
return model_meta
|
||||
|
||||
|
||||
def _get_arch_mapping():
|
||||
res = {}
|
||||
for model_type, model_meta in MODEL_MAPPING.items():
|
||||
architectures = model_meta.architectures
|
||||
if not architectures:
|
||||
architectures.append('null')
|
||||
for arch in architectures:
|
||||
if arch not in res:
|
||||
res[arch] = []
|
||||
res[arch].append(model_type)
|
||||
return res
|
||||
|
||||
|
||||
def get_matched_model_types(architectures: Optional[List[str]]) -> List[str]:
|
||||
"""Get possible model_type."""
|
||||
architectures = architectures or ['null']
|
||||
if architectures:
|
||||
architectures = architectures[0]
|
||||
arch_mapping = _get_arch_mapping()
|
||||
return arch_mapping.get(architectures) or []
|
||||
|
||||
|
||||
def _read_args_json_model_type(model_dir):
|
||||
if not os.path.exists(os.path.join(model_dir, 'args.json')):
|
||||
return
|
||||
from swift.arguments import BaseArguments
|
||||
args = BaseArguments.from_pretrained(model_dir)
|
||||
return args.model_type
|
||||
|
||||
|
||||
def _get_model_info(model_dir: str, model_type: Optional[str], quantization_config) -> ModelInfo:
|
||||
try:
|
||||
config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
|
||||
except Exception:
|
||||
config = PretrainedConfig.get_config_dict(model_dir)[0]
|
||||
if quantization_config is not None:
|
||||
HfConfigFactory.set_config_attr(config, 'quantization_config', quantization_config)
|
||||
quant_info = HfConfigFactory.get_quant_info(config) or {}
|
||||
torch_dtype = HfConfigFactory.get_torch_dtype(config, quant_info)
|
||||
max_model_len = HfConfigFactory.get_max_model_len(config)
|
||||
rope_scaling = HfConfigFactory.get_config_attr(config, 'rope_scaling')
|
||||
is_moe_model = HfConfigFactory.is_moe_model(config)
|
||||
is_multimodal = HfConfigFactory.is_multimodal(config)
|
||||
|
||||
if model_type is None:
|
||||
model_type = _read_args_json_model_type(model_dir)
|
||||
if model_type is None:
|
||||
architectures = HfConfigFactory.get_config_attr(config, 'architectures')
|
||||
model_types = get_matched_model_types(architectures)
|
||||
if len(model_types) > 1:
|
||||
raise ValueError(f'Failed to automatically match `model_type` for `{model_dir}`. '
|
||||
f'Multiple possible types found: {model_types}. '
|
||||
'Please specify `model_type` manually. See documentation: '
|
||||
'https://swift.readthedocs.io/en/latest/Instruction/Supported-models-and-datasets.html')
|
||||
elif len(model_types) == 1:
|
||||
model_type = model_types[0]
|
||||
elif model_type not in MODEL_MAPPING:
|
||||
raise ValueError(f"model_type: '{model_type}' not in {list(MODEL_MAPPING.keys())}")
|
||||
|
||||
res = ModelInfo(
|
||||
model_type,
|
||||
model_dir,
|
||||
torch_dtype,
|
||||
max_model_len,
|
||||
quant_info.get('quant_method'),
|
||||
quant_info.get('quant_bits'),
|
||||
rope_scaling=rope_scaling,
|
||||
is_moe_model=is_moe_model,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
def get_model_info_meta(
|
||||
model_id_or_path: str,
|
||||
*,
|
||||
torch_dtype: Optional[torch.dtype] = None,
|
||||
# hub
|
||||
use_hf: Optional[bool] = None,
|
||||
hub_token: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
download_model: bool = False,
|
||||
# model kwargs
|
||||
model_type: Optional[str] = None,
|
||||
quantization_config=None,
|
||||
task_type=None,
|
||||
num_labels=None,
|
||||
problem_type=None,
|
||||
**kwargs) -> Tuple[ModelInfo, ModelMeta]:
|
||||
from .register import ModelLoader
|
||||
model_meta = get_matched_model_meta(model_id_or_path)
|
||||
model_dir = safe_snapshot_download(
|
||||
model_id_or_path,
|
||||
revision=revision,
|
||||
download_model=download_model,
|
||||
use_hf=use_hf,
|
||||
ignore_patterns=getattr(model_meta, 'ignore_patterns', None),
|
||||
hub_token=hub_token)
|
||||
|
||||
model_type = model_type or getattr(model_meta, 'model_type', None)
|
||||
model_info = _get_model_info(model_dir, model_type, quantization_config=quantization_config)
|
||||
if model_type is None and model_info.model_type is not None:
|
||||
model_type = model_info.model_type
|
||||
logger.info(f'Setting model_type: {model_type}')
|
||||
if model_type is not None and (model_meta is None or model_meta.model_type != model_type):
|
||||
model_meta = MODEL_MAPPING[model_type]
|
||||
if model_meta is None: # not found
|
||||
if model_info.is_multimodal:
|
||||
raise ValueError(f'Model "{model_id_or_path}" is not supported because no suitable `model_type` was found. '
|
||||
'Please refer to the documentation and specify an appropriate `model_type` manually: '
|
||||
'https://swift.readthedocs.io/en/latest/Instruction/Supported-models-and-datasets.html')
|
||||
else:
|
||||
model_meta = ModelMeta(None, [], ModelLoader, template='dummy', model_arch=None)
|
||||
logger.info(f'Temporarily create model_meta: {model_meta}')
|
||||
if torch_dtype is None:
|
||||
torch_dtype = model_meta.torch_dtype or get_default_torch_dtype(model_info.torch_dtype)
|
||||
logger.info(f'Setting torch_dtype: {torch_dtype}')
|
||||
model_info.torch_dtype = torch_dtype
|
||||
if task_type is None:
|
||||
if model_meta.is_reward:
|
||||
num_labels = 1
|
||||
if num_labels is None:
|
||||
task_type = 'causal_lm'
|
||||
else:
|
||||
task_type = 'seq_cls'
|
||||
if model_meta.task_type is not None:
|
||||
task_type = model_meta.task_type
|
||||
|
||||
# Handle reranker task type
|
||||
if task_type == 'reranker':
|
||||
if num_labels is None:
|
||||
num_labels = 1 # Default to 1 for reranker tasks
|
||||
logger.info(f'Setting reranker task with num_labels={num_labels}')
|
||||
elif task_type == 'generative_reranker':
|
||||
# Generative reranker doesn't need num_labels as it uses CausalLM structure
|
||||
num_labels = None
|
||||
logger.info('Setting generative_reranker task (no num_labels needed)')
|
||||
elif task_type == 'seq_cls':
|
||||
assert num_labels is not None, 'Please pass the parameter `num_labels`.'
|
||||
if problem_type is None:
|
||||
if num_labels == 1:
|
||||
problem_type = 'regression'
|
||||
else:
|
||||
problem_type = 'single_label_classification'
|
||||
|
||||
model_info.task_type = task_type
|
||||
model_info.num_labels = num_labels
|
||||
model_info.problem_type = problem_type
|
||||
if model_meta.is_multimodal:
|
||||
model_info.is_multimodal = True
|
||||
model_meta.check_requires(model_info)
|
||||
return model_info, model_meta
|
||||
@@ -0,0 +1,3 @@
|
||||
from . import (baai, baichuan, baidu, bert, codefuse, deepseek, gemma, glm, internlm, llama, llava, llm, mamba,
|
||||
microsoft, minicpm, minimax, mistral, mllm, moonshot, mplug, openbuddy, qwen, seed, skywork, stepfun,
|
||||
telechat, tencent, valley, yi)
|
||||
@@ -0,0 +1,113 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import os
|
||||
import sys
|
||||
from transformers import AutoModel, AutoModelForSequenceClassification, PretrainedConfig, PreTrainedModel
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_device, git_clone_github, safe_snapshot_download
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class Emu3GenLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir, config) -> Processor:
|
||||
self.model_info.max_model_len = self.model_info.max_model_len + 40960
|
||||
config.image_area = int(os.environ.get('image_area', config.image_area))
|
||||
config.max_position_embeddings = int(os.environ.get('max_position_embeddings', config.max_position_embeddings))
|
||||
tokenizer = super().get_processor(model_dir, config)
|
||||
import sys
|
||||
sys.path.append(model_dir)
|
||||
from processing_emu3 import Emu3Processor
|
||||
vq_hub = safe_snapshot_download('BAAI/Emu3-VisionTokenizer', check_local=True)
|
||||
from transformers import AutoImageProcessor, AutoModel
|
||||
image_processor = AutoImageProcessor.from_pretrained(vq_hub, trust_remote_code=True)
|
||||
image_tokenizer = AutoModel.from_pretrained(vq_hub, trust_remote_code=True).eval().to(get_device())
|
||||
processor = Emu3Processor(image_processor, image_tokenizer, tokenizer)
|
||||
processor.image_area = config.image_area
|
||||
return processor
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs):
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.generation_config.do_sample = True
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.emu3_gen,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('BAAI/Emu3-Gen', 'BAAI/Emu3-Gen'),
|
||||
]),
|
||||
],
|
||||
Emu3GenLoader,
|
||||
template=TemplateType.emu3_gen,
|
||||
architectures=['Emu3ForCausalLM'],
|
||||
model_arch=ModelArch.emu3_chat,
|
||||
tags=['t2i'],
|
||||
))
|
||||
|
||||
|
||||
class Emu3ChatLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer = super().get_processor(model_dir, config)
|
||||
# download and load vision tokenizer
|
||||
from transformers import AutoImageProcessor
|
||||
vq_model = safe_snapshot_download('BAAI/Emu3-VisionTokenizer', check_local=True)
|
||||
image_processor = AutoImageProcessor.from_pretrained(vq_model, trust_remote_code=True)
|
||||
image_tokenizer = AutoModel.from_pretrained(
|
||||
vq_model, device_map=self.model_kwargs['device_map'], trust_remote_code=True)
|
||||
image_tokenizer.requires_grad_(False)
|
||||
image_tokenizer.to(get_device())
|
||||
# load processor
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/baaivision/Emu3.git')
|
||||
sys.path.append(local_repo_path)
|
||||
from emu3.mllm.processing_emu3 import Emu3Processor
|
||||
return Emu3Processor(image_processor, image_tokenizer, tokenizer)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.emu3_chat,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('BAAI/Emu3-Chat', 'BAAI/Emu3-Chat'),
|
||||
]),
|
||||
],
|
||||
Emu3ChatLoader,
|
||||
template=TemplateType.emu3_chat,
|
||||
architectures=['Emu3ForCausalLM'],
|
||||
model_arch=ModelArch.emu3_chat,
|
||||
tags=['vision'],
|
||||
requires=['transformers>=4.44.0'],
|
||||
))
|
||||
|
||||
|
||||
class BgeRerankerLoader(ModelLoader):
|
||||
|
||||
def get_model(self, *args, **kwargs) -> PreTrainedModel:
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForSequenceClassification
|
||||
return super().get_model(*args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.bge_reranker,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('BAAI/bge-reranker-base', 'BAAI/bge-reranker-base'),
|
||||
Model('BAAI/bge-reranker-v2-m3', 'BAAI/bge-reranker-v2-m3'),
|
||||
Model('BAAI/bge-reranker-large', 'BAAI/bge-reranker-large'),
|
||||
]),
|
||||
],
|
||||
BgeRerankerLoader,
|
||||
template=TemplateType.bge_reranker,
|
||||
task_type='reranker',
|
||||
architectures=['XLMRobertaForSequenceClassification'],
|
||||
))
|
||||
@@ -0,0 +1,131 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
from transformers import PreTrainedModel
|
||||
from types import MethodType
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_logger
|
||||
from ..constant import LLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class BaichuanLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
# baichuan-13b does not implement the `get_input_embeddings` function
|
||||
# fix gradient_checkpointing bug
|
||||
try:
|
||||
model.get_input_embeddings()
|
||||
except NotImplementedError:
|
||||
model.__class__.get_input_embeddings = lambda self: self.model.embed_tokens
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.baichuan, [
|
||||
ModelGroup([
|
||||
Model('baichuan-inc/Baichuan-13B-Chat', 'baichuan-inc/Baichuan-13B-Chat'),
|
||||
Model('baichuan-inc/Baichuan-13B-Base', 'baichuan-inc/Baichuan-13B-Base'),
|
||||
Model('baichuan-inc/baichuan-7B', 'baichuan-inc/Baichuan-7B'),
|
||||
]),
|
||||
],
|
||||
BaichuanLoader,
|
||||
template=TemplateType.baichuan,
|
||||
architectures=['BaichuanForCausalLM', 'BaiChuanForCausalLM'],
|
||||
model_arch=ModelArch.baichuan,
|
||||
requires=['transformers<4.34']))
|
||||
|
||||
|
||||
class BaichuanM1Loader(BaichuanLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
rotary_embedding = get_class_from_dynamic_module('modeling_baichuan.RotaryEmbedding', model_dir)
|
||||
_old_forward = rotary_embedding.forward
|
||||
|
||||
def _new_forward(self, q, k, seqlen_offset=None, cu_seqlens=None, max_seqlen=None):
|
||||
q = q.to(k.dtype)
|
||||
res = _old_forward(self, q, k, seqlen_offset, cu_seqlens, max_seqlen)
|
||||
return res
|
||||
|
||||
rotary_embedding.forward = _new_forward
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.baichuan_m1, [
|
||||
ModelGroup([
|
||||
Model('baichuan-inc/Baichuan-M1-14B-Instruct', 'baichuan-inc/Baichuan-M1-14B-Instruct'),
|
||||
]),
|
||||
],
|
||||
BaichuanM1Loader,
|
||||
template=TemplateType.baichuan_m1,
|
||||
architectures=['BaichuanM1ForCausalLM'],
|
||||
model_arch=ModelArch.baichuan,
|
||||
requires=['transformers>=4.48']))
|
||||
|
||||
|
||||
def patch_baichuan2_lm_head_forward(self, hidden_states: Tensor) -> Tensor:
|
||||
# patch: baichuan2 lm_head (fp32 bug)
|
||||
if self.training:
|
||||
norm_weight = F.normalize(self.weight).to(self.weight.dtype)
|
||||
elif self.first_flag:
|
||||
self.first_flag = False
|
||||
self.weight.data = F.normalize(self.weight).to(self.weight.dtype)
|
||||
norm_weight = self.weight
|
||||
else:
|
||||
norm_weight = self.weight
|
||||
return F.linear(hidden_states, norm_weight)
|
||||
|
||||
|
||||
class Baichuan2Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, *args, **kwargs) -> PreTrainedModel:
|
||||
if not hasattr(config, 'z_loss_weight'):
|
||||
config.z_loss_weight = 0
|
||||
# patch: baichuan2_13b configuration_baichuan.py bug
|
||||
if hasattr(config, 'gradient_checkpointing'):
|
||||
gradient_checkpointing = config.gradient_checkpointing
|
||||
if isinstance(gradient_checkpointing, (tuple, list)):
|
||||
config.gradient_checkpointing = gradient_checkpointing[0]
|
||||
model = super().get_model(model_dir, config, *args, **kwargs)
|
||||
model_ori = model
|
||||
if not hasattr(model, 'lm_head'): # fix awq
|
||||
model = model.model
|
||||
new_forward = MethodType(patch_baichuan2_lm_head_forward, model.lm_head)
|
||||
if hasattr(model, '_old_forward'): # device_map
|
||||
model.lm_head._old_forward = new_forward
|
||||
else:
|
||||
model.lm_head.forward = new_forward
|
||||
return model_ori
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.baichuan2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('baichuan-inc/Baichuan2-7B-Chat', 'baichuan-inc/Baichuan2-7B-Chat'),
|
||||
Model('baichuan-inc/Baichuan2-7B-Base', 'baichuan-inc/Baichuan2-7B-Base'),
|
||||
Model('baichuan-inc/Baichuan2-13B-Chat', 'baichuan-inc/Baichuan2-13B-Chat'),
|
||||
Model('baichuan-inc/Baichuan2-13B-Base', 'baichuan-inc/Baichuan2-13B-Base'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('baichuan-inc/Baichuan2-7B-Chat-4bits', 'baichuan-inc/Baichuan2-7B-Chat-4bits'),
|
||||
Model('baichuan-inc/Baichuan2-13B-Chat-4bits', 'baichuan-inc/Baichuan2-13B-Chat-4bits'),
|
||||
],
|
||||
requires=['bitsandbytes<0.41.2', 'accelerate<0.26'])
|
||||
],
|
||||
Baichuan2Loader,
|
||||
template=TemplateType.baichuan,
|
||||
architectures=['BaichuanForCausalLM', 'BaiChuanForCausalLM'],
|
||||
model_arch=ModelArch.baichuan,
|
||||
))
|
||||
@@ -0,0 +1,112 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_logger
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.ernie4_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/ERNIE-4.5-0.3B-Base-PT', 'baidu/ERNIE-4.5-0.3B-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-0.3B-PT', 'baidu/ERNIE-4.5-0.3B-PT'),
|
||||
], TemplateType.ernie),
|
||||
],
|
||||
architectures=['Ernie4_5_ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.ernie4_5_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/ERNIE-4.5-21B-A3B-Base-PT', 'baidu/ERNIE-4.5-21B-A3B-Base-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-21B-A3B-PT', 'baidu/ERNIE-4.5-21B-A3B-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-300B-A47B-Base-PT', 'baidu/ERNIE-4.5-300B-A47B-Base-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-300B-A47B-PT', 'baidu/ERNIE-4.5-300B-A47B-PT'),
|
||||
], TemplateType.ernie),
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/ERNIE-4.5-21B-A3B-Thinking', 'baidu/ERNIE-4.5-21B-A3B-Thinking'),
|
||||
], TemplateType.ernie_thinking),
|
||||
],
|
||||
architectures=['Ernie4_5_MoeForCausalLM'],
|
||||
))
|
||||
|
||||
|
||||
class ErnieVLLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
MOEAllGatherLayerV2 = get_class_from_dynamic_module('modeling_ernie4_5_vl.MOEAllGatherLayerV2', model_dir)
|
||||
self.leaf_modules = MOEAllGatherLayerV2
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.add_image_preprocess(processor)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.ernie_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/ERNIE-4.5-VL-28B-A3B-PT', 'baidu/ERNIE-4.5-VL-28B-A3B-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-VL-424B-A47B-PT', 'baidu/ERNIE-4.5-VL-424B-A47B-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-PT', 'baidu/ERNIE-4.5-VL-28B-A3B-Base-PT'),
|
||||
Model('PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-PT', 'baidu/ERNIE-4.5-VL-424B-A47B-Base-PT'),
|
||||
], TemplateType.ernie_vl),
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking', 'baidu/ERNIE-4.5-VL-28B-A3B-Thinking'),
|
||||
], TemplateType.ernie_vl_thinking),
|
||||
],
|
||||
ErnieVLLoader,
|
||||
model_arch=ModelArch.ernie_vl,
|
||||
architectures=['Ernie4_5_VLMoeForConditionalGeneration'],
|
||||
requires=['transformers>=4.52', 'moviepy'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.paddle_ocr,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/PaddleOCR-VL', 'PaddlePaddle/PaddleOCR-VL'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.paddle_ocr,
|
||||
model_arch=ModelArch.keye_vl,
|
||||
architectures=['PaddleOCRVLForConditionalGeneration'],
|
||||
requires=['transformers<5.0'],
|
||||
))
|
||||
|
||||
|
||||
class PaddleOCR1_5Loader(ModelLoader):
|
||||
default_trust_remote_code = False
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForImageTextToText
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.paddleocr_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/PaddleOCR-VL-1.5', 'PaddlePaddle/PaddleOCR-VL-1.5'),
|
||||
Model('PaddlePaddle/PaddleOCR-VL-1.6', 'PaddlePaddle/PaddleOCR-VL-1.6'),
|
||||
],
|
||||
template=TemplateType.paddle_ocr_1_5),
|
||||
],
|
||||
PaddleOCR1_5Loader,
|
||||
model_arch=ModelArch.paddleocr_vl,
|
||||
requires=['transformers>=5.0'],
|
||||
architectures=['PaddleOCRVLForConditionalGeneration'],
|
||||
))
|
||||
@@ -0,0 +1,89 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import torch.nn.functional as F
|
||||
from transformers import AutoModel, AutoModelForSequenceClassification, PreTrainedModel
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_logger
|
||||
from ..constant import BertModelType, LLMModelType
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class ModernBertLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, *args, **kwargs) -> PreTrainedModel:
|
||||
config.reference_compile = False
|
||||
return super().get_model(model_dir, config, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
BertModelType.modern_bert, [
|
||||
ModelGroup([
|
||||
Model('answerdotai/ModernBERT-base', 'answerdotai/ModernBERT-base'),
|
||||
Model('answerdotai/ModernBERT-large', 'answerdotai/ModernBERT-large'),
|
||||
])
|
||||
],
|
||||
ModernBertLoader,
|
||||
template=TemplateType.dummy,
|
||||
requires=['transformers>=4.48'],
|
||||
architectures=['ModernBertForMaskedLM'],
|
||||
tags=['bert']))
|
||||
|
||||
|
||||
class GTEBertLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModel
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
def _normalizer_hook(module, input, output):
|
||||
output.last_hidden_state = F.normalize(output.last_hidden_state[:, 0], p=2, dim=1)
|
||||
return output
|
||||
|
||||
model.register_forward_hook(_normalizer_hook)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
BertModelType.modern_bert_gte,
|
||||
[ModelGroup([
|
||||
Model('iic/gte-modernbert-base', 'Alibaba-NLP/gte-modernbert-base'),
|
||||
])],
|
||||
GTEBertLoader,
|
||||
template=TemplateType.dummy,
|
||||
requires=['transformers>=4.48'],
|
||||
architectures=['ModernBertModel'],
|
||||
tags=['bert', 'embedding']))
|
||||
|
||||
|
||||
class GTEBertReranker(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForSequenceClassification
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.modern_bert_gte_reranker,
|
||||
[ModelGroup([
|
||||
Model('iic/gte-reranker-modernbert-base', 'Alibaba-NLP/gte-reranker-modernbert-base'),
|
||||
])],
|
||||
GTEBertReranker,
|
||||
template=TemplateType.bert,
|
||||
requires=['transformers>=4.48'],
|
||||
architectures=['ModernBertForSequenceClassification'],
|
||||
task_type='reranker',
|
||||
tags=['bert', 'reranker']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
BertModelType.bert, [ModelGroup([
|
||||
Model('iic/nlp_structbert_backbone_base_std'),
|
||||
])],
|
||||
template=TemplateType.dummy,
|
||||
tags=['bert']))
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import AutoTokenizer, PretrainedConfig
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor
|
||||
from ..constant import LLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
from .glm import ChatGLMLoader
|
||||
from .qwen import QwenLoader
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.codefuse_qwen, [
|
||||
ModelGroup([
|
||||
Model('codefuse-ai/CodeFuse-QWen-14B', 'codefuse-ai/CodeFuse-QWen-14B'),
|
||||
]),
|
||||
],
|
||||
QwenLoader,
|
||||
template=TemplateType.codefuse,
|
||||
architectures=['QWenLMHeadModel'],
|
||||
model_arch=ModelArch.qwen,
|
||||
tags=['coding']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.codefuse_codegeex2,
|
||||
[
|
||||
ModelGroup([Model('codefuse-ai/CodeFuse-CodeGeeX2-6B', 'codefuse-ai/CodeFuse-CodeGeeX2-6B')], ),
|
||||
],
|
||||
ChatGLMLoader,
|
||||
template=TemplateType.codefuse,
|
||||
architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
|
||||
model_arch=ModelArch.chatglm,
|
||||
tags=['coding'],
|
||||
requires=['transformers<4.34'],
|
||||
))
|
||||
|
||||
|
||||
class CodeLlamaLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False, legacy=False)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.codefuse_codellama,
|
||||
[
|
||||
ModelGroup(
|
||||
[
|
||||
Model('codefuse-ai/CodeFuse-CodeLlama-34B', 'codefuse-ai/CodeFuse-CodeLlama-34B'),
|
||||
],
|
||||
tags=['coding'],
|
||||
),
|
||||
],
|
||||
CodeLlamaLoader,
|
||||
template=TemplateType.codefuse_codellama,
|
||||
model_arch=ModelArch.llama,
|
||||
mcore_model_type='gpt',
|
||||
architectures=['LlamaForCausalLM'],
|
||||
))
|
||||
@@ -0,0 +1,509 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import sys
|
||||
import torch
|
||||
from transformers import AutoModel, PretrainedConfig, PreTrainedModel
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_logger, git_clone_github
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_output_clone, patch_output_to_input_device
|
||||
from ..register import ModelLoader, register_model
|
||||
from ..utils import use_submodel_func
|
||||
|
||||
|
||||
class DeepseekLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
# fix dtype bug
|
||||
mlp_cls = model.model.layers[-1].mlp.__class__
|
||||
|
||||
for module in model.modules():
|
||||
if isinstance(module, mlp_cls):
|
||||
patch_output_to_input_device(module)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.deepseek,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/deepseek-moe-16b-chat', 'deepseek-ai/deepseek-moe-16b-chat'),
|
||||
Model('deepseek-ai/deepseek-moe-16b-base', 'deepseek-ai/deepseek-moe-16b-base'),
|
||||
], ),
|
||||
],
|
||||
DeepseekLoader,
|
||||
template=TemplateType.deepseek,
|
||||
architectures=['DeepseekForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.deepseek_v2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-Coder-V2-Instruct', 'deepseek-ai/DeepSeek-Coder-V2-Instruct'),
|
||||
Model('deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct', 'deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct'),
|
||||
Model('deepseek-ai/DeepSeek-Coder-V2-Base', 'deepseek-ai/DeepSeek-Coder-V2-Base'),
|
||||
Model('deepseek-ai/DeepSeek-Coder-V2-Lite-Base', 'deepseek-ai/DeepSeek-Coder-V2-Lite-Base'),
|
||||
Model('deepseek-ai/DeepSeek-V2-Lite', 'deepseek-ai/DeepSeek-V2-Lite'),
|
||||
Model('deepseek-ai/DeepSeek-V2-Lite-Chat', 'deepseek-ai/DeepSeek-V2-Lite-Chat'),
|
||||
Model('deepseek-ai/DeepSeek-V2', 'deepseek-ai/DeepSeek-V2'),
|
||||
Model('deepseek-ai/DeepSeek-V2-Chat', 'deepseek-ai/DeepSeek-V2-Chat'),
|
||||
], TemplateType.deepseek),
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-V2.5', 'deepseek-ai/DeepSeek-V2.5'),
|
||||
Model('deepseek-ai/DeepSeek-V2.5-1210', 'deepseek-ai/DeepSeek-V2.5-1210')
|
||||
], TemplateType.deepseek_v2_5)
|
||||
],
|
||||
DeepseekLoader,
|
||||
model_arch=ModelArch.deepseek_v2,
|
||||
architectures=['DeepseekV2ForCausalLM'],
|
||||
requires=['transformers>=4.39.3'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.deepseek_v3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-V3-Base', 'deepseek-ai/DeepSeek-V3-Base'),
|
||||
Model('deepseek-ai/DeepSeek-V3', 'deepseek-ai/DeepSeek-V3'),
|
||||
Model('deepseek-ai/DeepSeek-V3-0324', 'deepseek-ai/DeepSeek-V3-0324'),
|
||||
], TemplateType.deepseek_v2_5),
|
||||
ModelGroup([
|
||||
Model('cognitivecomputations/DeepSeek-V3-awq', 'cognitivecomputations/DeepSeek-V3-AWQ'),
|
||||
Model('cognitivecomputations/DeepSeek-V3-0324-AWQ', 'cognitivecomputations/DeepSeek-V3-0324-AWQ')
|
||||
], TemplateType.deepseek_v2_5),
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-Prover-V2-7B', 'deepseek-ai/DeepSeek-Prover-V2-7B'),
|
||||
Model('deepseek-ai/DeepSeek-Prover-V2-671B', 'deepseek-ai/DeepSeek-Prover-V2-671B'),
|
||||
], TemplateType.deepseek_v2_5),
|
||||
ModelGroup([
|
||||
Model('unsloth/DeepSeek-V3-bf16', 'unsloth/DeepSeek-V3-bf16'),
|
||||
Model('unsloth/DeepSeek-V3-0324-BF16', 'unsloth/DeepSeek-V3-0324-BF16'),
|
||||
Model('unsloth/DeepSeek-Prover-V2-671B-BF16', 'unsloth/DeepSeek-Prover-V2-671B-BF16'),
|
||||
], TemplateType.deepseek_v2_5),
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-R1', 'deepseek-ai/DeepSeek-R1'),
|
||||
Model('deepseek-ai/DeepSeek-R1-Zero', 'deepseek-ai/DeepSeek-R1-Zero'),
|
||||
Model('deepseek-ai/DeepSeek-R1-0528', 'deepseek-ai/DeepSeek-R1-0528'),
|
||||
], TemplateType.deepseek_r1),
|
||||
ModelGroup([
|
||||
Model('cognitivecomputations/DeepSeek-R1-awq', 'cognitivecomputations/DeepSeek-R1-AWQ'),
|
||||
Model('cognitivecomputations/DeepSeek-R1-0528-AWQ', 'cognitivecomputations/DeepSeek-R1-0528-AWQ'),
|
||||
], TemplateType.deepseek_r1),
|
||||
ModelGroup([
|
||||
Model('unsloth/DeepSeek-R1-BF16', 'unsloth/DeepSeek-R1-BF16'),
|
||||
Model('unsloth/DeepSeek-R1-Zero-BF16', 'unsloth/DeepSeek-R1-Zero-BF16'),
|
||||
Model('unsloth/DeepSeek-R1-0528-BF16', 'unsloth/DeepSeek-R1-0528-BF16'),
|
||||
], TemplateType.deepseek_r1),
|
||||
ModelGroup([
|
||||
Model('moonshotai/Moonlight-16B-A3B', 'moonshotai/Moonlight-16B-A3B'),
|
||||
Model('moonshotai/Moonlight-16B-A3B-Instruct', 'moonshotai/Moonlight-16B-A3B-Instruct'),
|
||||
],
|
||||
TemplateType.moonlight,
|
||||
requires=['transformers<4.49']),
|
||||
ModelGroup([
|
||||
Model('moonshotai/Kimi-K2-Base', 'moonshotai/Kimi-K2-Base'),
|
||||
Model('moonshotai/Kimi-K2-Instruct', 'moonshotai/Kimi-K2-Instruct'),
|
||||
Model('moonshotai/Kimi-K2-Instruct-0905', 'moonshotai/Kimi-K2-Instruct-0905'),
|
||||
Model('moonshotai/Kimi-K2-Thinking', 'moonshotai/Kimi-K2-Thinking'),
|
||||
], TemplateType.kimi_k2),
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-V3.1-Base', 'deepseek-ai/DeepSeek-V3.1-Base'),
|
||||
Model('deepseek-ai/DeepSeek-V3.1', 'deepseek-ai/DeepSeek-V3.1'),
|
||||
Model('deepseek-ai/DeepSeek-V3.1-Terminus', 'deepseek-ai/DeepSeek-V3.1-Terminus'),
|
||||
], TemplateType.deepseek_v3_1),
|
||||
],
|
||||
DeepseekLoader,
|
||||
model_arch=ModelArch.deepseek_v2,
|
||||
architectures=['DeepseekV3ForCausalLM'],
|
||||
requires=['transformers>=4.39.3'],
|
||||
))
|
||||
|
||||
|
||||
class DeepseekV32Loader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
try:
|
||||
from transformers.models.deepseek_v32 import DeepseekV32Config
|
||||
except ImportError:
|
||||
from transformers.models.deepseek_v3 import DeepseekV3Config as DeepseekV32Config
|
||||
return DeepseekV32Config.from_pretrained(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
try:
|
||||
from transformers.models.deepseek_v32 import DeepseekV32ForCausalLM
|
||||
except ImportError:
|
||||
# It’s only for compatibility with Megatron training or vllm/sglang infer,
|
||||
# while we wait for Transformers to support deepseek_v32.
|
||||
from transformers.models.deepseek_v3 import DeepseekV3ForCausalLM as DeepseekV32ForCausalLM
|
||||
if not self.return_dummy_model:
|
||||
raise ValueError('DeepSeek-V3.2 is not supported in transformers.')
|
||||
self.auto_model_cls = DeepseekV32ForCausalLM
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.deepseek_v32,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-V3.2', 'deepseek-ai/DeepSeek-V3.2'),
|
||||
Model('deepseek-ai/DeepSeek-V3.2-Speciale', 'deepseek-ai/DeepSeek-V3.2-Speciale'),
|
||||
Model('deepseek-ai/DeepSeek-V3.2-Exp', 'deepseek-ai/DeepSeek-V3.2-Exp'),
|
||||
Model('deepseek-ai/DeepSeek-V3.2-Exp-Base', 'deepseek-ai/DeepSeek-V3.2-Exp-Base'),
|
||||
Model('deepseek-ai/DeepSeek-Math-V2', 'deepseek-ai/DeepSeek-Math-V2'),
|
||||
]),
|
||||
],
|
||||
DeepseekV32Loader,
|
||||
template=TemplateType.deepseek_v3_1,
|
||||
architectures=['DeepseekV32ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.deepseek_v4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-V4-Flash', 'deepseek-ai/DeepSeek-V4-Flash'),
|
||||
Model('deepseek-ai/DeepSeek-V4-Flash-Base', 'deepseek-ai/DeepSeek-V4-Flash-Base'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-V4-Pro', 'deepseek-ai/DeepSeek-V4-Pro'),
|
||||
Model('deepseek-ai/DeepSeek-V4-Pro-Base', 'deepseek-ai/DeepSeek-V4-Pro-Base'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.deepseek_v4,
|
||||
architectures=['DeepseekV4ForCausalLM'],
|
||||
))
|
||||
|
||||
|
||||
class DeepseekVLLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
# compat with python==3.10
|
||||
if sys.version_info.minor >= 10:
|
||||
import collections
|
||||
import collections.abc
|
||||
for type_name in collections.abc.__all__:
|
||||
setattr(collections, type_name, getattr(collections.abc, type_name))
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/deepseek-ai/DeepSeek-VL')
|
||||
sys.path.append(local_repo_path)
|
||||
from deepseek_vl.models import VLChatProcessor
|
||||
self.auto_tokenizer_cls = VLChatProcessor
|
||||
return super().get_config(model_dir)
|
||||
|
||||
def _get_model(self, model_dir: str, llm_prefix, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
llm = getattr(model, llm_prefix)
|
||||
patch_output_clone(llm.model.embed_tokens)
|
||||
patch_output_to_input_device(llm.model.embed_tokens)
|
||||
use_submodel_func(model, llm_prefix)
|
||||
model.generation_config = llm.generation_config
|
||||
return model
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
return self._get_model(model_dir, 'language_model', *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.deepseek_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/deepseek-vl-1.3b-chat', 'deepseek-ai/deepseek-vl-1.3b-chat'),
|
||||
Model('deepseek-ai/deepseek-vl-7b-chat', 'deepseek-ai/deepseek-vl-7b-chat'),
|
||||
], ),
|
||||
],
|
||||
DeepseekVLLoader,
|
||||
template=TemplateType.deepseek_vl,
|
||||
architectures=['MultiModalityCausalLM'],
|
||||
model_arch=ModelArch.deepseek_vl,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class DeepseekJanusLoader(DeepseekVLLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
return self._get_model(model_dir, 'language_model', *args, **kwargs)
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/deepseek-ai/Janus')
|
||||
sys.path.append(local_repo_path)
|
||||
from janus.models import VLChatProcessor
|
||||
self.auto_tokenizer_cls = VLChatProcessor
|
||||
return super(DeepseekVLLoader, self).get_config(model_dir)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.deepseek_janus,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/Janus-1.3B', 'deepseek-ai/Janus-1.3B'),
|
||||
]),
|
||||
],
|
||||
DeepseekJanusLoader,
|
||||
template=TemplateType.deepseek_janus,
|
||||
model_arch=ModelArch.deepseek_janus,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.deepseek_janus_pro,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/Janus-Pro-1B', 'deepseek-ai/Janus-Pro-1B'),
|
||||
Model('deepseek-ai/Janus-Pro-7B', 'deepseek-ai/Janus-Pro-7B'),
|
||||
]),
|
||||
],
|
||||
DeepseekJanusLoader,
|
||||
template=TemplateType.deepseek_janus_pro,
|
||||
model_arch=ModelArch.deepseek_janus,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class DeepseekVL2Loader(DeepseekVLLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/deepseek-ai/DeepSeek-VL2')
|
||||
sys.path.append(local_repo_path)
|
||||
try:
|
||||
from deepseek_vl2.models import DeepseekVLV2Processor
|
||||
except ImportError:
|
||||
# compat transformers>=4.42
|
||||
import transformers
|
||||
transformers.models.llama.modeling_llama.LlamaFlashAttention2 = None
|
||||
from deepseek_vl2.models import DeepseekVLV2Processor
|
||||
self.auto_tokenizer_cls = DeepseekVLV2Processor
|
||||
return super(DeepseekVLLoader, self).get_config(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
return super()._get_model(model_dir, 'language', *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.deepseek_vl2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/deepseek-vl2-tiny', 'deepseek-ai/deepseek-vl2-tiny'),
|
||||
Model('deepseek-ai/deepseek-vl2-small', 'deepseek-ai/deepseek-vl2-small'),
|
||||
Model('deepseek-ai/deepseek-vl2', 'deepseek-ai/deepseek-vl2'),
|
||||
]),
|
||||
],
|
||||
DeepseekVL2Loader,
|
||||
template=TemplateType.deepseek_vl2,
|
||||
model_arch=ModelArch.deepseek_vl2,
|
||||
requires=['transformers<4.42'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class DeepseekOCRLoader(ModelLoader):
|
||||
visual_name = 'vision_model'
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModel
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.model.embed_tokens)
|
||||
patch_output_to_input_device(model.model.sam_model)
|
||||
patch_output_to_input_device(getattr(model.model, self.visual_name))
|
||||
patch_output_to_input_device(model.model.projector)
|
||||
return model
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
from transformers import AutoProcessor, AutoTokenizer
|
||||
|
||||
# When not loading model (e.g., vllm backend), avoid triggering AutoConfig which would execute
|
||||
# trust_remote_code and cause transformers version compatibility issues
|
||||
# For vllm backend, we only need the processor/tokenizer
|
||||
try:
|
||||
processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
|
||||
except Exception:
|
||||
# Fallback to AutoTokenizer if AutoProcessor is not available
|
||||
processor = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
|
||||
return processor
|
||||
|
||||
|
||||
class DeepseekOCR2Loader(DeepseekOCRLoader):
|
||||
visual_name = 'qwen2_model'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.deepseek_ocr,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-OCR', 'deepseek-ai/DeepSeek-OCR'),
|
||||
]),
|
||||
],
|
||||
DeepseekOCRLoader,
|
||||
template=TemplateType.deepseek_ocr,
|
||||
model_arch=ModelArch.deepseek_ocr,
|
||||
architectures=['DeepseekOCRForCausalLM'],
|
||||
requires=['transformers==4.46.3', 'easydict'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.deepseek_ocr2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-OCR-2', 'deepseek-ai/DeepSeek-OCR-2'),
|
||||
]),
|
||||
],
|
||||
DeepseekOCR2Loader,
|
||||
template=TemplateType.deepseek_ocr2,
|
||||
model_arch=ModelArch.deepseek_ocr2,
|
||||
architectures=['DeepseekOCR2ForCausalLM'],
|
||||
requires=['transformers==4.46.3', 'easydict'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class UnlimitedOCRLoader(DeepseekOCRLoader):
|
||||
visual_name = 'vision_model'
|
||||
|
||||
@staticmethod
|
||||
def _apply_multi_gpu_patch():
|
||||
"""
|
||||
Fixed two bugs affecting `UnlimitedOCRModel` in multi-GPU scenarios using `device_map='auto'`:
|
||||
|
||||
Bug 1 - Device mismatch in `torch.cat`:
|
||||
`image_newline` and `view_seperator` are `nn.Parameter`s;
|
||||
under `device_map='auto'`, their device placement might not align
|
||||
with the image features.
|
||||
|
||||
Bug 2 - Device mismatch in `masked_scatter_`:
|
||||
Hard-coded `.cuda()` usage caused a conflict where `images_in_this_batch`
|
||||
resided on the projector's device (e.g., `cuda:7`),
|
||||
while `inputs_embeds` resided on the device hosting `embed_tokens` (e.g., `cuda:0`).
|
||||
|
||||
Fix strategy: Temporarily replace `torch.cat` and `torch.Tensor.masked_scatter_` during the forward pass
|
||||
to handle device placement automatically, then restore the original methods after execution.
|
||||
"""
|
||||
modeling_module = None
|
||||
for mod_name, mod in sys.modules.items():
|
||||
if 'modeling_unlimitedocr' in mod_name:
|
||||
modeling_module = mod
|
||||
break
|
||||
|
||||
if modeling_module is None:
|
||||
return False
|
||||
|
||||
UnlimitedOCRModel = getattr(modeling_module, 'UnlimitedOCRModel', None)
|
||||
if UnlimitedOCRModel is None:
|
||||
return False
|
||||
|
||||
# Avoid redundant patching
|
||||
if getattr(UnlimitedOCRModel, '_swift_multi_gpu_patched', False):
|
||||
return True
|
||||
|
||||
_original_forward = UnlimitedOCRModel.forward
|
||||
|
||||
def _patched_forward(self, *args, **kwargs):
|
||||
_orig_cat = torch.cat
|
||||
_orig_masked_scatter_ = torch.Tensor.masked_scatter_
|
||||
|
||||
def _safe_cat(tensors, dim=0, **cat_kwargs):
|
||||
# Using the device of the first tensor as the reference, the others are aligned to it.
|
||||
ref_device = None
|
||||
for t in tensors:
|
||||
if isinstance(t, torch.Tensor):
|
||||
ref_device = t.device
|
||||
break
|
||||
if ref_device is None:
|
||||
return _orig_cat(tensors, dim, **cat_kwargs)
|
||||
aligned = [
|
||||
t.to(ref_device) if isinstance(t, torch.Tensor) and t.device != ref_device else t for t in tensors
|
||||
]
|
||||
return _orig_cat(aligned, dim, **cat_kwargs)
|
||||
|
||||
def _safe_masked_scatter_(tensor_self, mask, source):
|
||||
# Use the device of tensor_self (inputs_embeds[idx]) as the reference.
|
||||
dev = tensor_self.device
|
||||
if mask.device != dev:
|
||||
mask = mask.to(dev)
|
||||
if source.device != dev:
|
||||
source = source.to(dev)
|
||||
return _orig_masked_scatter_(tensor_self, mask, source)
|
||||
|
||||
# Simultaneously replace the module namespace and the global scope (double insurance).
|
||||
modeling_module.torch.cat = _safe_cat
|
||||
torch.cat = _safe_cat
|
||||
torch.Tensor.masked_scatter_ = _safe_masked_scatter_
|
||||
try:
|
||||
return _original_forward(self, *args, **kwargs)
|
||||
finally:
|
||||
# Restore the state to avoid contaminating other modules.
|
||||
modeling_module.torch.cat = _orig_cat
|
||||
torch.cat = _orig_cat
|
||||
torch.Tensor.masked_scatter_ = _orig_masked_scatter_
|
||||
|
||||
UnlimitedOCRModel.forward = _patched_forward
|
||||
UnlimitedOCRModel._swift_multi_gpu_patched = True
|
||||
return True
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
logger = get_logger()
|
||||
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModel
|
||||
model = super(DeepseekOCRLoader, self).get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.model.embed_tokens)
|
||||
patch_output_to_input_device(model.model.sam_model)
|
||||
patch_output_to_input_device(getattr(model.model, self.visual_name))
|
||||
patch_output_to_input_device(model.model.projector)
|
||||
patch_output_to_input_device(model.model)
|
||||
|
||||
_orig_sw = (getattr(model.config, 'sliding_window_size', None) or getattr(model.config, 'sliding_window', None))
|
||||
if _orig_sw is not None:
|
||||
model.config._ring_window = _orig_sw
|
||||
model.config.sliding_window = None
|
||||
logger.info('[UnlimitedOCR] R-SWA enabled: ring_window=%d', _orig_sw)
|
||||
else:
|
||||
logger.warning('[UnlimitedOCR] sliding_window config not found, R-SWA may not work.')
|
||||
|
||||
n_devices = len(set(str(p.device) for p in model.parameters() if p.device.type == 'cuda'))
|
||||
if n_devices > 1:
|
||||
if self._apply_multi_gpu_patch():
|
||||
logger.info('[UnlimitedOCR] Multi-GPU patch applied (%d GPUs).', n_devices)
|
||||
else:
|
||||
logger.warning('[UnlimitedOCR] Multi-GPU deployment failed to apply patch.'
|
||||
'If an inference error occurs, please check whether'
|
||||
' `modeling_unlimitedocr` has been loaded correctly.')
|
||||
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.unlimited_ocr,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('PaddlePaddle/Unlimited-OCR', 'PaddlePaddle/Unlimited-OCR'),
|
||||
]),
|
||||
],
|
||||
UnlimitedOCRLoader,
|
||||
template=TemplateType.unlimited_ocr,
|
||||
model_arch=ModelArch.unlimited_ocr,
|
||||
architectures=['UnlimitedOCRForCausalLM'],
|
||||
requires=['transformers==4.46.3', 'easydict'],
|
||||
tags=['vision'],
|
||||
))
|
||||
@@ -0,0 +1,508 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import inspect
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import transformers
|
||||
from packaging import version
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedModel
|
||||
from types import MethodType
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import is_deepspeed_enabled, to_device
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_output_to_input_device
|
||||
from ..register import ModelLoader, SentenceTransformersLoader, register_model
|
||||
|
||||
transformers_5_9 = version.parse(transformers.__version__) >= version.parse('5.9')
|
||||
|
||||
|
||||
class PaligemmaVisionLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import PaliGemmaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or PaliGemmaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.paligemma,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/paligemma-3b-pt-224', 'google/paligemma-3b-pt-224'),
|
||||
Model('AI-ModelScope/paligemma-3b-pt-448', 'google/paligemma-3b-pt-448'),
|
||||
Model('AI-ModelScope/paligemma-3b-pt-896', 'google/paligemma-3b-pt-896'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/paligemma-3b-mix-224', 'google/paligemma-3b-mix-224'),
|
||||
Model('AI-ModelScope/paligemma-3b-mix-448', 'google/paligemma-3b-mix-448'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/paligemma2-3b-pt-224', 'google/paligemma2-3b-pt-224'),
|
||||
Model('AI-ModelScope/paligemma2-3b-pt-448', 'google/paligemma2-3b-pt-448'),
|
||||
Model('AI-ModelScope/paligemma2-3b-pt-896', 'google/paligemma2-3b-pt-896'),
|
||||
Model('AI-ModelScope/paligemma2-10b-pt-224', 'google/paligemma2-10b-pt-224'),
|
||||
Model('AI-ModelScope/paligemma2-10b-pt-448', 'google/paligemma2-10b-pt-448'),
|
||||
Model('AI-ModelScope/paligemma2-10b-pt-896', 'google/paligemma2-10b-pt-896'),
|
||||
Model('AI-ModelScope/paligemma2-28b-pt-224', 'google/paligemma2-28b-pt-224'),
|
||||
Model('AI-ModelScope/paligemma2-28b-pt-448', 'google/paligemma2-28b-pt-448'),
|
||||
Model('AI-ModelScope/paligemma2-28b-pt-896', 'google/paligemma2-28b-pt-896'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/paligemma2-3b-ft-docci-448', 'google/paligemma2-3b-ft-docci-448'),
|
||||
Model('AI-ModelScope/paligemma2-10b-ft-docci-448', 'google/paligemma2-10b-ft-docci-448'),
|
||||
]),
|
||||
],
|
||||
PaligemmaVisionLoader,
|
||||
template=TemplateType.paligemma,
|
||||
architectures=['PaliGemmaForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.41'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.gemma,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/gemma-2b-it', 'google/gemma-2b-it'),
|
||||
Model('AI-ModelScope/gemma-2b', 'google/gemma-2b'),
|
||||
Model('AI-ModelScope/gemma-7b', 'google/gemma-7b'),
|
||||
Model('AI-ModelScope/gemma-7b-it', 'google/gemma-7b-it'),
|
||||
], ),
|
||||
],
|
||||
template=TemplateType.gemma,
|
||||
architectures=['GemmaForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.38'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.gemma2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/gemma-2-2b-it', 'google/gemma-2-2b-it'),
|
||||
Model('LLM-Research/gemma-2-2b', 'google/gemma-2-2b'),
|
||||
Model('LLM-Research/gemma-2-9b', 'google/gemma-2-9b'),
|
||||
Model('LLM-Research/gemma-2-9b-it', 'google/gemma-2-9b-it'),
|
||||
Model('LLM-Research/gemma-2-27b', 'google/gemma-2-27b'),
|
||||
Model('LLM-Research/gemma-2-27b-it', 'google/gemma-2-27b-it'),
|
||||
], ),
|
||||
],
|
||||
template=TemplateType.gemma,
|
||||
architectures=['Gemma2ForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.42'],
|
||||
))
|
||||
|
||||
|
||||
class Gemma3TextLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir):
|
||||
# It is strongly recommended to train Gemma3 models with the `eager` attention implementation instead of `sdpa`.
|
||||
self.attn_impl = self.attn_impl or 'eager'
|
||||
return super().get_config(model_dir)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.gemma3_text,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/gemma-3-1b-pt', 'google/gemma-3-1b-pt'),
|
||||
Model('LLM-Research/gemma-3-1b-it', 'google/gemma-3-1b-it'),
|
||||
Model('google/gemma-3-270m', 'google/gemma-3-270m'),
|
||||
Model('google/gemma-3-270m-it', 'google/gemma-3-270m-it'),
|
||||
Model('google/medgemma-27b-text-it', 'google/medgemma-27b-text-it'),
|
||||
], ),
|
||||
],
|
||||
Gemma3TextLoader,
|
||||
template=TemplateType.gemma3_text,
|
||||
architectures=['Gemma3ForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.49'],
|
||||
))
|
||||
|
||||
|
||||
class Gemma3VisionLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir):
|
||||
# It is strongly recommended to train Gemma3 models with the `eager` attention implementation instead of `sdpa`.
|
||||
self.attn_impl = self.attn_impl or 'eager'
|
||||
return super().get_config(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import Gemma3ForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Gemma3ForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.gemma3_vision,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/gemma-3-4b-pt', 'google/gemma-3-4b-pt'),
|
||||
Model('LLM-Research/gemma-3-4b-it', 'google/gemma-3-4b-it'),
|
||||
Model('LLM-Research/gemma-3-12b-pt', 'google/gemma-3-12b-pt'),
|
||||
Model('LLM-Research/gemma-3-12b-it', 'google/gemma-3-12b-it'),
|
||||
Model('LLM-Research/gemma-3-27b-pt', 'google/gemma-3-27b-pt'),
|
||||
Model('LLM-Research/gemma-3-27b-it', 'google/gemma-3-27b-it'),
|
||||
Model('google/medgemma-4b-pt', 'google/medgemma-4b-pt'),
|
||||
Model('google/medgemma-4b-it', 'google/medgemma-4b-it'),
|
||||
Model('google/medgemma-27b-it', 'google/medgemma-27b-it'),
|
||||
], ),
|
||||
],
|
||||
Gemma3VisionLoader,
|
||||
template=TemplateType.gemma3_vision,
|
||||
architectures=['Gemma3ForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.49'],
|
||||
))
|
||||
|
||||
|
||||
class Gemma3nLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import Gemma3nForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Gemma3nForConditionalGeneration
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_to_input_device(model.model.embed_vision)
|
||||
patch_output_to_input_device(model.model.embed_audio)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.gemma3n,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('google/gemma-3n-E2B', 'google/gemma-3n-E2B'),
|
||||
Model('google/gemma-3n-E4B', 'google/gemma-3n-E4B'),
|
||||
Model('google/gemma-3n-E2B-it', 'google/gemma-3n-E2B-it'),
|
||||
Model('google/gemma-3n-E4B-it', 'google/gemma-3n-E4B-it'),
|
||||
], ),
|
||||
],
|
||||
Gemma3nLoader,
|
||||
template=TemplateType.gemma3n,
|
||||
architectures=['Gemma3nForConditionalGeneration'],
|
||||
model_arch=ModelArch.gemma3n,
|
||||
requires=['transformers>=4.53.1'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.gemma_emb,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('google/embeddinggemma-300m', 'google/embeddinggemma-300m'),
|
||||
], ),
|
||||
],
|
||||
SentenceTransformersLoader,
|
||||
template=TemplateType.dummy,
|
||||
architectures=['Gemma3TextModel'],
|
||||
))
|
||||
|
||||
|
||||
def _patch_gemma4_forward(model, processor, is_gemma4_unified: bool = False):
|
||||
if is_gemma4_unified:
|
||||
from transformers.models.gemma4_unified.modeling_gemma4_unified import \
|
||||
Gemma4UnifiedModelOutputWithPast as Gemma4ModelOutputWithPast
|
||||
from transformers.models.gemma4_unified.modeling_gemma4_unified import (create_masks_for_generate,
|
||||
torch_compilable_check)
|
||||
else:
|
||||
from transformers.models.gemma4.modeling_gemma4 import (Gemma4ModelOutputWithPast, create_masks_for_generate,
|
||||
torch_compilable_check)
|
||||
if hasattr(model, 'origin_forward'):
|
||||
return
|
||||
|
||||
def _forward_dummy_image(model, inputs_embeds):
|
||||
images = [Image.new('RGB', (32, 32), (0, 0, 0))]
|
||||
image_inputs = processor.image_processor(images=images, return_tensors='pt')
|
||||
image_inputs = to_device(image_inputs, inputs_embeds.device)
|
||||
dummy_pixel = image_inputs['pixel_values'].to(model.dtype)
|
||||
dummy_pos_ids = image_inputs.get('image_position_ids')
|
||||
image_features = model.get_image_features(dummy_pixel, dummy_pos_ids, return_dict=True).pooler_output
|
||||
inputs_embeds = inputs_embeds + image_features.mean() * 0.
|
||||
return inputs_embeds
|
||||
|
||||
# transformers 5.6.2
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.LongTensor | None = None,
|
||||
pixel_values: torch.FloatTensor | None = None,
|
||||
pixel_values_videos: torch.FloatTensor | None = None,
|
||||
input_features: torch.FloatTensor | None = None,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
input_features_mask: torch.Tensor | None = None,
|
||||
position_ids: torch.LongTensor | None = None,
|
||||
past_key_values=None,
|
||||
mm_token_type_ids: torch.LongTensor | None = None,
|
||||
inputs_embeds: torch.FloatTensor | None = None,
|
||||
use_cache: bool | None = None,
|
||||
image_position_ids: torch.LongTensor | None = None,
|
||||
video_position_ids: torch.LongTensor | None = None,
|
||||
per_layer_inputs: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> Gemma4ModelOutputWithPast:
|
||||
r"""
|
||||
input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
|
||||
The attention mask for the input audio.
|
||||
image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
|
||||
2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
|
||||
Passed through to the vision encoder for positional embedding computation.
|
||||
video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
|
||||
2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
|
||||
Passed through to the vision encoder for positional embedding computation.
|
||||
"""
|
||||
if (input_ids is None) ^ (inputs_embeds is not None):
|
||||
raise ValueError('You must specify exactly one of input_ids or inputs_embeds')
|
||||
|
||||
image_mask, video_mask, audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds)
|
||||
multimodal_mask = image_mask | video_mask | audio_mask
|
||||
|
||||
# Replace image id with PAD if the image token if OOV, to avoid index-errors
|
||||
llm_input_ids = None
|
||||
if inputs_embeds is None:
|
||||
llm_input_ids = input_ids.clone()
|
||||
llm_input_ids[multimodal_mask] = self.config.text_config.pad_token_id
|
||||
inputs_embeds = self.get_input_embeddings()(llm_input_ids)
|
||||
|
||||
if per_layer_inputs is None and self.config.get_text_config().hidden_size_per_layer_input:
|
||||
pad_embedding = self.language_model.embed_tokens.weight[self.config.text_config.pad_token_id, :]
|
||||
pad_embedding = pad_embedding.to(device=multimodal_mask.device)
|
||||
llm_inputs_embeds = torch.where(multimodal_mask[..., None], pad_embedding.view(1, 1, -1), inputs_embeds)
|
||||
per_layer_inputs = self.language_model.get_per_layer_inputs(llm_input_ids, llm_inputs_embeds)
|
||||
else:
|
||||
per_layer_inputs = None
|
||||
|
||||
state = input_ids.new_tensor(
|
||||
[pixel_values is not None or pixel_values_videos is not None, input_features is not None], dtype=torch.bool)
|
||||
if dist.is_initialized() and is_deepspeed_enabled():
|
||||
dist.all_reduce(state, dist.ReduceOp.MAX)
|
||||
has_image, has_audio = state.tolist()
|
||||
|
||||
# Mixed modality training with both images and videos is not currently supported.
|
||||
if pixel_values is None and pixel_values_videos is None and has_image:
|
||||
inputs_embeds = _forward_dummy_image(self, inputs_embeds)
|
||||
|
||||
# Merge text and images
|
||||
if pixel_values is not None:
|
||||
image_features = self.get_image_features(pixel_values, image_position_ids, return_dict=True).pooler_output
|
||||
image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
|
||||
|
||||
# Confirm the number of soft tokens from the vision tower matches the number of slots in the embeddings.
|
||||
n_image_tokens = image_mask.sum()
|
||||
image_mask = image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
|
||||
torch_compilable_check(
|
||||
inputs_embeds[image_mask].numel() == image_features.numel(),
|
||||
f'Image features and image tokens do not match, tokens: {n_image_tokens}, features:'
|
||||
f' {image_features.shape[0]}',
|
||||
)
|
||||
|
||||
inputs_embeds = inputs_embeds.masked_scatter(
|
||||
image_mask.to(inputs_embeds.device), image_features.to(inputs_embeds.device))
|
||||
|
||||
if pixel_values_videos is not None:
|
||||
video_features = self.get_video_features(
|
||||
pixel_values_videos, video_position_ids, return_dict=True).pooler_output
|
||||
video_features = video_features.to(inputs_embeds.device, inputs_embeds.dtype)
|
||||
|
||||
# Confirm the number of soft tokens from the vision tower matches the number of slots in the embeddings.
|
||||
n_video_tokens = video_mask.sum()
|
||||
video_mask = video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
|
||||
torch_compilable_check(
|
||||
inputs_embeds[video_mask].numel() == video_features.numel(),
|
||||
f'Video features and video tokens do not match, tokens: {n_video_tokens}, features:'
|
||||
f' {video_features.shape[0]}',
|
||||
)
|
||||
|
||||
inputs_embeds = inputs_embeds.masked_scatter(
|
||||
video_mask.to(inputs_embeds.device), video_features.to(inputs_embeds.device))
|
||||
|
||||
# Merge text and audio
|
||||
if input_features is not None and input_features_mask is not None:
|
||||
audio_output = self.get_audio_features(input_features, input_features_mask, return_dict=True)
|
||||
audio_features = audio_output.pooler_output
|
||||
audio_mask_from_encoder = audio_output.attention_mask # True = valid
|
||||
|
||||
# Strip padding tokens: only keep real (non-padding) audio soft tokens.
|
||||
# audio_mask_from_encoder is True for valid positions, False for padding tokens.
|
||||
# This mirrors the vision encoder's padding stripping (see Gemma4VisionEncoder.forward).
|
||||
audio_features = audio_features[audio_mask_from_encoder]
|
||||
|
||||
n_audio_tokens = audio_mask.sum()
|
||||
audio_mask = audio_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
|
||||
torch_compilable_check(
|
||||
inputs_embeds[audio_mask].numel() == audio_features.numel(),
|
||||
f'Audio features and audio tokens do not match, tokens: {n_audio_tokens}, features:'
|
||||
f' {audio_features.shape[0] * audio_features.shape[1]}',
|
||||
)
|
||||
|
||||
inputs_embeds = inputs_embeds.masked_scatter(
|
||||
audio_mask.to(inputs_embeds.device), audio_features.to(inputs_embeds.device))
|
||||
elif has_audio and self.audio_tower is not None:
|
||||
feature_size = processor.feature_extractor.feature_size
|
||||
dummy_features = input_ids.new_zeros([1, 128, feature_size], dtype=self.audio_tower.dtype)
|
||||
dummy_mask = input_ids.new_ones([1, 128], dtype=torch.bool)
|
||||
audio_output = self.get_audio_features(dummy_features, dummy_mask, return_dict=True)
|
||||
audio_features = audio_output.pooler_output
|
||||
inputs_embeds = inputs_embeds + audio_features.mean() * 0.
|
||||
|
||||
# It may already have been prepared by, e.g., `generate`
|
||||
if position_ids is None:
|
||||
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
||||
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
|
||||
position_ids = position_ids.unsqueeze(0)
|
||||
|
||||
bi_vision_attn = self.config.get_text_config().use_bidirectional_attention == 'vision'
|
||||
if not isinstance(causal_mask_mapping := attention_mask, dict):
|
||||
if bi_vision_attn and not transformers_5_9:
|
||||
from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping
|
||||
|
||||
# Larger Gemma 4 models use Gemma 3's bidirectional attention mask for vision inputs
|
||||
causal_mask_mapping = create_causal_mask_mapping(
|
||||
self.config,
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=attention_mask,
|
||||
past_key_values=past_key_values,
|
||||
position_ids=position_ids,
|
||||
mm_token_type_ids=mm_token_type_ids,
|
||||
)
|
||||
else:
|
||||
mask_kwargs = {
|
||||
'config': self.config,
|
||||
'inputs_embeds': inputs_embeds,
|
||||
'attention_mask': attention_mask,
|
||||
'past_key_values': past_key_values,
|
||||
'position_ids': position_ids,
|
||||
}
|
||||
if bi_vision_attn:
|
||||
from transformers.models.gemma4.modeling_gemma4 import get_block_sequence_ids_for_mask
|
||||
block_sequence_ids = torch.full([*inputs_embeds.size()[:-1]], -1, device=inputs_embeds.device)
|
||||
if mm_token_type_ids is not None:
|
||||
kwargs = {
|
||||
'device': inputs_embeds.device
|
||||
} if 'device' in inspect.signature(get_block_sequence_ids_for_mask).parameters else {}
|
||||
block_sequence_ids = get_block_sequence_ids_for_mask(mm_token_type_ids, **kwargs)
|
||||
|
||||
mask_kwargs['block_sequence_ids'] = block_sequence_ids
|
||||
|
||||
causal_mask_mapping = create_masks_for_generate(**mask_kwargs)
|
||||
kwargs.pop('return_dict', None)
|
||||
outputs = self.language_model(
|
||||
per_layer_inputs=per_layer_inputs,
|
||||
attention_mask=causal_mask_mapping,
|
||||
position_ids=position_ids,
|
||||
past_key_values=past_key_values,
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=use_cache,
|
||||
return_dict=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
return Gemma4ModelOutputWithPast(
|
||||
last_hidden_state=outputs.last_hidden_state,
|
||||
past_key_values=outputs.past_key_values,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
image_hidden_states=image_features if pixel_values is not None else None,
|
||||
audio_hidden_states=audio_features if input_features is not None else None,
|
||||
)
|
||||
|
||||
model.origin_forward = model.forward
|
||||
model.forward = MethodType(forward, model)
|
||||
|
||||
|
||||
class Gemma4Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
from transformers import Gemma4ForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Gemma4ForConditionalGeneration
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
_patch_gemma4_forward(model.model, processor)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.gemma4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('google/gemma-4-E2B', 'google/gemma-4-E2B'),
|
||||
Model('google/gemma-4-E2B-it', 'google/gemma-4-E2B-it'),
|
||||
Model('google/gemma-4-E4B', 'google/gemma-4-E4B'),
|
||||
Model('google/gemma-4-E4B-it', 'google/gemma-4-E4B-it'),
|
||||
],
|
||||
template=TemplateType.gemma4_nothinking),
|
||||
ModelGroup([
|
||||
Model('google/gemma-4-31B', 'google/gemma-4-31B'),
|
||||
Model('google/gemma-4-31B-it', 'google/gemma-4-31B-it'),
|
||||
Model('google/gemma-4-26B-A4B', 'google/gemma-4-26B-A4B'),
|
||||
Model('google/gemma-4-26B-A4B-it', 'google/gemma-4-26B-A4B-it'),
|
||||
],
|
||||
template=TemplateType.gemma4),
|
||||
],
|
||||
Gemma4Loader,
|
||||
architectures=['Gemma4ForConditionalGeneration'],
|
||||
model_arch=ModelArch.gemma3n,
|
||||
))
|
||||
|
||||
|
||||
class Gemma4UnifiedLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
from transformers import Gemma4UnifiedForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Gemma4UnifiedForConditionalGeneration
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
_patch_gemma4_forward(model.model, processor, is_gemma4_unified=True)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.gemma4_unified,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('google/gemma-4-12B', 'google/gemma-4-12B'),
|
||||
Model('google/gemma-4-12B-it', 'google/gemma-4-12B-it'),
|
||||
],
|
||||
template=TemplateType.gemma4),
|
||||
],
|
||||
Gemma4UnifiedLoader,
|
||||
architectures=['Gemma4UnifiedForConditionalGeneration'],
|
||||
model_arch=ModelArch.gemma4_unified,
|
||||
requires=['transformers>=5.10.1'],
|
||||
))
|
||||
|
||||
|
||||
class DiffusionGemmaLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
from transformers import DiffusionGemmaForBlockDiffusion
|
||||
self.auto_model_cls = self.auto_model_cls or DiffusionGemmaForBlockDiffusion
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.prepare_inputs_for_generation = None
|
||||
model.config.use_cache = True
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.diffusion_gemma,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('google/diffusiongemma-26B-A4B-it', 'google/diffusiongemma-26B-A4B-it'),
|
||||
],
|
||||
template=TemplateType.diffusion_gemma),
|
||||
],
|
||||
DiffusionGemmaLoader,
|
||||
architectures=['DiffusionGemmaForBlockDiffusion'],
|
||||
model_arch=ModelArch.diffusion_gemma,
|
||||
requires=['transformers>=5.11'],
|
||||
))
|
||||
@@ -0,0 +1,518 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import inspect
|
||||
import torch
|
||||
import transformers
|
||||
from packaging import version
|
||||
from transformers import AutoTokenizer, PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
from transformers.models.auto.tokenization_auto import get_tokenizer_config
|
||||
from typing import Any, Dict, Type
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_device_count, get_dist_setting, get_logger, safe_snapshot_download
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_get_input_embeddings, patch_output_to_input_device
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def remove_property(tokenizer_cls: Type[PreTrainedTokenizerBase], tokenizer_config: Dict[str, Any]) -> None:
|
||||
for k, v in tokenizer_cls.__dict__.items():
|
||||
if k.endswith('_token') and isinstance(v, property) and k in tokenizer_config:
|
||||
setattr(tokenizer_cls, k, tokenizer_config[k])
|
||||
|
||||
|
||||
def _patch_tokenizer(tokenizer):
|
||||
tokenizer_cls = tokenizer.__class__
|
||||
if hasattr(tokenizer_cls, '_origin_pad'):
|
||||
return
|
||||
tokenizer_cls._origin_pad = tokenizer_cls._pad
|
||||
parameters = inspect.signature(tokenizer_cls._origin_pad).parameters
|
||||
|
||||
def _pad(self, *args, **kwargs):
|
||||
if 'padding_side' in kwargs and kwargs['padding_side'] is None and 'padding_side' not in parameters:
|
||||
kwargs.pop('padding_side')
|
||||
return tokenizer_cls._origin_pad(self, *args, **kwargs)
|
||||
|
||||
tokenizer_cls._pad = _pad
|
||||
|
||||
|
||||
class ChatGLMLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
if model_kwargs.get('quantization_config') is not None:
|
||||
model_kwargs['quantization_config'].llm_int8_skip_modules = ['output_layer']
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
from torch.nn import CrossEntropyLoss
|
||||
__old_forward = CrossEntropyLoss.forward
|
||||
|
||||
def cross_entropy_forward(self, inputs: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
|
||||
target = target.to(device=inputs.device)
|
||||
return __old_forward(self, inputs, target)
|
||||
|
||||
CrossEntropyLoss.forward = cross_entropy_forward
|
||||
return model
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
# fix transformers>=4.34 bug
|
||||
if version.parse(transformers.__version__) >= version.parse('4.34'):
|
||||
tokenizer_config = get_tokenizer_config(model_dir)
|
||||
class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
|
||||
tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
|
||||
tokenizer_cls._auto_class = 'AutoTokenizer'
|
||||
remove_property(tokenizer_cls, tokenizer_config)
|
||||
tokenizer = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
|
||||
else:
|
||||
tokenizer = super().get_processor(model_dir, config)
|
||||
_patch_tokenizer(tokenizer)
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.chatglm2, [
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/chatglm2-6b', 'zai-org/chatglm2-6b'),
|
||||
Model('ZhipuAI/chatglm2-6b-32k', 'zai-org/chatglm2-6b-32k')
|
||||
],
|
||||
requires=['transformers<4.42']),
|
||||
ModelGroup(
|
||||
[Model('ZhipuAI/codegeex2-6b', 'zai-org/codegeex2-6b')],
|
||||
requires=['transformers<4.34'],
|
||||
tags=['coding'],
|
||||
),
|
||||
],
|
||||
ChatGLMLoader,
|
||||
template=TemplateType.chatglm2,
|
||||
architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
|
||||
model_arch=ModelArch.chatglm))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.chatglm3, [
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/chatglm3-6b', 'zai-org/chatglm3-6b'),
|
||||
Model('ZhipuAI/chatglm3-6b-base', 'zai-org/chatglm3-6b-base'),
|
||||
Model('ZhipuAI/chatglm3-6b-32k', 'zai-org/chatglm3-6b-32k'),
|
||||
Model('ZhipuAI/chatglm3-6b-128k', 'zai-org/chatglm3-6b-128k'),
|
||||
])
|
||||
],
|
||||
ChatGLMLoader,
|
||||
template=TemplateType.chatglm4,
|
||||
architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
|
||||
requires=['transformers<4.42'],
|
||||
model_arch=ModelArch.chatglm))
|
||||
|
||||
|
||||
class ChatGLM4Loader(ChatGLMLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer = super().get_processor(model_dir, config)
|
||||
if len(tokenizer.encode('<|user|>', add_special_tokens=False)) > 1:
|
||||
for k in tokenizer.special_tokens.keys():
|
||||
tokenizer.add_tokens(k)
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.chatglm4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/glm-4-9b-chat', 'zai-org/glm-4-9b-chat'),
|
||||
Model('ZhipuAI/glm-4-9b', 'zai-org/glm-4-9b'),
|
||||
Model('ZhipuAI/glm-4-9b-chat-1m', 'zai-org/glm-4-9b-chat-1m'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/LongWriter-glm4-9b', 'zai-org/LongWriter-glm4-9b'),
|
||||
])
|
||||
],
|
||||
ChatGLM4Loader,
|
||||
template=TemplateType.chatglm4,
|
||||
architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
|
||||
model_arch=ModelArch.chatglm,
|
||||
requires=['transformers>=4.42'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.glm4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4-9B-0414', 'zai-org/GLM-4-9B-0414'),
|
||||
Model('ZhipuAI/GLM-4-32B-0414', 'zai-org/GLM-4-32B-0414'),
|
||||
Model('ZhipuAI/GLM-4-32B-Base-0414', 'zai-org/GLM-4-32B-Base-0414'),
|
||||
Model('ZhipuAI/GLM-Z1-9B-0414', 'zai-org/GLM-Z1-9B-0414'),
|
||||
Model('ZhipuAI/GLM-Z1-32B-0414', 'zai-org/GLM-Z1-32B-0414'),
|
||||
], TemplateType.glm4),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-Z1-Rumination-32B-0414', 'zai-org/GLM-Z1-Rumination-32B-0414'),
|
||||
], TemplateType.glm4_z1_rumination)
|
||||
],
|
||||
requires=['transformers>=4.51'],
|
||||
architectures=['Glm4ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.codegeex4,
|
||||
[ModelGroup([
|
||||
Model('ZhipuAI/codegeex4-all-9b', 'zai-org/codegeex4-all-9b'),
|
||||
])],
|
||||
ChatGLM4Loader,
|
||||
template=TemplateType.codegeex4,
|
||||
requires=['transformers<4.42'],
|
||||
architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
|
||||
model_arch=ModelArch.chatglm,
|
||||
tags=['coding'],
|
||||
))
|
||||
|
||||
|
||||
class ChatGLM4vLoader(ChatGLMLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
# fix device_map 4
|
||||
n_gpu = get_device_count()
|
||||
local_world_size = get_dist_setting()[3]
|
||||
if n_gpu // local_world_size >= 4:
|
||||
for layer in model.transformer.vision.transformer.layers:
|
||||
patch_output_to_input_device(layer.mlp)
|
||||
patch_output_to_input_device(layer.post_attention_layernorm)
|
||||
device = next(model.transformer.vision.linear_proj.parameters()).device
|
||||
model.transformer.vision.boi.data = model.transformer.vision.boi.to(device)
|
||||
model.transformer.vision.eoi.data = model.transformer.vision.eoi.to(device)
|
||||
return model
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
processor = super().get_processor(model_dir, config)
|
||||
processor.init_kwargs['image_size'] = 1120
|
||||
return processor
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.chatglm4v,
|
||||
[
|
||||
ModelGroup(
|
||||
[
|
||||
Model('ZhipuAI/glm-4v-9b', 'zai-org/glm-4v-9b'),
|
||||
],
|
||||
requires=['transformers>=4.42,<4.45'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('ZhipuAI/cogagent-9b-20241220', 'zai-org/cogagent-9b-20241220'),
|
||||
],
|
||||
requires=['transformers>=4.42'],
|
||||
)
|
||||
],
|
||||
ChatGLM4vLoader,
|
||||
template=TemplateType.chatglm4v,
|
||||
architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
|
||||
model_arch=ModelArch.chatglm4v,
|
||||
))
|
||||
|
||||
|
||||
class GLM4vLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import Glm4vForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Glm4vForConditionalGeneration
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
if hasattr(model, 'visual'):
|
||||
patch_get_input_embeddings(model.visual, 'patch_embed')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.glm4v,
|
||||
[
|
||||
ModelGroup(
|
||||
[
|
||||
Model('ZhipuAI/GLM-4.1V-9B-Base', 'zai-org/GLM-4.1V-9B-Base'),
|
||||
Model('ZhipuAI/GLM-4.1V-9B-Thinking', 'zai-org/GLM-4.1V-9B-Thinking'),
|
||||
Model('ZhipuAI/AutoGLM-Phone-9B', 'zai-org/AutoGLM-Phone-9B')
|
||||
],
|
||||
template=TemplateType.glm4v,
|
||||
requires=['transformers>=4.53'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('ZhipuAI/Glyph', 'zai-org/Glyph'),
|
||||
],
|
||||
template=TemplateType.glm4_5v,
|
||||
requires=['transformers>=4.57'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('ZhipuAI/GLM-4.6V-Flash', 'zai-org/GLM-4.6V-Flash'),
|
||||
],
|
||||
template=TemplateType.glm4_5v,
|
||||
requires=['transformers>=5.0.0.dev'],
|
||||
),
|
||||
],
|
||||
GLM4vLoader,
|
||||
model_arch=ModelArch.glm4v,
|
||||
architectures=['Glm4vForConditionalGeneration'],
|
||||
))
|
||||
|
||||
|
||||
class CogVLMLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
logger.warning('CogAgent with FusedLayerNorm will cause an training loss of NAN, '
|
||||
'to avoid this, please uninstall apex.')
|
||||
logger.info('Please ignore the unimported warning.')
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer_dir = safe_snapshot_download('AI-ModelScope/vicuna-7b-v1.5', download_model=False, check_local=True)
|
||||
tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, trust_remote_code=True)
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.cogvlm, [
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/cogvlm-chat', 'zai-org/cogvlm-chat-hf'),
|
||||
]),
|
||||
],
|
||||
CogVLMLoader,
|
||||
template=TemplateType.cogvlm,
|
||||
architectures=['CogVLMForCausalLM'],
|
||||
requires=['transformers<4.42'],
|
||||
model_arch=ModelArch.cogvlm))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.cogagent_chat, [
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/cogagent-chat', 'zai-org/cogagent-chat-hf'),
|
||||
]),
|
||||
],
|
||||
CogVLMLoader,
|
||||
template=TemplateType.cogagent_chat,
|
||||
architectures=['CogAgentForCausalLM'],
|
||||
requires=['transformers<4.42', 'timm'],
|
||||
model_arch=ModelArch.cogvlm))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.cogagent_vqa, [ModelGroup([
|
||||
Model('ZhipuAI/cogagent-vqa', 'zai-org/cogagent-vqa-hf'),
|
||||
])],
|
||||
CogVLMLoader,
|
||||
template=TemplateType.cogagent_vqa,
|
||||
architectures=['CogAgentForCausalLM'],
|
||||
requires=['transformers<4.42'],
|
||||
model_arch=ModelArch.cogvlm))
|
||||
|
||||
|
||||
class CogVLM2Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
# fix device map 4
|
||||
for layer in model.model.vision.transformer.layers:
|
||||
patch_output_to_input_device(layer.mlp)
|
||||
patch_output_to_input_device(layer.post_attention_layernorm)
|
||||
|
||||
device = next(model.model.vision.linear_proj.parameters()).device
|
||||
model.model.vision.boi.data = model.model.vision.boi.to(device)
|
||||
model.model.vision.eoi.data = model.model.vision.eoi.to(device)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.cogvlm2, [
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/cogvlm2-llama3-chat-19B', 'zai-org/cogvlm2-llama3-chat-19B'),
|
||||
Model('ZhipuAI/cogvlm2-llama3-chinese-chat-19B', 'zai-org/cogvlm2-llama3-chinese-chat-19B'),
|
||||
]),
|
||||
],
|
||||
CogVLM2Loader,
|
||||
template=TemplateType.cogvlm2,
|
||||
architectures=['CogVLMForCausalLM'],
|
||||
requires=['transformers<4.42'],
|
||||
model_arch=ModelArch.cogvlm))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.cogvlm2_video,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/cogvlm2-video-llama3-chat', 'zai-org/cogvlm2-video-llama3-chat'),
|
||||
]),
|
||||
],
|
||||
CogVLM2Loader,
|
||||
template=TemplateType.cogvlm2_video,
|
||||
architectures=['CogVLMVideoForCausalLM'],
|
||||
requires=['decord', 'pytorchvideo', 'transformers>=4.42'],
|
||||
model_arch=ModelArch.cogvlm,
|
||||
tags=['video'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.glm_edge,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/glm-edge-1.5b-chat', 'zai-org/glm-edge-1.5b-chat'),
|
||||
Model('ZhipuAI/glm-edge-4b-chat', 'zai-org/glm-edge-4b-chat'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.chatglm4,
|
||||
architectures=['GlmForCausalLM'],
|
||||
requires=['transformers>=4.46'],
|
||||
))
|
||||
|
||||
|
||||
class GLMEdgeVLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
from transformers import AutoImageProcessor
|
||||
self.auto_tokenizer_cls = AutoImageProcessor
|
||||
return super().get_processor(model_dir, config)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.glm_edge_v,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/glm-edge-v-2b', 'zai-org/glm-edge-v-2b'),
|
||||
Model('ZhipuAI/glm-edge-4b-chat', 'zai-org/glm-edge-4b-chat'),
|
||||
]),
|
||||
],
|
||||
GLMEdgeVLoader,
|
||||
template=TemplateType.glm_edge_v,
|
||||
architectures=['GlmForCausalLM'],
|
||||
requires=['transformers>=4.46'],
|
||||
model_arch=ModelArch.glm_edge_v,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.glm4_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4.5-Air-Base', 'zai-org/GLM-4.5-Air-Base'),
|
||||
Model('ZhipuAI/GLM-4.5-Air', 'zai-org/GLM-4.5-Air'),
|
||||
Model('ZhipuAI/GLM-4.5-Air-FP8', 'zai-org/GLM-4.5-Air-FP8'),
|
||||
Model('ZhipuAI/GLM-4.5-Base', 'zai-org/GLM-4.5-Base'),
|
||||
Model('ZhipuAI/GLM-4.5', 'zai-org/GLM-4.5'),
|
||||
Model('ZhipuAI/GLM-4.5-FP8', 'zai-org/GLM-4.5-FP8'),
|
||||
], TemplateType.glm4_5),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4.6', 'zai-org/GLM-4.6'),
|
||||
Model('ZhipuAI/GLM-4.6-FP8', 'zai-org/GLM-4.6-FP8'),
|
||||
], TemplateType.glm4_5),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4.7', 'zai-org/GLM-4.7'),
|
||||
Model('ZhipuAI/GLM-4.7-FP8', 'zai-org/GLM-4.7-FP8'),
|
||||
], TemplateType.glm4_7),
|
||||
],
|
||||
requires=['transformers>=4.54'],
|
||||
architectures=['Glm4MoeForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.glm4_moe_lite,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4.7-Flash', 'zai-org/GLM-4.7-Flash'),
|
||||
], TemplateType.glm4_7),
|
||||
],
|
||||
requires=['transformers>=5.0.0.dev'],
|
||||
architectures=['Glm4MoeLiteForCausalLM'],
|
||||
))
|
||||
|
||||
|
||||
class Glm4vMoeLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import Glm4vMoeForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Glm4vMoeForConditionalGeneration
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_get_input_embeddings(model.visual, 'patch_embed')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.glm4v_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4.5V', 'zai-org/GLM-4.5V'),
|
||||
Model('ZhipuAI/GLM-4.5V-FP8', 'zai-org/GLM-4.5V-FP8'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-4.6V', 'zai-org/GLM-4.6V'),
|
||||
Model('ZhipuAI/GLM-4.6V-FP8', 'zai-org/GLM-4.6V-FP8'),
|
||||
],
|
||||
requires=['transformers>=5.0.0.dev']),
|
||||
],
|
||||
Glm4vMoeLoader,
|
||||
template=TemplateType.glm4_5v,
|
||||
model_arch=ModelArch.glm4v,
|
||||
architectures=['Glm4vMoeForConditionalGeneration'],
|
||||
requires=['transformers>=4.56'],
|
||||
))
|
||||
|
||||
|
||||
class GLMOCRLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForImageTextToText
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
if hasattr(model, 'visual'):
|
||||
patch_get_input_embeddings(model.visual, 'patch_embed')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.glm_ocr,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-OCR', 'zai-org/GLM-OCR'),
|
||||
]),
|
||||
],
|
||||
GLMOCRLoader,
|
||||
template=TemplateType.glm_ocr,
|
||||
model_arch=ModelArch.glm4v,
|
||||
architectures=['GlmOcrForConditionalGeneration'],
|
||||
requires=['transformers>=5.0.1dev0'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.glm_moe_dsa,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-5', 'zai-org/GLM-5'),
|
||||
], template=TemplateType.glm4_7),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-5.1', 'zai-org/GLM-5.1'),
|
||||
Model('ZhipuAI/GLM-5.1-FP8', 'ZhipuAI/GLM-5.1-FP8'),
|
||||
],
|
||||
template=TemplateType.glm5_1),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/GLM-5.2', 'ZhipuAI/GLM-5.2'),
|
||||
Model('ZhipuAI/GLM-5.2-FP8', 'ZhipuAI/GLM-5.2-FP8'),
|
||||
],
|
||||
template=TemplateType.glm5_2),
|
||||
],
|
||||
architectures=['GlmMoeDsaForCausalLM'],
|
||||
requires=['transformers>=5.2.0'],
|
||||
))
|
||||
@@ -0,0 +1,507 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import AutoTokenizer, PretrainedConfig, PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, safe_snapshot_download
|
||||
from ..constant import LLMModelType, MLLMModelType, RMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_output_clone, patch_output_to_input_device
|
||||
from ..register import ModelLoader, RewardModelLoader, register_model
|
||||
from ..utils import use_submodel_func
|
||||
from .qwen import Qwen2AudioLoader
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.internlm,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm-chat-7b', 'internlm/internlm-chat-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm-7b', 'internlm/internlm-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm-chat-7b-8k'),
|
||||
Model('Shanghai_AI_Laboratory/internlm-20b', 'internlm/internlm-20b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm-chat-20b', 'internlm/internlm-chat-20b'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.internlm,
|
||||
architectures=['InternLMForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.internlm2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm2-chat-1_8b', 'internlm/internlm2-chat-1_8b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-1_8b', 'internlm/internlm2-1_8b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-chat-1_8b-sft', 'internlm/internlm2-chat-1_8b-sft'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-base-7b', 'internlm/internlm2-base-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-7b', 'internlm/internlm2-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-chat-7b', 'internlm/internlm2-chat-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-chat-7b-sft', 'internlm/internlm2-chat-7b-sft'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-base-20b', 'internlm/internlm2-base-20b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-20b', 'internlm/internlm2-20b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-chat-20b', 'internlm/internlm2-chat-20b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-chat-20b-sft', 'internlm/internlm2-chat-20b-sft'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm2-math-7b', 'internlm/internlm2-math-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-math-base-7b', 'internlm/internlm2-math-base-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-math-base-20b', 'internlm/internlm2-math-base-20b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-math-20b', 'internlm/internlm2-math-20b'),
|
||||
],
|
||||
tags=['math']),
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-1_8b-chat', 'internlm/internlm2_5-1_8b-chat'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-1_8b', 'internlm/internlm2_5-1_8b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-7b', 'internlm/internlm2_5-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-7b-chat', 'internlm/internlm2_5-7b-chat'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-7b-chat-1m', 'internlm/internlm2_5-7b-chat-1m'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-20b', 'internlm/internlm2_5-20b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2_5-20b-chat', 'internlm/internlm2_5-20b-chat'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.internlm2,
|
||||
requires=['transformers>=4.38'],
|
||||
architectures=['InternLM2ForCausalLM'],
|
||||
model_arch=ModelArch.internlm2,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.internlm3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm3-8b-instruct', 'internlm/internlm3-8b-instruct'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.internlm2,
|
||||
requires=['transformers>=4.48'],
|
||||
architectures=['InternLM3ForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
))
|
||||
|
||||
|
||||
class InternVLLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
self.auto_tokenizer_cls = AutoTokenizer
|
||||
return super().get_processor(model_dir, config)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
if self.model_info.quant_method == 'bnb': # 'is_training'
|
||||
# patch: bnb backward shape mismatch bug
|
||||
if model is not None and model.language_model is not None:
|
||||
model.language_model.output.state.force_no_igemmlt = True
|
||||
use_submodel_func(model, 'language_model')
|
||||
patch_output_clone(model.language_model.get_input_embeddings())
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.internvl_chat,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenGVLab/Mini-InternVL-Chat-2B-V1-5', 'OpenGVLab/Mini-InternVL-Chat-2B-V1-5'),
|
||||
Model('AI-ModelScope/InternVL-Chat-V1-5', 'OpenGVLab/InternVL-Chat-V1-5'),
|
||||
Model('AI-ModelScope/InternVL-Chat-V1-5-int8', 'OpenGVLab/InternVL-Chat-V1-5-int8'),
|
||||
],
|
||||
template=TemplateType.internvl,
|
||||
requires=['transformers>=4.35', 'timm'],
|
||||
tags=['vision']),
|
||||
ModelGroup([
|
||||
Model('OpenGVLab/Mini-InternVL-Chat-4B-V1-5', 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'),
|
||||
],
|
||||
template=TemplateType.internvl_phi3,
|
||||
requires=['transformers>=4.35,<4.42', 'timm'],
|
||||
tags=['vision']),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('OpenGVLab/InternVL2-1B', 'OpenGVLab/InternVL2-1B'),
|
||||
Model('OpenGVLab/InternVL2-2B', 'OpenGVLab/InternVL2-2B'),
|
||||
Model('OpenGVLab/InternVL2-8B', 'OpenGVLab/InternVL2-8B'),
|
||||
Model('OpenGVLab/InternVL2-26B', 'OpenGVLab/InternVL2-26B'),
|
||||
Model('OpenGVLab/InternVL2-40B', 'OpenGVLab/InternVL2-40B'),
|
||||
Model('OpenGVLab/InternVL2-Llama3-76B', 'OpenGVLab/InternVL2-Llama3-76B'),
|
||||
# (infer use lmdeploy)
|
||||
Model('OpenGVLab/InternVL2-2B-AWQ', 'OpenGVLab/InternVL2-2B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2-8B-AWQ', 'OpenGVLab/InternVL2-8B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2-26B-AWQ', 'OpenGVLab/InternVL2-26B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2-40B-AWQ', 'OpenGVLab/InternVL2-40B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2-Llama3-76B-AWQ', 'OpenGVLab/InternVL2-Llama3-76B-AWQ'),
|
||||
# mpo
|
||||
Model('OpenGVLab/InternVL2-8B-MPO', 'OpenGVLab/InternVL2-8B-MPO'),
|
||||
# pretrain
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-1B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-1B-Pretrain'),
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-2B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-2B-Pretrain'),
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-4B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-4B-Pretrain'),
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-8B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-8B-Pretrain'),
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-26B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-26B-Pretrain'),
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-40B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-40B-Pretrain'),
|
||||
Model('OpenGVLab/InternVL2-Pretrain-Models:InternVL2-Llama3-76B-Pretrain',
|
||||
'OpenGVLab/InternVL2-Pretrain-Models:InternVL2-Llama3-76B-Pretrain'),
|
||||
],
|
||||
template=TemplateType.internvl2,
|
||||
requires=['transformers>=4.36', 'timm'],
|
||||
tags=['vision', 'video'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('OpenGVLab/InternVL2-4B', 'OpenGVLab/InternVL2-4B'),
|
||||
],
|
||||
template=TemplateType.internvl2_phi3,
|
||||
requires=['transformers>=4.36,<4.42', 'timm'],
|
||||
tags=['vision', 'video'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('OpenGVLab/InternVL2_5-1B', 'OpenGVLab/InternVL2_5-1B'),
|
||||
Model('OpenGVLab/InternVL2_5-2B', 'OpenGVLab/InternVL2_5-2B'),
|
||||
Model('OpenGVLab/InternVL2_5-4B', 'OpenGVLab/InternVL2_5-4B'),
|
||||
Model('OpenGVLab/InternVL2_5-8B', 'OpenGVLab/InternVL2_5-8B'),
|
||||
Model('OpenGVLab/InternVL2_5-26B', 'OpenGVLab/InternVL2_5-26B'),
|
||||
Model('OpenGVLab/InternVL2_5-38B', 'OpenGVLab/InternVL2_5-38B'),
|
||||
Model('OpenGVLab/InternVL2_5-78B', 'OpenGVLab/InternVL2_5-78B'),
|
||||
# quant (infer use lmdeploy)
|
||||
Model('OpenGVLab/InternVL2_5-4B-AWQ', 'OpenGVLab/InternVL2_5-4B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2_5-8B-AWQ', 'OpenGVLab/InternVL2_5-8B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2_5-26B-AWQ', 'OpenGVLab/InternVL2_5-26B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2_5-38B-AWQ', 'OpenGVLab/InternVL2_5-38B-AWQ'),
|
||||
Model('OpenGVLab/InternVL2_5-78B-AWQ', 'OpenGVLab/InternVL2_5-78B-AWQ'),
|
||||
# mpo
|
||||
Model('OpenGVLab/InternVL2_5-1B-MPO', 'OpenGVLab/InternVL2_5-1B-MPO'),
|
||||
Model('OpenGVLab/InternVL2_5-2B-MPO', 'OpenGVLab/InternVL2_5-2B-MPO'),
|
||||
Model('OpenGVLab/InternVL2_5-4B-MPO', 'OpenGVLab/InternVL2_5-4B-MPO'),
|
||||
Model('OpenGVLab/InternVL2_5-8B-MPO', 'OpenGVLab/InternVL2_5-8B-MPO'),
|
||||
Model('OpenGVLab/InternVL2_5-26B-MPO', 'OpenGVLab/InternVL2_5-26B-MPO'),
|
||||
Model('OpenGVLab/InternVL2_5-38B-MPO', 'OpenGVLab/InternVL2_5-38B-MPO'),
|
||||
Model('OpenGVLab/InternVL2_5-78B-MPO', 'OpenGVLab/InternVL2_5-78B-MPO'),
|
||||
],
|
||||
template=TemplateType.internvl2_5,
|
||||
requires=['transformers>=4.36', 'timm'],
|
||||
tags=['vision', 'video'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
# pretrain
|
||||
Model('OpenGVLab/InternVL3-1B-Pretrained', 'OpenGVLab/InternVL3-1B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3-2B-Pretrained', 'OpenGVLab/InternVL3-2B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3-8B-Pretrained', 'OpenGVLab/InternVL3-8B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3-9B-Pretrained', 'OpenGVLab/InternVL3-9B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3-14B-Pretrained', 'OpenGVLab/InternVL3-14B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3-38B-Pretrained', 'OpenGVLab/InternVL3-38B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3-78B-Pretrained', 'OpenGVLab/InternVL3-78B-Pretrained'),
|
||||
# instruct
|
||||
Model('OpenGVLab/InternVL3-1B-Instruct', 'OpenGVLab/InternVL3-1B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3-2B-Instruct', 'OpenGVLab/InternVL3-2B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3-8B-Instruct', 'OpenGVLab/InternVL3-8B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3-9B-Instruct', 'OpenGVLab/InternVL3-9B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3-14B-Instruct', 'OpenGVLab/InternVL3-14B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3-38B-Instruct', 'OpenGVLab/InternVL3-38B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3-78B-Instruct', 'OpenGVLab/InternVL3-78B-Instruct'),
|
||||
# mpo
|
||||
Model('OpenGVLab/InternVL3-1B', 'OpenGVLab/InternVL3-1B'),
|
||||
Model('OpenGVLab/InternVL3-2B', 'OpenGVLab/InternVL3-2B'),
|
||||
Model('OpenGVLab/InternVL3-8B', 'OpenGVLab/InternVL3-8B'),
|
||||
Model('OpenGVLab/InternVL3-9B', 'OpenGVLab/InternVL3-9B'),
|
||||
Model('OpenGVLab/InternVL3-14B', 'OpenGVLab/InternVL3-14B'),
|
||||
Model('OpenGVLab/InternVL3-38B', 'OpenGVLab/InternVL3-38B'),
|
||||
Model('OpenGVLab/InternVL3-78B', 'OpenGVLab/InternVL3-78B'),
|
||||
# awq (Use lmdeploy for inference.)
|
||||
Model('OpenGVLab/InternVL3-1B-AWQ', 'OpenGVLab/InternVL3-1B-AWQ'),
|
||||
Model('OpenGVLab/InternVL3-2B-AWQ', 'OpenGVLab/InternVL3-2B-AWQ'),
|
||||
Model('OpenGVLab/InternVL3-8B-AWQ', 'OpenGVLab/InternVL3-8B-AWQ'),
|
||||
Model('OpenGVLab/InternVL3-9B-AWQ', 'OpenGVLab/InternVL3-9B-AWQ'),
|
||||
Model('OpenGVLab/InternVL3-14B-AWQ', 'OpenGVLab/InternVL3-14B-AWQ'),
|
||||
Model('OpenGVLab/InternVL3-38B-AWQ', 'OpenGVLab/InternVL3-38B-AWQ'),
|
||||
Model('OpenGVLab/InternVL3-78B-AWQ', 'OpenGVLab/InternVL3-78B-AWQ'),
|
||||
# SenseNova-SI
|
||||
Model('SenseNova/SenseNova-SI-InternVL3-2B', 'sensenova/SenseNova-SI-InternVL3-2B'),
|
||||
Model('SenseNova/SenseNova-SI-InternVL3-8B', 'sensenova/SenseNova-SI-InternVL3-8B'),
|
||||
Model('SenseNova/SenseNova-SI-1.1-InternVL3-2B', 'sensenova/SenseNova-SI-1.1-InternVL3-2B'),
|
||||
Model('SenseNova/SenseNova-SI-1.1-InternVL3-8B', 'sensenova/SenseNova-SI-1.1-InternVL3-8B'),
|
||||
],
|
||||
template=TemplateType.internvl2_5,
|
||||
requires=['transformers>=4.37.2', 'timm'],
|
||||
tags=['vision', 'video'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
# pretrain
|
||||
Model('OpenGVLab/InternVL3_5-1B-Pretrained', 'OpenGVLab/InternVL3_5-1B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-2B-Pretrained', 'OpenGVLab/InternVL3_5-2B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-4B-Pretrained', 'OpenGVLab/InternVL3_5-4B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-8B-Pretrained', 'OpenGVLab/InternVL3_5-8B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-14B-Pretrained', 'OpenGVLab/InternVL3_5-14B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-38B-Pretrained', 'OpenGVLab/InternVL3_5-38B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-30B-A3B-Pretrained', 'OpenGVLab/InternVL3_5-30B-A3B-Pretrained'),
|
||||
Model('OpenGVLab/InternVL3_5-241B-A28B-Pretrained', 'OpenGVLab/InternVL3_5-241B-A28B-Pretrained'),
|
||||
# Instruct
|
||||
Model('OpenGVLab/InternVL3_5-1B-Instruct', 'OpenGVLab/InternVL3_5-1B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-2B-Instruct', 'OpenGVLab/InternVL3_5-2B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-4B-Instruct', 'OpenGVLab/InternVL3_5-4B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-8B-Instruct', 'OpenGVLab/InternVL3_5-8B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-14B-Instruct', 'OpenGVLab/InternVL3_5-14B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-38B-Instruct', 'OpenGVLab/InternVL3_5-38B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-30B-A3B-Instruct', 'OpenGVLab/InternVL3_5-30B-A3B-Instruct'),
|
||||
Model('OpenGVLab/InternVL3_5-241B-A28B-Instruct', 'OpenGVLab/InternVL3_5-241B-A28B-Instruct'),
|
||||
# MPO
|
||||
Model('OpenGVLab/InternVL3_5-1B-MPO', 'OpenGVLab/InternVL3_5-1B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-2B-MPO', 'OpenGVLab/InternVL3_5-2B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-4B-MPO', 'OpenGVLab/InternVL3_5-4B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-8B-MPO', 'OpenGVLab/InternVL3_5-8B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-14B-MPO', 'OpenGVLab/InternVL3_5-14B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-38B-MPO', 'OpenGVLab/InternVL3_5-38B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-30B-A3B-MPO', 'OpenGVLab/InternVL3_5-30B-A3B-MPO'),
|
||||
Model('OpenGVLab/InternVL3_5-241B-A28B-MPO', 'OpenGVLab/InternVL3_5-241B-A28B-MPO'),
|
||||
#
|
||||
Model('OpenGVLab/InternVL3_5-1B', 'OpenGVLab/InternVL3_5-1B'),
|
||||
Model('OpenGVLab/InternVL3_5-2B', 'OpenGVLab/InternVL3_5-2B'),
|
||||
Model('OpenGVLab/InternVL3_5-4B', 'OpenGVLab/InternVL3_5-4B'),
|
||||
Model('OpenGVLab/InternVL3_5-8B', 'OpenGVLab/InternVL3_5-8B'),
|
||||
Model('OpenGVLab/InternVL3_5-14B', 'OpenGVLab/InternVL3_5-14B'),
|
||||
Model('OpenGVLab/InternVL3_5-38B', 'OpenGVLab/InternVL3_5-38B'),
|
||||
Model('OpenGVLab/InternVL3_5-30B-A3B', 'OpenGVLab/InternVL3_5-30B-A3B'),
|
||||
Model('OpenGVLab/InternVL3_5-241B-A28B', 'OpenGVLab/InternVL3_5-241B-A28B'),
|
||||
],
|
||||
template=TemplateType.internvl3_5,
|
||||
requires=['transformers>=4.37.2', 'timm'],
|
||||
tags=['vision', 'video'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview',
|
||||
'OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview'),
|
||||
],
|
||||
template=TemplateType.internvl3_5_gpt,
|
||||
requires=['transformers>=4.37.2', 'timm'],
|
||||
tags=['vision', 'video'],
|
||||
),
|
||||
],
|
||||
InternVLLoader,
|
||||
architectures=['InternVLChatModel'],
|
||||
model_arch=ModelArch.internvl,
|
||||
))
|
||||
|
||||
|
||||
class Interns1Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers.modeling_utils import PreTrainedModel
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
if not hasattr(PreTrainedModel, '_old_enable_input_require_grads'):
|
||||
old_enable_input_require_grads = PreTrainedModel.enable_input_require_grads
|
||||
|
||||
def patched_enable_input_require_grads(self):
|
||||
|
||||
def make_inputs_require_grads(module, input, output):
|
||||
if isinstance(output, tuple):
|
||||
output[0].requires_grad_(True)
|
||||
else:
|
||||
output.requires_grad_(True)
|
||||
|
||||
self._require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)
|
||||
|
||||
PreTrainedModel.enable_input_require_grads = patched_enable_input_require_grads
|
||||
PreTrainedModel._old_enable_input_require_grads = old_enable_input_require_grads
|
||||
return model
|
||||
|
||||
|
||||
class InternVLHfLoader(Interns1Loader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForImageTextToText
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.internvl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenGVLab/InternVL3-1B-hf', 'OpenGVLab/InternVL3-1B-hf'),
|
||||
Model('OpenGVLab/InternVL3-2B-hf', 'OpenGVLab/InternVL3-2B-hf'),
|
||||
Model('OpenGVLab/InternVL3-8B-hf', 'OpenGVLab/InternVL3-8B-hf'),
|
||||
Model('OpenGVLab/InternVL3-9B-hf', 'OpenGVLab/InternVL3-9B-hf'),
|
||||
Model('OpenGVLab/InternVL3-14B-hf', 'OpenGVLab/InternVL3-14B-hf'),
|
||||
Model('OpenGVLab/InternVL3-38B-hf', 'OpenGVLab/InternVL3-38B-hf'),
|
||||
Model('OpenGVLab/InternVL3-78B-hf', 'OpenGVLab/InternVL3-78B-hf'),
|
||||
],
|
||||
template=TemplateType.internvl_hf,
|
||||
requires=['transformers>=4.52.1', 'timm']),
|
||||
ModelGroup([
|
||||
Model('OpenGVLab/InternVL3_5-1B-HF', 'OpenGVLab/InternVL3_5-1B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-2B-HF', 'OpenGVLab/InternVL3_5-2B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-4B-HF', 'OpenGVLab/InternVL3_5-4B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-8B-HF', 'OpenGVLab/InternVL3_5-8B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-14B-HF', 'OpenGVLab/InternVL3_5-14B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-38B-HF', 'OpenGVLab/InternVL3_5-38B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-30B-A3B-HF', 'OpenGVLab/InternVL3_5-30B-A3B-HF'),
|
||||
Model('OpenGVLab/InternVL3_5-241B-A28B-HF', 'OpenGVLab/InternVL3_5-241B-A28B-HF'),
|
||||
],
|
||||
template=TemplateType.internvl_hf,
|
||||
requires=['transformers>=4.52.1', 'timm']),
|
||||
ModelGroup([
|
||||
Model('OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF',
|
||||
'OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF'),
|
||||
],
|
||||
template=TemplateType.internvl_hf,
|
||||
requires=['transformers>=4.55.0', 'timm']),
|
||||
],
|
||||
InternVLHfLoader,
|
||||
architectures=['InternVLForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.interns1,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/Intern-S1-mini', 'internlm/Intern-S1-mini'),
|
||||
Model('Shanghai_AI_Laboratory/Intern-S1', 'internlm/Intern-S1'),
|
||||
Model('Shanghai_AI_Laboratory/Intern-S1-mini-FP8', 'internlm/Intern-S1-mini-FP8'),
|
||||
Model('Shanghai_AI_Laboratory/Intern-S1-FP8', 'internlm/Intern-S1-FP8'),
|
||||
]),
|
||||
],
|
||||
Interns1Loader,
|
||||
template=TemplateType.interns1,
|
||||
architectures=['InternS1ForConditionalGeneration'],
|
||||
model_arch=ModelArch.interns1,
|
||||
requires=['transformers>=4.55.2,<4.56'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
|
||||
class Xcomposer2Loader(ModelLoader):
|
||||
version = 'v2'
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
if self.version == 'v2-4khd':
|
||||
from transformers import CLIPVisionModel
|
||||
|
||||
def load_model(self):
|
||||
self.vision_tower_name = safe_snapshot_download(
|
||||
'AI-ModelScope/clip-vit-large-patch14-336', check_local=True)
|
||||
self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name)
|
||||
self.vision_tower.requires_grad_(False)
|
||||
self.is_loaded = True
|
||||
|
||||
CLIPVisionTower = get_class_from_dynamic_module('build_mlp.CLIPVisionTower', model_dir)
|
||||
CLIPVisionTower.load_model = load_model
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
model.vit.vision_tower.gradient_checkpointing_enable()
|
||||
if self.version == 'v2':
|
||||
# fix AttributeError: no attribute 'attention_dropout'
|
||||
model.model.layers[0].attention.__class__.attention_dropout = 0.
|
||||
|
||||
if self.version == 'v2.5':
|
||||
patch_output_to_input_device(model.vit)
|
||||
patch_output_to_input_device(model.vision_proj)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.xcomposer2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm-xcomposer2-7b', 'internlm/internlm-xcomposer2-7b'),
|
||||
], ),
|
||||
],
|
||||
Xcomposer2Loader,
|
||||
template=TemplateType.xcomposer2,
|
||||
architectures=['InternLMXComposer2ForCausalLM'],
|
||||
model_arch=ModelArch.xcomposer,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class Xcomposer2_4khdLoader(Xcomposer2Loader):
|
||||
version = 'v2-4khd'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.xcomposer2_4khd,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm-xcomposer2-4khd-7b', 'internlm/internlm-xcomposer2-4khd-7b'),
|
||||
], ),
|
||||
],
|
||||
Xcomposer2_4khdLoader,
|
||||
template=TemplateType.xcomposer2,
|
||||
architectures=['InternLM2ForCausalLM', 'InternLMXComposer2ForCausalLM'],
|
||||
model_arch=ModelArch.xcomposer,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class Xcomposer2_5Loader(Xcomposer2Loader):
|
||||
version = 'v2.5'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.xcomposer2_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm-xcomposer2d5-7b', 'internlm/internlm-xcomposer2d5-7b'),
|
||||
Model('Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:base',
|
||||
'internlm/internlm-xcomposer2d5-ol-7b:base')
|
||||
]),
|
||||
],
|
||||
Xcomposer2_5Loader,
|
||||
template=TemplateType.xcomposer2_5,
|
||||
architectures=['InternLMXComposer2ForCausalLM'],
|
||||
model_arch=ModelArch.xcomposer,
|
||||
tags=['vision'],
|
||||
requires=['decord'],
|
||||
# target_modules: attention.wqkv attention.wo feed_forward.w1 feed_forward.w2 feed_forward.w3
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.xcomposer2_5_ol_audio,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:audio',
|
||||
'internlm/internlm-xcomposer2d5-ol-7b:audio'),
|
||||
]),
|
||||
],
|
||||
Qwen2AudioLoader,
|
||||
template=TemplateType.qwen2_audio,
|
||||
requires=['transformers>=4.45'],
|
||||
architectures=['Qwen2AudioForConditionalGeneration'],
|
||||
model_arch=ModelArch.qwen2_audio,
|
||||
tags=['audio'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
RMModelType.internlm2_reward,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Shanghai_AI_Laboratory/internlm2-1_8b-reward', 'internlm/internlm2-1_8b-reward'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-7b-reward', 'internlm/internlm2-7b-reward'),
|
||||
Model('Shanghai_AI_Laboratory/internlm2-20b-reward', 'internlm/internlm2-20b-reward'),
|
||||
]),
|
||||
],
|
||||
RewardModelLoader,
|
||||
template=TemplateType.internlm2_reward,
|
||||
is_reward=True,
|
||||
requires=['transformers>=4.38'],
|
||||
architectures=['InternLM2ForRewardModel'],
|
||||
))
|
||||
@@ -0,0 +1,348 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import os
|
||||
import sys
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_device, git_clone_github
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class LlamaLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir):
|
||||
config = super().get_config(model_dir)
|
||||
if getattr(config, 'pretraining_tp', 1) > 1:
|
||||
config.pretraining_tp = 1
|
||||
return config
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.llama,
|
||||
[
|
||||
# llama2
|
||||
ModelGroup(
|
||||
[
|
||||
# base
|
||||
Model('modelscope/Llama-2-7b-ms', 'meta-llama/Llama-2-7b-hf'),
|
||||
Model('modelscope/Llama-2-13b-ms', 'meta-llama/Llama-2-13b-hf'),
|
||||
Model('modelscope/Llama-2-70b-ms', 'meta-llama/Llama-2-70b-hf'),
|
||||
# chat
|
||||
Model('modelscope/Llama-2-7b-chat-ms', 'meta-llama/Llama-2-7b-chat-hf'),
|
||||
Model('modelscope/Llama-2-13b-chat-ms', 'meta-llama/Llama-2-13b-chat-hf'),
|
||||
Model('modelscope/Llama-2-70b-chat-ms', 'meta-llama/Llama-2-70b-chat-hf'),
|
||||
],
|
||||
TemplateType.llama,
|
||||
ignore_patterns=[r'.+\.bin$']),
|
||||
# chinese-llama2
|
||||
ModelGroup(
|
||||
[
|
||||
# base
|
||||
Model('AI-ModelScope/chinese-llama-2-1.3b', 'hfl/chinese-llama-2-1.3b'),
|
||||
Model('AI-ModelScope/chinese-llama-2-7b', 'hfl/chinese-llama-2-7b'),
|
||||
Model('AI-ModelScope/chinese-llama-2-7b-16k', 'hfl/chinese-llama-2-7b-16k'),
|
||||
Model('AI-ModelScope/chinese-llama-2-7b-64k', 'hfl/chinese-llama-2-7b-64k'),
|
||||
Model('AI-ModelScope/chinese-llama-2-13b', 'hfl/chinese-llama-2-13b'),
|
||||
Model('AI-ModelScope/chinese-llama-2-13b-16k', 'hfl/chinese-llama-2-13b-16k'),
|
||||
# chat
|
||||
Model('AI-ModelScope/chinese-alpaca-2-1.3b', 'hfl/chinese-alpaca-2-1.3b'),
|
||||
Model('AI-ModelScope/chinese-alpaca-2-7b', 'hfl/chinese-alpaca-2-7b'),
|
||||
Model('AI-ModelScope/chinese-alpaca-2-7b-16k', 'hfl/chinese-alpaca-2-7b-16k'),
|
||||
Model('AI-ModelScope/chinese-alpaca-2-7b-64k', 'hfl/chinese-alpaca-2-7b-64k'),
|
||||
Model('AI-ModelScope/chinese-alpaca-2-13b', 'hfl/chinese-alpaca-2-13b'),
|
||||
Model('AI-ModelScope/chinese-alpaca-2-13b-16k', 'hfl/chinese-alpaca-2-13b-16k'),
|
||||
],
|
||||
TemplateType.llama),
|
||||
# base quant
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Llama-2-7b-AQLM-2Bit-1x16-hf', 'ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf'),
|
||||
],
|
||||
TemplateType.llama,
|
||||
requires=['transformers>=4.38', 'aqlm', 'torch>=2.2.0']),
|
||||
ModelGroup([
|
||||
Model('FlagAlpha/Atom-7B', 'FlagAlpha/Atom-7B'),
|
||||
Model('FlagAlpha/Atom-7B-Chat', 'FlagAlpha/Atom-7B-Chat'),
|
||||
],
|
||||
template=TemplateType.atom),
|
||||
ModelGroup([
|
||||
Model('langboat/Mengzi3-13B-Base', 'Langboat/Mengzi3-13B-Base'),
|
||||
],
|
||||
template=TemplateType.mengzi),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/NuminaMath-7B-TIR', 'AI-MO/NuminaMath-7B-TIR'),
|
||||
],
|
||||
template=TemplateType.numina,
|
||||
tags=['math']),
|
||||
ModelGroup([
|
||||
Model('Fengshenbang/Ziya2-13B-Base', 'IDEA-CCNL/Ziya2-13B-Base'),
|
||||
Model('Fengshenbang/Ziya2-13B-Chat', 'IDEA-CCNL/Ziya2-13B-Chat'),
|
||||
],
|
||||
template=TemplateType.ziya),
|
||||
ModelGroup([
|
||||
Model('InfiniAI/Megrez-3b-Instruct', 'Infinigence/Megrez-3B-Instruct'),
|
||||
], TemplateType.megrez),
|
||||
# deepseek
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/deepseek-llm-7b-base', 'deepseek-ai/deepseek-llm-7b-base'),
|
||||
Model('deepseek-ai/deepseek-llm-7b-chat', 'deepseek-ai/deepseek-llm-7b-chat'),
|
||||
Model('deepseek-ai/deepseek-llm-67b-base', 'deepseek-ai/deepseek-llm-67b-base'),
|
||||
Model('deepseek-ai/deepseek-llm-67b-chat', 'deepseek-ai/deepseek-llm-67b-chat'),
|
||||
], TemplateType.deepseek),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('deepseek-ai/deepseek-math-7b-base', 'deepseek-ai/deepseek-math-7b-base'),
|
||||
Model('deepseek-ai/deepseek-math-7b-instruct', 'deepseek-ai/deepseek-math-7b-instruct'),
|
||||
Model('deepseek-ai/deepseek-math-7b-rl', 'deepseek-ai/deepseek-math-7b-rl'),
|
||||
],
|
||||
TemplateType.deepseek,
|
||||
tags=['math'],
|
||||
),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('deepseek-ai/deepseek-coder-1.3b-base', 'deepseek-ai/deepseek-coder-1.3b-base'),
|
||||
Model('deepseek-ai/deepseek-coder-1.3b-instruct', 'deepseek-ai/deepseek-coder-1.3b-instruct'),
|
||||
Model('deepseek-ai/deepseek-coder-6.7b-base', 'deepseek-ai/deepseek-coder-6.7b-base'),
|
||||
Model('deepseek-ai/deepseek-coder-6.7b-instruct', 'deepseek-ai/deepseek-coder-6.7b-instruct'),
|
||||
Model('deepseek-ai/deepseek-coder-33b-base', 'deepseek-ai/deepseek-coder-33b-base'),
|
||||
Model('deepseek-ai/deepseek-coder-33b-instruct', 'deepseek-ai/deepseek-coder-33b-instruct'),
|
||||
],
|
||||
TemplateType.deepseek,
|
||||
tags=['coding'],
|
||||
),
|
||||
# MiniMind2
|
||||
ModelGroup(
|
||||
[
|
||||
# MiniMind2
|
||||
Model('gongjy/MiniMind2', 'jingyaogong/MiniMind2'),
|
||||
# MiniMind2-Small
|
||||
Model(None, 'jingyaogong/MiniMind2-Small'),
|
||||
],
|
||||
TemplateType.minimind,
|
||||
requires=['transformers>=4.57.1']),
|
||||
# llama3
|
||||
ModelGroup(
|
||||
[
|
||||
# chat
|
||||
Model('LLM-Research/Meta-Llama-3-8B-Instruct', 'meta-llama/Meta-Llama-3-8B-Instruct'),
|
||||
Model('LLM-Research/Meta-Llama-3-70B-Instruct', 'meta-llama/Meta-Llama-3-70B-Instruct'),
|
||||
# base
|
||||
Model('LLM-Research/Meta-Llama-3-8B', 'meta-llama/Meta-Llama-3-8B'),
|
||||
Model('LLM-Research/Meta-Llama-3-70B', 'meta-llama/Meta-Llama-3-70B'),
|
||||
],
|
||||
TemplateType.llama3),
|
||||
# llama3-quant
|
||||
ModelGroup([
|
||||
Model('swift/Meta-Llama-3-8B-Instruct-GPTQ-Int4', 'study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int4'),
|
||||
Model('swift/Meta-Llama-3-8B-Instruct-GPTQ-Int8', 'study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int8'),
|
||||
Model('swift/Meta-Llama-3-8B-Instruct-AWQ', 'study-hjt/Meta-Llama-3-8B-Instruct-AWQ'),
|
||||
Model('swift/Meta-Llama-3-70B-Instruct-GPTQ-Int4', 'study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int4'),
|
||||
Model('swift/Meta-Llama-3-70B-Instruct-GPTQ-Int8', 'study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int8'),
|
||||
Model('swift/Meta-Llama-3-70B-Instruct-AWQ', 'study-hjt/Meta-Llama-3-70B-Instruct-AWQ'),
|
||||
], TemplateType.llama3),
|
||||
# chinese-llama3
|
||||
ModelGroup([
|
||||
Model('ChineseAlpacaGroup/llama-3-chinese-8b-instruct', 'hfl/llama-3-chinese-8b-instruct'),
|
||||
Model('ChineseAlpacaGroup/llama-3-chinese-8b', 'hfl/llama-3-chinese-8b'),
|
||||
], TemplateType.llama3),
|
||||
# llama3.1
|
||||
ModelGroup(
|
||||
[
|
||||
# chat
|
||||
Model('LLM-Research/Meta-Llama-3.1-8B-Instruct', 'meta-llama/Meta-Llama-3.1-8B-Instruct'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-70B-Instruct', 'meta-llama/Meta-Llama-3.1-70B-Instruct'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-405B-Instruct', 'meta-llama/Meta-Llama-3.1-405B-Instruct'),
|
||||
# base
|
||||
Model('LLM-Research/Meta-Llama-3.1-8B', 'meta-llama/Meta-Llama-3.1-8B'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-70B', 'meta-llama/Meta-Llama-3.1-70B'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-405B', 'meta-llama/Meta-Llama-3.1-405B'),
|
||||
# fp8
|
||||
Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', 'meta-llama/Meta-Llama-3.1-70B-Instruct-FP8'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8',
|
||||
'meta-llama/Meta-Llama-3.1-405B-Instruct-FP8'),
|
||||
],
|
||||
TemplateType.llama3_2,
|
||||
requires=['transformers>=4.43']),
|
||||
# llama3.1-quant
|
||||
ModelGroup(
|
||||
[
|
||||
# bnb-nf4
|
||||
Model('LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4',
|
||||
'hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit',
|
||||
'unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4',
|
||||
'hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4'),
|
||||
# gptq-int4
|
||||
Model('LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4',
|
||||
'hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4',
|
||||
'hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4',
|
||||
'hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4'),
|
||||
# awq-int4
|
||||
Model('LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4',
|
||||
'hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4',
|
||||
'hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4'),
|
||||
Model('LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4',
|
||||
'hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4'),
|
||||
],
|
||||
TemplateType.llama3_2,
|
||||
requires=['transformers>=4.43']),
|
||||
# nvidia Nemotron
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', 'nvidia/Llama-3.1-Nemotron-70B-Instruct-HF'),
|
||||
],
|
||||
TemplateType.llama3_2,
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B', 'Skywork/Skywork-o1-Open-Llama-3.1-8B'),
|
||||
],
|
||||
TemplateType.skywork_o1,
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Llama-3.2-1B', 'meta-llama/Llama-3.2-1B'),
|
||||
Model('LLM-Research/Llama-3.2-3B', 'meta-llama/Llama-3.2-3B'),
|
||||
Model('LLM-Research/Llama-3.2-1B-Instruct', 'meta-llama/Llama-3.2-1B-Instruct'),
|
||||
Model('LLM-Research/Llama-3.2-3B-Instruct', 'meta-llama/Llama-3.2-3B-Instruct'),
|
||||
],
|
||||
template=TemplateType.llama3_2,
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Llama-3.3-70B-Instruct', 'meta-llama/Llama-3.3-70B-Instruct'),
|
||||
Model('unsloth/Llama-3.3-70B-Instruct-bnb-4bit', 'unsloth/Llama-3.3-70B-Instruct-bnb-4bit'),
|
||||
],
|
||||
template=TemplateType.llama3_2,
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup([
|
||||
Model('ZhipuAI/LongWriter-llama3.1-8b', 'zai-org/LongWriter-llama3.1-8b'),
|
||||
],
|
||||
TemplateType.longwriter_llama,
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup([
|
||||
Model('deepseek-ai/DeepSeek-R1-Distill-Llama-8B', 'deepseek-ai/DeepSeek-R1-Distill-Llama-8B'),
|
||||
Model('deepseek-ai/DeepSeek-R1-Distill-Llama-70B', 'deepseek-ai/DeepSeek-R1-Distill-Llama-70B'),
|
||||
], TemplateType.deepseek_r1),
|
||||
# MiniCPM5
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM5-1B', 'openbmb/MiniCPM5-1B'),
|
||||
Model('OpenBMB/MiniCPM5-1B-Base', 'openbmb/MiniCPM5-1B-Base'),
|
||||
Model('OpenBMB/MiniCPM5-1B-SFT', 'openbmb/MiniCPM5-1B-SFT'),
|
||||
],
|
||||
TemplateType.minicpm5,
|
||||
requires=['transformers>=5.6']),
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Reflection-Llama-3.1-70B', 'mattshumer/Reflection-Llama-3.1-70B'),
|
||||
],
|
||||
TemplateType.reflection,
|
||||
requires=['transformers>=4.43']),
|
||||
],
|
||||
LlamaLoader,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['LlamaForCausalLM'],
|
||||
))
|
||||
|
||||
|
||||
class Llama3_2VisionLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import MllamaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or MllamaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama3_2_vision,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Llama-3.2-11B-Vision-Instruct', 'meta-llama/Llama-3.2-11B-Vision-Instruct'),
|
||||
Model('LLM-Research/Llama-3.2-90B-Vision-Instruct', 'meta-llama/Llama-3.2-90B-Vision-Instruct'),
|
||||
Model('LLM-Research/Llama-3.2-11B-Vision', 'meta-llama/Llama-3.2-11B-Vision'),
|
||||
Model('LLM-Research/Llama-3.2-90B-Vision', 'meta-llama/Llama-3.2-90B-Vision'),
|
||||
])
|
||||
],
|
||||
Llama3_2VisionLoader,
|
||||
template=TemplateType.llama3_2_vision,
|
||||
requires=['transformers>=4.45'],
|
||||
architectures=['MllamaForConditionalGeneration'],
|
||||
model_arch=ModelArch.llama3_2_vision,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class Llama4Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import Llama4ForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Llama4ForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Llama-4-Scout-17B-16E', 'meta-llama/Llama-4-Scout-17B-16E'),
|
||||
Model('LLM-Research/Llama-4-Maverick-17B-128E', 'meta-llama/Llama-4-Maverick-17B-128E'),
|
||||
Model('LLM-Research/Llama-4-Scout-17B-16E-Instruct', 'meta-llama/Llama-4-Scout-17B-16E-Instruct'),
|
||||
Model('LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8',
|
||||
'meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8'),
|
||||
Model('LLM-Research/Llama-4-Maverick-17B-128E-Instruct',
|
||||
'meta-llama/Llama-4-Maverick-17B-128E-Instruct'),
|
||||
])
|
||||
],
|
||||
Llama4Loader,
|
||||
template=TemplateType.llama4,
|
||||
requires=['transformers>=4.51'],
|
||||
model_arch=ModelArch.llama4,
|
||||
architectures=['Llama4ForConditionalGeneration'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class Llama3OmniLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/ictnlp/LLaMA-Omni')
|
||||
sys.path.append(self.local_repo_path)
|
||||
import whisper
|
||||
from omni_speech.model import OmniSpeech2SLlamaForCausalLM, OmniSpeechLlamaForCausalLM
|
||||
config.speech_encoder = os.path.join(model_dir, 'large-v3.pt')
|
||||
if not os.path.exists(config.speech_encoder):
|
||||
whisper.load_model('large-v3', download_root=model_dir)
|
||||
self.auto_model_cls = self.auto_model_cls or OmniSpeech2SLlamaForCausalLM
|
||||
for key in ['forward', 'generate']:
|
||||
try:
|
||||
delattr(OmniSpeech2SLlamaForCausalLM, key)
|
||||
delattr(OmniSpeechLlamaForCausalLM, key)
|
||||
except AttributeError:
|
||||
pass
|
||||
# not support device_map='auto'
|
||||
device_map = model_kwargs['device_map']
|
||||
model_kwargs['device_map'] = None
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.to(get_device() if device_map == 'auto' else device_map)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama3_1_omni,
|
||||
[ModelGroup([
|
||||
Model('ICTNLP/Llama-3.1-8B-Omni', 'ICTNLP/Llama-3.1-8B-Omni'),
|
||||
], )],
|
||||
Llama3OmniLoader,
|
||||
template=TemplateType.llama3_1_omni,
|
||||
architectures=['OmniSpeech2SLlamaForCausalLM'],
|
||||
model_arch=ModelArch.llama3_1_omni,
|
||||
requires=['openai-whisper'],
|
||||
tags=['audio'],
|
||||
))
|
||||
@@ -0,0 +1,455 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import sys
|
||||
from functools import wraps
|
||||
from transformers import PretrainedConfig, PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import git_clone_github, safe_snapshot_download
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_get_input_embeddings
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class LlavaLlamaHfLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
from transformers import LlavaConfig
|
||||
self.auto_config_cls = LlavaConfig
|
||||
return super().get_config(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_llama3_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-llama-3-8b-v1_1-transformers', 'xtuner/llava-llama-3-8b-v1_1-transformers'),
|
||||
]),
|
||||
],
|
||||
LlavaLlamaHfLoader,
|
||||
template=TemplateType.llava_llama3_hf,
|
||||
architectures=['LlavaForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.36'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
def _patch_llava(model):
|
||||
if hasattr(model, '__old_generate'):
|
||||
return
|
||||
generate = model.generate
|
||||
model.__old_generate = generate
|
||||
|
||||
@wraps(generate)
|
||||
def _new_generate(inputs=None, *args, **kwargs):
|
||||
input_ids = kwargs.pop('input_ids', None)
|
||||
if inputs is None and input_ids is not None:
|
||||
inputs = input_ids
|
||||
return generate(inputs, *args, **kwargs)
|
||||
|
||||
model.generate = _new_generate
|
||||
|
||||
|
||||
class LlavahfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_5_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-1.5-7b-hf', 'llava-hf/llava-1.5-7b-hf'),
|
||||
Model('llava-hf/llava-1.5-13b-hf', 'llava-hf/llava-1.5-13b-hf'),
|
||||
]),
|
||||
],
|
||||
LlavahfLoader,
|
||||
template=TemplateType.llava1_5_hf,
|
||||
architectures=['LlavaForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.36'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaOnevisionHfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaOnevisionForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaOnevisionForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_onevision_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-onevision-qwen2-0.5b-ov-hf', 'llava-hf/llava-onevision-qwen2-0.5b-ov-hf'),
|
||||
Model('llava-hf/llava-onevision-qwen2-7b-ov-hf', 'llava-hf/llava-onevision-qwen2-7b-ov-hf'),
|
||||
Model('llava-hf/llava-onevision-qwen2-72b-ov-hf', 'llava-hf/llava-onevision-qwen2-72b-ov-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaOnevisionHfLoader,
|
||||
template=TemplateType.llava_onevision_hf,
|
||||
architectures=['LlavaOnevisionForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.45'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextHfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaNextForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaNextForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_qwen_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-next-72b-hf', 'llava-hf/llava-next-72b-hf'),
|
||||
Model('llava-hf/llava-next-110b-hf', 'llava-hf/llava-next-110b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava_next_qwen_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama3_llava_next_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llama3-llava-next-8b-hf', 'llava-hf/llama3-llava-next-8b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llama3_llava_next_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_vicuna_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-v1.6-vicuna-7b-hf', 'llava-hf/llava-v1.6-vicuna-7b-hf'),
|
||||
Model('llava-hf/llava-v1.6-vicuna-13b-hf', 'llava-hf/llava-v1.6-vicuna-13b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava1_6_vicuna_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_mistral_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-v1.6-mistral-7b-hf', 'llava-hf/llava-v1.6-mistral-7b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava1_6_mistral_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_llama3_1_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('swift/llava-llama3.1-8b'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava_llama3_1_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.41'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextYiHfLoader(LlavaNextHfLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.image_token_index = 64003
|
||||
return config
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_yi_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-v1.6-34b-hf', 'llava-hf/llava-v1.6-34b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava1_6_yi_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextVideoHfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaNextVideoForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaNextVideoForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_video_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/LLaVA-NeXT-Video-7B-DPO-hf', 'llava-hf/LLaVA-NeXT-Video-7B-DPO-hf'),
|
||||
Model('llava-hf/LLaVA-NeXT-Video-7B-32K-hf', 'llava-hf/LLaVA-NeXT-Video-7B-32K-hf'),
|
||||
Model('llava-hf/LLaVA-NeXT-Video-7B-hf', 'llava-hf/LLaVA-NeXT-Video-7B-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextVideoHfLoader,
|
||||
template=TemplateType.llava_next_video_hf,
|
||||
architectures=['LlavaNextVideoForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_next_video_hf,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['video'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextVideoYiHfLoader(LlavaNextVideoHfLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.video_token_index = 64003
|
||||
config.image_token_index = 64004
|
||||
return config
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_video_yi_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/LLaVA-NeXT-Video-34B-hf', 'llava-hf/LLaVA-NeXT-Video-34B-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextVideoYiHfLoader,
|
||||
template=TemplateType.llava_next_video_hf,
|
||||
architectures=['LlavaNextVideoForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_next_video_hf,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['video'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaLoader(ModelLoader):
|
||||
llm_model_type = None
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
if 'next' in self.llm_model_type:
|
||||
repo_path = 'https://github.com/LLaVA-VL/LLaVA-NeXT'
|
||||
else:
|
||||
repo_path = 'https://github.com/haotian-liu/LLaVA'
|
||||
local_repo_path = git_clone_github(repo_path)
|
||||
sys.path.append(local_repo_path)
|
||||
if self.llm_model_type == 'mistral':
|
||||
from llava.model import LlavaMistralConfig
|
||||
self.auto_config_cls = LlavaMistralConfig
|
||||
elif 'llama' in self.llm_model_type: # llama
|
||||
from llava.model import LlavaConfig
|
||||
self.auto_config_cls = LlavaConfig
|
||||
config = super().get_config(model_dir)
|
||||
if not hasattr(config, 'max_sequence_length'):
|
||||
config.max_sequence_length = 2048
|
||||
return config
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
if self.llm_model_type == 'mistral':
|
||||
from llava.model import LlavaMistralForCausalLM
|
||||
auto_model_cls = LlavaMistralForCausalLM
|
||||
elif 'llama' in self.llm_model_type: # llama
|
||||
from llava.model import LlavaLlamaForCausalLM
|
||||
if not hasattr(LlavaLlamaForCausalLM, '__old_forward'): # Avoid double patching
|
||||
forward = LlavaLlamaForCausalLM.forward
|
||||
LlavaLlamaForCausalLM.__old_forward = forward
|
||||
|
||||
@wraps(forward)
|
||||
def _new_forward(*args, **kwargs):
|
||||
kwargs.pop('cache_position', None)
|
||||
return forward(*args, **kwargs)
|
||||
|
||||
LlavaLlamaForCausalLM.forward = _new_forward
|
||||
auto_model_cls = LlavaLlamaForCausalLM
|
||||
else: # qwen
|
||||
from llava.model import LlavaQwenForCausalLM
|
||||
auto_model_cls = LlavaQwenForCausalLM
|
||||
|
||||
config.mm_vision_tower = safe_snapshot_download('AI-ModelScope/clip-vit-large-patch14-336', check_local=True)
|
||||
self.auto_model_cls = self.auto_model_cls or auto_model_cls
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
vision_tower = model.get_vision_tower()
|
||||
device_map = str(model_kwargs.get('device_map', str(model.device)))
|
||||
if not vision_tower.is_loaded:
|
||||
vision_tower.load_model(device_map=device_map)
|
||||
_patch_llava(model)
|
||||
model.resize_token_embeddings(len(processor))
|
||||
processor.image_processor = vision_tower.image_processor
|
||||
return model
|
||||
|
||||
|
||||
class Llama3LlavaNextLoader(LlavaLoader):
|
||||
llm_model_type = 'next_llama'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama3_llava_next,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llama3-llava-next-8b', 'lmms-lab/llama3-llava-next-8b'),
|
||||
], ),
|
||||
],
|
||||
Llama3LlavaNextLoader,
|
||||
template=TemplateType.llama3_llava_next,
|
||||
architectures=['LlavaLlamaForCausalLM'],
|
||||
model_arch=ModelArch.llava_llama,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaMistralLoader(LlavaLoader):
|
||||
llm_model_type = 'next_llama'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_mistral,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-v1.6-mistral-7b', 'liuhaotian/llava-v1.6-mistral-7b'),
|
||||
], ),
|
||||
],
|
||||
LlavaMistralLoader,
|
||||
template=TemplateType.llava1_6_mistral,
|
||||
requires=['transformers>=4.34'],
|
||||
architectures=['LlavaMistralForCausalLM'],
|
||||
model_arch=ModelArch.llava_mistral,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaLlamaLoader(LlavaLoader):
|
||||
llm_model_type = 'llama'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_yi, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-v1.6-34b', 'liuhaotian/llava-v1.6-34b'),
|
||||
], ),
|
||||
],
|
||||
LlavaLlamaLoader,
|
||||
template=TemplateType.llava1_6_yi,
|
||||
requires=['transformers>=4.34'],
|
||||
architectures=['LlavaLlamaForCausalLM'],
|
||||
tags=['vision'],
|
||||
model_arch=None))
|
||||
|
||||
|
||||
class LlavaNextQwenLoader(LlavaLoader):
|
||||
llm_model_type = 'next_qwen'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_qwen, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-next-72b', 'lmms-lab/llava-next-72b'),
|
||||
Model('AI-ModelScope/llava-next-110b', 'lmms-lab/llava-next-110b'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextQwenLoader,
|
||||
template=TemplateType.llava_next_qwen,
|
||||
architectures=['LlavaQwenForCausalLM'],
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['vision'],
|
||||
model_arch=None))
|
||||
|
||||
|
||||
class LlavaOnevisionLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.vision_start_token_id = 151652
|
||||
return config
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model_cls = get_class_from_dynamic_module(
|
||||
'modeling_llavaonevision1_5.LLaVAOneVision1_5_ForConditionalGeneration', model_dir)
|
||||
model_cls._no_split_modules = ['LLaVAOneVision1_5_DecoderLayer', 'RiceBlock']
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_get_input_embeddings(model.visual, 'patch_embed')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_onevision1_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-4B-Instruct', 'lmms-lab/LLaVA-OneVision-1.5-4B-Instruct'),
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-8B-Instruct', 'lmms-lab/LLaVA-OneVision-1.5-8B-Instruct'),
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-4B-Base', 'lmms-lab/LLaVA-OneVision-1.5-4B-Base'),
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-8B-Base', 'lmms-lab/LLaVA-OneVision-1.5-8B-Base'),
|
||||
], ),
|
||||
],
|
||||
LlavaOnevisionLoader,
|
||||
template=TemplateType.llava_onevision1_5,
|
||||
architectures=['LLaVAOneVision1_5_ForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_onevision1_5,
|
||||
requires=['transformers>=4.53.0', 'qwen_vl_utils'],
|
||||
tags=['vision'],
|
||||
))
|
||||
@@ -0,0 +1,441 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import AutoTokenizer, PretrainedConfig
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_logger, safe_snapshot_download
|
||||
from ..constant import LLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, SentenceTransformersLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class GrokLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer_dir = safe_snapshot_download('AI-ModelScope/grok-1-tokenizer', download_model=False, check_local=True)
|
||||
tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, trust_remote_code=True)
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.grok, [
|
||||
ModelGroup([
|
||||
Model('colossalai/grok-1-pytorch', 'hpcai-tech/grok-1'),
|
||||
]),
|
||||
],
|
||||
GrokLoader,
|
||||
template=TemplateType.default,
|
||||
architectures=['Grok1ModelForCausalLM'],
|
||||
model_arch=ModelArch.llama))
|
||||
|
||||
|
||||
class PolyLMLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False, legacy=True)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.polylm,
|
||||
[
|
||||
ModelGroup(
|
||||
[
|
||||
# base
|
||||
Model('damo/nlp_polylm_13b_text_generation', 'DAMO-NLP-MT/polylm-13b'),
|
||||
], ),
|
||||
],
|
||||
PolyLMLoader,
|
||||
template=TemplateType.default,
|
||||
architectures=['GPT2LMHeadModel'],
|
||||
model_arch=ModelArch.qwen))
|
||||
|
||||
|
||||
class YuanLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_dir, add_eos_token=False, add_bos_token=False, eos_token='<eod>', legacy=True)
|
||||
addi_tokens = [
|
||||
'<sep>', '<pad>', '<mask>', '<predict>', '<FIM_SUFFIX>', '<FIM_PREFIX>', '<FIM_MIDDLE>', '<commit_before>',
|
||||
'<commit_msg>', '<commit_after>', '<jupyter_start>', '<jupyter_text>', '<jupyter_code>', '<jupyter_output>',
|
||||
'<empty_output>'
|
||||
]
|
||||
tokenizer.add_tokens(addi_tokens, special_tokens=True)
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.yuan2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('IEITYuan/Yuan2.0-2B-hf', 'IEITYuan/Yuan2-2B-hf'),
|
||||
Model('IEITYuan/Yuan2.0-51B-hf', 'IEITYuan/Yuan2-51B-hf'),
|
||||
Model('IEITYuan/Yuan2.0-102B-hf', 'IEITYuan/Yuan2-102B-hf'),
|
||||
Model('IEITYuan/Yuan2-2B-Janus-hf', 'IEITYuan/Yuan2-2B-Janus-hf'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('IEITYuan/Yuan2-M32-hf', 'IEITYuan/Yuan2-M32-hf'),
|
||||
]),
|
||||
],
|
||||
YuanLoader,
|
||||
template=TemplateType.yuan,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['YuanForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.orion,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OrionStarAI/Orion-14B-Chat', 'OrionStarAI/Orion-14B-Chat'),
|
||||
Model('OrionStarAI/Orion-14B-Base', 'OrionStarAI/Orion-14B-Base'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.orion,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['OrionForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.dbrx, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/dbrx-base', 'databricks/dbrx-base'),
|
||||
Model('AI-ModelScope/dbrx-instruct', 'databricks/dbrx-instruct'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.dbrx,
|
||||
model_arch=ModelArch.dbrx,
|
||||
architectures=['DbrxForCausalLM'],
|
||||
requires=['transformers>=4.36']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.bluelm,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('vivo-ai/BlueLM-7B-Chat-32K', 'vivo-ai/BlueLM-7B-Chat-32K'),
|
||||
Model('vivo-ai/BlueLM-7B-Chat', 'vivo-ai/BlueLM-7B-Chat'),
|
||||
Model('vivo-ai/BlueLM-7B-Base-32K', 'vivo-ai/BlueLM-7B-Base-32K'),
|
||||
Model('vivo-ai/BlueLM-7B-Base', 'vivo-ai/BlueLM-7B-Base'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.bluelm,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['BlueLMForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.seggpt,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('damo/nlp_seqgpt-560m', 'DAMO-NLP/SeqGPT-560M'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.default,
|
||||
model_arch=None,
|
||||
architectures=['BloomForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.xverse,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('xverse/XVERSE-7B-Chat', 'xverse/XVERSE-7B-Chat'),
|
||||
Model('xverse/XVERSE-7B', 'xverse/XVERSE-7B'),
|
||||
Model('xverse/XVERSE-13B', 'xverse/XVERSE-13B'),
|
||||
Model('xverse/XVERSE-13B-Chat', 'xverse/XVERSE-13B-Chat'),
|
||||
Model('xverse/XVERSE-65B', 'xverse/XVERSE-65B'),
|
||||
Model('xverse/XVERSE-65B-2', 'xverse/XVERSE-65B-2'),
|
||||
Model('xverse/XVERSE-65B-Chat', 'xverse/XVERSE-65B-Chat'),
|
||||
Model('xverse/XVERSE-13B-256K', 'xverse/XVERSE-13B-256K', ms_revision='v1.0.0'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.xverse,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['XverseForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.xverse_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('xverse/XVERSE-MoE-A4.2B', 'xverse/XVERSE-MoE-A4.2B'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.xverse,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['XverseForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.c4ai,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/c4ai-command-r-v01', 'CohereForAI/c4ai-command-r-v01'),
|
||||
Model('AI-ModelScope/c4ai-command-r-plus', 'CohereForAI/c4ai-command-r-plus'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.c4ai,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['CohereForCausalLM'],
|
||||
requires=['transformers>=4.39'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.aya, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/aya-expanse-8b', 'CohereForAI/aya-expanse-8b'),
|
||||
Model('AI-ModelScope/aya-expanse-32b', 'CohereForAI/aya-expanse-32b'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.aya,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['CohereForCausalLM'],
|
||||
requires=['transformers>=4.44.0']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.ling,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('inclusionAI/Ling-lite', 'inclusionAI/Ling-lite'),
|
||||
Model('inclusionAI/Ling-plus', 'inclusionAI/Ling-plus'),
|
||||
Model('inclusionAI/Ling-lite-base', 'inclusionAI/Ling-lite-base'),
|
||||
Model('inclusionAI/Ling-plus-base', 'inclusionAI/Ling-plus-base'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.ling,
|
||||
architectures=['BailingMoeForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.qwen2_gte, [
|
||||
ModelGroup([
|
||||
Model('iic/gte_Qwen2-1.5B-instruct', 'Alibaba-NLP/gte-Qwen2-1.5B-instruct'),
|
||||
Model('iic/gte_Qwen2-7B-instruct', 'Alibaba-NLP/gte-Qwen2-7B-instruct'),
|
||||
]),
|
||||
],
|
||||
SentenceTransformersLoader,
|
||||
template=TemplateType.dummy,
|
||||
architectures=['Qwen2ForCausalLM']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.mimo, [
|
||||
ModelGroup([
|
||||
Model('XiaomiMiMo/MiMo-7B-Base', 'XiaomiMiMo/MiMo-7B-Base'),
|
||||
Model('XiaomiMiMo/MiMo-7B-SFT', 'XiaomiMiMo/MiMo-7B-SFT'),
|
||||
Model('XiaomiMiMo/MiMo-7B-RL-Zero', 'XiaomiMiMo/MiMo-7B-RL-Zero'),
|
||||
Model('XiaomiMiMo/MiMo-7B-RL', 'XiaomiMiMo/MiMo-7B-RL'),
|
||||
], TemplateType.qwen),
|
||||
ModelGroup([
|
||||
Model('XiaomiMiMo/MiMo-7B-RL-0530', 'XiaomiMiMo/MiMo-7B-RL-0530'),
|
||||
], TemplateType.mimo_rl),
|
||||
],
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['MiMoForCausalLM'],
|
||||
requires=['transformers>=4.37']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.dots1,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('rednote-hilab/dots.llm1.base', 'rednote-hilab/dots.llm1.base'),
|
||||
Model('rednote-hilab/dots.llm1.inst', 'rednote-hilab/dots.llm1.inst'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.dots1,
|
||||
architectures=['Dots1ForCausalLM'],
|
||||
requires=['transformers>=4.53'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.hunyuan,
|
||||
[ModelGroup([
|
||||
Model('Tencent-Hunyuan/Hunyuan-A13B-Instruct', 'tencent/Hunyuan-A13B-Instruct'),
|
||||
])],
|
||||
template=TemplateType.hunyuan_moe,
|
||||
architectures=['HunYuanMoEV1ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.hunyuan_v1_dense,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct', 'tencent/Hunyuan-0.5B-Instruct'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct', 'tencent/Hunyuan-1.8B-Instruct'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-4B-Instruct', 'tencent/Hunyuan-4B-Instruct'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-7B-Instruct', 'tencent/Hunyuan-7B-Instruct'),
|
||||
# pretrain
|
||||
Model('Tencent-Hunyuan/Hunyuan-0.5B-Pretrain', 'tencent/Hunyuan-0.5B-Pretrain'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-1.8B-Pretrain', 'tencent/Hunyuan-1.8B-Pretrain'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-4B-Pretrain', 'tencent/Hunyuan-4B-Pretrain'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-7B-Pretrain', 'tencent/Hunyuan-7B-Pretrain'),
|
||||
# fp8
|
||||
Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct-FP8', 'tencent/Hunyuan-0.5B-Instruct-FP8'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct-FP8', 'tencent/Hunyuan-1.8B-Instruct-FP8'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-4B-Instruct-FP8', 'tencent/Hunyuan-4B-Instruct-FP8'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-7B-Instruct-FP8', 'tencent/Hunyuan-7B-Instruct-FP8'),
|
||||
# awq
|
||||
Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct-AWQ-Int4', 'tencent/Hunyuan-0.5B-Instruct-AWQ-Int4'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct-AWQ-Int4', 'tencent/Hunyuan-1.8B-Instruct-AWQ-Int4'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-4B-Instruct-AWQ-Int4', 'tencent/Hunyuan-4B-Instruct-AWQ-Int4'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-7B-Instruct-AWQ-Int4', 'tencent/Hunyuan-7B-Instruct-AWQ-Int4'),
|
||||
# gptq
|
||||
Model('Tencent-Hunyuan/Hunyuan-0.5B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-0.5B-Instruct-GPTQ-Int4'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-1.8B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-1.8B-Instruct-GPTQ-Int4'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-4B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-4B-Instruct-GPTQ-Int4'),
|
||||
Model('Tencent-Hunyuan/Hunyuan-7B-Instruct-GPTQ-Int4', 'tencent/Hunyuan-7B-Instruct-GPTQ-Int4'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.hunyuan,
|
||||
requires=['transformers>=4.55.0.dev0'],
|
||||
architectures=['HunYuanDenseV1ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.hy_v3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Tencent-Hunyuan/Hy3-preview', 'tencent/Hy3-preview'),
|
||||
Model('Tencent-Hunyuan/Hy3-preview-Base', 'tencent/Hy3-preview-Base'),
|
||||
],
|
||||
template=TemplateType.hy_v3_preview),
|
||||
ModelGroup([
|
||||
Model('Tencent-Hunyuan/Hy3', 'tencent/Hy3'),
|
||||
Model('Tencent-Hunyuan/Hy3-FP8', 'tencent/Hy3-FP8'),
|
||||
],
|
||||
template=TemplateType.hy_v3),
|
||||
],
|
||||
requires=['transformers>=5.6.0'],
|
||||
architectures=['HYV3ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.gpt_oss, [
|
||||
ModelGroup([
|
||||
Model('openai-mirror/gpt-oss-20b', 'openai/gpt-oss-20b'),
|
||||
Model('openai-mirror/gpt-oss-120b', 'openai/gpt-oss-120b'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.gpt_oss,
|
||||
ignore_patterns=['metal/', 'original/'],
|
||||
architectures=['GptOssForCausalLM'],
|
||||
requires=['transformers>=4.55']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.longchat,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('meituan-longcat/LongCat-Flash-Chat', 'meituan-longcat/LongCat-Flash-Chat'),
|
||||
Model('meituan-longcat/LongCat-Flash-Chat-FP8', 'meituan-longcat/LongCat-Flash-Chat-FP8'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.longchat,
|
||||
architectures=['LongcatFlashForCausalLM'],
|
||||
requires=['transformers>=4.54,<4.56'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.bailing_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('inclusionAI/Ling-mini-2.0', 'inclusionAI/Ling-mini-2.0'),
|
||||
Model('inclusionAI/Ling-mini-base-2.0', 'inclusionAI/Ling-mini-base-2.0'),
|
||||
Model('inclusionAI/Ling-1T', 'inclusionAI/Ling-1T'),
|
||||
],
|
||||
template=TemplateType.ling2),
|
||||
ModelGroup([
|
||||
Model('inclusionAI/Ring-mini-2.0', 'inclusionAI/Ring-mini-2.0'),
|
||||
], template=TemplateType.ring2)
|
||||
],
|
||||
architectures=['BailingMoeV2ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.bailing_hybrid,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('inclusionAI/Ling-2.5-1T', 'inclusionAI/Ling-2.5-1T'),
|
||||
Model('inclusionAI/Ling-2.6-1T', 'inclusionAI/Ling-2.6-1T'),
|
||||
Model('inclusionAI/Ling-2.6-flash', 'inclusionAI/Ling-2.6-flash'),
|
||||
],
|
||||
template=TemplateType.ling2),
|
||||
ModelGroup([
|
||||
Model('inclusionAI/Ring-2.5-1T', 'inclusionAI/Ring-2.5-1T'),
|
||||
Model('inclusionAI/Ring-2.6-1T', 'inclusionAI/Ring-2.6-1T'),
|
||||
],
|
||||
template=TemplateType.ring2_5),
|
||||
],
|
||||
architectures=['BailingMoeV2_5ForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.iquestcoder,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('IQuestLab/IQuest-Coder-V1-40B-Base-Stage1', 'IQuestLab/IQuest-Coder-V1-40B-Base-Stage1'),
|
||||
Model('IQuestLab/IQuest-Coder-V1-40B-Base', 'IQuestLab/IQuest-Coder-V1-40B-Base'),
|
||||
Model('IQuestLab/IQuest-Coder-V1-40B-Instruct', 'IQuestLab/IQuest-Coder-V1-40B-Instruct'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.iquestcoder,
|
||||
requires=['transformers==4.52.4'],
|
||||
architectures=['IQuestCoderForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.youtu_llm,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Tencent-YouTu-Research/Youtu-LLM-2B', 'tencent/Youtu-LLM-2B'),
|
||||
Model('Tencent-YouTu-Research/Youtu-LLM-2B-Base', 'tencent/Youtu-LLM-2B-Base'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.youtu_llm,
|
||||
architectures=['YoutuForCausalLM'],
|
||||
requires=['transformers>=4.56'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.olmoe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('allenai/OLMoE-1B-7B-0125', 'allenai/OLMoE-1B-7B-0125'),
|
||||
Model('allenai/OLMoE-1B-7B-0125-Instruct', 'allenai/OLMoE-1B-7B-0125-Instruct'),
|
||||
],
|
||||
template=TemplateType.olmoe),
|
||||
ModelGroup([
|
||||
Model('allenai/OLMoE-1B-7B-0924', 'allenai/OLMoE-1B-7B-0924'),
|
||||
Model('allenai/OLMoE-1B-7B-0924-Instruct', 'allenai/OLMoE-1B-7B-0924-Instruct'),
|
||||
Model('allenai/OLMoE-1B-7B-0924-SFT', 'allenai/OLMoE-1B-7B-0924-SFT'),
|
||||
],
|
||||
template=TemplateType.olmoe_0924)
|
||||
],
|
||||
architectures=['OlmoeForCausalLM'],
|
||||
))
|
||||
@@ -0,0 +1,40 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_logger
|
||||
from ..constant import LLMModelType
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class MambaLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
logger.info(
|
||||
'[IMPORTANT] Remember installing causal-conv1d>=1.2.0 and mamba-ssm, or you training and inference will'
|
||||
'be really slow!')
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.mamba,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/mamba-130m-hf', 'state-spaces/mamba-130m-hf'),
|
||||
Model('AI-ModelScope/mamba-370m-hf', 'state-spaces/mamba-370m-hf'),
|
||||
Model('AI-ModelScope/mamba-390m-hf', 'state-spaces/mamba-390m-hf'),
|
||||
Model('AI-ModelScope/mamba-790m-hf', 'state-spaces/mamba-790m-hf'),
|
||||
Model('AI-ModelScope/mamba-1.4b-hf', 'state-spaces/mamba-1.4b-hf'),
|
||||
Model('AI-ModelScope/mamba-2.8b-hf', 'state-spaces/mamba-2.8b-hf'),
|
||||
])
|
||||
],
|
||||
MambaLoader,
|
||||
template=TemplateType.default,
|
||||
architectures=['MambaForCausalLM'],
|
||||
model_arch=None,
|
||||
requires=['transformers>=4.39.0'],
|
||||
))
|
||||
@@ -0,0 +1,210 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import PretrainedConfig, PreTrainedModel
|
||||
from types import MethodType
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_device, get_env_args
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_ignore_check_imports, patch_output_clone
|
||||
from ..register import ModelLoader, register_model
|
||||
from ..utils import use_submodel_func
|
||||
|
||||
|
||||
class Phi3VisionLoader(ModelLoader):
|
||||
num_crops = 4
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
processor_kwargs = {'num_crops': get_env_args('num_crops', int, self.num_crops)}
|
||||
from transformers import AutoProcessor
|
||||
processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True, **processor_kwargs)
|
||||
return processor
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.model.vision_embed_tokens.wte)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.phi3_vision,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Phi-3-vision-128k-instruct', 'microsoft/Phi-3-vision-128k-instruct'),
|
||||
Model('LLM-Research/Phi-3.5-vision-instruct', 'microsoft/Phi-3.5-vision-instruct'),
|
||||
])
|
||||
],
|
||||
Phi3VisionLoader,
|
||||
template=TemplateType.phi3_vision,
|
||||
architectures=['Phi3VForCausalLM'],
|
||||
model_arch=ModelArch.phi3_vision,
|
||||
requires=['transformers>=4.36'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class Phi4MultimodalLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
processor = super().get_processor(model_dir, config)
|
||||
processor.audio_processor.audio_compression_rate = processor.audio_processor.compression_rate
|
||||
processor.audio_processor.audio_downsample_rate = processor.audio_processor.qformer_compression_rate
|
||||
processor.audio_processor.audio_feat_stride = processor.audio_processor.feat_stride
|
||||
del processor.audio_processor.feature_size
|
||||
del processor.audio_processor.sampling_rate
|
||||
del processor.audio_processor.padding_value
|
||||
del processor.__class__.chat_template
|
||||
processor.chat_template = None
|
||||
return processor
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
model.set_lora_adapter(['vision', 'speech'])
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.phi4_multimodal,
|
||||
[ModelGroup([
|
||||
Model('LLM-Research/Phi-4-multimodal-instruct', 'microsoft/Phi-4-multimodal-instruct'),
|
||||
])],
|
||||
Phi4MultimodalLoader,
|
||||
template=TemplateType.phi4_multimodal,
|
||||
architectures=['Phi4MMForCausalLM'],
|
||||
model_arch=ModelArch.phi4_multimodal,
|
||||
requires=['transformers>=4.36,<4.49', 'backoff', 'soundfile'],
|
||||
tags=['vision', 'audio'],
|
||||
))
|
||||
|
||||
|
||||
class FlorenceLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
config.vision_config.model_type = 'davit' # fix merge-lora
|
||||
if model_kwargs['device_map'] == 'auto':
|
||||
model_kwargs['device_map'] = get_device()
|
||||
with patch_ignore_check_imports():
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.vision_tower.enable_checkpoint = True
|
||||
use_submodel_func(model, 'language_model', ['generate', 'forward'])
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.florence,
|
||||
[
|
||||
# llama2
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Florence-2-base-ft', 'microsoft/Florence-2-base-ft'),
|
||||
Model('AI-ModelScope/Florence-2-base', 'microsoft/Florence-2-base'),
|
||||
Model('AI-ModelScope/Florence-2-large', 'microsoft/Florence-2-large'),
|
||||
Model('AI-ModelScope/Florence-2-large-ft', 'microsoft/Florence-2-large-ft'),
|
||||
]),
|
||||
],
|
||||
FlorenceLoader,
|
||||
template=TemplateType.florence,
|
||||
architectures=['Florence2ForConditionalGeneration'],
|
||||
model_arch=ModelArch.florence,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class Phi3SmallLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
def rotary_emb(self, query_states, key_states, **kwargs):
|
||||
q_type = query_states.dtype
|
||||
k_type = key_states.dtype
|
||||
query_states, key_states = self.rotory_emb_origin(query_states, key_states, **kwargs)
|
||||
query_states = query_states.to(q_type)
|
||||
key_states = key_states.to(k_type)
|
||||
return query_states, key_states
|
||||
|
||||
for i in range(32): # TODO: 32
|
||||
re = model.model.layers[i].self_attn.rotary_emb
|
||||
re.rotory_emb_origin = re.forward
|
||||
re.forward = MethodType(rotary_emb, re)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.phi3_small,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Phi-3-small-8k-instruct', 'microsoft/Phi-3-small-8k-instruct'),
|
||||
Model('LLM-Research/Phi-3-small-128k-instruct', 'microsoft/Phi-3-small-128k-instruct'),
|
||||
]),
|
||||
],
|
||||
Phi3SmallLoader,
|
||||
template=TemplateType.phi3,
|
||||
architectures=['Phi3SmallForCausalLM'],
|
||||
model_arch=ModelArch.phi3_small,
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.phi2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/phi-2', 'microsoft/phi-2'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.default,
|
||||
architectures=['PhiForCausalLM'],
|
||||
model_arch=ModelArch.phi2,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.phi3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Phi-3-mini-4k-instruct', 'microsoft/Phi-3-mini-4k-instruct'),
|
||||
Model('LLM-Research/Phi-3-mini-128k-instruct', 'microsoft/Phi-3-mini-128k-instruct'),
|
||||
Model('LLM-Research/Phi-3-medium-4k-instruct', 'microsoft/Phi-3-medium-4k-instruct'),
|
||||
Model('LLM-Research/Phi-3-medium-128k-instruct', 'microsoft/Phi-3-medium-128k-instruct'),
|
||||
Model('LLM-Research/Phi-3.5-mini-instruct', 'microsoft/Phi-3.5-mini-instruct'),
|
||||
]),
|
||||
ModelGroup([Model('LLM-Research/Phi-4-mini-instruct', 'microsoft/Phi-4-mini-instruct')])
|
||||
],
|
||||
template=TemplateType.phi3,
|
||||
architectures=['Phi3ForCausalLM'],
|
||||
requires=['transformers>=4.36'],
|
||||
model_arch=ModelArch.phi3,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.phi4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/phi-4', 'microsoft/phi-4'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.phi4,
|
||||
architectures=['Phi3ForCausalLM'],
|
||||
requires=['transformers>=4.36'],
|
||||
model_arch=ModelArch.phi3,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.phi3_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Phi-3.5-MoE-instruct', 'microsoft/Phi-3.5-MoE-instruct'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.phi3,
|
||||
architectures=['PhiMoEForCausalLM'],
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
@@ -0,0 +1,268 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import torch
|
||||
from transformers import PreTrainedModel
|
||||
from transformers.utils import strtobool
|
||||
from types import MethodType
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_env_args
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_device_map, patch_fixed_device, patch_output_clone
|
||||
from ..register import ModelLoader, register_model
|
||||
from ..utils import use_submodel_func
|
||||
from .deepseek import DeepseekLoader
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minicpm_moe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-MoE-8x2B', 'openbmb/MiniCPM-MoE-8x2B'),
|
||||
]),
|
||||
],
|
||||
DeepseekLoader,
|
||||
template=TemplateType.minicpm,
|
||||
architectures=['MiniCPMForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
|
||||
|
||||
def _patch_minicpmv_device_map(model) -> None:
|
||||
if not hasattr(model, 'hf_device_map') or len(model.hf_device_map.values()) == 1:
|
||||
return
|
||||
|
||||
device = list(model.hf_device_map.values())[0]
|
||||
if hasattr(model, 'get_vision_embedding') and not hasattr(model, '_old_get_vision_embedding'):
|
||||
# minicpm-v-v2-chat; avoid double patching
|
||||
_old_get_vision_embedding = model.__class__.get_vision_embedding
|
||||
|
||||
def _get_vision_embedding(self, pixel_values):
|
||||
output = _old_get_vision_embedding(self, pixel_values)
|
||||
if len(pixel_values) == 0:
|
||||
return output
|
||||
if isinstance(output, list):
|
||||
return [x.to(device=device) if isinstance(x, torch.Tensor) else x for x in output]
|
||||
else:
|
||||
return output.to(device=device)
|
||||
|
||||
model.__class__._old_get_vision_embedding = _old_get_vision_embedding
|
||||
model.__class__.get_vision_embedding = _get_vision_embedding
|
||||
|
||||
if hasattr(model, 'resampler'): # minicpm-v-v2_5-chat
|
||||
patch_fixed_device(model.resampler, device)
|
||||
|
||||
|
||||
class MiniCPMVLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.resampler.to(self.torch_dtype) # fix float32
|
||||
_patch_minicpmv_device_map(model)
|
||||
func_list = ['generate', 'get_input_embeddings', 'forward']
|
||||
use_submodel_func(model, 'llm', func_list)
|
||||
if hasattr(model, 'get_slice_image_placeholder'):
|
||||
processor.get_slice_image_placeholder = MethodType(model.get_slice_image_placeholder, processor)
|
||||
processor.transform = MethodType(model.transform, processor)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmv,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-V', 'openbmb/MiniCPM-V'),
|
||||
Model('OpenBMB/MiniCPM-V-2', 'openbmb/MiniCPM-V-2'),
|
||||
], ),
|
||||
],
|
||||
MiniCPMVLoader,
|
||||
template=TemplateType.minicpmv,
|
||||
architectures=['MiniCPMV'],
|
||||
model_arch=ModelArch.minicpmv,
|
||||
requires=['timm', 'transformers<4.42'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class MiniCPMV2Loader(MiniCPMVLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
with patch_device_map():
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
embedding = model.get_input_embeddings()
|
||||
patch_output_clone(embedding)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmv2_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-Llama3-V-2_5', 'openbmb/MiniCPM-Llama3-V-2_5'),
|
||||
], ),
|
||||
],
|
||||
MiniCPMV2Loader,
|
||||
template=TemplateType.minicpmv2_5,
|
||||
architectures=['MiniCPMV'],
|
||||
model_arch=ModelArch.minicpmv,
|
||||
requires=['timm', 'transformers>=4.36'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmv2_6,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-V-2_6', 'openbmb/MiniCPM-V-2_6'),
|
||||
], ),
|
||||
],
|
||||
MiniCPMV2Loader,
|
||||
template=TemplateType.minicpmv2_6,
|
||||
architectures=['MiniCPMV'],
|
||||
model_arch=ModelArch.minicpmv,
|
||||
requires=['timm', 'transformers>=4.36', 'decord'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
|
||||
class MiniCPMO2Loader(MiniCPMV2Loader):
|
||||
|
||||
def get_model(self, model_dir: str, config, *args, **kwargs) -> PreTrainedModel:
|
||||
config.init_tts = strtobool(get_env_args('init_tts', str, 'false'))
|
||||
config.init_audio = strtobool(get_env_args('init_audio', str, 'true'))
|
||||
return super().get_model(model_dir, config, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmo,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-o-2_6', 'openbmb/MiniCPM-o-2_6'),
|
||||
], template=TemplateType.minicpmo),
|
||||
ModelGroup(
|
||||
[
|
||||
Model('OpenBMB/MiniCPM-o-4_5', 'openbmb/MiniCPM-o-4_5'),
|
||||
],
|
||||
template=TemplateType.minicpmo4_5,
|
||||
requires=['timm', 'transformers==4.51.3', 'decord', 'soundfile', 'minicpmo-utils==1.0.6'],
|
||||
),
|
||||
],
|
||||
MiniCPMO2Loader,
|
||||
architectures=['MiniCPMO'],
|
||||
model_arch=ModelArch.minicpmo,
|
||||
requires=['timm', 'transformers>=4.36', 'decord', 'soundfile'],
|
||||
tags=['vision', 'video', 'omni', 'audio'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmv4,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-V-4', 'openbmb/MiniCPM-V-4'),
|
||||
], ),
|
||||
],
|
||||
MiniCPMV2Loader,
|
||||
template=TemplateType.minicpmv4,
|
||||
architectures=['MiniCPMV'],
|
||||
model_arch=ModelArch.minicpmv,
|
||||
requires=['timm', 'transformers>=4.36', 'decord'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmv4_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-V-4_5', 'openbmb/MiniCPM-V-4_5'),
|
||||
], ),
|
||||
],
|
||||
MiniCPMV2Loader,
|
||||
template=TemplateType.minicpmv4_5,
|
||||
architectures=['MiniCPMV'],
|
||||
model_arch=ModelArch.minicpmv,
|
||||
requires=['timm', 'transformers>=4.36', 'decord'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
|
||||
class MiniCPMV4_6Loader(ModelLoader):
|
||||
|
||||
def get_model(self, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForImageTextToText
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
|
||||
from .qwen import _patch_qwen3_5_linear_attention_sequence_parallel
|
||||
_patch_qwen3_5_linear_attention_sequence_parallel()
|
||||
return super().get_model(*args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minicpmv4_6,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-V-4.6', 'openbmb/MiniCPM-V-4.6'),
|
||||
], ),
|
||||
],
|
||||
MiniCPMV4_6Loader,
|
||||
template=TemplateType.minicpmv4_6,
|
||||
architectures=['MiniCPMV4_6ForConditionalGeneration'],
|
||||
model_arch=ModelArch.minicpmv4_6,
|
||||
requires=['transformers>=5.7.0'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minicpm,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-2B-sft-fp32', 'openbmb/MiniCPM-2B-sft-fp32'),
|
||||
Model('OpenBMB/MiniCPM-2B-dpo-fp32', 'openbmb/MiniCPM-2B-dpo-fp32'),
|
||||
Model('OpenBMB/MiniCPM-1B-sft-bf16', 'openbmb/MiniCPM-1B-sft-bf16'),
|
||||
], ),
|
||||
],
|
||||
template=TemplateType.minicpm,
|
||||
architectures=['MiniCPMForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.36.0'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minicpm_chatml,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM-2B-128k', 'openbmb/MiniCPM-2B-128k'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM4-0.5B', 'openbmb/MiniCPM4-0.5B'),
|
||||
Model('OpenBMB/MiniCPM4-8B', 'openbmb/MiniCPM4-8B'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.chatml,
|
||||
architectures=['MiniCPMForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minicpm3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBMB/MiniCPM3-4B', 'openbmb/MiniCPM3-4B'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.chatml,
|
||||
architectures=['MiniCPM3ForCausalLM'],
|
||||
model_arch=ModelArch.deepseek_v2,
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
@@ -0,0 +1,193 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import json
|
||||
import os
|
||||
from transformers import AutoProcessor, PretrainedConfig, PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_device, get_device_count, get_dist_setting, get_logger
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_ignore_check_imports
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class MiniMaxVLLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
logger.warn('NOTE: minimax-vl-01 model does not support training.')
|
||||
n_gpu = get_device_count()
|
||||
_, local_rank, _, local_world_size = get_dist_setting()
|
||||
device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
|
||||
if 'quantization_config' in model_kwargs:
|
||||
quantization_config = model_kwargs['quantization_config']
|
||||
from transformers import QuantoConfig
|
||||
if isinstance(quantization_config, QuantoConfig):
|
||||
quantization_config.modules_to_not_convert = (
|
||||
[
|
||||
'vision_tower',
|
||||
'image_newline',
|
||||
'multi_modal_projector',
|
||||
'lm_head',
|
||||
'embed_tokens',
|
||||
] + [f'model.layers.{i}.coefficient' for i in range(config.text_config.num_hidden_layers)]
|
||||
+ [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.text_config.num_hidden_layers)])
|
||||
|
||||
if len(device_ids) > 1:
|
||||
model_safetensors_index_path = os.path.join(model_dir, 'model.safetensors.index.json')
|
||||
with open(model_safetensors_index_path, 'r') as f:
|
||||
model_safetensors_index = json.load(f)
|
||||
weight_map = model_safetensors_index['weight_map']
|
||||
vision_map = {}
|
||||
for key, value in weight_map.items():
|
||||
if 'vision_tower' in key or 'image_newline' in key or 'multi_modal_projector' in key:
|
||||
new_key = key.replace('.weight', '').replace('.bias', '')
|
||||
if new_key not in vision_map:
|
||||
vision_map[new_key] = value
|
||||
|
||||
device_map = {
|
||||
'language_model.model.embed_tokens': get_device(device_ids[0]),
|
||||
'language_model.model.norm': get_device(device_ids[len(device_ids) - 1]),
|
||||
'language_model.lm_head': get_device(device_ids[len(device_ids) - 1])
|
||||
}
|
||||
for key, value in vision_map.items():
|
||||
device_map[key] = get_device(device_ids[0])
|
||||
device_map['vision_tower.vision_model.post_layernorm'] = get_device(device_ids[0])
|
||||
layers_per_device = config.text_config.num_hidden_layers // len(device_ids)
|
||||
for i in range(len(device_ids)):
|
||||
for j in range(layers_per_device):
|
||||
device_map[f'language_model.model.layers.{i * layers_per_device + j}'] = get_device(device_ids[i])
|
||||
model_kwargs['device_map'] = device_map
|
||||
with patch_ignore_check_imports():
|
||||
return super().get_model(model_dir, config, processor, model_kwargs)
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
MiniMaxVL01ProcessorKwargs = get_class_from_dynamic_module(
|
||||
'processing_minimax_vl_01.MiniMaxVL01ProcessorKwargs', model_dir)
|
||||
get_hw_multiple_of = get_class_from_dynamic_module('processing_minimax_vl_01.get_hw_multiple_of', model_dir)
|
||||
get_num_token = get_class_from_dynamic_module('processing_minimax_vl_01.get_num_token', model_dir)
|
||||
|
||||
processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
|
||||
processor.MiniMaxVL01ProcessorKwargs = MiniMaxVL01ProcessorKwargs
|
||||
processor.get_hw_multiple_of = get_hw_multiple_of
|
||||
processor.get_num_token = get_num_token
|
||||
return processor
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minimax_vl, [
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-VL-01', 'MiniMaxAI/MiniMax-VL-01'),
|
||||
]),
|
||||
],
|
||||
MiniMaxVLLoader,
|
||||
template=TemplateType.minimax_vl,
|
||||
architectures=['MiniMaxVL01ForConditionalGeneration'],
|
||||
tags=['vision']))
|
||||
|
||||
|
||||
class MinimaxTextLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
logger.warn('NOTE: minimax-text-01 model does not support training.')
|
||||
n_gpu = get_device_count()
|
||||
_, local_rank, _, local_world_size = get_dist_setting()
|
||||
device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
|
||||
if 'quantization_config' in model_kwargs:
|
||||
quantization_config = model_kwargs['quantization_config']
|
||||
from transformers import QuantoConfig
|
||||
if isinstance(quantization_config, QuantoConfig):
|
||||
quantization_config.modules_to_not_convert = (
|
||||
[
|
||||
'lm_head',
|
||||
'embed_tokens',
|
||||
] + [f'model.layers.{i}.coefficient' for i in range(config.num_hidden_layers)]
|
||||
+ [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.num_hidden_layers)])
|
||||
|
||||
if len(device_ids) > 1:
|
||||
layers_per_device = config.num_hidden_layers // len(device_ids)
|
||||
# set device map
|
||||
device_map = {
|
||||
'model.embed_tokens': get_device(0),
|
||||
'model.norm': get_device(len(device_ids) - 1),
|
||||
'lm_head': get_device(len(device_ids) - 1)
|
||||
}
|
||||
for i in range(len(device_ids)):
|
||||
for j in range(layers_per_device):
|
||||
device_map[f'model.layers.{i * layers_per_device + j}'] = get_device(i)
|
||||
model_kwargs['device_map'] = device_map
|
||||
with patch_ignore_check_imports():
|
||||
return super().get_model(model_dir, config, processor, model_kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minimax, [
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-Text-01', 'MiniMaxAI/MiniMax-Text-01'),
|
||||
]),
|
||||
],
|
||||
MinimaxTextLoader,
|
||||
template=TemplateType.minimax,
|
||||
architectures=['MiniMaxText01ForCausalLM']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minimax_m1, [
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-M1-40k', 'MiniMaxAI/MiniMax-M1-40k'),
|
||||
Model('MiniMax/MiniMax-M1-80k', 'MiniMaxAI/MiniMax-M1-80k'),
|
||||
]),
|
||||
],
|
||||
MinimaxTextLoader,
|
||||
template=TemplateType.minimax_m1,
|
||||
architectures=['MiniMaxM1ForCausalLM']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.minimax_m2, [
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-M2', 'MiniMaxAI/MiniMax-M2'),
|
||||
], TemplateType.minimax_m2),
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-M2.1', 'MiniMaxAI/MiniMax-M2.1'),
|
||||
], TemplateType.minimax_m2_1),
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-M2.5', 'MiniMaxAI/MiniMax-M2.5'),
|
||||
], TemplateType.minimax_m2_5),
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-M2.7', 'MiniMaxAI/MiniMax-M2.7'),
|
||||
], TemplateType.minimax_m2_7),
|
||||
],
|
||||
requires=['transformers==4.57.1'],
|
||||
architectures=['MiniMaxM2ForCausalLM']))
|
||||
|
||||
|
||||
class MinimaxM3VLLoader(ModelLoader):
|
||||
default_trust_remote_code = False
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
return AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForImageTextToText
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
|
||||
return super().get_model(model_dir, config, processor, model_kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.minimax_m3_vl, [
|
||||
ModelGroup([
|
||||
Model('MiniMax/MiniMax-M3', 'MiniMaxAI/MiniMax-M3'),
|
||||
]),
|
||||
],
|
||||
MinimaxM3VLLoader,
|
||||
template=TemplateType.minimax_m3_vl,
|
||||
model_arch=ModelArch.minimax_m3_vl,
|
||||
architectures=['MiniMaxM3SparseForConditionalGeneration'],
|
||||
tags=['vision', 'video']))
|
||||
@@ -0,0 +1,210 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import AutoProcessor, AutoTokenizer, PretrainedConfig, PreTrainedModel
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, safe_snapshot_download
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.mistral,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Mistral-7B-Instruct-v0.1', 'mistralai/Mistral-7B-Instruct-v0.1'),
|
||||
Model('AI-ModelScope/Mistral-7B-Instruct-v0.2', 'mistralai/Mistral-7B-Instruct-v0.2'),
|
||||
Model('LLM-Research/Mistral-7B-Instruct-v0.3', 'mistralai/Mistral-7B-Instruct-v0.3'),
|
||||
Model('AI-ModelScope/Mistral-7B-v0.1', 'mistralai/Mistral-7B-v0.1'),
|
||||
Model('AI-ModelScope/Mistral-7B-v0.2-hf', 'alpindale/Mistral-7B-v0.2-hf'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('swift/Codestral-22B-v0.1', 'mistralai/Codestral-22B-v0.1'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.llama,
|
||||
architectures=['MistralForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.34'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.mixtral, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Mixtral-8x7B-Instruct-v0.1', 'mistralai/Mixtral-8x7B-Instruct-v0.1'),
|
||||
Model('AI-ModelScope/Mixtral-8x7B-v0.1', 'mistralai/Mixtral-8x7B-v0.1'),
|
||||
Model('AI-ModelScope/Mixtral-8x22B-v0.1', 'mistral-community/Mixtral-8x22B-v0.1'),
|
||||
],
|
||||
requires=['transformers>=4.36']),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Mixtral-8x7b-AQLM-2Bit-1x16-hf', 'ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf'),
|
||||
],
|
||||
requires=['transformers>=4.38', 'aqlm', 'torch>=2.2.0']),
|
||||
],
|
||||
template=TemplateType.llama,
|
||||
architectures=['MixtralForCausalLM'],
|
||||
model_arch=ModelArch.llama))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.mistral_nemo, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Mistral-Small-Instruct-2409', 'mistralai/Mistral-Small-Instruct-2409'),
|
||||
Model('LLM-Research/Mistral-Large-Instruct-2407', 'mistralai/Mistral-Large-Instruct-2407'),
|
||||
Model('AI-ModelScope/Mistral-Nemo-Base-2407', 'mistralai/Mistral-Nemo-Base-2407'),
|
||||
Model('AI-ModelScope/Mistral-Nemo-Instruct-2407', 'mistralai/Mistral-Nemo-Instruct-2407'),
|
||||
],
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Ministral-8B-Instruct-2410', 'mistralai/Ministral-8B-Instruct-2410'),
|
||||
],
|
||||
requires=['transformers>=4.46']),
|
||||
],
|
||||
template=TemplateType.mistral_nemo,
|
||||
architectures=['MistralForCausalLM'],
|
||||
model_arch=ModelArch.llama))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.mistral_2501, [
|
||||
ModelGroup([
|
||||
Model('mistralai/Mistral-Small-24B-Base-2501', 'mistralai/Mistral-Small-24B-Base-2501'),
|
||||
Model('mistralai/Mistral-Small-24B-Instruct-2501', 'mistralai/Mistral-Small-24B-Instruct-2501'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.mistral_2501,
|
||||
architectures=['MistralForCausalLM'],
|
||||
model_arch=ModelArch.llama))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.zephyr,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('modelscope/zephyr-7b-beta', 'HuggingFaceH4/zephyr-7b-beta'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.zephyr,
|
||||
model_arch=ModelArch.llama,
|
||||
architectures=['MistralForCausalLM'],
|
||||
requires=['transformers>=4.34'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.wizardlm2_moe,
|
||||
[ModelGroup([
|
||||
Model('AI-ModelScope/WizardLM-2-8x22B', 'alpindale/WizardLM-2-8x22B'),
|
||||
])],
|
||||
template=TemplateType.wizardlm2_moe,
|
||||
architectures=['MixtralForCausalLM'],
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.wizardlm2,
|
||||
[ModelGroup([
|
||||
Model('AI-ModelScope/WizardLM-2-7B-AWQ', 'MaziyarPanahi/WizardLM-2-7B-AWQ'),
|
||||
])],
|
||||
template=TemplateType.wizardlm2,
|
||||
architectures=['MistralForCausalLM'],
|
||||
requires=['transformers>=4.34'],
|
||||
))
|
||||
|
||||
|
||||
class DevstralLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
# src: sglang did the same (https://github.com/sgl-project/sglang/pull/6547)
|
||||
tokenizer_dir = safe_snapshot_download('mistralai/Mistral-Small-3.1-24B-Instruct-2503', download_model=False)
|
||||
tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.devstral, [
|
||||
ModelGroup([
|
||||
Model('mistralai/Devstral-Small-2505', 'mistralai/Devstral-Small-2505'),
|
||||
],
|
||||
requires=['transformers>=4.43', 'mistral-common>=1.5.5'])
|
||||
],
|
||||
DevstralLoader,
|
||||
template=TemplateType.devstral,
|
||||
architectures=['MistralForCausalLM'],
|
||||
model_arch=ModelArch.llama))
|
||||
|
||||
|
||||
class Mistral3Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import Mistral3ForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or Mistral3ForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.mistral3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('mistralai/Mistral-Small-3.1-24B-Base-2503', 'mistralai/Mistral-Small-3.1-24B-Base-2503'),
|
||||
Model('mistralai/Mistral-Small-3.1-24B-Instruct-2503', 'mistralai/Mistral-Small-3.1-24B-Instruct-2503'),
|
||||
],
|
||||
requires=['transformers>=4.49']),
|
||||
ModelGroup([
|
||||
Model('mistralai/Ministral-3-3B-Base-2512', 'mistralai/Ministral-3-3B-Base-2512'),
|
||||
Model('mistralai/Ministral-3-3B-Instruct-2512', 'mistralai/Ministral-3-3B-Instruct-2512'),
|
||||
Model('mistralai/Ministral-3-3B-Instruct-2512-BF16', 'mistralai/Ministral-3-3B-Instruct-2512-BF16'),
|
||||
Model('mistralai/Ministral-3-8B-Base-2512', 'mistralai/Ministral-3-8B-Base-2512'),
|
||||
Model('mistralai/Ministral-3-8B-Instruct-2512', 'mistralai/Ministral-3-8B-Instruct-2512'),
|
||||
Model('mistralai/Ministral-3-8B-Instruct-2512-BF16', 'mistralai/Ministral-3-8B-Instruct-2512-BF16'),
|
||||
Model('mistralai/Ministral-3-14B-Base-2512', 'mistralai/Ministral-3-14B-Base-2512'),
|
||||
Model('mistralai/Ministral-3-14B-Instruct-2512', 'mistralai/Ministral-3-14B-Instruct-2512'),
|
||||
Model('mistralai/Ministral-3-14B-Instruct-2512-BF16', 'mistralai/Ministral-3-14B-Instruct-2512-BF16'),
|
||||
],
|
||||
TemplateType.mistral_2512,
|
||||
requires=['transformers>=5.0.0.dev0', 'mistral-common>=1.8.6']),
|
||||
ModelGroup([
|
||||
Model('mistralai/Ministral-3-3B-Reasoning-2512', 'mistralai/Ministral-3-3B-Reasoning-2512'),
|
||||
Model('mistralai/Ministral-3-8B-Reasoning-2512', 'mistralai/Ministral-3-8B-Reasoning-2512'),
|
||||
Model('mistralai/Ministral-3-14B-Reasoning-2512', 'mistralai/Ministral-3-14B-Reasoning-2512'),
|
||||
],
|
||||
TemplateType.mistral_2512_thinking,
|
||||
requires=['transformers>=5.0.0.dev0', 'mistral-common>=1.8.6']),
|
||||
],
|
||||
Mistral3Loader,
|
||||
template=TemplateType.mistral_2503,
|
||||
model_arch=ModelArch.llava_hf,
|
||||
architectures=['Mistral3ForConditionalGeneration'],
|
||||
tags=['vision'],
|
||||
ignore_patterns=[],
|
||||
))
|
||||
|
||||
|
||||
class Mistral3_2506Loader(Mistral3Loader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer_dir = safe_snapshot_download('mistralai/Mistral-Small-3.1-24B-Instruct-2503', download_model=False)
|
||||
processor = AutoProcessor.from_pretrained(tokenizer_dir)
|
||||
return processor
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.mistral3_2506,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('mistralai/Mistral-Small-3.2-24B-Instruct-2506', 'mistralai/Mistral-Small-3.2-24B-Instruct-2506'),
|
||||
]),
|
||||
],
|
||||
Mistral3_2506Loader,
|
||||
template=TemplateType.mistral_2506,
|
||||
architectures=['Mistral3ForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.49'],
|
||||
))
|
||||
@@ -0,0 +1,390 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import torch
|
||||
from transformers import PretrainedConfig, PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
from types import MethodType
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_logger
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_output_clone
|
||||
from ..register import ModelLoader, register_model
|
||||
from ..utils import use_submodel_func
|
||||
from .qwen import Qwen2VLLoader, patch_qwen_vl_utils
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class Idefics3Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForVision2Seq
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForVision2Seq
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.idefics3,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Idefics3-8B-Llama3', 'HuggingFaceM4/Idefics3-8B-Llama3'),
|
||||
]),
|
||||
],
|
||||
Idefics3Loader,
|
||||
template=TemplateType.idefics3,
|
||||
model_arch=ModelArch.idefics3,
|
||||
architectures=['Idefics3ForConditionalGeneration'],
|
||||
tags=['vision'],
|
||||
requires=['transformers>=4.45'],
|
||||
))
|
||||
|
||||
|
||||
class PixtralLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.pixtral,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/pixtral-12b', 'mistral-community/pixtral-12b'),
|
||||
]),
|
||||
],
|
||||
PixtralLoader,
|
||||
template=TemplateType.pixtral,
|
||||
model_arch=ModelArch.llava_hf,
|
||||
architectures=['LlavaForConditionalGeneration'],
|
||||
requires=['transformers>=4.45'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class MolMoeLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
# fix bug for molmoe-1b
|
||||
def to_dict(self, *args, **kwargs):
|
||||
res = self._to_dict(*args, **kwargs)
|
||||
res['vision_backbone'] = self.vision_backbone.__dict__
|
||||
res.pop('to_dict')
|
||||
res.pop('_to_dict')
|
||||
return res
|
||||
|
||||
model.config._to_dict = model.config.to_dict
|
||||
model.config.to_dict = MethodType(to_dict, model.config)
|
||||
patch_output_clone(model.model.transformer.wte)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.molmoe,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/MolmoE-1B-0924', 'allenai/MolmoE-1B-0924'),
|
||||
]),
|
||||
],
|
||||
MolMoeLoader,
|
||||
template=TemplateType.molmo,
|
||||
model_arch=ModelArch.molmo,
|
||||
torch_dtype=torch.float32,
|
||||
architectures=['OLMoForCausalLM'],
|
||||
tags=['vision'],
|
||||
requires=['transformers>=4.45'],
|
||||
))
|
||||
|
||||
|
||||
class MolmoLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model_cls = get_class_from_dynamic_module('modeling_molmo.MolmoForCausalLM', model_dir)
|
||||
model_cls._no_split_modules = ['MolmoSequentialBlock']
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.model.transformer.wte)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.molmo,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('LLM-Research/Molmo-7B-O-0924', 'allenai/Molmo-7B-O-0924'),
|
||||
Model('LLM-Research/Molmo-7B-D-0924', 'allenai/Molmo-7B-D-0924'),
|
||||
Model('LLM-Research/Molmo-72B-0924', 'allenai/Molmo-72B-0924'),
|
||||
]),
|
||||
],
|
||||
MolmoLoader,
|
||||
template=TemplateType.molmo,
|
||||
model_arch=ModelArch.molmo,
|
||||
architectures=['MolmoForCausalLM'],
|
||||
tags=['vision'],
|
||||
requires=['transformers>=4.45'],
|
||||
))
|
||||
|
||||
|
||||
class Molmo2Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import AutoModelForImageTextToText
|
||||
model_cls = get_class_from_dynamic_module('modeling_molmo2.Molmo2ForConditionalGeneration', model_dir)
|
||||
no_split_modules = getattr(model_cls, '_no_split_modules', []) or []
|
||||
if 'MolmoSequentialBlock' not in no_split_modules:
|
||||
model_cls._no_split_modules = no_split_modules + ['MolmoSequentialBlock']
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModelForImageTextToText
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.model.transformer.wte)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.molmo2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('allenai/Molmo2-4B', 'allenai/Molmo2-4B'),
|
||||
Model('allenai/Molmo2-8B', 'allenai/Molmo2-8B'),
|
||||
Model('allenai/Molmo2-O-7B', 'allenai/Molmo2-O-7B'),
|
||||
]),
|
||||
],
|
||||
Molmo2Loader,
|
||||
template=TemplateType.molmo2,
|
||||
model_arch=ModelArch.molmo,
|
||||
architectures=['Molmo2ForConditionalGeneration'],
|
||||
tags=['vision', 'video'],
|
||||
requires=['transformers>=4.57.1,<5', 'decord'],
|
||||
))
|
||||
|
||||
|
||||
class MegrezOmniLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model_cls = get_class_from_dynamic_module('modeling_megrezo.MegrezO', model_dir)
|
||||
model_cls._no_split_modules = ['ResidualAttentionBlock', 'LlamaDecoderLayer']
|
||||
model_cls = get_class_from_dynamic_module('modeling_megrezo.SiglipVisionTransformer', model_dir)
|
||||
model_cls._no_split_modules = ['SiglipEncoderLayer']
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.llm.model.embed_tokens)
|
||||
use_submodel_func(model, 'llm')
|
||||
return model
|
||||
|
||||
def _get_model_processor(self, model_dir, config):
|
||||
model, processor = super().get_processor(model_dir, config)
|
||||
if model:
|
||||
processor = model._get_or_init_processor()
|
||||
return model, processor
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.megrez_omni,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('InfiniAI/Megrez-3B-Omni', 'Infinigence/Megrez-3B-Omni'),
|
||||
]),
|
||||
],
|
||||
MegrezOmniLoader,
|
||||
template=TemplateType.megrez_omni,
|
||||
model_arch=ModelArch.megrez_omni,
|
||||
architectures=['MegrezO'],
|
||||
tags=['vision', 'audio'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.qwen2_gme, [
|
||||
ModelGroup([
|
||||
Model('iic/gme-Qwen2-VL-2B-Instruct', 'Alibaba-NLP/gme-Qwen2-VL-2B-Instruct'),
|
||||
Model('iic/gme-Qwen2-VL-7B-Instruct', 'Alibaba-NLP/gme-Qwen2-VL-7B-Instruct'),
|
||||
]),
|
||||
],
|
||||
Qwen2VLLoader,
|
||||
template=TemplateType.qwen2_gme,
|
||||
model_arch=ModelArch.qwen2_vl,
|
||||
architectures=['Qwen2VLForConditionalGeneration'],
|
||||
tags=['vision']))
|
||||
|
||||
|
||||
class JinaRerankerM0Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
# Use AutoModel to respect the model repo's dynamic class mapping
|
||||
# and load the custom Jina reranker head via trust_remote_code.
|
||||
from transformers import AutoModel
|
||||
from transformers.modeling_outputs import SequenceClassifierOutputWithPast
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModel
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
# Patch forward to return a sequence-classification-style output with `.logits`
|
||||
# Use the model's own head (already present in jina-reranker-m0), just wrap outputs.
|
||||
|
||||
if not hasattr(model, '_forward_origin'):
|
||||
model._forward_origin = model.forward
|
||||
model.logit_bias = 2.65
|
||||
|
||||
def forward(self,
|
||||
input_ids=None,
|
||||
attention_mask=None,
|
||||
position_ids=None,
|
||||
inputs_embeds=None,
|
||||
pixel_values=None,
|
||||
image_grid_thw=None,
|
||||
video_grid_thw=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
**kwargs):
|
||||
# Remove labels to avoid upstream asserts in ranking models
|
||||
kwargs.pop('labels', None)
|
||||
if return_dict is None:
|
||||
return_dict = True
|
||||
|
||||
out = self._forward_origin(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
position_ids=position_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
pixel_values=pixel_values,
|
||||
image_grid_thw=image_grid_thw,
|
||||
video_grid_thw=video_grid_thw,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
**kwargs)
|
||||
|
||||
logits = out.unsqueeze(-1) - self.logit_bias
|
||||
|
||||
if not return_dict:
|
||||
return (logits, )
|
||||
|
||||
return SequenceClassifierOutputWithPast(logits=logits)
|
||||
|
||||
model.forward = MethodType(forward, model)
|
||||
|
||||
def padding_free_fn(self, output, kwargs, padding_side):
|
||||
return_dict = kwargs.get('return_dict', None)
|
||||
|
||||
output.logits = output['last_hidden_state'][:, -1]
|
||||
logits = self.score(output.logits)
|
||||
logits = logits - self.logit_bias
|
||||
|
||||
if not return_dict:
|
||||
return (logits, )
|
||||
|
||||
return SequenceClassifierOutputWithPast(logits=logits)
|
||||
|
||||
model.padding_free_fn = MethodType(padding_free_fn, model)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.jina_reranker_m0,
|
||||
[ModelGroup([Model('JinaAI/jina-reranker-m0', 'JinaAI/jina-reranker-m0')])],
|
||||
JinaRerankerM0Loader,
|
||||
template=TemplateType.jina_reranker_m0,
|
||||
model_arch=ModelArch.qwen2_vl,
|
||||
architectures=['JinaRerankerM0ForConditionalGeneration'],
|
||||
task_type='reranker',
|
||||
tags=['reranker', 'vision'],
|
||||
))
|
||||
|
||||
|
||||
class KeyeVLLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
processor = super().get_processor(model_dir, config)
|
||||
from keye_vl_utils import vision_process
|
||||
global_vars = patch_qwen_vl_utils(vision_process)
|
||||
processor.global_vars = global_vars
|
||||
return processor
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.keye_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Kwai-Keye/Keye-VL-8B-Preview', 'Kwai-Keye/Keye-VL-8B-Preview'),
|
||||
]),
|
||||
],
|
||||
KeyeVLLoader,
|
||||
template=TemplateType.keye_vl,
|
||||
model_arch=ModelArch.keye_vl,
|
||||
architectures=['KeyeForConditionalGeneration'],
|
||||
tags=['vision'],
|
||||
requires=['keye_vl_utils'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.keye_vl_1_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Kwai-Keye/Keye-VL-1_5-8B', 'Kwai-Keye/Keye-VL-1_5-8B'),
|
||||
]),
|
||||
],
|
||||
KeyeVLLoader,
|
||||
template=TemplateType.keye_vl_1_5,
|
||||
model_arch=ModelArch.keye_vl,
|
||||
architectures=['KeyeVL1_5ForConditionalGeneration'],
|
||||
tags=['vision'],
|
||||
requires=['keye_vl_utils>=1.5.2', 'transformers==4.52.4'],
|
||||
))
|
||||
|
||||
|
||||
class DotsOCRLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model_cls = get_class_from_dynamic_module('modeling_dots_vision.DotsVisionTransformer', model_dir)
|
||||
model_cls._no_split_modules = ['DotsVisionBlock']
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.dots_ocr,
|
||||
[ModelGroup([
|
||||
Model('rednote-hilab/dots.ocr', 'rednote-hilab/dots.ocr'),
|
||||
])],
|
||||
DotsOCRLoader,
|
||||
template=TemplateType.dots_ocr,
|
||||
model_arch=ModelArch.dots_ocr,
|
||||
architectures=['DotsOCRForCausalLM'],
|
||||
requires=['transformers>=4.51.0'],
|
||||
))
|
||||
|
||||
|
||||
class Sail2VLLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
use_submodel_func(model, 'language_model')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.sail_vl2, [
|
||||
ModelGroup([
|
||||
Model('BytedanceDouyinContent/SAIL-VL2-2B', 'BytedanceDouyinContent/SAIL-VL2-2B'),
|
||||
Model('BytedanceDouyinContent/SAIL-VL2-2B-Thinking', 'BytedanceDouyinContent/SAIL-VL2-2B-Thinking'),
|
||||
Model('BytedanceDouyinContent/SAIL-VL2-8B', 'BytedanceDouyinContent/SAIL-VL2-8B'),
|
||||
Model('BytedanceDouyinContent/SAIL-VL2-8B-Thinking', 'BytedanceDouyinContent/SAIL-VL2-8B-Thinking'),
|
||||
])
|
||||
],
|
||||
Sail2VLLoader,
|
||||
template=TemplateType.sail_vl2,
|
||||
model_arch=ModelArch.internvl,
|
||||
architectures=['SAILVLModel'],
|
||||
requires=['transformers<=4.51.3'],
|
||||
tags=['vision']))
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from transformers import PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from swift.template import TemplateType
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_get_input_embeddings
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class KimiVLLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
KimiVLPreTrainedModel = get_class_from_dynamic_module('modeling_kimi_vl.KimiVLPreTrainedModel', model_dir)
|
||||
try:
|
||||
del KimiVLPreTrainedModel._supports_sdpa
|
||||
except AttributeError:
|
||||
pass
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_get_input_embeddings(model.vision_tower, 'patch_embed')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.kimi_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('moonshotai/Kimi-VL-A3B-Instruct', 'moonshotai/Kimi-VL-A3B-Instruct'),
|
||||
Model('moonshotai/Kimi-VL-A3B-Thinking', 'moonshotai/Kimi-VL-A3B-Thinking'),
|
||||
Model('moonshotai/Kimi-VL-A3B-Thinking-2506', 'moonshotai/Kimi-VL-A3B-Thinking-2506'),
|
||||
])
|
||||
],
|
||||
KimiVLLoader,
|
||||
template=TemplateType.kimi_vl,
|
||||
model_arch=ModelArch.llava_hf_legacy,
|
||||
architectures=['KimiVLForConditionalGeneration'],
|
||||
requires=['transformers<4.49'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.kimi_k25,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('moonshotai/Kimi-K2.5', 'moonshotai/Kimi-K2.5'),
|
||||
Model('moonshotai/Kimi-K2.6', 'moonshotai/Kimi-K2.6'),
|
||||
Model('moonshotai/Kimi-K2.7-Code', 'moonshotai/Kimi-K2.7-Code'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.kimi_k25,
|
||||
model_arch=ModelArch.kimi_k25,
|
||||
architectures=['KimiK25ForConditionalGeneration'],
|
||||
requires=['transformers>=4.57.1,<5.0.0'],
|
||||
))
|
||||
@@ -0,0 +1,163 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import os
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from transformers import PretrainedConfig, PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_logger, git_clone_github
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
from ..utils import use_submodel_func
|
||||
from .qwen import QwenLoader
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class MplugOwl2Loader(ModelLoader):
|
||||
|
||||
def _get_model(self, model_dir: str, vocab_size, *args, **kwargs) -> PreTrainedModel:
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/X-PLUG/mPLUG-Owl')
|
||||
local_repo_path = os.path.join(local_repo_path, 'mPLUG-Owl2')
|
||||
sys.path.append(local_repo_path)
|
||||
# register
|
||||
# https://github.com/X-PLUG/mPLUG-Owl/blob/main/mPLUG-Owl2/mplug_owl2/model/modeling_mplug_owl2.py#L447
|
||||
from mplug_owl2 import MPLUGOwl2LlamaForCausalLM
|
||||
if vocab_size is not None:
|
||||
config.vocab_size = vocab_size
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
logger.info('Please ignore the unimported warning.')
|
||||
return model
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
return self._get_model(model_dir, None, *args, **kwargs)
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
from transformers.models.clip.image_processing_clip import CLIPImageProcessor
|
||||
processor = CLIPImageProcessor.from_pretrained(model_dir)
|
||||
return processor
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.mplug_owl2, [ModelGroup([
|
||||
Model('iic/mPLUG-Owl2', 'MAGAer13/mplug-owl2-llama2-7b'),
|
||||
])],
|
||||
MplugOwl2Loader,
|
||||
template=TemplateType.mplug_owl2,
|
||||
model_arch=ModelArch.mplug_owl2,
|
||||
requires=['transformers<4.35', 'icecream'],
|
||||
tags=['vision']), )
|
||||
|
||||
|
||||
class MplugOwl2_1Loader(QwenLoader, MplugOwl2Loader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
return self._get_model(model_dir, 151851, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.mplug_owl2_1, [ModelGroup([
|
||||
Model('iic/mPLUG-Owl2.1', 'Mizukiluke/mplug_owl_2_1'),
|
||||
])],
|
||||
MplugOwl2_1Loader,
|
||||
template=TemplateType.mplug_owl2,
|
||||
model_arch=ModelArch.mplug_owl2_1,
|
||||
requires=['transformers<4.35', 'icecream'],
|
||||
tags=['vision']))
|
||||
|
||||
|
||||
class MplugOwl3Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
get_class_from_dynamic_module('configuration_hyper_qwen2.HyperQwen2Config', model_dir)
|
||||
model_cls = get_class_from_dynamic_module('modeling_mplugowl3.mPLUGOwl3Model', model_dir)
|
||||
model_cls._no_split_modules = ['SiglipEncoderLayer']
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
func_list = ['generate', 'forward']
|
||||
use_submodel_func(model, 'language_model', func_list)
|
||||
|
||||
all_hooks = OrderedDict()
|
||||
hooks_with_kwargs = OrderedDict()
|
||||
|
||||
def append_hooks(sub_module, inc_id=0):
|
||||
for id, hook in sub_module._forward_hooks.items():
|
||||
all_hooks[inc_id] = hook
|
||||
if id in sub_module._forward_hooks_with_kwargs:
|
||||
hooks_with_kwargs[inc_id] = sub_module._forward_hooks_with_kwargs[id]
|
||||
inc_id += 1
|
||||
return inc_id
|
||||
|
||||
inc_id = append_hooks(model.language_model)
|
||||
append_hooks(model, inc_id)
|
||||
model._forward_hooks = all_hooks
|
||||
model._forward_hooks_with_kwargs = hooks_with_kwargs
|
||||
return model
|
||||
|
||||
def _get_model_processor(self, model_dir, config):
|
||||
model, tokenizer = super()._get_model_processor(model_dir, config)
|
||||
if model:
|
||||
tokenizer = model.init_processor(tokenizer)
|
||||
return model, tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.mplug_owl3, [
|
||||
ModelGroup([
|
||||
Model('iic/mPLUG-Owl3-1B-241014', 'mPLUG/mPLUG-Owl3-1B-241014'),
|
||||
Model('iic/mPLUG-Owl3-2B-241014', 'mPLUG/mPLUG-Owl3-2B-241014'),
|
||||
Model('iic/mPLUG-Owl3-7B-240728', 'mPLUG/mPLUG-Owl3-7B-240728'),
|
||||
]),
|
||||
],
|
||||
MplugOwl3Loader,
|
||||
template=TemplateType.mplug_owl3,
|
||||
architectures=['mPLUGOwl3Model'],
|
||||
model_arch=ModelArch.mplug_owl3,
|
||||
requires=['transformers>=4.36', 'icecream', 'decord'],
|
||||
tags=['vision', 'video']))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.mplug_owl3_241101, [
|
||||
ModelGroup([
|
||||
Model('iic/mPLUG-Owl3-7B-241101', 'mPLUG/mPLUG-Owl3-7B-241101'),
|
||||
]),
|
||||
],
|
||||
MplugOwl3Loader,
|
||||
template=TemplateType.mplug_owl3_241101,
|
||||
architectures=['mPLUGOwl3Model'],
|
||||
model_arch=ModelArch.mplug_owl3,
|
||||
requires=['transformers>=4.36', 'icecream'],
|
||||
tags=['vision', 'video']))
|
||||
|
||||
|
||||
class DocOwl2Loader(ModelLoader):
|
||||
|
||||
def _get_model_processor(self, model_dir, config):
|
||||
model, tokenizer = super()._get_model_processor(model_dir, config)
|
||||
if model:
|
||||
tokenizer = model.init_processor(tokenizer, basic_image_size=504, crop_anchors='grid_12')
|
||||
return model, tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.doc_owl2, [
|
||||
ModelGroup([
|
||||
Model('iic/DocOwl2', 'mPLUG/DocOwl2'),
|
||||
]),
|
||||
],
|
||||
DocOwl2Loader,
|
||||
template=TemplateType.doc_owl2,
|
||||
architectures=['mPLUGDocOwl2'],
|
||||
model_arch=ModelArch.doc_owl2,
|
||||
requires=['transformers>=4.36', 'icecream'],
|
||||
tags=['vision']))
|
||||
@@ -0,0 +1,74 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_logger
|
||||
from ..constant import LLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.openbuddy_llama,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-llama-65b-v8-bf16', 'OpenBuddy/openbuddy-llama-65b-v8-bf16'),
|
||||
], TemplateType.openbuddy),
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-llama2-13b-v8.1-fp16', 'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16'),
|
||||
Model('OpenBuddy/openbuddy-llama2-70b-v10.1-bf16', 'OpenBuddy/openbuddy-llama2-70b-v10.1-bf16'),
|
||||
], TemplateType.openbuddy),
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-deepseek-67b-v15.2', 'OpenBuddy/openbuddy-deepseek-67b-v15.2'),
|
||||
], TemplateType.openbuddy),
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-llama3-8b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-8b-v21.1-8k'),
|
||||
Model('OpenBuddy/openbuddy-llama3-70b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-70b-v21.1-8k'),
|
||||
Model('OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k', 'OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k'),
|
||||
], TemplateType.openbuddy2),
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k', 'OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k'),
|
||||
Model('OpenBuddy/openbuddy-nemotron-70b-v23.2-131k', 'OpenBuddy/openbuddy-nemotron-70b-v23.2-131k'),
|
||||
],
|
||||
TemplateType.openbuddy2,
|
||||
requires=['transformers>=4.43']),
|
||||
ModelGroup(
|
||||
[Model('OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k', 'OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k')],
|
||||
TemplateType.openbuddy2,
|
||||
requires=['transformers>=4.45']),
|
||||
],
|
||||
model_arch=ModelArch.llama,
|
||||
mcore_model_type='gpt',
|
||||
architectures=['LlamaForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.openbuddy_mistral,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-mistral-7b-v17.1-32k', 'OpenBuddy/openbuddy-mistral-7b-v17.1-32k'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-zephyr-7b-v14.1', 'OpenBuddy/openbuddy-zephyr-7b-v14.1'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.openbuddy,
|
||||
model_arch=ModelArch.llama,
|
||||
requires=['transformers>=4.34'],
|
||||
architectures=['MistralForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.openbuddy_mixtral,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k', 'OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k'),
|
||||
], ),
|
||||
],
|
||||
template=TemplateType.openbuddy,
|
||||
architectures=['MixtralForCausalLM'],
|
||||
requires=['transformers>=4.36'],
|
||||
))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import get_logger
|
||||
from ..constant import LLMModelType
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.seed_oss, [
|
||||
ModelGroup([
|
||||
Model('ByteDance-Seed/Seed-OSS-36B-Instruct', 'ByteDance-Seed/Seed-OSS-36B-Instruct'),
|
||||
Model('ByteDance-Seed/Seed-OSS-36B-Base', 'ByteDance-Seed/Seed-OSS-36B-Base'),
|
||||
Model('ByteDance-Seed/Seed-OSS-36B-Base-woSyn', 'ByteDance-Seed/Seed-OSS-36B-Base-woSyn'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.seed_oss,
|
||||
architectures=['SeedOssForCausalLM'],
|
||||
requires=['transformers>=4.56']))
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor
|
||||
from ..constant import LLMModelType, RMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class SkyworkLoader(ModelLoader):
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
tokenizer = super().get_processor(model_dir, config)
|
||||
tokenizer.add_tokens('[USER]')
|
||||
tokenizer.add_tokens('[BOT]')
|
||||
tokenizer.add_tokens('[SEP]')
|
||||
return tokenizer
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.skywork,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('skywork/Skywork-13B-base', 'skywork/Skywork-13B-base'),
|
||||
Model('skywork/Skywork-13B-chat'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.skywork,
|
||||
architectures=['SkyworkForCausalLM'],
|
||||
model_arch=ModelArch.llama,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
RMModelType.llama3_2_reward,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B', 'Skywork/Skywork-Reward-Llama-3.1-8B'),
|
||||
Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2', 'Skywork/Skywork-Reward-Llama-3.1-8B-v0.2'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft', 'Ray2333/GRM_Llama3.1_8B_rewardmodel-ft'),
|
||||
Model('AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft', 'Ray2333/GRM-llama3.2-3B-rewardmodel-ft'),
|
||||
])
|
||||
],
|
||||
template=TemplateType.llama3_2,
|
||||
requires=['transformers>=4.43'],
|
||||
architectures=['LlamaForSequenceClassification'],
|
||||
model_arch=ModelArch.llama,
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
RMModelType.gemma_reward,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B', 'Skywork/Skywork-Reward-Gemma-2-27B'),
|
||||
Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2', 'Skywork/Skywork-Reward-Gemma-2-27B-v0.2'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.gemma,
|
||||
requires=['transformers>=4.42'],
|
||||
architectures=['Gemma2ForSequenceClassification'],
|
||||
model_arch=ModelArch.llama,
|
||||
))
|
||||
@@ -0,0 +1,167 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import sys
|
||||
from functools import wraps
|
||||
from transformers import AutoModel, PretrainedConfig, PreTrainedModel
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, git_clone_github, safe_snapshot_download
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_output_clone
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class GotOCR2Loader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
self.auto_model_cls = AutoModel
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.got_ocr2, [
|
||||
ModelGroup([
|
||||
Model('stepfun-ai/GOT-OCR2_0', 'stepfun-ai/GOT-OCR2_0'),
|
||||
]),
|
||||
],
|
||||
GotOCR2Loader,
|
||||
template=TemplateType.got_ocr2,
|
||||
model_arch=ModelArch.got_ocr2,
|
||||
architectures=['GOTQwenForCausalLM'],
|
||||
tags=['vision']))
|
||||
|
||||
|
||||
class GotOCR2HfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers.models.got_ocr2 import GotOcr2ForConditionalGeneration
|
||||
GotOcr2ForConditionalGeneration._no_split_modules = ['GotOcr2VisionLayer']
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.got_ocr2_hf, [
|
||||
ModelGroup([
|
||||
Model('stepfun-ai/GOT-OCR-2.0-hf', 'stepfun-ai/GOT-OCR-2.0-hf'),
|
||||
]),
|
||||
],
|
||||
GotOCR2HfLoader,
|
||||
template=TemplateType.got_ocr2_hf,
|
||||
model_arch=ModelArch.llava_hf,
|
||||
architectures=['GotOcr2ForConditionalGeneration'],
|
||||
tags=['vision']))
|
||||
|
||||
|
||||
class StepAudioLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/stepfun-ai/Step-Audio.git')
|
||||
sys.path.append(local_repo_path)
|
||||
from tokenizer import StepAudioTokenizer
|
||||
encoder_path = safe_snapshot_download('stepfun-ai/Step-Audio-Tokenizer', check_local=True)
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
model.encoder = StepAudioTokenizer(encoder_path)
|
||||
# from tts import StepAudioTTS
|
||||
# if not os.path.exists('speakers'):
|
||||
# shutil.copytree(os.path.join(local_repo_path, 'speakers'), 'speakers')
|
||||
# decoder_path = safe_snapshot_download('stepfun-ai/Step-Audio-TTS-3B', check_local=True)
|
||||
# model.decoder = StepAudioTTS(decoder_path, model.encoder)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.step_audio, [
|
||||
ModelGroup([
|
||||
Model('stepfun-ai/Step-Audio-Chat', 'stepfun-ai/Step-Audio-Chat'),
|
||||
]),
|
||||
],
|
||||
StepAudioLoader,
|
||||
template=TemplateType.step_audio,
|
||||
architectures=['Step1ForCausalLM'],
|
||||
requires=['funasr', 'sox', 'conformer', 'openai-whisper', 'librosa'],
|
||||
tags=['audio']))
|
||||
|
||||
|
||||
def _patch_step_audio2_mini(model):
|
||||
if hasattr(model.__class__, 'origin_forward'):
|
||||
return
|
||||
|
||||
model.__class__.origin_forward = model.__class__.forward
|
||||
|
||||
@wraps(model.__class__.origin_forward)
|
||||
def _forward(self, *args, **kwargs):
|
||||
labels = kwargs.get('labels')
|
||||
output = self.origin_forward(*args, **kwargs)
|
||||
if labels is not None and output.loss is None:
|
||||
output['loss'] = self.loss_function(
|
||||
logits=output.logits, labels=labels, vocab_size=self.config.get_text_config().vocab_size)
|
||||
return output
|
||||
|
||||
model.__class__.forward = _forward
|
||||
|
||||
|
||||
class StepAudio2MiniLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_output_clone(model.model.embed_tokens)
|
||||
_patch_step_audio2_mini(model)
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.step_audio2_mini,
|
||||
[ModelGroup([
|
||||
Model('stepfun-ai/Step-Audio-2-mini', 'stepfun-ai/Step-Audio-2-mini'),
|
||||
])],
|
||||
StepAudio2MiniLoader,
|
||||
template=TemplateType.step_audio2_mini,
|
||||
model_arch=ModelArch.step_audio2_mini,
|
||||
architectures=['StepAudio2ForCausalLM'],
|
||||
requires=['transformers==4.53.3', 'torchaudio', 'librosa'],
|
||||
tags=['audio'],
|
||||
))
|
||||
|
||||
|
||||
class Step3VLLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.vocab_size = config.text_config.vocab_size
|
||||
return config
|
||||
|
||||
def get_model(self, model_dir: str, config: PretrainedConfig, processor: Processor,
|
||||
model_kwargs) -> PreTrainedModel:
|
||||
key_mapping = {
|
||||
'^vision_model': 'model.vision_model',
|
||||
r'^model(?!\.(language_model|vision_model))': 'model.language_model',
|
||||
'vit_large_projector': 'model.vit_large_projector',
|
||||
}
|
||||
model_kwargs = model_kwargs.copy()
|
||||
model_kwargs['key_mapping'] = key_mapping
|
||||
return super().get_model(model_dir, config, processor, model_kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.step3_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('stepfun-ai/Step3-VL-10B-Base', 'stepfun-ai/Step3-VL-10B-Base'),
|
||||
Model('stepfun-ai/Step3-VL-10B', 'stepfun-ai/Step3-VL-10B'),
|
||||
])
|
||||
],
|
||||
Step3VLLoader,
|
||||
template=TemplateType.step3_vl,
|
||||
model_arch=ModelArch.step3_vl,
|
||||
architectures=['StepVLForConditionalGeneration'],
|
||||
requires=['transformers>=4.57.0'],
|
||||
tags=['vision'],
|
||||
))
|
||||
@@ -0,0 +1,60 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
|
||||
from transformers import GenerationConfig, PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
|
||||
|
||||
from swift.template import TemplateType
|
||||
from ..constant import LLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class TeleChatLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, **kwargs) -> PreTrainedModel:
|
||||
model = super().get_model(model_dir, config, processor, **kwargs)
|
||||
generation_config = GenerationConfig.from_pretrained(model_dir)
|
||||
for k in ['bos_token_id', 'eos_token_id', 'pad_token_id', 'user_token_id', 'bot_token_id']:
|
||||
setattr(processor, k, getattr(generation_config, k))
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.telechat,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('TeleAI/TeleChat-7B', 'Tele-AI/telechat-7B'),
|
||||
Model('TeleAI/TeleChat-12B', 'Tele-AI/TeleChat-12B'),
|
||||
Model('TeleAI/TeleChat-12B-v2', 'Tele-AI/TeleChat-12B-v2'),
|
||||
Model('TeleAI/TeleChat-52B', 'TeleAI/TeleChat-52B'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('swift/TeleChat-12B-V2-GPTQ-Int4'),
|
||||
]),
|
||||
ModelGroup([
|
||||
Model('TeleAI/TeleChat2-35B', 'Tele-AI/TeleChat2-35B'),
|
||||
Model('TeleAI/TeleChat2-115B', 'Tele-AI/TeleChat2-115B'),
|
||||
]),
|
||||
],
|
||||
TeleChatLoader,
|
||||
template=TemplateType.telechat,
|
||||
model_arch=ModelArch.telechat,
|
||||
architectures=['TelechatForCausalLM', 'TeleChatForCausalLM'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.telechat2,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('TeleAI/TeleChat2-3B', 'Tele-AI/TeleChat2-3B'),
|
||||
Model('TeleAI/TeleChat2-7B-32K', 'Tele-AI/TeleChat2-7B-32K'),
|
||||
Model('TeleAI/TeleChat2-35B-32K', 'Tele-AI/TeleChat2-35B-32K'),
|
||||
Model('TeleAI/TeleChat2-35B-Nov', 'Tele-AI/TeleChat2-35B-Nov'),
|
||||
]),
|
||||
],
|
||||
template=TemplateType.telechat2,
|
||||
model_arch=ModelArch.telechat,
|
||||
architectures=['TeleChat2ForCausalLM'],
|
||||
))
|
||||
@@ -0,0 +1,37 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from swift.template import TemplateType
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class HunyuanVLLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
self.attn_impl = self.attn_impl or 'eager'
|
||||
return super().get_config(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import HunYuanVLForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or HunYuanVLForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.hunyuan_ocr,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('Tencent-Hunyuan/HunyuanOCR', 'tencent/HunyuanOCR'),
|
||||
]),
|
||||
],
|
||||
HunyuanVLLoader,
|
||||
template=TemplateType.hunyuan_ocr,
|
||||
architectures=['HunYuanVLForConditionalGeneration'],
|
||||
model_arch=ModelArch.hunyuan_vl,
|
||||
requires=['transformers>=4.49.0'],
|
||||
))
|
||||
@@ -0,0 +1,80 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import sys
|
||||
from functools import wraps
|
||||
from transformers import PreTrainedModel
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import git_clone_github, safe_snapshot_download
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class ValleyLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
repo_path = 'https://github.com/bytedance/Valley.git'
|
||||
local_repo_path = git_clone_github(repo_path)
|
||||
sys.path.append(local_repo_path)
|
||||
from valley_eagle.model.language_model.valley_qwen2 import ValleyConfig
|
||||
self.auto_config_cls = ValleyConfig
|
||||
return super().get_config(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
from transformers.modeling_outputs import CausalLMOutputWithPast
|
||||
from valley_eagle.model.language_model.valley_qwen2 import ValleyQwen2ForCausalLM
|
||||
config.mm_vision_tower = safe_snapshot_download('AI-ModelScope/siglip-so400m-patch14-384', check_local=True)
|
||||
config.eagle_vision_tower = safe_snapshot_download('Qwen/Qwen2-VL-7B-Instruct', check_local=True)
|
||||
auto_model_cls = ValleyQwen2ForCausalLM
|
||||
|
||||
if not hasattr(ValleyQwen2ForCausalLM, '_origin_forward'):
|
||||
forward = ValleyQwen2ForCausalLM.forward
|
||||
ValleyQwen2ForCausalLM._origin_forward = forward
|
||||
|
||||
@wraps(forward)
|
||||
def new_forward(*args, **kwargs):
|
||||
import torch
|
||||
outputs = forward(*args, **kwargs)
|
||||
loss = outputs.loss
|
||||
if loss is not None and loss.shape[-1] > 0:
|
||||
loss = torch.mean(loss, dim=-1)
|
||||
return CausalLMOutputWithPast(
|
||||
loss=loss,
|
||||
logits=outputs.logits,
|
||||
past_key_values=outputs.past_key_values,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
)
|
||||
|
||||
ValleyQwen2ForCausalLM.forward = new_forward
|
||||
self.auto_model_cls = auto_model_cls
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
model.generation_config.repetition_penalty = 1.0 # Otherwise, Error. Same for original code.
|
||||
|
||||
from transformers import AutoProcessor, SiglipImageProcessor
|
||||
processor.image_processor = SiglipImageProcessor.from_pretrained(model.config.mm_vision_tower)
|
||||
processor.qwen2vl_processor = AutoProcessor.from_pretrained(
|
||||
model.config.eagle_vision_tower, max_pixels=1280 * 28 * 28)
|
||||
processor.image_processor.crop_size = processor.image_processor.size['height']
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.valley,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('bytedance-research/Valley-Eagle-7B'),
|
||||
], ),
|
||||
],
|
||||
ValleyLoader,
|
||||
template=TemplateType.valley,
|
||||
architectures=['ValleyQwen2ForCausalLM'],
|
||||
model_arch=ModelArch.valley,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['vision'],
|
||||
))
|
||||
@@ -0,0 +1,123 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import os
|
||||
import sys
|
||||
from transformers import AutoTokenizer, PretrainedConfig, PreTrainedModel
|
||||
from typing import Any, Dict
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import Processor, get_logger, git_clone_github
|
||||
from ..constant import LLMModelType, MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class YiVLLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
local_repo_path = git_clone_github('https://github.com/01-ai/Yi')
|
||||
sys.path.append(os.path.join(local_repo_path, 'VL'))
|
||||
from llava.model import LlavaConfig
|
||||
config = LlavaConfig.from_pretrained(model_dir)
|
||||
mm_vision_tower = config.mm_vision_tower
|
||||
config.mm_vision_tower = os.path.join(model_dir, *mm_vision_tower.rsplit('/', maxsplit=2)[-2:])
|
||||
config.attention_dropout = 0.
|
||||
if not hasattr(config, 'max_sequence_length'):
|
||||
config.max_sequence_length = 2048
|
||||
return config
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False)
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, **kwargs) -> PreTrainedModel:
|
||||
from llava.model import LlavaLlamaForCausalLM
|
||||
from llava.model.constants import key_info
|
||||
key_info['model_path'] = model_dir
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaLlamaForCausalLM
|
||||
model = super().get_model(model_dir, config, processor, **kwargs)
|
||||
vision_tower = model.get_vision_tower()
|
||||
vision_tower.load_model()
|
||||
vision_tower.to(device=model.device, dtype=config.torch_dtype)
|
||||
|
||||
logger.info('Please ignore the above warning.')
|
||||
logger.info('Loading the parameters of vision_tower...')
|
||||
model.resize_token_embeddings(len(processor))
|
||||
processor.image_processor = vision_tower.image_processor
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.yi_vl,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('01ai/Yi-VL-6B', '01-ai/Yi-VL-6B'),
|
||||
Model('01ai/Yi-VL-34B', '01-ai/Yi-VL-34B'),
|
||||
], ),
|
||||
],
|
||||
YiVLLoader,
|
||||
template=TemplateType.yi_vl,
|
||||
model_arch=ModelArch.llava_llama,
|
||||
architectures=['LlavaLlamaForCausalLM'],
|
||||
requires=['transformers>=4.34'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
LLMModelType.yi,
|
||||
[ # yi
|
||||
ModelGroup([
|
||||
Model('01ai/Yi-6B', '01-ai/Yi-6B'),
|
||||
Model('01ai/Yi-6B-200K', '01-ai/Yi-6B-200K'),
|
||||
Model('01ai/Yi-6B-Chat', '01-ai/Yi-6B-Chat'),
|
||||
Model('01ai/Yi-6B-Chat-4bits', '01-ai/Yi-6B-Chat-4bits'),
|
||||
Model('01ai/Yi-6B-Chat-8bits', '01-ai/Yi-6B-Chat-8bits'),
|
||||
Model('01ai/Yi-9B', '01-ai/Yi-9B'),
|
||||
Model('01ai/Yi-9B-200K', '01-ai/Yi-9B-200K'),
|
||||
Model('01ai/Yi-34B', '01-ai/Yi-34B'),
|
||||
Model('01ai/Yi-34B-200K', '01-ai/Yi-34B-200K'),
|
||||
Model('01ai/Yi-34B-Chat', '01-ai/Yi-34B-Chat'),
|
||||
Model('01ai/Yi-34B-Chat-4bits', '01-ai/Yi-34B-Chat-4bits'),
|
||||
Model('01ai/Yi-34B-Chat-8bits', '01-ai/Yi-34B-Chat-8bits'),
|
||||
], TemplateType.chatml),
|
||||
# yi1.5
|
||||
ModelGroup([
|
||||
Model('01ai/Yi-1.5-6B', '01-ai/Yi-1.5-6B'),
|
||||
Model('01ai/Yi-1.5-6B-Chat', '01-ai/Yi-1.5-6B-Chat'),
|
||||
Model('01ai/Yi-1.5-9B', '01-ai/Yi-1.5-9B'),
|
||||
Model('01ai/Yi-1.5-9B-Chat', '01-ai/Yi-1.5-9B-Chat'),
|
||||
Model('01ai/Yi-1.5-9B-Chat-16K', '01-ai/Yi-1.5-9B-Chat-16K'),
|
||||
Model('01ai/Yi-1.5-34B', '01-ai/Yi-1.5-34B'),
|
||||
Model('01ai/Yi-1.5-34B-Chat', '01-ai/Yi-1.5-34B-Chat'),
|
||||
Model('01ai/Yi-1.5-34B-Chat-16K', '01-ai/Yi-1.5-34B-Chat-16K'),
|
||||
], TemplateType.chatml),
|
||||
# yi1.5-quant
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/Yi-1.5-6B-Chat-GPTQ', 'modelscope/Yi-1.5-6B-Chat-GPTQ'),
|
||||
Model('AI-ModelScope/Yi-1.5-6B-Chat-AWQ', 'modelscope/Yi-1.5-6B-Chat-AWQ'),
|
||||
Model('AI-ModelScope/Yi-1.5-9B-Chat-GPTQ', 'modelscope/Yi-1.5-9B-Chat-GPTQ'),
|
||||
Model('AI-ModelScope/Yi-1.5-9B-Chat-AWQ', 'modelscope/Yi-1.5-9B-Chat-AWQ'),
|
||||
Model('AI-ModelScope/Yi-1.5-34B-Chat-GPTQ', 'modelscope/Yi-1.5-34B-Chat-GPTQ'),
|
||||
Model('AI-ModelScope/Yi-1.5-34B-Chat-AWQ', 'modelscope/Yi-1.5-34B-Chat-AWQ'),
|
||||
], TemplateType.chatml),
|
||||
ModelGroup([
|
||||
Model('01ai/Yi-Coder-1.5B', '01-ai/Yi-Coder-1.5B'),
|
||||
Model('01ai/Yi-Coder-9B', '01-ai/Yi-Coder-9B'),
|
||||
Model('01ai/Yi-Coder-1.5B-Chat', '01-ai/Yi-Coder-1.5B-Chat'),
|
||||
Model('01ai/Yi-Coder-9B-Chat', '01-ai/Yi-Coder-9B-Chat'),
|
||||
],
|
||||
TemplateType.yi_coder,
|
||||
tags=['coding']),
|
||||
ModelGroup([
|
||||
Model('SUSTC/SUS-Chat-34B', 'SUSTech/SUS-Chat-34B'),
|
||||
], TemplateType.sus),
|
||||
],
|
||||
architectures=['LlamaForCausalLM'],
|
||||
mcore_model_type='gpt',
|
||||
model_arch=ModelArch.llama,
|
||||
))
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from transformers.utils import strtobool
|
||||
|
||||
from .fsdp import NPUCastError
|
||||
from .mindspeed import patch_mindspeed_te_cp_implementation
|
||||
|
||||
_APPLIED = False
|
||||
_ENABLE_NPU_MODEL_PATCH_ARGS = ('--enable_npu_model_patch', '--enable-npu-model-patch')
|
||||
|
||||
|
||||
def _parse_model_patch_enabled(value: str) -> bool:
|
||||
try:
|
||||
return bool(strtobool(value))
|
||||
except ValueError as exc:
|
||||
raise ValueError('--enable_npu_model_patch must be true or false.') from exc
|
||||
|
||||
|
||||
def _is_model_patch_enabled_from_argv() -> bool:
|
||||
for i, arg in enumerate(sys.argv):
|
||||
if arg in _ENABLE_NPU_MODEL_PATCH_ARGS:
|
||||
if i + 1 >= len(sys.argv) or sys.argv[i + 1].startswith('--'):
|
||||
raise ValueError('--enable_npu_model_patch requires a value: true or false.')
|
||||
return _parse_model_patch_enabled(sys.argv[i + 1])
|
||||
if any(arg.startswith(f'{name}=') for name in _ENABLE_NPU_MODEL_PATCH_ARGS):
|
||||
value = arg.split('=', 1)[1]
|
||||
return _parse_model_patch_enabled(value)
|
||||
return True
|
||||
|
||||
|
||||
def apply_all_patches() -> None:
|
||||
global _APPLIED
|
||||
if _APPLIED:
|
||||
return
|
||||
|
||||
from . import env, fsdp
|
||||
|
||||
env.apply_patch()
|
||||
fsdp.apply_patch()
|
||||
# The model patch switch is checked only on the first import; monkey patches are not reversible.
|
||||
if _is_model_patch_enabled_from_argv():
|
||||
from . import model
|
||||
model.apply_patch()
|
||||
_APPLIED = True
|
||||
|
||||
|
||||
__all__ = ['NPUCastError', 'apply_all_patches', 'patch_mindspeed_te_cp_implementation']
|
||||
@@ -0,0 +1,51 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from swift.utils.logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
_DEFAULT_NPU_HCCL_CONNECT_TIMEOUT = '600'
|
||||
_TORCH_NPU_GETENV_MODULE = 'torch_npu.utils.patch_getenv'
|
||||
|
||||
|
||||
def _patch_torch_npu_getenv() -> None:
|
||||
try:
|
||||
from torch_npu.utils import patch_getenv
|
||||
except Exception:
|
||||
return
|
||||
|
||||
orig_environ_get = getattr(patch_getenv, '_orig_environ_get', None)
|
||||
current_get = os.environ.get
|
||||
current_getenv = os.getenv
|
||||
getenv_module = getattr(current_getenv, '__module__', None)
|
||||
environ_get_module = getattr(current_get, '__module__', None)
|
||||
if not (getenv_module == _TORCH_NPU_GETENV_MODULE or environ_get_module == _TORCH_NPU_GETENV_MODULE):
|
||||
return
|
||||
if getattr(orig_environ_get, '__self__', None) is None:
|
||||
return
|
||||
|
||||
log_once = getattr(patch_getenv, '_log_once', None)
|
||||
|
||||
def _get_from_current_environ(key, default=None):
|
||||
hit = key in os.environ
|
||||
value = os.environ[key] if hit else default
|
||||
if hit and isinstance(value, str) and value != '' and log_once is not None:
|
||||
log_once(key, value)
|
||||
return value
|
||||
|
||||
os.getenv = _get_from_current_environ
|
||||
os.environ.get = _get_from_current_environ
|
||||
logger.info('Patched torch_npu getenv to read from current os.environ.')
|
||||
|
||||
|
||||
def apply_patch() -> None:
|
||||
_patch_torch_npu_getenv()
|
||||
|
||||
if 'HCCL_CONNECT_TIMEOUT' in os.environ:
|
||||
return
|
||||
|
||||
os.environ['HCCL_CONNECT_TIMEOUT'] = _DEFAULT_NPU_HCCL_CONNECT_TIMEOUT
|
||||
logger.info(f'Set HCCL_CONNECT_TIMEOUT={_DEFAULT_NPU_HCCL_CONNECT_TIMEOUT} by default for NPU.')
|
||||
@@ -0,0 +1,86 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
import accelerate.utils.fsdp_utils as fsdp_utils
|
||||
import torch
|
||||
from accelerate.accelerator import Accelerator
|
||||
from functools import wraps
|
||||
|
||||
|
||||
class NPUCastError(RuntimeError):
|
||||
"""Raised when fp32 casting fails during NPU FSDP2 preparation."""
|
||||
|
||||
|
||||
def _cast_module_to_fp32_for_npu_if_needed(module: torch.nn.Module, accelerator: Accelerator) -> torch.nn.Module:
|
||||
if accelerator.device.type != 'npu':
|
||||
return module
|
||||
|
||||
param = next(module.parameters(recurse=True), None)
|
||||
if param is None:
|
||||
return module
|
||||
|
||||
if not param.is_floating_point() or param.dtype == torch.float32:
|
||||
return module
|
||||
|
||||
# Accelerate FSDP2 flattens and shards parameters during prepare. On NPU,
|
||||
# entering that path with bf16/fp16 parameters can fail before mixed
|
||||
# precision policy has a chance to manage runtime compute dtype. Cast early
|
||||
# while parameters are still on CPU or meta, so only dtype changes here.
|
||||
|
||||
# GRPO with vLLM colocate mode may preload the model onto NPU before
|
||||
# Accelerator.prepare() is called. In that case, casting fp32 on NPU
|
||||
# would temporarily duplicate the full model (bf16 + fp32), causing OOM.
|
||||
# We move the model back to CPU first to free NPU memory, then cast.
|
||||
try:
|
||||
if param.device.type == 'npu':
|
||||
import torch_npu
|
||||
module = module.cpu()
|
||||
torch_npu.npu.synchronize()
|
||||
torch_npu.npu.empty_cache()
|
||||
return module.to(torch.float32)
|
||||
except Exception as exc:
|
||||
raise NPUCastError(f'Failed to cast {module.__class__.__name__} to fp32.') from exc
|
||||
|
||||
|
||||
_original_fsdp2_prepare_model = fsdp_utils.fsdp2_prepare_model
|
||||
|
||||
|
||||
@wraps(_original_fsdp2_prepare_model)
|
||||
def wrapped_fsdp2_prepare_model(
|
||||
accelerator: Accelerator,
|
||||
model: torch.nn.Module,
|
||||
):
|
||||
# Public utility entry used by some code paths before Accelerator.prepare.
|
||||
model = _cast_module_to_fp32_for_npu_if_needed(model, accelerator)
|
||||
return _original_fsdp2_prepare_model(accelerator, model)
|
||||
|
||||
|
||||
_original_prepare_fsdp2 = Accelerator._prepare_fsdp2
|
||||
|
||||
|
||||
@wraps(_original_prepare_fsdp2)
|
||||
def wrapped_prepare_fsdp2(
|
||||
self: Accelerator,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
# Accelerator.prepare may receive one or more modules directly; patch this
|
||||
# private entry too so all FSDP2 NPU preparation paths get the same fp32 cast.
|
||||
patched_args = [
|
||||
_cast_module_to_fp32_for_npu_if_needed(obj, self) if isinstance(obj, torch.nn.Module) else obj for obj in args
|
||||
]
|
||||
|
||||
return _original_prepare_fsdp2(self, *patched_args, **kwargs)
|
||||
|
||||
|
||||
_APPLIED = False
|
||||
|
||||
|
||||
def apply_patch() -> None:
|
||||
global _APPLIED
|
||||
if _APPLIED:
|
||||
return
|
||||
|
||||
fsdp_utils.fsdp2_prepare_model = wrapped_fsdp2_prepare_model
|
||||
Accelerator._prepare_fsdp2 = wrapped_prepare_fsdp2
|
||||
_APPLIED = True
|
||||
@@ -0,0 +1,209 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
"""NPU-only Megatron checkpoint compatibility helpers.
|
||||
|
||||
MindSpeed patches Megatron's distributed optimizer on NPU, but some Megatron-Core
|
||||
checkpoint formats still need the native Megatron param_state loaders.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from contextlib import contextmanager
|
||||
|
||||
from swift.utils import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def _iter_optimizer_param_groups(optimizer):
|
||||
visited = set()
|
||||
|
||||
def visit(obj):
|
||||
if obj is None or id(obj) in visited:
|
||||
return
|
||||
visited.add(id(obj))
|
||||
|
||||
param_groups = getattr(obj, 'param_groups', None)
|
||||
if param_groups is not None:
|
||||
yield param_groups
|
||||
|
||||
inner_optimizer = getattr(obj, 'optimizer', None)
|
||||
if inner_optimizer is not obj:
|
||||
yield from visit(inner_optimizer)
|
||||
|
||||
for child in getattr(obj, 'chained_optimizers', []) or []:
|
||||
yield from visit(child)
|
||||
for child in getattr(obj, 'sub_optimizers', []) or []:
|
||||
yield from visit(child)
|
||||
|
||||
yield from visit(optimizer)
|
||||
|
||||
|
||||
def _step_to_int(step):
|
||||
if isinstance(step, torch.Tensor):
|
||||
if step.numel() != 1:
|
||||
raise RuntimeError(f'Optimizer step tensor must be scalar, got shape: {tuple(step.shape)}')
|
||||
return int(step.item())
|
||||
return int(step)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _canonicalize_optimizer_steps_for_checkpoint(optimizer):
|
||||
"""Normalize NPU scalar step tensors while Megatron builds optimizer checkpoint state.
|
||||
|
||||
Megatron-Core deduplicates param-group steps with set(). Equal NPU scalar
|
||||
tensors can still hash as distinct objects, so use their numeric value only
|
||||
while sharded_state_dict() is being built and restore the optimizer in place.
|
||||
"""
|
||||
saved_steps = []
|
||||
numeric_steps = set()
|
||||
for param_groups in _iter_optimizer_param_groups(optimizer):
|
||||
for param_group in param_groups:
|
||||
if len(param_group.get('params', [])) == 0 or 'step' not in param_group:
|
||||
continue
|
||||
step = param_group['step']
|
||||
numeric_step = _step_to_int(step)
|
||||
saved_steps.append((param_group, step))
|
||||
numeric_steps.add(numeric_step)
|
||||
|
||||
if len(numeric_steps) > 1:
|
||||
raise RuntimeError(f'Inconsistent optimizer steps before checkpoint save: {sorted(numeric_steps)}')
|
||||
|
||||
canonical_step = next(iter(numeric_steps), None)
|
||||
try:
|
||||
if canonical_step is not None:
|
||||
for param_group, _step in saved_steps:
|
||||
param_group['step'] = canonical_step
|
||||
if any(isinstance(step, torch.Tensor) for _param_group, step in saved_steps):
|
||||
logger.warning(f'Canonicalized optimizer param-group step to {canonical_step} for checkpoint save.')
|
||||
yield
|
||||
finally:
|
||||
for param_group, step in saved_steps:
|
||||
param_group['step'] = step
|
||||
|
||||
|
||||
def optimizer_sharded_state_dict(optimizer, state_dict, **optim_sd_kwargs):
|
||||
with _canonicalize_optimizer_steps_for_checkpoint(optimizer):
|
||||
return optimizer.sharded_state_dict(state_dict, **optim_sd_kwargs)
|
||||
|
||||
|
||||
def _iter_distributed_optimizers(optimizer):
|
||||
visited = set()
|
||||
|
||||
def visit(obj):
|
||||
if obj is None or id(obj) in visited:
|
||||
return
|
||||
visited.add(id(obj))
|
||||
|
||||
if hasattr(obj, 'load_parameter_state_from_dp_reshardable') or hasattr(
|
||||
obj, 'load_parameter_state_from_fully_reshardable'):
|
||||
yield obj
|
||||
return
|
||||
|
||||
for child in getattr(obj, 'chained_optimizers', []) or []:
|
||||
yield from visit(child)
|
||||
for child in getattr(obj, 'sub_optimizers', []) or []:
|
||||
yield from visit(child)
|
||||
|
||||
yield from visit(optimizer)
|
||||
|
||||
|
||||
def _has_mindspeed_patched_load_state_dict(distributed_optimizer):
|
||||
load_state_dict = getattr(type(distributed_optimizer), 'load_state_dict', None)
|
||||
return getattr(load_state_dict, '__module__', '').startswith('mindspeed.')
|
||||
|
||||
|
||||
_MEGATRON_RESHARDABLE_PARAM_STATE_LOADERS = {
|
||||
'dp_reshardable': 'load_parameter_state_from_dp_reshardable',
|
||||
'fully_reshardable': 'load_parameter_state_from_fully_reshardable',
|
||||
}
|
||||
|
||||
|
||||
def _current_npu_device():
|
||||
if hasattr(torch, 'npu'):
|
||||
return torch.device('npu', torch.npu.current_device())
|
||||
return torch.cuda.current_device()
|
||||
|
||||
|
||||
def _restore_mindspeed_optimizer_step_tensors(optimizer):
|
||||
restored_count = 0
|
||||
for param_groups in _iter_optimizer_param_groups(optimizer):
|
||||
for param_group in param_groups:
|
||||
step = param_group.get('step')
|
||||
if isinstance(step, torch.Tensor):
|
||||
continue
|
||||
if isinstance(step, (int, float)):
|
||||
param_group['step'] = torch.tensor(int(step), dtype=torch.int64, device=_current_npu_device())
|
||||
restored_count += 1
|
||||
if restored_count:
|
||||
logger.warning(f'Restored {restored_count} MindSpeed optimizer param-group step values to NPU tensors.')
|
||||
|
||||
|
||||
def _split_chained_optimizer_state_dict(chained_optimizers, state_dict):
|
||||
if isinstance(state_dict, dict):
|
||||
state_dicts = [v for _k, v in sorted(state_dict.items())]
|
||||
else:
|
||||
state_dicts = list(state_dict)
|
||||
if len(chained_optimizers) != len(state_dicts):
|
||||
raise RuntimeError(
|
||||
f'Expected {len(chained_optimizers)} entries in optimizer state dict, but got {len(state_dicts)}.')
|
||||
return state_dicts
|
||||
|
||||
|
||||
def _load_chained_optimizer_state_dict(optimizer, state_dict):
|
||||
chained_optimizers = getattr(optimizer, 'chained_optimizers', None)
|
||||
if not chained_optimizers or len(chained_optimizers) <= 1:
|
||||
return False
|
||||
|
||||
state_dicts = _split_chained_optimizer_state_dict(chained_optimizers, state_dict)
|
||||
for child_optimizer, child_state_dict in zip(chained_optimizers, state_dicts):
|
||||
load_optimizer_state_dict(child_optimizer, child_state_dict)
|
||||
synchronize_steps = getattr(optimizer, '_synchronize_steps', None)
|
||||
if synchronize_steps is not None:
|
||||
synchronize_steps()
|
||||
return True
|
||||
|
||||
|
||||
def load_optimizer_state_dict(optimizer, state_dict):
|
||||
if _load_chained_optimizer_state_dict(optimizer, state_dict):
|
||||
return
|
||||
|
||||
distributed_optimizers = list(_iter_distributed_optimizers(optimizer))
|
||||
mindspeed_patched = any(
|
||||
_has_mindspeed_patched_load_state_dict(distributed_optimizer)
|
||||
for distributed_optimizer in distributed_optimizers)
|
||||
sharding_type = state_dict.get('param_state_sharding_type') if isinstance(state_dict, dict) else None
|
||||
native_loader_name = _MEGATRON_RESHARDABLE_PARAM_STATE_LOADERS.get(sharding_type)
|
||||
if native_loader_name is None:
|
||||
optimizer.load_state_dict(state_dict)
|
||||
if mindspeed_patched:
|
||||
_restore_mindspeed_optimizer_step_tensors(optimizer)
|
||||
return
|
||||
|
||||
if not mindspeed_patched:
|
||||
optimizer.load_state_dict(state_dict)
|
||||
return
|
||||
|
||||
if len(distributed_optimizers) != 1:
|
||||
raise RuntimeError(f'MindSpeed optimizer checkpoint compatibility supports exactly one distributed optimizer, '
|
||||
f'got {len(distributed_optimizers)}.')
|
||||
distributed_optimizer = distributed_optimizers[0]
|
||||
if not hasattr(distributed_optimizer, native_loader_name):
|
||||
raise RuntimeError(f'Distributed optimizer does not support sharding type {sharding_type}.')
|
||||
|
||||
state_dict_without_param_state = dict(state_dict)
|
||||
param_state = state_dict_without_param_state.pop('param_state', None)
|
||||
state_dict_without_param_state.pop('param_state_sharding_type', None)
|
||||
if param_state is None:
|
||||
raise RuntimeError(f'Optimizer checkpoint missing param_state for sharding type {sharding_type}.')
|
||||
|
||||
logger.warning(f'Loading optimizer param_state with ms-swift compatibility path because MindSpeed '
|
||||
f'DistributedOptimizer.load_state_dict does not support {sharding_type}.')
|
||||
# Let MindSpeed restore the generic optimizer state; load the missing
|
||||
# reshardable param_state with Megatron-Core's native implementation.
|
||||
optimizer.load_state_dict(state_dict_without_param_state)
|
||||
_restore_mindspeed_optimizer_step_tensors(optimizer)
|
||||
getattr(distributed_optimizer, native_loader_name)(param_state)
|
||||
|
||||
|
||||
__all__ = ['load_optimizer_state_dict', 'optimizer_sharded_state_dict']
|
||||
@@ -0,0 +1,52 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from swift.utils.logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
_ORIGINAL_MINDSPEED_TE_CP_CLASS = None
|
||||
|
||||
|
||||
def patch_mindspeed_te_cp_implementation(megatron_args: dict[str, Any]) -> None:
|
||||
"""
|
||||
Route NPU CP to the legacy MindSpeed TE adaptor when the new strategy factory
|
||||
only supports kvallgather.
|
||||
"""
|
||||
# MindSpeed 0.15.3 replaced the TE context-parallel attention class with a
|
||||
# new implementation. That new class does not yet cover all CP algorithms,
|
||||
# so the default non-kvallgather path can fail during Megatron training.
|
||||
# For those algorithms, temporarily route TE attention back to the legacy
|
||||
# MindSpeedCPDotProductAttention adaptor. Once MindSpeed's new CP class has
|
||||
# feature parity, this compatibility patch can be removed.
|
||||
try:
|
||||
import mindspeed.te.pytorch.attention.dot_product_attention.dot_product_attention as ms_te_dpa
|
||||
from mindspeed.core.context_parallel.adaptor import MindSpeedCPDotProductAttention
|
||||
except ImportError as e:
|
||||
logger.warning(f'Failed to import MindSpeed CP modules before repatch: {e}')
|
||||
return
|
||||
|
||||
global _ORIGINAL_MINDSPEED_TE_CP_CLASS
|
||||
if _ORIGINAL_MINDSPEED_TE_CP_CLASS is None:
|
||||
_ORIGINAL_MINDSPEED_TE_CP_CLASS = getattr(ms_te_dpa, 'MindSpeedTEDotProductAttention', None)
|
||||
|
||||
if _ORIGINAL_MINDSPEED_TE_CP_CLASS is None:
|
||||
logger.warning('MindSpeedTEDotProductAttention is unavailable before repatch; skip CP workaround.')
|
||||
return
|
||||
|
||||
cp_algo = megatron_args.get('context_parallel_algo', 'megatron_cp_algo')
|
||||
use_legacy_cp_te = int(megatron_args.get('context_parallel_size', 1)) > 1 and cp_algo != 'kvallgather_cp_algo'
|
||||
target_cls = MindSpeedCPDotProductAttention if use_legacy_cp_te else _ORIGINAL_MINDSPEED_TE_CP_CLASS
|
||||
|
||||
if getattr(ms_te_dpa, 'MindSpeedTEDotProductAttention', None) is target_cls:
|
||||
return
|
||||
|
||||
ms_te_dpa.MindSpeedTEDotProductAttention = target_cls
|
||||
logger.info(
|
||||
'Patched MindSpeedTEDotProductAttention to %s for context_parallel_size=%s, context_parallel_algo=%s.',
|
||||
target_cls.__name__,
|
||||
megatron_args.get('context_parallel_size', 1),
|
||||
cp_algo,
|
||||
)
|
||||
@@ -0,0 +1,552 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import torch_npu
|
||||
from torch import nn
|
||||
from transformers.models.qwen2 import modeling_qwen2
|
||||
from transformers.models.qwen3 import modeling_qwen3
|
||||
from transformers.models.qwen3_moe import modeling_qwen3_moe
|
||||
from transformers.models.qwen3_vl_moe import modeling_qwen3_vl_moe
|
||||
|
||||
from swift.utils.logger import get_logger
|
||||
from .utils import apply_patch_map, import_optional_module
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Common NPU helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_unsqueeze_dim(position_ids=None, unsqueeze_dim=1):
|
||||
if isinstance(position_ids, int) and unsqueeze_dim == 1:
|
||||
return position_ids
|
||||
return unsqueeze_dim
|
||||
|
||||
|
||||
def _get_hidden_size(module, hidden_states: torch.Tensor) -> int:
|
||||
return getattr(module, 'hidden_size', getattr(module, 'hidden_dim', hidden_states.shape[-1]))
|
||||
|
||||
|
||||
class NpuRMSNorm(nn.Module):
|
||||
|
||||
def __init__(self, hidden_size, eps=1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return torch_npu.npu_rms_norm(hidden_states, self.weight, epsilon=self.variance_epsilon)[0]
|
||||
|
||||
def extra_repr(self):
|
||||
return f'{tuple(self.weight.shape)}, eps={self.variance_epsilon}'
|
||||
|
||||
|
||||
class NpuGmmFunction(torch.autograd.Function):
|
||||
|
||||
@staticmethod
|
||||
def forward(ctx, x, weight, group_list, split_size):
|
||||
ctx.save_for_backward(x, weight)
|
||||
ctx.group_list = group_list
|
||||
ctx.split_size = split_size
|
||||
|
||||
outputs = torch_npu.npu_grouped_matmul([x], [weight], group_list=group_list, group_type=0, split_item=2)
|
||||
return outputs[0]
|
||||
|
||||
@staticmethod
|
||||
def backward(ctx, grad_outputs):
|
||||
x, weight = ctx.saved_tensors
|
||||
group_list = ctx.group_list
|
||||
wt = weight.permute(0, 2, 1)
|
||||
xt = x.permute(1, 0)
|
||||
dx = torch_npu.npu_grouped_matmul([grad_outputs], [wt], group_list=group_list, group_type=0, split_item=2)
|
||||
split_size = ctx.split_size
|
||||
xt_list = torch.split(xt, split_size, dim=1)
|
||||
grad_outputs_list = torch.split(grad_outputs, split_size, dim=0)
|
||||
with torch.npu.amp.autocast(enabled=False):
|
||||
dw = torch.stack([torch.matmul(xt_list[i], grad_outputs_list[i]) for i in range(len(xt_list))])
|
||||
|
||||
return dx[0], dw, None, None
|
||||
|
||||
|
||||
class GmmFunction(torch.autograd.Function):
|
||||
|
||||
@staticmethod
|
||||
def forward(ctx, x, weight, group_list):
|
||||
ctx.save_for_backward(x, weight)
|
||||
ctx.group_list = group_list
|
||||
|
||||
fwd_output = torch_npu.npu_grouped_matmul([x], [weight],
|
||||
bias=None,
|
||||
group_list=group_list,
|
||||
split_item=2,
|
||||
group_type=0,
|
||||
group_list_type=1)[0]
|
||||
return fwd_output
|
||||
|
||||
@staticmethod
|
||||
def backward(ctx, grad_output):
|
||||
input_tensor, weight = ctx.saved_tensors
|
||||
group_list = ctx.group_list
|
||||
|
||||
weight = torch.transpose(weight, 1, 2)
|
||||
grad_input = torch_npu.npu_grouped_matmul([grad_output], [weight],
|
||||
bias=None,
|
||||
group_list=group_list,
|
||||
split_item=2,
|
||||
group_type=0,
|
||||
group_list_type=1)[0]
|
||||
grad_weight = torch_npu.npu_grouped_matmul(
|
||||
[input_tensor.T],
|
||||
[grad_output],
|
||||
bias=None,
|
||||
group_list=group_list,
|
||||
split_item=3,
|
||||
group_type=2,
|
||||
group_list_type=1,
|
||||
)[0]
|
||||
return grad_input, grad_weight, None
|
||||
|
||||
|
||||
def _normalize_packed_expert_weights(module, input_dtype: torch.dtype, hidden_dim: int):
|
||||
gate_up_proj = module.gate_up_proj.to(input_dtype)
|
||||
down_proj = module.down_proj.to(input_dtype)
|
||||
|
||||
if gate_up_proj.shape[1] == hidden_dim:
|
||||
gate_up_weight = gate_up_proj
|
||||
elif gate_up_proj.shape[2] == hidden_dim:
|
||||
gate_up_weight = gate_up_proj.transpose(1, 2)
|
||||
else:
|
||||
raise RuntimeError(f'Unsupported gate_up_proj shape for NPU MoE patch: {tuple(gate_up_proj.shape)}.')
|
||||
|
||||
if down_proj.shape[2] == hidden_dim:
|
||||
down_weight = down_proj
|
||||
elif down_proj.shape[1] == hidden_dim:
|
||||
down_weight = down_proj.transpose(1, 2)
|
||||
else:
|
||||
raise RuntimeError(f'Unsupported down_proj shape for NPU MoE patch: {tuple(down_proj.shape)}.')
|
||||
|
||||
return gate_up_weight, down_weight
|
||||
|
||||
|
||||
def npu_packed_moe_experts_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_indices_or_routing_weights: torch.Tensor,
|
||||
routing_weights_or_router_indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if router_indices_or_routing_weights.dtype in {torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8}:
|
||||
router_indices = router_indices_or_routing_weights
|
||||
routing_weights = routing_weights_or_router_indices
|
||||
else:
|
||||
routing_weights = router_indices_or_routing_weights
|
||||
router_indices = routing_weights_or_router_indices
|
||||
|
||||
output_shape = hidden_states.shape
|
||||
hidden_dim = output_shape[-1]
|
||||
hidden_states = hidden_states.reshape(-1, hidden_dim)
|
||||
|
||||
if routing_weights.shape != router_indices.shape:
|
||||
routing_weights = torch.gather(routing_weights, dim=-1, index=router_indices.to(torch.long))
|
||||
routing_weights = routing_weights.to(hidden_states.dtype)
|
||||
router_indices = router_indices.to(torch.int32)
|
||||
|
||||
permuted_hidden_states, row_ids_map = torch_npu.npu_moe_token_permute(hidden_states, router_indices)
|
||||
tokens_per_expert = torch.histc(
|
||||
router_indices.to(torch.float), bins=self.num_experts, min=0, max=self.num_experts).to(torch.int64)
|
||||
gate_up_weight, down_weight = _normalize_packed_expert_weights(self, hidden_states.dtype, hidden_dim)
|
||||
|
||||
intermediate_hidden_states = GmmFunction.apply(permuted_hidden_states, gate_up_weight, tokens_per_expert)
|
||||
intermediate_activations = torch_npu.npu_swiglu(intermediate_hidden_states, dim=-1)
|
||||
output = GmmFunction.apply(intermediate_activations, down_weight, tokens_per_expert)
|
||||
next_states = torch_npu.npu_moe_token_unpermute(output, row_ids_map, probs=routing_weights)
|
||||
return next_states.view(*output_shape)
|
||||
|
||||
|
||||
def _topk_from_router_logits(module, hidden_states: torch.Tensor, router_logits: torch.Tensor):
|
||||
routing_weights = F.softmax(router_logits, dim=-1, dtype=torch.float)
|
||||
routing_weights, router_indices = torch.topk(routing_weights, module.top_k, dim=-1)
|
||||
if getattr(module, 'norm_topk_prob', True):
|
||||
routing_weights = routing_weights / routing_weights.sum(dim=-1, keepdim=True)
|
||||
routing_weights = routing_weights.to(hidden_states.dtype)
|
||||
return routing_weights, router_indices
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Qwen2/Qwen3 dense patch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def npu_apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
||||
"""Applies Rotary Position Embedding to the query and key tensors."""
|
||||
unsqueeze_dim = _resolve_unsqueeze_dim(position_ids, unsqueeze_dim)
|
||||
cos = cos.unsqueeze(unsqueeze_dim)
|
||||
sin = sin.unsqueeze(unsqueeze_dim)
|
||||
q_embed = torch_npu.npu_rotary_mul(q, cos, sin)
|
||||
k_embed = torch_npu.npu_rotary_mul(k, cos, sin)
|
||||
return q_embed, k_embed
|
||||
|
||||
|
||||
def npu_swiglu_forward(self, hidden_state):
|
||||
return self.down_proj(
|
||||
torch_npu.npu_swiglu(torch.cat((self.gate_proj(hidden_state), self.up_proj(hidden_state)), dim=-1), dim=-1))
|
||||
|
||||
|
||||
QWEN2_PATCHES = {
|
||||
'Qwen2RMSNorm': NpuRMSNorm,
|
||||
'apply_rotary_pos_emb': npu_apply_rotary_pos_emb,
|
||||
'Qwen2MLP.forward': npu_swiglu_forward,
|
||||
}
|
||||
|
||||
QWEN3_PATCHES = {
|
||||
'Qwen3RMSNorm': NpuRMSNorm,
|
||||
'apply_rotary_pos_emb': npu_apply_rotary_pos_emb,
|
||||
'Qwen3MLP.forward': npu_swiglu_forward,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Qwen3.5 dense patch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NpuQwen3_5RMSNorm(nn.Module):
|
||||
|
||||
def __init__(self, dim, eps=1e-6):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.zeros(dim))
|
||||
|
||||
def forward(self, x):
|
||||
scale = (1.0 + self.weight).to(dtype=x.dtype)
|
||||
return torch_npu.npu_rms_norm(x, scale, epsilon=self.eps)[0]
|
||||
|
||||
def extra_repr(self):
|
||||
return f'{tuple(self.weight.shape)}, eps={self.eps}'
|
||||
|
||||
|
||||
def npu_apply_rotary_pos_emb_qwen3_5(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
||||
unsqueeze_dim = _resolve_unsqueeze_dim(position_ids, unsqueeze_dim)
|
||||
cos = cos.unsqueeze(unsqueeze_dim)
|
||||
sin = sin.unsqueeze(unsqueeze_dim)
|
||||
|
||||
rotary_dim = cos.shape[-1]
|
||||
q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
|
||||
k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
|
||||
|
||||
q_rot = torch_npu.npu_rotary_mul(q_rot, cos, sin)
|
||||
k_rot = torch_npu.npu_rotary_mul(k_rot, cos, sin)
|
||||
|
||||
q_embed = torch.cat([q_rot, q_pass], dim=-1)
|
||||
k_embed = torch.cat([k_rot, k_pass], dim=-1)
|
||||
return q_embed, k_embed
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
_TRANSFORMERS_FLA_PROBE_MODULES = ('transformers.utils', 'transformers.utils.import_utils')
|
||||
|
||||
|
||||
def _patch_transformers_flash_linear_attention_available(available: bool) -> dict[str, object]:
|
||||
|
||||
def _is_flash_linear_attention_available() -> bool:
|
||||
return available
|
||||
|
||||
originals = {}
|
||||
for module_name in _TRANSFORMERS_FLA_PROBE_MODULES:
|
||||
module = import_optional_module(module_name)
|
||||
if module is None:
|
||||
continue
|
||||
originals[module_name] = getattr(module, 'is_flash_linear_attention_available', _MISSING)
|
||||
setattr(module, 'is_flash_linear_attention_available', _is_flash_linear_attention_available)
|
||||
return originals
|
||||
|
||||
|
||||
def _restore_transformers_flash_linear_attention_available(originals: dict[str, object]) -> None:
|
||||
for module_name, original in originals.items():
|
||||
module = import_optional_module(module_name)
|
||||
if module is None:
|
||||
continue
|
||||
if original is _MISSING:
|
||||
delattr(module, 'is_flash_linear_attention_available')
|
||||
else:
|
||||
setattr(module, 'is_flash_linear_attention_available', original)
|
||||
|
||||
|
||||
def patch_qwen3_5_chunk_gated_delta_rule_with_mindspeed() -> None:
|
||||
try:
|
||||
from ..chunk_gated_delta_rule import chunk_gated_delta_rule
|
||||
except ImportError as exc:
|
||||
logger.warning('Failed to import embedded MindSpeed chunk_gated_delta_rule: %s', exc)
|
||||
return
|
||||
|
||||
patched_modules = []
|
||||
for module_name in ('transformers.models.qwen3_5.modeling_qwen3_5',
|
||||
'transformers.models.qwen3_5_moe.modeling_qwen3_5_moe'):
|
||||
module = import_optional_module(module_name)
|
||||
if module is None:
|
||||
continue
|
||||
|
||||
setattr(module, 'is_flash_linear_attention_available', lambda: True)
|
||||
setattr(module, 'is_fast_path_available', True)
|
||||
# FLA's fused RMSNormGated initializes with torch.cuda.current_device(),
|
||||
# so keep the native Qwen3.5 torch implementation on NPU.
|
||||
setattr(module, 'FusedRMSNormGated', None)
|
||||
setattr(module, 'chunk_gated_delta_rule', chunk_gated_delta_rule)
|
||||
patched_modules.append(module_name)
|
||||
|
||||
if patched_modules:
|
||||
logger.info('Patched Qwen3.5 chunk_gated_delta_rule to embedded MindSpeed implementation: %s.',
|
||||
', '.join(patched_modules))
|
||||
|
||||
|
||||
QWEN3_5_PATCHES = {
|
||||
'Qwen3_5RMSNorm': NpuQwen3_5RMSNorm,
|
||||
'apply_rotary_pos_emb': npu_apply_rotary_pos_emb_qwen3_5,
|
||||
'Qwen3_5MLP.forward': npu_swiglu_forward,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Qwen3-MoE patch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _qwen3_moe_forward_transformers_457(self, hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_size, sequence_length, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
|
||||
routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
|
||||
if getattr(self, 'norm_topk_prob', False):
|
||||
routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
|
||||
routing_weights = routing_weights.to(hidden_states.dtype)
|
||||
|
||||
expert_mask = F.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
|
||||
|
||||
input_dtype = hidden_states.dtype
|
||||
up_weight_list = [expert.up_proj.weight.t().to(input_dtype) for expert in self.experts]
|
||||
gate_weight_list = [expert.gate_proj.weight.t().to(input_dtype) for expert in self.experts]
|
||||
down_weight_list = [expert.down_proj.weight.t().to(input_dtype) for expert in self.experts]
|
||||
w1 = torch.stack(up_weight_list)
|
||||
w2 = torch.stack(gate_weight_list)
|
||||
w3 = torch.stack(down_weight_list)
|
||||
|
||||
routing_map = selected_experts
|
||||
flatten_indices = routing_map.view(-1)
|
||||
sorted_indices = torch.sort(flatten_indices.float(), stable=True)[1]
|
||||
permuted_tokens = hidden_states.index_select(0, sorted_indices // self.top_k)
|
||||
|
||||
tokens_per_experts = torch.sum(expert_mask, dim=(1, 2))
|
||||
group_list = torch.cumsum(tokens_per_experts, dim=0)
|
||||
|
||||
cpu_group_list = group_list.to('cpu', non_blocking=False)
|
||||
cpu_group_list = [0] + cpu_group_list.tolist()
|
||||
split_size = [cpu_group_list[i + 1] - cpu_group_list[i] for i in range(len(cpu_group_list) - 1)]
|
||||
|
||||
up_res = NpuGmmFunction.apply(permuted_tokens, w1, group_list, split_size)
|
||||
gate_res = NpuGmmFunction.apply(permuted_tokens, w2, group_list, split_size)
|
||||
act_res = torch_npu.npu_swiglu(torch.cat([gate_res, up_res], dim=-1))
|
||||
down_res = NpuGmmFunction.apply(act_res, w3, group_list, split_size)
|
||||
|
||||
num_unpermuted_tokens = routing_weights.numel()
|
||||
unpermuted_tokens = torch.zeros(
|
||||
[num_unpermuted_tokens, down_res.shape[-1]],
|
||||
dtype=down_res.dtype,
|
||||
device=down_res.device,
|
||||
)
|
||||
unpermuted_tokens.index_copy_(0, sorted_indices, down_res)
|
||||
unpermuted_tokens = unpermuted_tokens.reshape(-1, self.top_k, down_res.size(-1))
|
||||
unpermuted_tokens = unpermuted_tokens * routing_weights.unsqueeze(-1)
|
||||
final_hidden_states = unpermuted_tokens.sum(dim=1).to(hidden_states.dtype)
|
||||
final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
|
||||
|
||||
return final_hidden_states, router_logits
|
||||
|
||||
|
||||
def _qwen3_moe_forward_transformers_5(self, hidden_states: torch.Tensor, routing_weights: torch.Tensor,
|
||||
selected_experts: torch.Tensor) -> torch.Tensor:
|
||||
batch_size, sequence_length, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
final_hidden_states = self.experts(hidden_states, selected_experts, routing_weights)
|
||||
return final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
|
||||
|
||||
|
||||
def npu_qwen3_moe_sparse_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
hidden_dim = hidden_states.shape[-1]
|
||||
gate_output = self.gate(hidden_states.view(-1, hidden_dim))
|
||||
|
||||
if isinstance(gate_output, tuple):
|
||||
# Transformers 5.x: gate is a router module and returns
|
||||
# (router_logits, routing_weights, selected_experts).
|
||||
_, routing_weights, selected_experts = gate_output
|
||||
return _qwen3_moe_forward_transformers_5(self, hidden_states, routing_weights, selected_experts)
|
||||
|
||||
# Transformers 4.57.x: gate is nn.Linear and returns router logits.
|
||||
return _qwen3_moe_forward_transformers_457(self, hidden_states, gate_output)
|
||||
|
||||
|
||||
QWEN3_MOE_PATCHES = {
|
||||
'Qwen3MoeRMSNorm': NpuRMSNorm,
|
||||
'apply_rotary_pos_emb': npu_apply_rotary_pos_emb,
|
||||
'Qwen3MoeSparseMoeBlock.forward': npu_qwen3_moe_sparse_block_forward,
|
||||
}
|
||||
|
||||
QWEN3_MOE_TRANSFORMERS_5_PATCHES = {
|
||||
'Qwen3MoeExperts.forward': npu_packed_moe_experts_forward,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Qwen3-VL-MoE patch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _qwen3_vl_moe_forward_transformers_457(self, hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor) -> torch.Tensor:
|
||||
batch_size = hidden_states.shape[0]
|
||||
hidden_size = _get_hidden_size(self, hidden_states)
|
||||
hidden_states = hidden_states.reshape(-1, hidden_size)
|
||||
|
||||
routing_weights, router_indices = _topk_from_router_logits(self, hidden_states, router_logits)
|
||||
hidden_states = hidden_states.reshape(batch_size, -1, hidden_size)
|
||||
return self.experts(hidden_states, routing_weights, router_indices)
|
||||
|
||||
|
||||
def _qwen3_vl_moe_forward_transformers_5(self, hidden_states: torch.Tensor, routing_weights: torch.Tensor,
|
||||
selected_experts: torch.Tensor) -> torch.Tensor:
|
||||
batch_size, sequence_length, hidden_size = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, hidden_size)
|
||||
routed_out = self.experts(hidden_states, selected_experts, routing_weights)
|
||||
return routed_out.reshape(batch_size, sequence_length, hidden_size)
|
||||
|
||||
|
||||
def npu_qwen3_vl_moe_sparse_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
hidden_size = _get_hidden_size(self, hidden_states)
|
||||
gate_output = self.gate(hidden_states.reshape(-1, hidden_size))
|
||||
|
||||
if isinstance(gate_output, tuple):
|
||||
# Transformers 5.x: gate is a router module and returns
|
||||
# (router_logits, routing_weights, selected_experts).
|
||||
_, routing_weights, selected_experts = gate_output
|
||||
return _qwen3_vl_moe_forward_transformers_5(self, hidden_states, routing_weights, selected_experts)
|
||||
|
||||
# Transformers 4.57.x: gate is nn.Linear and experts use the old
|
||||
# (hidden_states, routing_weights, router_indices) call order.
|
||||
return _qwen3_vl_moe_forward_transformers_457(self, hidden_states, gate_output)
|
||||
|
||||
|
||||
QWEN3_VL_MOE_PATCHES = {
|
||||
'Qwen3VLMoeTextExperts.forward': npu_packed_moe_experts_forward,
|
||||
'Qwen3VLMoeTextSparseMoeBlock.forward': npu_qwen3_vl_moe_sparse_block_forward,
|
||||
'Qwen3VLMoeTextRMSNorm': NpuRMSNorm,
|
||||
'apply_rotary_pos_emb': npu_apply_rotary_pos_emb,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Qwen3.5-MoE patch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _add_shared_expert(self, hidden_states: torch.Tensor, expert_output: torch.Tensor) -> torch.Tensor:
|
||||
if not (hasattr(self, 'shared_expert') and hasattr(self, 'shared_expert_gate')):
|
||||
return expert_output
|
||||
|
||||
shared_expert_output = self.shared_expert(hidden_states)
|
||||
shared_expert_output = F.sigmoid(self.shared_expert_gate(hidden_states)) * shared_expert_output
|
||||
return expert_output + shared_expert_output
|
||||
|
||||
|
||||
def _qwen3_5_moe_forward_transformers_5(self, hidden_states: torch.Tensor, routing_weights: torch.Tensor,
|
||||
selected_experts: torch.Tensor) -> torch.Tensor:
|
||||
batch_size, sequence_length, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
expert_output = self.experts(hidden_states, selected_experts, routing_weights)
|
||||
expert_output = _add_shared_expert(self, hidden_states, expert_output)
|
||||
return expert_output.reshape(batch_size, sequence_length, hidden_dim)
|
||||
|
||||
|
||||
def _qwen3_5_moe_forward_linear_gate(self, hidden_states: torch.Tensor, router_logits: torch.Tensor) -> torch.Tensor:
|
||||
batch_size, sequence_length, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
routing_weights, selected_experts = _topk_from_router_logits(self, hidden_states, router_logits)
|
||||
expert_output = self.experts(hidden_states, selected_experts, routing_weights)
|
||||
expert_output = _add_shared_expert(self, hidden_states, expert_output)
|
||||
return expert_output.reshape(batch_size, sequence_length, hidden_dim)
|
||||
|
||||
|
||||
def npu_qwen3_5_moe_sparse_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
hidden_dim = hidden_states.shape[-1]
|
||||
gate_output = self.gate(hidden_states.view(-1, hidden_dim))
|
||||
|
||||
if isinstance(gate_output, tuple):
|
||||
# Transformers 5.x: Qwen3.5-MoE has packed experts plus shared expert.
|
||||
_, routing_weights, selected_experts = gate_output
|
||||
return _qwen3_5_moe_forward_transformers_5(self, hidden_states, routing_weights, selected_experts)
|
||||
|
||||
return _qwen3_5_moe_forward_linear_gate(self, hidden_states, gate_output)
|
||||
|
||||
|
||||
QWEN3_5_MOE_PATCHES = {
|
||||
'Qwen3_5MoeRMSNorm': NpuQwen3_5RMSNorm,
|
||||
'apply_rotary_pos_emb': npu_apply_rotary_pos_emb_qwen3_5,
|
||||
'Qwen3_5MoeMLP.forward': npu_swiglu_forward,
|
||||
'Qwen3_5MoeExperts.forward': npu_packed_moe_experts_forward,
|
||||
'Qwen3_5MoeSparseMoeBlock.forward': npu_qwen3_5_moe_sparse_block_forward,
|
||||
}
|
||||
|
||||
QWEN3_5_MOE_OPTIONAL_PATCHES = {}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Patch table and apply entry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_patch_map(root, patches: dict[str, object], optional_patches: dict[str, object] | None = None):
|
||||
patch_map = dict(patches)
|
||||
for path, value in (optional_patches or {}).items():
|
||||
current = root
|
||||
for part in path.split('.'):
|
||||
if not hasattr(current, part):
|
||||
break
|
||||
current = getattr(current, part)
|
||||
else:
|
||||
patch_map[path] = value
|
||||
return patch_map
|
||||
|
||||
|
||||
_APPLIED = False
|
||||
|
||||
|
||||
def apply_patch() -> None:
|
||||
global _APPLIED
|
||||
if _APPLIED:
|
||||
return
|
||||
|
||||
patch_groups = [
|
||||
('qwen2', modeling_qwen2, QWEN2_PATCHES, {}),
|
||||
('qwen3', modeling_qwen3, QWEN3_PATCHES, {}),
|
||||
('qwen3_moe', modeling_qwen3_moe, QWEN3_MOE_PATCHES, QWEN3_MOE_TRANSFORMERS_5_PATCHES),
|
||||
('qwen3_vl_moe', modeling_qwen3_vl_moe, QWEN3_VL_MOE_PATCHES, {}),
|
||||
]
|
||||
|
||||
# Qwen3.5 GDN is patched to swift's embedded MindSpeed implementation below.
|
||||
# Skip Transformers' import-time FLA probe so FLA is not a hard dependency.
|
||||
fla_probe_originals = _patch_transformers_flash_linear_attention_available(False)
|
||||
try:
|
||||
modeling_qwen3_5 = import_optional_module('transformers.models.qwen3_5.modeling_qwen3_5')
|
||||
modeling_qwen3_5_moe = import_optional_module('transformers.models.qwen3_5_moe.modeling_qwen3_5_moe')
|
||||
finally:
|
||||
_restore_transformers_flash_linear_attention_available(fla_probe_originals)
|
||||
if modeling_qwen3_5 is not None:
|
||||
patch_qwen3_5_chunk_gated_delta_rule_with_mindspeed()
|
||||
|
||||
if modeling_qwen3_5 is not None:
|
||||
patch_groups.append(('qwen3_5', modeling_qwen3_5, QWEN3_5_PATCHES, {}))
|
||||
|
||||
if modeling_qwen3_5_moe is not None:
|
||||
patch_groups.append(('qwen3_5_moe', modeling_qwen3_5_moe, QWEN3_5_MOE_PATCHES, QWEN3_5_MOE_OPTIONAL_PATCHES))
|
||||
|
||||
for _group_name, module, patches, optional_patches in patch_groups:
|
||||
apply_patch_map(module, _build_patch_map(module, patches, optional_patches))
|
||||
|
||||
_APPLIED = True
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import Any
|
||||
|
||||
from swift.utils.logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def import_optional_module(module_name: str) -> Any | None:
|
||||
try:
|
||||
return importlib.import_module(module_name)
|
||||
except ImportError as exc:
|
||||
logger.debug('Failed to import optional module %s: %s', module_name, exc)
|
||||
return None
|
||||
|
||||
|
||||
def apply_patch_map(root: Any, patch_map: dict[str, Any]) -> None:
|
||||
for path, value in patch_map.items():
|
||||
current = root
|
||||
parts = path.split('.')
|
||||
for part in parts[:-1]:
|
||||
current = getattr(current, part)
|
||||
setattr(current, parts[-1], value)
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
"""Facade for SWIFT's vLLM-Ascend NPU compatibility patches.
|
||||
|
||||
Keep this file thin. The real patches are split by responsibility:
|
||||
|
||||
* ``vllm_ascend_moe``: MoE routing and GRPO weight-sync layout handling.
|
||||
* ``vllm_ascend_memory``: small torch-npu/vLLM-Ascend memory API compatibility.
|
||||
Callers should import from this module so the public entrypoints stay stable,
|
||||
while reviewers can audit each patch family in its own file. The caller is
|
||||
still responsible for guarding these entrypoints with an NPU/device check.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from swift.model.npu_patch.vllm_ascend_memory import patch_vllm_ascend_memory_runtime
|
||||
from swift.model.npu_patch.vllm_ascend_moe import (patch_vllm_ascend_moe_expert_weight_loader,
|
||||
patch_vllm_ascend_moe_runtime, should_skip_vllm_ascend_moe_post_load,
|
||||
use_vllm_ascend_moe_preprocessed_weight)
|
||||
from swift.utils.logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def _patch_flash_attn_optional_import() -> None:
|
||||
"""Clear a stub ``flash_attn`` module that can block optional imports.
|
||||
|
||||
Some stacks insert a non-package ``flash_attn`` placeholder into
|
||||
``sys.modules``. vLLM import paths then treat it as the real package and
|
||||
fail on submodule imports. Removing the placeholder lets normal optional
|
||||
dependency checks proceed.
|
||||
"""
|
||||
module = sys.modules.get('flash_attn')
|
||||
if module is None or hasattr(module, '__path__'):
|
||||
return
|
||||
for module_name in list(sys.modules):
|
||||
if module_name == 'flash_attn' or module_name.startswith('flash_attn.'):
|
||||
sys.modules.pop(module_name, None)
|
||||
|
||||
|
||||
def patch_vllm_ascend_runtime(*, colocate: bool = False) -> None:
|
||||
"""Apply vLLM-Ascend patches needed by SWIFT NPU rollout.
|
||||
|
||||
``colocate=False`` covers patches that are also safe for standalone
|
||||
vLLM-Ascend server/native inference, such as optional import cleanup, MoE
|
||||
routing, and ``mem_get_info`` binding compatibility.
|
||||
|
||||
``colocate`` is kept in the public signature for callers that share this
|
||||
entrypoint between server and colocate modes. Process-group creation is
|
||||
left to upstream vLLM/vLLM-Ascend; SWIFT only keeps the narrow runtime
|
||||
compatibility patches below.
|
||||
"""
|
||||
_patch_flash_attn_optional_import()
|
||||
patch_vllm_ascend_moe_runtime()
|
||||
patch_vllm_ascend_memory_runtime()
|
||||
|
||||
|
||||
__all__ = [
|
||||
'patch_vllm_ascend_moe_expert_weight_loader',
|
||||
'patch_vllm_ascend_runtime',
|
||||
'should_skip_vllm_ascend_moe_post_load',
|
||||
'use_vllm_ascend_moe_preprocessed_weight',
|
||||
]
|
||||
@@ -0,0 +1,91 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
"""Small vLLM-Ascend memory API compatibility patches.
|
||||
|
||||
This module intentionally avoids colocate memory-policy changes. It only
|
||||
normalizes API differences that are safe for both standalone vLLM-Ascend
|
||||
inference and SWIFT GRPO rollout.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
|
||||
_ORIGIN_TORCH_NPU_MEM_GET_INFO = None
|
||||
_BOUND_TORCH_NPU_MEM_GET_INFO_DEVICE = None
|
||||
|
||||
|
||||
def _patch_vllm_ascend_mem_get_info() -> None:
|
||||
"""Patch ``NPUPlatform.mem_get_info`` for torch-npu binding differences.
|
||||
|
||||
vLLM-Ascend calls ``current_platform.mem_get_info(device)`` during worker
|
||||
initialization. Without this wrapper, some versions expose
|
||||
``NPUPlatform.mem_get_info`` in a way that gets Python method binding plus
|
||||
the explicit device argument at the same time, producing:
|
||||
|
||||
TypeError: mem_get_info() got multiple values for argument 'device'
|
||||
|
||||
Defining a classmethod here gives vLLM-Ascend one stable call surface. It
|
||||
keeps the device-aware torch-npu query when available and falls back to the
|
||||
no-argument query only when torch-npu rejects the keyword. This does not
|
||||
change memory profiling policy.
|
||||
"""
|
||||
try:
|
||||
from vllm_ascend.platform import NPUPlatform
|
||||
except (ImportError, AttributeError):
|
||||
return
|
||||
if getattr(NPUPlatform, '_swift_mem_get_info_patched', False):
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def mem_get_info(cls, device=None):
|
||||
if device is None:
|
||||
return torch.npu.mem_get_info()
|
||||
try:
|
||||
return torch.npu.mem_get_info(device=device)
|
||||
except TypeError:
|
||||
return torch.npu.mem_get_info()
|
||||
|
||||
NPUPlatform.mem_get_info = mem_get_info
|
||||
NPUPlatform._swift_mem_get_info_patched = True
|
||||
|
||||
|
||||
def patch_vllm_ascend_memory_runtime() -> None:
|
||||
"""Apply memory patches that do not depend on colocated training."""
|
||||
_patch_vllm_ascend_mem_get_info()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def vllm_ascend_mem_get_info_context(vllm_device: str):
|
||||
"""Bind bare ``torch.npu.mem_get_info()`` calls to vLLM's device.
|
||||
|
||||
Most vLLM memory accounting goes through ``NPUPlatform.mem_get_info`` and is
|
||||
handled by ``patch_vllm_ascend_memory_runtime`` above. Some vLLM-Ascend
|
||||
paths still call ``torch.npu.mem_get_info()`` directly, or assign it to
|
||||
``torch.cuda.mem_get_info`` for CUDA-compatible worker code.
|
||||
|
||||
Keep this binding for the process lifetime after the context exits. vLLM
|
||||
sleep/wake paths can call bare ``torch.npu.mem_get_info()`` after engine
|
||||
construction, so restoring here would regress the original behavior in
|
||||
``swift.infer_engine.utils.patch_npu_vllm``. Re-entering with another device
|
||||
rebinds from the original function instead of stacking nested partials.
|
||||
"""
|
||||
global _ORIGIN_TORCH_NPU_MEM_GET_INFO, _BOUND_TORCH_NPU_MEM_GET_INFO_DEVICE
|
||||
|
||||
if (_ORIGIN_TORCH_NPU_MEM_GET_INFO is None
|
||||
or getattr(torch.npu.mem_get_info, '_swift_bound_mem_get_info_device', None) is None):
|
||||
_ORIGIN_TORCH_NPU_MEM_GET_INFO = torch.npu.mem_get_info
|
||||
|
||||
if _BOUND_TORCH_NPU_MEM_GET_INFO_DEVICE != vllm_device:
|
||||
mem_get_info = partial(_ORIGIN_TORCH_NPU_MEM_GET_INFO, device=vllm_device)
|
||||
mem_get_info._swift_bound_mem_get_info_device = vllm_device
|
||||
torch.npu.mem_get_info = mem_get_info
|
||||
_BOUND_TORCH_NPU_MEM_GET_INFO_DEVICE = vllm_device
|
||||
|
||||
yield
|
||||
|
||||
|
||||
__all__ = [
|
||||
'patch_vllm_ascend_memory_runtime',
|
||||
'vllm_ascend_mem_get_info_context',
|
||||
]
|
||||
@@ -0,0 +1,394 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
"""vLLM-Ascend MoE patches used by SWIFT NPU rollout.
|
||||
|
||||
There are two independent responsibilities in this file:
|
||||
|
||||
* runtime routing: avoid the unstable custom non-quantized MoE routing op on
|
||||
stacks where vLLM-Ascend still dispatches that branch to
|
||||
``aclnnMoeInitRoutingCustom``;
|
||||
* weight sync: adapt 2D HF/Megatron MoE expert weights to the already-processed
|
||||
3D vLLM-Ascend expert parameter layout during GRPO colocate updates.
|
||||
|
||||
Both patches are guarded by vLLM-Ascend implementation checks and only touch the
|
||||
specific MoE paths they need.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import torch
|
||||
|
||||
from swift.utils.logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
_VLLM_ASCEND_MOE_SYNC_LAYOUT_ATTR = '_swift_vllm_ascend_moe_weight_sync_layout'
|
||||
_VLLM_ASCEND_MOE_SKIP_POST_LOAD_ATTR = '_swift_vllm_ascend_moe_skip_post_load'
|
||||
_VLLM_ASCEND_MOE_PROCESSED_LAYOUT = 'megatron_processed'
|
||||
_VLLM_ASCEND_MOE_PREPROCESSED_LAYOUT = 'fsdp2_preprocessed'
|
||||
_QWEN_MOE_MODEL_TYPES = {'qwen3_moe', 'qwen3_5_moe'}
|
||||
|
||||
|
||||
def _patch_vllm_ascend_device_op_nonquant_routing() -> None:
|
||||
"""Use the stable torch-npu routing op for non-quantized MoE when needed.
|
||||
|
||||
Some released vLLM-Ascend versions route the non-quantized MoE case
|
||||
(``scale is None`` and ``quant_mode == -1``) through
|
||||
``npu_moe_init_routing_custom`` / ``aclnnMoeInitRoutingCustom``, which is
|
||||
not stable for the parameter combination used by Qwen-style MoE rollout.
|
||||
|
||||
This is intentionally gated by implementation detection instead of a fixed
|
||||
version threshold: source builds or future/backported versions may already
|
||||
dispatch the non-quantized path to ``torch_npu.npu_moe_init_routing_v2``.
|
||||
When that fixed branch is present, skip patching and keep the upstream
|
||||
implementation intact.
|
||||
|
||||
Do not probe the custom op by calling it first. On Ascend, a missing custom
|
||||
binary can be reported asynchronously: even if Python catches the immediate
|
||||
RuntimeError and falls back, the failed launch can poison the stream and hang
|
||||
later at an unrelated event synchronization. Therefore, when source
|
||||
inspection shows that the non-quantized branch still routes to the custom op,
|
||||
dispatch that branch directly to ``torch_npu.npu_moe_init_routing_v2``.
|
||||
"""
|
||||
try:
|
||||
import torch_npu
|
||||
from vllm_ascend.device import device_op
|
||||
except (ImportError, AttributeError):
|
||||
return
|
||||
|
||||
adaptor_cls = getattr(device_op, 'BaseDeviceAdaptor', None)
|
||||
if adaptor_cls is None:
|
||||
return
|
||||
origin_routing = getattr(adaptor_cls, 'npu_moe_init_routing', None)
|
||||
if origin_routing is None or getattr(origin_routing, '_swift_nonquant_routing_patched', False):
|
||||
return
|
||||
try:
|
||||
origin_source = inspect.getsource(origin_routing)
|
||||
except (OSError, TypeError):
|
||||
origin_source = ''
|
||||
if 'npu_moe_init_routing_v2' in origin_source and 'quant_mode == -1' in origin_source:
|
||||
return
|
||||
origin_signature = inspect.signature(origin_routing)
|
||||
routing_defaults = {
|
||||
'scale': None,
|
||||
'active_num': None,
|
||||
'expert_num': None,
|
||||
'expert_tokens_num_type': 1,
|
||||
'expert_tokens_num_flag': True,
|
||||
'active_expert_range': None,
|
||||
'quant_mode': -1,
|
||||
}
|
||||
missing_params = set(routing_defaults).difference(origin_signature.parameters)
|
||||
if missing_params:
|
||||
raise RuntimeError('Unsupported vLLM-Ascend npu_moe_init_routing signature: '
|
||||
f'signature={origin_signature}, missing={sorted(missing_params)}.')
|
||||
|
||||
def is_nonquant_routing(routing_kwargs) -> bool:
|
||||
return routing_kwargs['scale'] is None and routing_kwargs['quant_mode'] == -1
|
||||
|
||||
def npu_moe_init_routing_v2(hidden_states, topk_ids, routing_kwargs):
|
||||
active_num = routing_kwargs['active_num']
|
||||
expert_num = routing_kwargs['expert_num']
|
||||
active_expert_range = routing_kwargs['active_expert_range']
|
||||
return torch_npu.npu_moe_init_routing_v2(
|
||||
hidden_states,
|
||||
topk_ids,
|
||||
scale=None,
|
||||
offset=None,
|
||||
active_num=0 if active_num is None else active_num,
|
||||
expert_capacity=-1,
|
||||
expert_num=expert_num,
|
||||
drop_pad_mode=0,
|
||||
expert_tokens_num_type=routing_kwargs['expert_tokens_num_type'],
|
||||
expert_tokens_num_flag=routing_kwargs['expert_tokens_num_flag'],
|
||||
active_expert_range=[0, expert_num] if active_expert_range is None else active_expert_range,
|
||||
quant_mode=routing_kwargs['quant_mode'],
|
||||
row_idx_type=0,
|
||||
)
|
||||
|
||||
def patched_npu_moe_init_routing(hidden_states, topk_ids, *args, **kwargs):
|
||||
try:
|
||||
bound = origin_signature.bind(hidden_states, topk_ids, *args, **kwargs)
|
||||
except TypeError as e:
|
||||
raise RuntimeError('Failed to bind vLLM-Ascend npu_moe_init_routing arguments: '
|
||||
f'signature={origin_signature}, args={args}, kwargs={kwargs}.') from e
|
||||
bound.apply_defaults()
|
||||
routing_kwargs = {key: bound.arguments.get(key, default) for key, default in routing_defaults.items()}
|
||||
|
||||
if not is_nonquant_routing(routing_kwargs):
|
||||
return origin_routing(hidden_states, topk_ids, *args, **kwargs)
|
||||
logger.warning_once(
|
||||
'Using torch_npu.npu_moe_init_routing_v2 for vLLM-Ascend non-quantized MoE routing. '
|
||||
'The installed vLLM-Ascend implementation still dispatches this branch to '
|
||||
'npu_moe_init_routing_custom, whose missing custom-op binary fails asynchronously on this stack.')
|
||||
return npu_moe_init_routing_v2(hidden_states, topk_ids, routing_kwargs)
|
||||
|
||||
patched_npu_moe_init_routing._swift_nonquant_routing_patched = True
|
||||
patched_npu_moe_init_routing._swift_origin = origin_routing
|
||||
adaptor_cls.npu_moe_init_routing = staticmethod(patched_npu_moe_init_routing)
|
||||
|
||||
|
||||
def patch_vllm_ascend_moe_runtime() -> None:
|
||||
"""Apply MoE runtime patches that are independent of GRPO weight sync."""
|
||||
_patch_vllm_ascend_device_op_nonquant_routing()
|
||||
|
||||
|
||||
def _is_qwen_moe_model(model) -> bool:
|
||||
return getattr(getattr(model, 'config', None), 'model_type', None) in _QWEN_MOE_MODEL_TYPES
|
||||
|
||||
|
||||
def configure_vllm_ascend_moe_weight_sync(vllm_model, train_model, *, is_fsdp2: bool) -> None:
|
||||
"""Record the vLLM-Ascend MoE sync layout required by this training backend."""
|
||||
fsdp2_qwen_moe = is_fsdp2 and _is_qwen_moe_model(train_model)
|
||||
layout = _VLLM_ASCEND_MOE_PROCESSED_LAYOUT
|
||||
# Current vLLM-Ascend 0.18 non-quantized Qwen MoE forward keeps
|
||||
# ``need_trans=False`` and feeds ``w13_weight`` directly to
|
||||
# ``npu_grouped_matmul``. After FSDP2 runtime sync, write Qwen MoE weights
|
||||
# directly into the runtime [hidden, I_tp] direction and skip checkpoint
|
||||
# post-load processing; otherwise post-load transposes them back to
|
||||
# [I_tp, hidden] and the first rollout fails with a hidden-size mismatch
|
||||
# such as 2048 vs 192/384.
|
||||
setattr(vllm_model, _VLLM_ASCEND_MOE_SYNC_LAYOUT_ATTR, layout)
|
||||
setattr(vllm_model, _VLLM_ASCEND_MOE_SKIP_POST_LOAD_ATTR, fsdp2_qwen_moe)
|
||||
|
||||
|
||||
def configure_vllm_ascend_moe_preprocessed_weight_sync(vllm_model) -> None:
|
||||
"""Record that reload writes the layout expected before vLLM-Ascend post-processing."""
|
||||
setattr(vllm_model, _VLLM_ASCEND_MOE_SYNC_LAYOUT_ATTR, _VLLM_ASCEND_MOE_PREPROCESSED_LAYOUT)
|
||||
setattr(vllm_model, _VLLM_ASCEND_MOE_SKIP_POST_LOAD_ATTR, False)
|
||||
|
||||
|
||||
def use_vllm_ascend_moe_preprocessed_weight(vllm_model) -> bool:
|
||||
"""Return whether runtime sync should write the pre-process MoE layout."""
|
||||
return getattr(vllm_model, _VLLM_ASCEND_MOE_SYNC_LAYOUT_ATTR,
|
||||
_VLLM_ASCEND_MOE_PROCESSED_LAYOUT) == _VLLM_ASCEND_MOE_PREPROCESSED_LAYOUT
|
||||
|
||||
|
||||
def should_skip_vllm_ascend_moe_post_load(vllm_model) -> bool:
|
||||
"""Return whether vLLM post-load processing should be skipped after sync."""
|
||||
return bool(getattr(vllm_model, _VLLM_ASCEND_MOE_SKIP_POST_LOAD_ATTR, False))
|
||||
|
||||
|
||||
def expand_fused_moe_expert_names_for_vllm_ascend(name: str):
|
||||
"""Map Transformers fused Qwen MoE expert names to vLLM checkpoint names.
|
||||
|
||||
FSDP2 can expose Qwen-style MoE expert weights as fused tensors:
|
||||
|
||||
mlp.experts.gate_up_proj: [experts, 2 * intermediate, hidden]
|
||||
mlp.experts.down_proj : [experts, hidden, intermediate]
|
||||
|
||||
vLLM's Qwen MoE ``load_weights`` path expects checkpoint-style names such as
|
||||
``mlp.experts.0.gate_proj.weight`` / ``up_proj`` / ``down_proj`` and maps
|
||||
those names onto its internal ``w13_weight`` / ``w2_weight`` parameters.
|
||||
Use expert 0 only as a name anchor; the paired vLLM-Ascend weight-loader
|
||||
patch below copies all local experts from the full 3D tensor.
|
||||
"""
|
||||
gate_up_suffix = '.mlp.experts.gate_up_proj'
|
||||
down_suffix = '.mlp.experts.down_proj'
|
||||
if name.endswith(gate_up_suffix):
|
||||
prefix = name[:-len('gate_up_proj')]
|
||||
return [
|
||||
f'{prefix}0.gate_proj.weight',
|
||||
f'{prefix}0.up_proj.weight',
|
||||
]
|
||||
if name.endswith(down_suffix):
|
||||
prefix = name[:-len('down_proj')]
|
||||
return [f'{prefix}0.down_proj.weight']
|
||||
return None
|
||||
|
||||
|
||||
def expand_fused_moe_expert_weight_for_vllm_ascend(name: str, param):
|
||||
"""Expand one FSDP2 fused Qwen MoE expert tensor for vLLM-Ascend weight sync."""
|
||||
if not isinstance(param, torch.Tensor) or param.dim() != 3:
|
||||
return None
|
||||
expanded_names = expand_fused_moe_expert_names_for_vllm_ascend(name)
|
||||
if expanded_names is None:
|
||||
return None
|
||||
if name.endswith('.mlp.experts.gate_up_proj'):
|
||||
gate_proj, up_proj = param.chunk(2, dim=1)
|
||||
return [
|
||||
(expanded_names[0], gate_proj.contiguous()),
|
||||
(expanded_names[1], up_proj.contiguous()),
|
||||
]
|
||||
if name.endswith('.mlp.experts.down_proj'):
|
||||
return [(expanded_names[0], param)]
|
||||
return None
|
||||
|
||||
|
||||
def patch_vllm_ascend_moe_expert_weight_loader(experts,
|
||||
name: str,
|
||||
param,
|
||||
*,
|
||||
load_preprocessed_weight: bool = False) -> None:
|
||||
"""Patch one processed vLLM-Ascend MoE expert parameter loader.
|
||||
|
||||
vLLM-Ascend transposes unquantized MoE weights after each model load
|
||||
so grouped matmul can consume them efficiently. During GRPO weight sync,
|
||||
however, SWIFT can send regular HF/Megatron expert weights, for example:
|
||||
|
||||
gate_proj/up_proj: [intermediate, hidden] -> w13_weight
|
||||
down_proj : [hidden, intermediate] -> w2_weight
|
||||
|
||||
FSDP2 Qwen MoE may expose the same weights as fused 3D tensors. SWIFT
|
||||
expands those tensors to checkpoint-style gate/up/down names before calling
|
||||
vLLM ``load_weights``:
|
||||
|
||||
gate_proj/up_proj: [experts, intermediate, hidden]
|
||||
down_proj : [experts, hidden, intermediate]
|
||||
|
||||
Full-weight server reload still writes the pre-processed layout and then
|
||||
calls ``process_weights_after_loading`` once, letting vLLM-Ascend transpose
|
||||
complete weights afterwards:
|
||||
|
||||
w13_weight before process: [local_experts, 2 * intermediate_per_tp, hidden]
|
||||
w2_weight before process : [local_experts, hidden, intermediate_per_tp]
|
||||
|
||||
Megatron colocate runtime sync loads into the already-processed layout used
|
||||
by the existing Megatron rollout path:
|
||||
|
||||
w13_weight after process: [local_experts, hidden, 2 * intermediate_per_tp]
|
||||
w2_weight after process : [local_experts, intermediate_per_tp, hidden]
|
||||
|
||||
``load_preprocessed_weight`` selects the server full-reload target. FSDP2
|
||||
Qwen MoE colocate runtime sync keeps the processed target and deliberately
|
||||
skips the post-load transpose because current vLLM-Ascend non-quantized
|
||||
grouped matmul consumes the [hidden, I_tp] direction in this path.
|
||||
|
||||
This wrapper keeps the normal vLLM loader for initial checkpoint load,
|
||||
quantized experts, and non-Ascend backends. It only handles the 3D
|
||||
vLLM-Ascend expert tensors when a 2D or fused 3D runtime-sync tensor is
|
||||
loaded into ``w13_weight`` or ``w2_weight``.
|
||||
"""
|
||||
if 'w13_weight' not in name and 'w2_weight' not in name:
|
||||
return
|
||||
quant_method = getattr(experts, 'quant_method', None)
|
||||
quant_method_module = type(quant_method).__module__ if quant_method is not None else ''
|
||||
if not quant_method_module.startswith('vllm_ascend'):
|
||||
return
|
||||
|
||||
def make_ascend_moe_weight_loader(experts, origin_weight_loader):
|
||||
|
||||
def load_processed_ascend_weight(param, loaded_weight, weight_name, shard_id, expert_id, return_success=False):
|
||||
quant_method = getattr(experts, 'quant_method', None)
|
||||
quant_method_module = type(quant_method).__module__ if quant_method is not None else ''
|
||||
# Only the GRPO runtime-sync path needs special handling here.
|
||||
# SWIFT provides HF/Megatron tensors, while vLLM-Ascend stores MoE
|
||||
# experts as 3D per-local-expert tensors. Initial checkpoint load
|
||||
# and other layouts continue to use the original vLLM loader.
|
||||
is_runtime_sync_into_processed_param = (
|
||||
param.data.dim() == 3 and loaded_weight.dim() in {2, 3}
|
||||
and quant_method_module.startswith('vllm_ascend'))
|
||||
if not is_runtime_sync_into_processed_param:
|
||||
return origin_weight_loader(param, loaded_weight, weight_name, shard_id, expert_id, return_success)
|
||||
|
||||
is_w13_shard = shard_id in {'w1', 'w3'} and 'w13_weight' in weight_name
|
||||
is_w2_shard = shard_id == 'w2' and 'w2_weight' in weight_name
|
||||
|
||||
loaded_expert_sample = loaded_weight[0] if loaded_weight.dim() == 3 else loaded_weight
|
||||
|
||||
def prepare_fsdp2_preprocessed_target_layout():
|
||||
"""FSDP2 path: write weights before vLLM-Ascend post-load processing."""
|
||||
if is_w13_shard and param.data.shape[1] == loaded_expert_sample.shape[-1]:
|
||||
param.data = param.data.transpose(1, 2).contiguous()
|
||||
elif is_w2_shard and param.data.shape[2] == loaded_expert_sample.shape[0]:
|
||||
param.data = param.data.transpose(1, 2).contiguous()
|
||||
|
||||
def prepare_megatron_processed_target_layout():
|
||||
"""Megatron path: write weights into vLLM-Ascend runtime layout."""
|
||||
if (is_w13_shard and param.data.shape[-1] == loaded_expert_sample.shape[-1]
|
||||
and param.data.shape[-2] != loaded_expert_sample.shape[-1]):
|
||||
param.data = param.data.transpose(1, 2).contiguous()
|
||||
elif (is_w2_shard and param.data.shape[-2] == loaded_expert_sample.shape[0]
|
||||
and param.data.shape[-1] != loaded_expert_sample.shape[0]):
|
||||
param.data = param.data.transpose(1, 2).contiguous()
|
||||
|
||||
tp_rank = experts.tp_rank
|
||||
|
||||
def copy_fsdp2_preprocessed_expert(local_expert_id: int, loaded_expert_weight) -> bool:
|
||||
"""Copy FSDP2 fused expert weights into pre-process vLLM-Ascend layout."""
|
||||
param_data = param.data[local_expert_id]
|
||||
if is_w13_shard:
|
||||
# Target: [2 * intermediate_per_tp, hidden].
|
||||
shard_size = param_data.shape[0] // 2
|
||||
loaded_expert_weight = loaded_expert_weight.narrow(0, shard_size * tp_rank, shard_size)
|
||||
offset = 0 if shard_id == 'w1' else shard_size
|
||||
param_data[offset:offset + shard_size].copy_(loaded_expert_weight.contiguous())
|
||||
return True
|
||||
|
||||
if is_w2_shard:
|
||||
# Target: [hidden, intermediate_per_tp].
|
||||
shard_size = param_data.shape[1]
|
||||
loaded_expert_weight = loaded_expert_weight.narrow(1, shard_size * tp_rank, shard_size)
|
||||
param_data.copy_(loaded_expert_weight.contiguous())
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def copy_megatron_processed_expert(local_expert_id: int, loaded_expert_weight) -> bool:
|
||||
"""Copy Megatron/HF expert shards into processed vLLM-Ascend layout."""
|
||||
param_data = param.data[local_expert_id]
|
||||
if is_w13_shard:
|
||||
# Target: [hidden, 2 * intermediate_per_tp].
|
||||
shard_size = param_data.shape[1] // 2
|
||||
loaded_expert_weight = loaded_expert_weight.narrow(0, shard_size * tp_rank, shard_size)
|
||||
offset = 0 if shard_id == 'w1' else shard_size
|
||||
param_data[:, offset:offset + shard_size].copy_(loaded_expert_weight.transpose(0, 1).contiguous())
|
||||
return True
|
||||
|
||||
if is_w2_shard:
|
||||
# Target: [intermediate_per_tp, hidden].
|
||||
shard_size = param_data.shape[0]
|
||||
loaded_expert_weight = loaded_expert_weight.narrow(1, shard_size * tp_rank, shard_size)
|
||||
param_data.copy_(loaded_expert_weight.transpose(0, 1).contiguous())
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
if load_preprocessed_weight:
|
||||
prepare_fsdp2_preprocessed_target_layout()
|
||||
copy_one_expert = copy_fsdp2_preprocessed_expert
|
||||
else:
|
||||
prepare_megatron_processed_target_layout()
|
||||
copy_one_expert = copy_megatron_processed_expert
|
||||
|
||||
if loaded_weight.dim() == 3:
|
||||
copied = False
|
||||
for global_expert_id, loaded_expert_weight in enumerate(loaded_weight):
|
||||
local_expert_id = experts._map_global_expert_id_to_local_expert_id(global_expert_id)
|
||||
if local_expert_id == -1:
|
||||
continue
|
||||
copied = copy_one_expert(local_expert_id, loaded_expert_weight) or copied
|
||||
return copied if return_success else None
|
||||
|
||||
local_expert_id = experts._map_global_expert_id_to_local_expert_id(expert_id)
|
||||
if local_expert_id == -1:
|
||||
return False if return_success else None
|
||||
|
||||
if copy_one_expert(local_expert_id, loaded_weight):
|
||||
return True if return_success else None
|
||||
|
||||
return origin_weight_loader(param, loaded_weight, weight_name, shard_id, expert_id, return_success)
|
||||
|
||||
load_processed_ascend_weight._swift_ascend_moe_weight_loader = True
|
||||
load_processed_ascend_weight._swift_origin_weight_loader = origin_weight_loader
|
||||
load_processed_ascend_weight._swift_load_preprocessed_weight = load_preprocessed_weight
|
||||
return load_processed_ascend_weight
|
||||
|
||||
if not hasattr(experts, 'weight_loader'):
|
||||
return
|
||||
weight_loader = getattr(param, 'weight_loader', experts.weight_loader)
|
||||
origin_weight_loader = getattr(weight_loader, '_swift_origin_weight_loader', weight_loader)
|
||||
if (not getattr(weight_loader, '_swift_ascend_moe_weight_loader', False)
|
||||
or getattr(weight_loader, '_swift_load_preprocessed_weight', None) != load_preprocessed_weight):
|
||||
param.weight_loader = make_ascend_moe_weight_loader(experts, origin_weight_loader)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'configure_vllm_ascend_moe_preprocessed_weight_sync',
|
||||
'configure_vllm_ascend_moe_weight_sync',
|
||||
'expand_fused_moe_expert_names_for_vllm_ascend',
|
||||
'expand_fused_moe_expert_weight_for_vllm_ascend',
|
||||
'patch_vllm_ascend_moe_expert_weight_loader',
|
||||
'patch_vllm_ascend_moe_runtime',
|
||||
'should_skip_vllm_ascend_moe_post_load',
|
||||
'use_vllm_ascend_moe_preprocessed_weight',
|
||||
]
|
||||
@@ -0,0 +1,8 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from __future__ import annotations
|
||||
|
||||
from .npu_patch import NPUCastError, apply_all_patches, patch_mindspeed_te_cp_implementation
|
||||
|
||||
apply_all_patches()
|
||||
|
||||
__all__ = ['NPUCastError', 'apply_all_patches', 'patch_mindspeed_te_cp_implementation']
|
||||
@@ -0,0 +1,623 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import accelerate
|
||||
import copy
|
||||
import os
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import transformers
|
||||
from accelerate.utils import find_device
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from packaging import version
|
||||
from peft import PeftModel
|
||||
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
|
||||
from torch.nn.parallel import DistributedDataParallel as DDP
|
||||
from transformers import PreTrainedModel, dynamic_module_utils, trainer
|
||||
from transformers.modeling_outputs import SequenceClassifierOutputWithPast
|
||||
from types import MethodType
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from swift.utils import (HfConfigFactory, deep_getattr, get_device_count, get_dist_setting, get_last_valid_indices,
|
||||
get_logger, get_position_ids_from_cu_seqlens, is_mp, is_mp_ddp, safe_ddp_context, to_device,
|
||||
to_float_dtype)
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
transformers_version = version.parse(transformers.__version__)
|
||||
transformers_5 = transformers_version >= version.parse('5.0.0')
|
||||
|
||||
|
||||
def patch_fixed_float_dtype(module: torch.nn.Module, dtype):
|
||||
"""Patch the module, to make sure the consisitent dtype."""
|
||||
|
||||
def get_float_dtype_hook(dtype):
|
||||
|
||||
def _float_dtype_hook(module, input, output):
|
||||
return to_float_dtype(output, dtype)
|
||||
|
||||
return _float_dtype_hook
|
||||
|
||||
module.register_forward_hook(get_float_dtype_hook(dtype))
|
||||
|
||||
|
||||
def patch_fixed_device(module: torch.nn.Module, device):
|
||||
"""Move the output to the specific device"""
|
||||
|
||||
def get_device_hook(device):
|
||||
|
||||
def _device_hook(module, input, output):
|
||||
return to_device(output, device)
|
||||
|
||||
return _device_hook
|
||||
|
||||
module.register_forward_hook(get_device_hook(device))
|
||||
|
||||
|
||||
def patch_output_clone(module: torch.nn.Module):
|
||||
"""Clone the output, to avoid the inplace problem"""
|
||||
|
||||
def _clone_hook(module, input, output):
|
||||
return output.requires_grad_(True).clone()
|
||||
|
||||
module.register_forward_hook(_clone_hook)
|
||||
|
||||
|
||||
def patch_get_input_embeddings(model, embedding_keys: str):
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return deep_getattr(model, embedding_keys)
|
||||
|
||||
model.get_input_embeddings = MethodType(get_input_embeddings, model)
|
||||
|
||||
|
||||
def patch_output_normalizer(module: torch.nn.Module, model_meta):
|
||||
|
||||
def lm_head_forward(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
lm_heads = ['lm_head', 'output', 'embed_out', 'output_layer']
|
||||
lm_head_model = get_lm_head_model(module, model_meta=model_meta, lm_heads=lm_heads)
|
||||
|
||||
found = False
|
||||
for lm_head in lm_heads:
|
||||
if hasattr(lm_head_model, lm_head):
|
||||
getattr(lm_head_model, lm_head).forward = MethodType(lm_head_forward, getattr(lm_head_model, lm_head))
|
||||
found = True
|
||||
break
|
||||
|
||||
assert found, 'Cannot find the proper lm_head name'
|
||||
|
||||
def _output_embedding_hook(module, args, kwargs, output):
|
||||
attention_mask = kwargs.get('attention_mask', None)
|
||||
hidden_states = output.logits
|
||||
sequence_lengths = -1 if attention_mask is None else get_last_valid_indices(attention_mask)
|
||||
embeddings = hidden_states[torch.arange(hidden_states.shape[0], device=hidden_states.device), sequence_lengths]
|
||||
embeddings = F.normalize(embeddings, p=2, dim=1)
|
||||
return {
|
||||
'last_hidden_state': embeddings.contiguous(),
|
||||
}
|
||||
|
||||
lm_head_model.register_forward_hook(_output_embedding_hook, with_kwargs=True)
|
||||
|
||||
|
||||
def patch_output_to_input_device(module: torch.nn.Module):
|
||||
"""Patch the module, to make sure the output is in the same device with the input.
|
||||
|
||||
Args:
|
||||
module: The module to be patched
|
||||
"""
|
||||
|
||||
def _output_to_input_device_hook(module, args, kwargs, output):
|
||||
device = find_device(args) or find_device(kwargs)
|
||||
return to_device(output, device)
|
||||
|
||||
module.register_forward_hook(_output_to_input_device_hook, with_kwargs=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_device_map():
|
||||
|
||||
if not hasattr(PreTrainedModel, '_get_no_split_modules'):
|
||||
yield
|
||||
return
|
||||
|
||||
_get_no_split_modules = PreTrainedModel._get_no_split_modules
|
||||
|
||||
def _new_get_no_split_modules(self, device_map: str):
|
||||
for module in self.modules():
|
||||
if isinstance(module, PreTrainedModel) and module._no_split_modules is None:
|
||||
module.__class__._no_split_modules = []
|
||||
return _get_no_split_modules(self, device_map)
|
||||
|
||||
PreTrainedModel._get_no_split_modules = _new_get_no_split_modules
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
PreTrainedModel._get_no_split_modules = _get_no_split_modules
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_ignore_check_imports():
|
||||
import transformers.dynamic_module_utils as td
|
||||
|
||||
def _check_imports(filename) -> List[str]:
|
||||
return td.get_relative_imports(filename)
|
||||
|
||||
_old_check_imports = td.check_imports
|
||||
td.check_imports = _check_imports
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
td.check_imports = _old_check_imports
|
||||
|
||||
|
||||
def get_lm_head_model(model, model_meta=None, lm_heads=None):
|
||||
if isinstance(model, PeftModel):
|
||||
model = model.model
|
||||
model_meta = model_meta or model.model_meta
|
||||
if lm_heads is None:
|
||||
lm_heads = ['lm_head', 'output', 'embed_out', 'output_layer']
|
||||
llm_prefix_list = getattr(model_meta.model_arch, 'language_model', None)
|
||||
prefix_list = []
|
||||
if llm_prefix_list:
|
||||
prefix_list = llm_prefix_list[0].split('.')
|
||||
|
||||
current_model = model
|
||||
for prefix in prefix_list:
|
||||
current_model = getattr(current_model, prefix)
|
||||
for lm_head in lm_heads:
|
||||
if hasattr(current_model, lm_head):
|
||||
return current_model
|
||||
return model
|
||||
|
||||
|
||||
def transformers_seq_cls_forward(self, *args, origin_forward, padding_side=None, **kwargs):
|
||||
labels = kwargs.pop('labels', None)
|
||||
return_dict = kwargs.pop('return_dict', None)
|
||||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
input_ids = kwargs.get('input_ids')
|
||||
inputs_embeds = kwargs.get('inputs_embeds')
|
||||
|
||||
output = origin_forward(*args, **kwargs)
|
||||
if hasattr(output, 'logits'):
|
||||
output.logits = output.logits.to(self.score.weight.dtype)
|
||||
elif 'last_hidden_state' in output:
|
||||
output.logits = output['last_hidden_state'].to(self.score.weight.dtype)
|
||||
logits = self.score(output.logits)
|
||||
if input_ids is not None:
|
||||
batch_size = input_ids.shape[0]
|
||||
else:
|
||||
batch_size = inputs_embeds.shape[0]
|
||||
|
||||
if padding_side == 'left':
|
||||
pooled_logits = logits[:, -1]
|
||||
else:
|
||||
pad_token_id = HfConfigFactory.get_config_attr(self.config, 'pad_token_id')
|
||||
if pad_token_id is None and batch_size != 1:
|
||||
raise ValueError('Cannot handle batch sizes > 1 if no padding token is defined.')
|
||||
if pad_token_id is None:
|
||||
sequence_lengths = -1
|
||||
else:
|
||||
if input_ids is not None:
|
||||
# if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
|
||||
sequence_lengths = torch.eq(input_ids, pad_token_id).int().argmax(-1) - 1
|
||||
sequence_lengths = sequence_lengths % input_ids.shape[-1]
|
||||
elif kwargs.get('attention_mask') is not None:
|
||||
sequence_lengths = get_last_valid_indices(kwargs['attention_mask'])
|
||||
else:
|
||||
sequence_lengths = -1
|
||||
if isinstance(sequence_lengths, torch.Tensor):
|
||||
sequence_lengths = sequence_lengths.to(logits.device)
|
||||
|
||||
pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
|
||||
|
||||
loss = None
|
||||
if labels is not None:
|
||||
labels = labels.to(logits.device)
|
||||
if self.config.problem_type is None:
|
||||
if self.num_labels == 1:
|
||||
self.config.problem_type = 'regression'
|
||||
elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
|
||||
self.config.problem_type = 'single_label_classification'
|
||||
else:
|
||||
self.config.problem_type = 'multi_label_classification'
|
||||
|
||||
if self.config.problem_type == 'regression':
|
||||
loss_fct = MSELoss()
|
||||
if self.num_labels == 1:
|
||||
loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
|
||||
else:
|
||||
loss = loss_fct(pooled_logits, labels)
|
||||
elif self.config.problem_type == 'single_label_classification':
|
||||
loss_fct = CrossEntropyLoss()
|
||||
loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
|
||||
elif self.config.problem_type == 'multi_label_classification':
|
||||
loss_fct = BCEWithLogitsLoss()
|
||||
loss = loss_fct(pooled_logits, labels)
|
||||
if not return_dict:
|
||||
output = (pooled_logits, ) + output[1:]
|
||||
return ((loss, ) + output) if loss is not None else output
|
||||
|
||||
return SequenceClassifierOutputWithPast(
|
||||
loss=loss,
|
||||
logits=pooled_logits,
|
||||
past_key_values=output.past_key_values,
|
||||
hidden_states=output.hidden_states,
|
||||
attentions=output.attentions,
|
||||
)
|
||||
|
||||
|
||||
def _patch_sequence_classification(model, model_meta):
|
||||
hidden_size = HfConfigFactory.get_config_attr(model.config, 'hidden_size')
|
||||
initializer_range = HfConfigFactory.get_config_attr(model.config, 'initializer_range')
|
||||
|
||||
lm_heads = ['lm_head', 'output', 'embed_out', 'output_layer']
|
||||
lm_head_model = get_lm_head_model(model, model_meta, lm_heads)
|
||||
lm_head_model.num_labels = model.config.num_labels
|
||||
for lm_head in lm_heads:
|
||||
if hasattr(lm_head_model, lm_head):
|
||||
hidden_size = getattr(lm_head_model, lm_head).in_features
|
||||
setattr(lm_head_model, lm_head, nn.Identity())
|
||||
break
|
||||
lm_head_model.score = nn.Linear(hidden_size, lm_head_model.num_labels, bias=False, dtype=lm_head_model.dtype)
|
||||
if lm_head_model.score.weight.device == torch.device('meta'):
|
||||
lm_head_model.score.to_empty(device='cpu')
|
||||
lm_head_model.score.weight.data.normal_(mean=0.0, std=initializer_range)
|
||||
|
||||
origin_forward = lm_head_model.forward
|
||||
|
||||
@wraps(origin_forward.__func__)
|
||||
def new_forward(self, *args, **kwargs):
|
||||
return transformers_seq_cls_forward(self, *args, origin_forward=origin_forward, **kwargs)
|
||||
|
||||
lm_head_model.forward = MethodType(new_forward, lm_head_model)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_automodel_for_sequence_classification(model_info=None,
|
||||
model_meta=None,
|
||||
patch_from_pretrained=True,
|
||||
patch_missing_init=True,
|
||||
**kwargs):
|
||||
"""
|
||||
Context manager for patching AutoModel sequence classification.
|
||||
|
||||
Args:
|
||||
model_info: Model information
|
||||
model_meta: Model metadata
|
||||
patch_from_pretrained (bool): Whether to patch PreTrainedModel.from_pretrained
|
||||
patch_missing_init (bool): Whether to patch missing __init__ methods
|
||||
**kwargs: Additional keyword arguments
|
||||
"""
|
||||
model_config = kwargs.get('model_config', None)
|
||||
from_pretrained = PreTrainedModel.from_pretrained.__func__
|
||||
|
||||
# Patch 1: from_pretrained method
|
||||
if patch_from_pretrained:
|
||||
|
||||
@classmethod
|
||||
def _new_from_pretrained(cls, *args, **kwargs):
|
||||
__init__ = cls.__init__
|
||||
|
||||
def __new_init__(self, *args, **kwargs):
|
||||
__init__(self, *args, **kwargs)
|
||||
_patch_sequence_classification(self, model_meta)
|
||||
|
||||
cls.__init__ = __new_init__
|
||||
if hasattr(cls, '_tp_plan'): # fix tp_plan
|
||||
cls._tp_plan = cls._tp_plan or {}
|
||||
res = from_pretrained(cls, *args, **kwargs)
|
||||
cls.__init__ = __init__
|
||||
return res
|
||||
else:
|
||||
_new_from_pretrained = None
|
||||
|
||||
# Patch 2: missing __init__ methods
|
||||
# https://github.com/modelscope/ms-swift/pull/5820
|
||||
patched_classes = []
|
||||
if patch_missing_init:
|
||||
|
||||
def get_all_subclasses(cls, include_root=True):
|
||||
subclass_list = []
|
||||
|
||||
def recurse(cl):
|
||||
for subclass in cl.__subclasses__():
|
||||
subclass_list.append(subclass)
|
||||
recurse(subclass)
|
||||
|
||||
recurse(cls)
|
||||
|
||||
ret = set(subclass_list)
|
||||
if include_root:
|
||||
ret.add(cls)
|
||||
return ret
|
||||
|
||||
def create_default_init(cls):
|
||||
"""Create a default __init__ method that calls super().__init__"""
|
||||
|
||||
def default_init(self, *args, **kwargs):
|
||||
super(cls, self).__init__(*args, **kwargs)
|
||||
|
||||
return default_init
|
||||
|
||||
if model_config is not None:
|
||||
# we should import in advance so that get_all_subclasses can find the class
|
||||
archs = model_config.architectures
|
||||
for arch in archs:
|
||||
try:
|
||||
getattr(transformers, arch)
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
for subclass in get_all_subclasses(torch.nn.modules.module.Module):
|
||||
if '__init__' not in subclass.__dict__:
|
||||
subclass.__init__ = create_default_init(subclass)
|
||||
patched_classes.append(subclass)
|
||||
|
||||
if patch_from_pretrained:
|
||||
PreTrainedModel.from_pretrained = _new_from_pretrained
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# Restore patches
|
||||
if patch_from_pretrained:
|
||||
PreTrainedModel.from_pretrained = classmethod(from_pretrained)
|
||||
|
||||
if patch_missing_init:
|
||||
for subclass in patched_classes:
|
||||
try:
|
||||
if '__init__' in subclass.__dict__:
|
||||
del subclass.__init__
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_automodel(model_info, model_meta, auto_model_cls, return_dummy_model, **kwargs):
|
||||
from_pretrained = PreTrainedModel.from_pretrained.__func__
|
||||
|
||||
@classmethod
|
||||
def _new_from_pretrained(cls, *args, **kwargs):
|
||||
if 'AutoAWQFor' in auto_model_cls.__name__:
|
||||
kwargs.pop('use_cache', None)
|
||||
if model_info.quant_method == 'gptq':
|
||||
cls.main_input_name = 'input_ids'
|
||||
if hasattr(cls, '_tp_plan'): # fix tp_plan
|
||||
cls._tp_plan = cls._tp_plan or {}
|
||||
if return_dummy_model:
|
||||
origin_torch_dtype = torch.get_default_dtype()
|
||||
torch.set_default_dtype(kwargs['config'].torch_dtype)
|
||||
model = cls(copy.deepcopy(kwargs['config']))
|
||||
torch.set_default_dtype(origin_torch_dtype)
|
||||
else:
|
||||
model = from_pretrained(cls, *args, **kwargs)
|
||||
return model
|
||||
|
||||
PreTrainedModel.from_pretrained = _new_from_pretrained
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
PreTrainedModel.from_pretrained = classmethod(from_pretrained)
|
||||
|
||||
|
||||
def _get_max_memory(device_ids: List[int]) -> Dict[Union[int, str], int]:
|
||||
"""add feat in accelerate to support MP + DDP"""
|
||||
import psutil
|
||||
|
||||
# Make sure CUDA is initialized on each GPU to have the right memory info.
|
||||
for i in device_ids:
|
||||
_ = torch.tensor([0], device=i)
|
||||
|
||||
device_ids_set = set(device_ids)
|
||||
max_memory = {}
|
||||
for i in range(get_device_count()):
|
||||
max_memory[i] = 0
|
||||
if i in device_ids_set:
|
||||
max_memory[i] = torch.cuda.mem_get_info(i)[0]
|
||||
max_memory['cpu'] = psutil.virtual_memory().available
|
||||
return max_memory
|
||||
|
||||
|
||||
def _sync_max_memory(max_memory: Dict[Union[int, str], int]) -> Dict[Union[int, str], int]:
|
||||
"""Make sure that the model structure of MP(device_map) is the same, when using DDP."""
|
||||
max_memory_list = [v for k, v in max_memory.items() if (v > 0 and k != 'cpu')]
|
||||
_, local_rank, world_size, _ = get_dist_setting()
|
||||
src_tensor = torch.tensor(max_memory_list).to(local_rank)
|
||||
tgt_tensor_list = [torch.zeros_like(src_tensor) for _ in range(world_size)]
|
||||
dist.all_gather(tgt_tensor_list, src_tensor)
|
||||
tgt_tensor = torch.stack(tgt_tensor_list, dim=0)
|
||||
new_max_memory_iter = iter(tgt_tensor.min(dim=0)[0].tolist())
|
||||
new_max_memory = {}
|
||||
for k, v in max_memory.items():
|
||||
new_max_memory[k] = v
|
||||
if v > 0 and k != 'cpu':
|
||||
new_max_memory[k] = next(new_max_memory_iter)
|
||||
return new_max_memory
|
||||
|
||||
|
||||
_mp_ddp_patched = False
|
||||
|
||||
|
||||
def patch_mp_ddp():
|
||||
"""Patch ddp with device_map.
|
||||
After patching, the ddp can run with the device_map.
|
||||
This should be called before any training starts.
|
||||
"""
|
||||
global _mp_ddp_patched
|
||||
if _mp_ddp_patched:
|
||||
return
|
||||
_mp_ddp_patched = True
|
||||
if is_mp_ddp():
|
||||
if transformers_5:
|
||||
from transformers.integrations import accelerate as tf_accelerate
|
||||
get_balanced_memory = tf_accelerate.get_balanced_memory
|
||||
infer_auto_device_map = tf_accelerate.infer_auto_device_map
|
||||
else:
|
||||
from accelerate.utils.modeling import get_balanced_memory, infer_auto_device_map
|
||||
|
||||
@wraps(infer_auto_device_map)
|
||||
def _infer_auto_device_map_patch(model: nn.Module,
|
||||
max_memory: Optional[Dict[Union[int, str], Union[int, str]]] = None,
|
||||
**kwargs) -> Dict[str, Union[int, str, torch.device]]:
|
||||
"""The auxiliary function for supports MP + DDP. Monkey Patching.
|
||||
add feat in accelerate to support MP + DDP"""
|
||||
verbose = kwargs.pop('verbose', False)
|
||||
n_gpu = get_device_count()
|
||||
_, local_rank, _, local_world_size = get_dist_setting()
|
||||
device_ids = list(range(local_rank, n_gpu, local_world_size))
|
||||
max_memory = _get_max_memory(device_ids)
|
||||
max_memory = _sync_max_memory(max_memory)
|
||||
max_memory = get_balanced_memory(model, max_memory, low_zero=False, **kwargs)
|
||||
max_memory = {k: v for k, v in max_memory.items() if v > 0}
|
||||
return infer_auto_device_map(model, max_memory, verbose=verbose, **kwargs)
|
||||
|
||||
_old_ddp_init = DDP.__init__
|
||||
accelerate.accelerator.torch.nn.parallel.DistributedDataParallel.__init__ = (
|
||||
lambda self, model, device_ids, output_device, *args, **kwargs: _old_ddp_init(self, model, *args, **kwargs))
|
||||
if transformers_5:
|
||||
tf_accelerate.infer_auto_device_map = _infer_auto_device_map_patch
|
||||
else:
|
||||
transformers.modeling_utils.infer_auto_device_map = _infer_auto_device_map_patch
|
||||
transformers.modeling_utils.get_balanced_memory = lambda *args, **kwargs: {}
|
||||
_old_accelerator_init = trainer.Accelerator.__init__
|
||||
trainer.Accelerator.__init__ = (lambda self, device_placement=False, *args, **kwargs: _old_accelerator_init(
|
||||
self, device_placement=device_placement, *args, **kwargs))
|
||||
trainer.Accelerator.verify_device_map = lambda *args, **kwargs: False
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_get_dynamic_module():
|
||||
origin_get_cached_module_file = dynamic_module_utils.get_cached_module_file
|
||||
|
||||
def new_get_cached_module_file(pretrained_model_name_or_path, *args, **kwargs):
|
||||
with safe_ddp_context(hash_id=str(pretrained_model_name_or_path)):
|
||||
return origin_get_cached_module_file(pretrained_model_name_or_path, *args, **kwargs)
|
||||
|
||||
dynamic_module_utils.get_cached_module_file = new_get_cached_module_file
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
dynamic_module_utils.get_cached_module_file = origin_get_cached_module_file
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_tp_plan(load_model: bool):
|
||||
if not load_model or not is_mp() or transformers_version < version.parse('4.50') or 'WORLD_SIZE' not in os.environ:
|
||||
yield
|
||||
return
|
||||
logger.info_once('Patch tp_plan.')
|
||||
WORLD_SIZE = os.environ.get('WORLD_SIZE')
|
||||
os.environ['_PATCH_WORLD_SIZE'] = WORLD_SIZE
|
||||
os.environ.pop('WORLD_SIZE')
|
||||
yield
|
||||
os.environ['WORLD_SIZE'] = WORLD_SIZE
|
||||
|
||||
|
||||
def revert_padding_free(outputs: Dict[str, Any], inputs: Dict[str, Any], padding_side='left'):
|
||||
hidden_state_key = None
|
||||
if 'last_hidden_state' in outputs:
|
||||
hidden_state_key = 'last_hidden_state'
|
||||
elif 'logits' in outputs:
|
||||
hidden_state_key = 'logits'
|
||||
elif 'token_embeddings' in outputs:
|
||||
hidden_state_key = 'token_embeddings'
|
||||
|
||||
if hidden_state_key is None:
|
||||
raise NotImplementedError()
|
||||
last_hidden_state = outputs[hidden_state_key]
|
||||
last_hidden_state = last_hidden_state.squeeze(dim=0)
|
||||
if 'cu_seq_lens_q' in inputs:
|
||||
position_ids = get_position_ids_from_cu_seqlens(inputs['cu_seq_lens_q'])
|
||||
elif 'position_ids' in inputs and inputs['position_ids'].shape[0] == 1:
|
||||
position_ids = inputs['position_ids']
|
||||
else:
|
||||
raise ValueError(
|
||||
"revert_padding_free requires 'cu_seq_lens_q' or 'position_ids' in inputs, but neither was found.")
|
||||
|
||||
seq_lengths = []
|
||||
pos = position_ids[0]
|
||||
resets = torch.where(pos[1:] < pos[:-1])[0] + 1
|
||||
|
||||
if len(resets) == 0:
|
||||
# Only one sequence in this batch item
|
||||
seq_lengths = [pos.max().item() + 1]
|
||||
else:
|
||||
# Multiple sequences
|
||||
start = 0
|
||||
for end in resets:
|
||||
seq_lengths.append(end - start)
|
||||
start = end
|
||||
seq_lengths.append(pos.shape[0] - start)
|
||||
|
||||
max_length = max(seq_lengths)
|
||||
unpacked_logits = []
|
||||
|
||||
start = 0
|
||||
for length in seq_lengths:
|
||||
seq_state = last_hidden_state[start:start + length]
|
||||
padding = torch.zeros(
|
||||
(max_length - length, last_hidden_state.shape[-1])).to(last_hidden_state.dtype).to(last_hidden_state.device)
|
||||
# re-padding
|
||||
if padding_side == 'left':
|
||||
seq_state = torch.cat((padding, seq_state), dim=0)
|
||||
else:
|
||||
seq_state = torch.cat((seq_state, padding), dim=0)
|
||||
unpacked_logits.append(seq_state)
|
||||
start += length
|
||||
outputs[hidden_state_key] = torch.stack(unpacked_logits, dim=0)
|
||||
return outputs
|
||||
|
||||
|
||||
def gather_sequence_parallel_outputs(
|
||||
outputs: Dict[str, Any],
|
||||
tensor_keys: Optional[List[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Gather split tensors produced by sequence parallel training so that downstream
|
||||
components (loss, metrics, etc.) can operate on full-length sequences.
|
||||
"""
|
||||
from swift.sequence_parallel import GatherTensor, sequence_parallel
|
||||
|
||||
tensor_keys = tensor_keys or ['logits', 'last_hidden_state', 'hidden_states']
|
||||
|
||||
position_ids = None
|
||||
|
||||
if sequence_parallel.rp_world_size and sequence_parallel.rp_world_size > 1:
|
||||
position_ids = sequence_parallel.real_position_ids
|
||||
position_ids = sequence_parallel.pad(position_ids, padding_value=-1, position_ids=position_ids)
|
||||
|
||||
for key in tensor_keys:
|
||||
if key in outputs:
|
||||
outputs[key] = GatherTensor.apply(outputs[key], 1, position_ids)
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
@contextmanager
|
||||
def patch_attach_align_device_hook_on_blocks():
|
||||
from accelerate import big_modeling
|
||||
origin_attach_align_device_hook_on_blocks = big_modeling.attach_align_device_hook_on_blocks
|
||||
|
||||
def attach_align_device_hook_on_blocks(*args, **kwargs):
|
||||
return
|
||||
|
||||
big_modeling.attach_align_device_hook_on_blocks = attach_align_device_hook_on_blocks
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
big_modeling.attach_align_device_hook_on_blocks = origin_attach_align_device_hook_on_blocks
|
||||
|
||||
|
||||
def patch_module_forward(module, new_forward):
|
||||
if getattr(module, '_patched', False):
|
||||
return
|
||||
module._patched = True
|
||||
new_forward_wrapped = MethodType(new_forward, module)
|
||||
if hasattr(module, '_old_forward'): # device_map
|
||||
module._old_forward = new_forward_wrapped
|
||||
else:
|
||||
module.forward = new_forward_wrapped
|
||||
@@ -0,0 +1,664 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import math
|
||||
import os
|
||||
import torch
|
||||
import transformers
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from functools import partial
|
||||
from packaging import version
|
||||
from peft import PeftModel
|
||||
from transformers import (AutoConfig, AutoModel, AutoModelForCausalLM, AutoModelForSequenceClassification,
|
||||
AutoTokenizer, GenerationConfig, PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase)
|
||||
from transformers.integrations import is_deepspeed_zero3_enabled
|
||||
from transformers.utils import strtobool
|
||||
from types import MethodType
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
|
||||
from swift.utils import (HfConfigFactory, Processor, get_generative_reranker_logits, get_logger, is_unsloth_available,
|
||||
patch_getattr)
|
||||
from .constant import ModelType
|
||||
from .model_meta import MODEL_MAPPING, BaseModelLoader, ModelInfo, ModelMeta, get_model_info_meta
|
||||
from .patcher import (get_lm_head_model, patch_attach_align_device_hook_on_blocks, patch_automodel,
|
||||
patch_automodel_for_sequence_classification, patch_get_dynamic_module, patch_module_forward,
|
||||
patch_mp_ddp, patch_tp_plan)
|
||||
from .utils import AttnImpl, InitModelStrategy, get_default_device_map
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
transformers_5 = version.parse(transformers.__version__) >= version.parse('5.0.0.dev')
|
||||
|
||||
|
||||
def register_model(model_meta: ModelMeta, *, exist_ok: bool = False) -> None:
|
||||
"""
|
||||
model_type: The unique ID for the model type. Models with the same model_type share
|
||||
the same architectures, template, get_function, etc.
|
||||
"""
|
||||
from .model_arch import get_model_arch
|
||||
model_type = model_meta.model_type
|
||||
if not exist_ok and model_type in MODEL_MAPPING:
|
||||
raise ValueError(f'The `{model_type}` has already been registered in the MODEL_MAPPING.')
|
||||
if model_meta.model_arch:
|
||||
model_meta.model_arch = get_model_arch(model_meta.model_arch)
|
||||
MODEL_MAPPING[model_type] = model_meta
|
||||
|
||||
|
||||
def load_by_unsloth(args):
|
||||
"""Load model by unsloth"""
|
||||
assert is_unsloth_available(), 'please install unsloth if using `--tuner_backend unsloth`: `pip install unsloth`'
|
||||
os.environ['UNSLOTH_RETURN_LOGITS'] = '1'
|
||||
os.environ['UNSLOTH_DISABLE_STATISTICS'] = '1'
|
||||
model_info = args.model_info
|
||||
model_meta = args.model_meta
|
||||
|
||||
os.environ['UNSLOTH_IS_PRESENT'] = '1'
|
||||
|
||||
@contextmanager
|
||||
def _patch_distributed_function():
|
||||
from unsloth_zoo import compiler, utils
|
||||
|
||||
def distributed_function(n=1, function=None, *args, **kwargs):
|
||||
return function(*args, **kwargs)
|
||||
|
||||
_origin_distributed_function = utils.distributed_function
|
||||
utils.distributed_function = distributed_function
|
||||
compiler.distributed_function = distributed_function
|
||||
yield
|
||||
utils.distributed_function = _origin_distributed_function
|
||||
compiler.distributed_function = _origin_distributed_function
|
||||
|
||||
with _patch_distributed_function():
|
||||
if model_meta.is_multimodal:
|
||||
from unsloth import FastVisionModel as UnslothModel
|
||||
elif model_info.is_moe_model:
|
||||
from unsloth import FastModel as UnslothModel
|
||||
else:
|
||||
from unsloth import FastLanguageModel as UnslothModel
|
||||
|
||||
model, processor = UnslothModel.from_pretrained(
|
||||
model_name=args.adapters and args.adapters[0] or args.model_dir,
|
||||
dtype=args.torch_dtype,
|
||||
max_seq_length=args.max_length,
|
||||
full_finetuning=args.tuner_type == 'full',
|
||||
load_in_4bit=args.quant_bits == 4,
|
||||
load_in_8bit=args.quant_bits == 8,
|
||||
device_map=args.device_map,
|
||||
)
|
||||
if isinstance(model, PeftModel):
|
||||
base_model = model.model
|
||||
else:
|
||||
base_model = model
|
||||
base_model.model_dir = args.model_dir
|
||||
base_model.model_info = model_info
|
||||
base_model.model_meta = model_meta
|
||||
processor.model_info = model_info
|
||||
processor.model_meta = model_meta
|
||||
return model, processor
|
||||
|
||||
|
||||
def _patch_awq_compat(model_info):
|
||||
if version.parse(transformers.__version__) < version.parse('4.50') or model_info.quant_method != 'awq':
|
||||
return
|
||||
|
||||
try:
|
||||
# compat transformers>=4.50 (autoawq)
|
||||
from transformers.integrations import get_keys_to_not_convert
|
||||
from transformers.quantizers.quantizer_awq import AwqQuantizer
|
||||
_process_model_before_weight_loading = AwqQuantizer._process_model_before_weight_loading
|
||||
|
||||
def _new_process_model_before_weight_loading(self, model, *args, **kwargs):
|
||||
modules_to_not_convert = self.quantization_config.modules_to_not_convert
|
||||
if modules_to_not_convert is not None:
|
||||
self.quantization_config.modules_to_not_convert = list(
|
||||
modules_to_not_convert) + get_keys_to_not_convert(model)
|
||||
return _process_model_before_weight_loading(self, model, *args, **kwargs)
|
||||
|
||||
AwqQuantizer._process_model_before_weight_loading = _new_process_model_before_weight_loading
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _set_property(model, key):
|
||||
if not hasattr(model, 'model'):
|
||||
return
|
||||
text_model = model.model
|
||||
if not hasattr(text_model, key) or hasattr(model.__class__, key):
|
||||
return
|
||||
|
||||
def _value(self):
|
||||
return getattr(self.model, key)
|
||||
|
||||
setattr(model.__class__, key, property(_value))
|
||||
|
||||
|
||||
def fix_do_sample_warning(generation_config: GenerationConfig) -> None:
|
||||
# Use the default values of temperature/top_p/top_k in generation_config.
|
||||
if generation_config.temperature == 0:
|
||||
generation_config.do_sample = False
|
||||
if generation_config.do_sample is False:
|
||||
generation_config.temperature = 1.
|
||||
generation_config.top_p = 1.
|
||||
generation_config.top_k = 50
|
||||
|
||||
|
||||
def get_model_list() -> List[str]:
|
||||
use_hf = strtobool(os.environ.get('USE_HF', 'False'))
|
||||
models = []
|
||||
for model_type in ModelType.get_model_name_list():
|
||||
model_meta = MODEL_MAPPING.get(model_type)
|
||||
if model_meta:
|
||||
for group in model_meta.model_groups:
|
||||
for model in group.models:
|
||||
if use_hf:
|
||||
if model.hf_model_id:
|
||||
models.append(model.hf_model_id)
|
||||
else:
|
||||
if model.ms_model_id:
|
||||
models.append(model.ms_model_id)
|
||||
return models
|
||||
|
||||
|
||||
class ModelLoader(BaseModelLoader):
|
||||
default_trust_remote_code = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_info: ModelInfo,
|
||||
model_meta: ModelMeta,
|
||||
*,
|
||||
load_model: bool = False,
|
||||
# model kwargs
|
||||
attn_impl: Optional[str] = None,
|
||||
experts_impl: Optional[str] = None,
|
||||
rope_scaling: Optional[Dict[str, Any]] = None,
|
||||
max_model_len: Optional[int] = None,
|
||||
auto_model_cls=None,
|
||||
return_dummy_model: bool = False,
|
||||
new_special_tokens: Optional[List[str]] = None,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.model_info = model_info
|
||||
self.model_meta = model_meta
|
||||
self.load_model = load_model
|
||||
attn_impl = attn_impl or kwargs.get('attn_implementation')
|
||||
self.attn_impl = attn_impl
|
||||
self.attn_impl_keys = None
|
||||
experts_impl = experts_impl or kwargs.get('experts_implementation')
|
||||
if experts_impl is not None and not transformers_5:
|
||||
if experts_impl == 'eager':
|
||||
experts_impl = None
|
||||
else:
|
||||
raise ValueError('experts_impl is only supported in "transformers>=5.0".')
|
||||
self.experts_impl = experts_impl
|
||||
self.rope_scaling = rope_scaling
|
||||
self.max_model_len = max_model_len
|
||||
self.auto_model_cls = auto_model_cls
|
||||
self.auto_config_cls = None
|
||||
self.auto_tokenizer_cls = None
|
||||
self.return_dummy_model = return_dummy_model
|
||||
self.new_special_tokens = new_special_tokens
|
||||
self.model_kwargs = model_kwargs
|
||||
self.patch_offload = kwargs.pop('patch_offload', False)
|
||||
self.init_strategy = kwargs.get('init_strategy')
|
||||
self.local_repo_path = kwargs.get('local_repo_path')
|
||||
self.leaf_modules = None
|
||||
self.pad_token = None
|
||||
if model_info.quant_method == 'fp8':
|
||||
self.torch_dtype = 'auto'
|
||||
else:
|
||||
self.torch_dtype = model_info.torch_dtype
|
||||
if version.parse(transformers.__version__) >= version.parse('4.56'):
|
||||
model_kwargs['dtype'] = self.torch_dtype
|
||||
else:
|
||||
model_kwargs['torch_dtype'] = self.torch_dtype
|
||||
_patch_awq_compat(model_info)
|
||||
|
||||
def _postprocess_config(self, config):
|
||||
# fix prediction_step (internvl2, ovis, ...)
|
||||
if not hasattr(config, 'keys_to_ignore_at_inference'):
|
||||
config.keys_to_ignore_at_inference = []
|
||||
if 'past_key_values' not in config.keys_to_ignore_at_inference:
|
||||
config.keys_to_ignore_at_inference.append('past_key_values')
|
||||
torch_dtype = self.model_info.torch_dtype
|
||||
HfConfigFactory.set_config_attr(config, 'torch_dtype', torch_dtype, include_vit=True)
|
||||
HfConfigFactory.compat_zero3(config)
|
||||
|
||||
if self.rope_scaling:
|
||||
if transformers_5:
|
||||
rope_parameters = HfConfigFactory.get_config_attr(config, 'rope_parameters') or {}
|
||||
for key in ['rope_theta', 'partial_rotary_factor']:
|
||||
if self.rope_scaling.get(key) is None and rope_parameters.get(key) is not None:
|
||||
self.rope_scaling[key] = rope_parameters[key]
|
||||
HfConfigFactory.set_config_attr(config, 'rope_scaling', self.rope_scaling)
|
||||
if self.max_model_len:
|
||||
HfConfigFactory.set_max_model_len(config, self.max_model_len)
|
||||
num_labels = self.model_info.num_labels or getattr(config, 'num_labels', None)
|
||||
if num_labels and self.model_info.task_type in ['seq_cls', 'reranker']:
|
||||
self.model_info.num_labels = num_labels
|
||||
config.num_labels = num_labels
|
||||
problem_type = self.model_info.problem_type or getattr(config, 'problem_type', None)
|
||||
if problem_type and self.model_info.task_type == 'seq_cls':
|
||||
self.model_info.problem_type = problem_type
|
||||
config.problem_type = problem_type
|
||||
self._update_attn_impl(config)
|
||||
self.model_info.config = config
|
||||
return config
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
auto_config_cls = self.auto_config_cls or AutoConfig
|
||||
return auto_config_cls.from_pretrained(model_dir, trust_remote_code=self.default_trust_remote_code)
|
||||
|
||||
def _get_tokenizer(self, processor):
|
||||
if not isinstance(processor, PreTrainedTokenizerBase) and hasattr(processor, 'tokenizer'):
|
||||
tokenizer = processor.tokenizer
|
||||
patch_getattr(processor.__class__, 'tokenizer')
|
||||
else:
|
||||
tokenizer = processor
|
||||
return tokenizer
|
||||
|
||||
def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
|
||||
auto_tokenizer_cls = self.auto_tokenizer_cls
|
||||
if auto_tokenizer_cls is None:
|
||||
if os.path.exists(os.path.join(model_dir, 'preprocessor_config.json')) or os.path.exists(
|
||||
os.path.join(model_dir, 'processor_config.json')):
|
||||
from transformers import AutoProcessor
|
||||
auto_tokenizer_cls = AutoProcessor
|
||||
else:
|
||||
auto_tokenizer_cls = AutoTokenizer
|
||||
return auto_tokenizer_cls.from_pretrained(model_dir, trust_remote_code=self.default_trust_remote_code)
|
||||
|
||||
def get_model(self, model_dir: str, config: PretrainedConfig, processor: Processor,
|
||||
model_kwargs) -> PreTrainedModel:
|
||||
if self.experts_impl is not None:
|
||||
model_kwargs['experts_implementation'] = self.experts_impl
|
||||
logger.info(f'model_kwargs: {model_kwargs}')
|
||||
model_info = self.model_info
|
||||
model_meta = self.model_meta
|
||||
auto_model_cls = self.auto_model_cls
|
||||
model = None
|
||||
if model_info.task_type in {'seq_cls', 'reranker'}:
|
||||
HfConfigFactory.set_config_attr(config, 'tie_word_embeddings', False)
|
||||
if model_info.task_type in {'seq_cls', 'reranker'} and auto_model_cls in {
|
||||
None, AutoModelForSequenceClassification
|
||||
} and not self.return_dummy_model:
|
||||
with patch_automodel_for_sequence_classification(model_config=config, patch_from_pretrained=False):
|
||||
try:
|
||||
model = AutoModelForSequenceClassification.from_pretrained(
|
||||
model_dir, config=config, trust_remote_code=self.default_trust_remote_code, **self.model_kwargs)
|
||||
auto_model_cls = AutoModelForSequenceClassification
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
auto_model_cls = auto_model_cls or AutoModelForCausalLM
|
||||
context_kwargs = {
|
||||
'model_info': model_info,
|
||||
'model_meta': model_meta,
|
||||
'auto_model_cls': auto_model_cls,
|
||||
'return_dummy_model': self.return_dummy_model,
|
||||
}
|
||||
if model is None:
|
||||
if self.return_dummy_model:
|
||||
context = partial(patch_automodel, **context_kwargs)
|
||||
elif model_info.task_type == 'seq_cls' and not model_meta.is_reward:
|
||||
context = partial(patch_automodel_for_sequence_classification, **context_kwargs)
|
||||
elif model_info.task_type == 'seq_cls' and model_meta.is_reward and config.num_labels > 1:
|
||||
logger.warning('You are using a reward model for seq_cls task and num_labels > 1, '
|
||||
'ignore_mismatched_sizes will be set to True')
|
||||
model_kwargs['ignore_mismatched_sizes'] = True
|
||||
context = partial(patch_automodel_for_sequence_classification, **context_kwargs)
|
||||
elif model_info.task_type == 'reranker':
|
||||
# For reranker task, patch CausalLM to SequenceClassification with num_labels=1
|
||||
logger.info('Converting CausalLM to SequenceClassification for reranker task with num_labels=1')
|
||||
context = partial(patch_automodel_for_sequence_classification, **context_kwargs)
|
||||
else:
|
||||
context = partial(patch_automodel, **context_kwargs)
|
||||
with context():
|
||||
model = auto_model_cls.from_pretrained(
|
||||
model_dir, config=config, trust_remote_code=self.default_trust_remote_code, **model_kwargs)
|
||||
# fix not save modeling_xxx.py (transformers 4.45)
|
||||
# https://github.com/huggingface/transformers/issues/24737
|
||||
has_remote_code = hasattr(config, 'auto_map') and auto_model_cls.__name__ in config.auto_map
|
||||
if has_remote_code and model._auto_class is None:
|
||||
model._auto_class = auto_model_cls.__name__
|
||||
|
||||
if model_info.task_type == 'embedding' and auto_model_cls.__name__ != 'AutoModel':
|
||||
from swift.model.patcher import patch_output_normalizer
|
||||
patch_output_normalizer(model, model_meta=model_meta)
|
||||
elif model_info.task_type == 'generative_reranker':
|
||||
self._patch_generative_reranker(model, processor)
|
||||
if transformers_5:
|
||||
self._compat_transformers5(model)
|
||||
return model
|
||||
|
||||
def _patch_generative_reranker(self, model, processor):
|
||||
tokenizer = self._get_tokenizer(processor)
|
||||
lm_head_model = get_lm_head_model(model, self.model_meta).lm_head
|
||||
|
||||
def lm_head_forward(module, hidden_states):
|
||||
return get_generative_reranker_logits(module.weight, tokenizer, hidden_states)
|
||||
|
||||
patch_module_forward(lm_head_model, lm_head_forward)
|
||||
|
||||
def _postprocess_model(self, model_dir, model):
|
||||
model_info = self.model_info
|
||||
|
||||
if self.init_strategy is not None:
|
||||
InitModelStrategy.init_parameters(model, self.init_strategy)
|
||||
# fix seq classification task
|
||||
if self.leaf_modules is not None or model_info.is_moe_model:
|
||||
# deepspeed zero3
|
||||
self._deepspeed_set_z3_leaf_modules(model, self.leaf_modules)
|
||||
model.model_info = self.model_info
|
||||
model.model_meta = self.model_meta
|
||||
model.model_dir = model_dir
|
||||
self._init_generation_config(model, model_dir)
|
||||
HfConfigFactory.set_config_attr(model.config, 'pad_token_id', self.pad_token)
|
||||
|
||||
def _add_new_special_tokens(self, model, processor, config):
|
||||
if not self.new_special_tokens:
|
||||
return
|
||||
tokenizer = self._get_tokenizer(processor)
|
||||
num_new_tokens = tokenizer.add_special_tokens({'additional_special_tokens': self.new_special_tokens})
|
||||
if num_new_tokens > 0:
|
||||
logger.info(f'Added {num_new_tokens} new special tokens.')
|
||||
origin_vocab_size = HfConfigFactory.get_config_attr(config, 'vocab_size')
|
||||
if origin_vocab_size < len(tokenizer):
|
||||
vocab_size = math.ceil(len(tokenizer) / 128) * 128
|
||||
# fix transformers==4.52.4 qwen2.5-vl
|
||||
HfConfigFactory.set_config_attr(config, 'vocab_size', vocab_size)
|
||||
if model is not None and not self.return_dummy_model:
|
||||
llm_model = get_lm_head_model(model, self.model_meta)
|
||||
llm_model.resize_token_embeddings(vocab_size)
|
||||
|
||||
def _postprocess_processor(self, processor: Processor):
|
||||
tokenizer = self._get_tokenizer(processor)
|
||||
pad_token = tokenizer.pad_token_id
|
||||
if pad_token is None:
|
||||
pad_token = tokenizer.eos_token_id
|
||||
if tokenizer.eos_token_id is None:
|
||||
tokenizer.eos_token_id = pad_token
|
||||
if tokenizer.pad_token_id is None:
|
||||
tokenizer.pad_token_id = pad_token
|
||||
assert tokenizer.eos_token_id is not None
|
||||
assert tokenizer.pad_token_id is not None
|
||||
self.pad_token = pad_token
|
||||
tokenizer.model_info = self.model_info
|
||||
tokenizer.model_meta = self.model_meta
|
||||
|
||||
def _compat_transformers5(self, model):
|
||||
if self.model_meta.is_multimodal:
|
||||
for key in ['language_model', 'vision_tower', 'multi_modal_projector', 'visual', 'vision_model']:
|
||||
_set_property(model, key)
|
||||
|
||||
def _update_attn_impl(self, config):
|
||||
AttnImpl.update_attn_impl(config, self.attn_impl, self.attn_impl_keys)
|
||||
|
||||
def _deepspeed_set_z3_leaf_modules(self, model, z3_leaf_modules):
|
||||
if not is_deepspeed_zero3_enabled():
|
||||
return
|
||||
try:
|
||||
hf_model_type = model.config.model_type
|
||||
except Exception:
|
||||
return
|
||||
if z3_leaf_modules is None:
|
||||
if hf_model_type == 'qwen3_vl_moe':
|
||||
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeTextSparseMoeBlock
|
||||
z3_leaf_modules = [Qwen3VLMoeTextSparseMoeBlock]
|
||||
elif hf_model_type == 'qwen3_omni_moe':
|
||||
from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import \
|
||||
Qwen3OmniMoeThinkerTextSparseMoeBlock
|
||||
z3_leaf_modules = [Qwen3OmniMoeThinkerTextSparseMoeBlock]
|
||||
elif hf_model_type == 'qwen2_moe':
|
||||
from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock
|
||||
z3_leaf_modules = [Qwen2MoeSparseMoeBlock]
|
||||
elif hf_model_type == 'qwen3_moe':
|
||||
from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock
|
||||
z3_leaf_modules = [Qwen3MoeSparseMoeBlock]
|
||||
elif hf_model_type == 'gemma4':
|
||||
from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts
|
||||
z3_leaf_modules = [Gemma4TextExperts]
|
||||
elif hf_model_type == 'glm4_moe':
|
||||
from transformers.models.glm4_moe.modeling_glm4_moe import Glm4MoeMoE
|
||||
z3_leaf_modules = [Glm4MoeMoE]
|
||||
elif hf_model_type == 'glm4_moe_lite':
|
||||
from transformers.models.glm4_moe_lite.modeling_glm4_moe_lite import Glm4MoeLiteMoE
|
||||
z3_leaf_modules = [Glm4MoeLiteMoE]
|
||||
elif hf_model_type == 'glm4v_moe':
|
||||
from transformers.models.glm4v_moe.modeling_glm4v_moe import Glm4vMoeTextMoE
|
||||
z3_leaf_modules = [Glm4vMoeTextMoE]
|
||||
elif hf_model_type == 'gpt_oss':
|
||||
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
|
||||
z3_leaf_modules = [GptOssMLP]
|
||||
elif hf_model_type == 'llama4':
|
||||
from transformers.models.llama4.modeling_llama4 import Llama4TextMoe
|
||||
z3_leaf_modules = [Llama4TextMoe]
|
||||
elif hf_model_type == 'qwen3_next':
|
||||
from transformers.models.qwen3_next.modeling_qwen3_next import Qwen3NextSparseMoeBlock
|
||||
z3_leaf_modules = [Qwen3NextSparseMoeBlock]
|
||||
elif hf_model_type == 'olmoe':
|
||||
from transformers.models.olmoe.modeling_olmoe import OlmoeSparseMoeBlock
|
||||
z3_leaf_modules = [OlmoeSparseMoeBlock]
|
||||
elif hf_model_type == 'qwen3_5_moe':
|
||||
from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeSparseMoeBlock
|
||||
z3_leaf_modules = [Qwen3_5MoeSparseMoeBlock]
|
||||
elif hf_model_type == 'glm_moe_dsa':
|
||||
from transformers.models.glm_moe_dsa.modeling_glm_moe_dsa import GlmMoeDsaMoE
|
||||
z3_leaf_modules = [GlmMoeDsaMoE]
|
||||
|
||||
if z3_leaf_modules:
|
||||
from deepspeed.utils import set_z3_leaf_modules
|
||||
set_z3_leaf_modules(model, z3_leaf_modules)
|
||||
logger.info(f'Setting z3_leaf_modules: {z3_leaf_modules}')
|
||||
|
||||
def _init_generation_config(self, model, model_dir):
|
||||
# generation_config
|
||||
generation_config_path = os.path.join(model_dir, 'generation_config.json')
|
||||
if getattr(model, 'generation_config', None) is None:
|
||||
model.generation_config = GenerationConfig.from_pretrained(model_dir) if os.path.isfile(
|
||||
generation_config_path) else None
|
||||
# fix llama2 warning
|
||||
if getattr(model, 'generation_config', None) and hasattr(model.generation_config, 'do_sample'):
|
||||
fix_do_sample_warning(model.generation_config)
|
||||
|
||||
def _get_model_processor(self, model_dir, config):
|
||||
processor = self.get_processor(model_dir, config)
|
||||
model = None
|
||||
if self.load_model:
|
||||
model = self.get_model(model_dir, config, processor, self.model_kwargs.copy())
|
||||
return model, processor
|
||||
|
||||
def load(self) -> Tuple[Optional[PreTrainedModel], Processor]:
|
||||
patch_offload_context = patch_attach_align_device_hook_on_blocks() if self.patch_offload else nullcontext()
|
||||
model_dir = self.model_info.model_dir
|
||||
with patch_get_dynamic_module(), patch_tp_plan(self.load_model), patch_offload_context:
|
||||
config = self.get_config(model_dir)
|
||||
config.name_or_path = model_dir
|
||||
self._postprocess_config(config)
|
||||
model, processor = self._get_model_processor(model_dir, config)
|
||||
self._postprocess_processor(processor)
|
||||
if model:
|
||||
self._postprocess_model(model_dir, model)
|
||||
self._add_new_special_tokens(model, processor, config)
|
||||
return model, processor
|
||||
|
||||
|
||||
class SentenceTransformersLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
model = SentenceTransformer(
|
||||
model_dir,
|
||||
trust_remote_code=self.default_trust_remote_code,
|
||||
model_kwargs={
|
||||
'torch_dtype': self.torch_dtype,
|
||||
})
|
||||
model.config = config
|
||||
|
||||
def enable_input_require_grads(self):
|
||||
|
||||
def make_inputs_require_grads(module, input, output):
|
||||
output.requires_grad_(True)
|
||||
|
||||
self._require_grads_hook = self[0].auto_model.embed_tokens.register_forward_hook(make_inputs_require_grads)
|
||||
|
||||
model.enable_input_require_grads = MethodType(enable_input_require_grads, model)
|
||||
return model
|
||||
|
||||
|
||||
class RewardModelLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
if 'AutoModel' in (getattr(config, 'auto_map', None) or {}):
|
||||
self.auto_model_cls = self.auto_model_cls or AutoModel
|
||||
return super().get_model(model_dir, config, processor, model_kwargs)
|
||||
|
||||
|
||||
def get_model_processor(
|
||||
model_id_or_path: str,
|
||||
*,
|
||||
torch_dtype: Optional[torch.dtype] = None,
|
||||
device_map: Union[str, Dict[str, Any], None] = None,
|
||||
load_model: bool = True,
|
||||
# hub
|
||||
use_hf: Optional[bool] = None,
|
||||
hub_token: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
download_model: Optional[bool] = None,
|
||||
# model kwargs
|
||||
model_type: Optional[str] = None,
|
||||
quantization_config=None,
|
||||
max_memory: Union[str, Dict[str, Any]] = None,
|
||||
attn_impl: Optional[str] = None,
|
||||
experts_impl: Optional[str] = None,
|
||||
rope_scaling: Optional[Dict[str, Any]] = None,
|
||||
max_model_len: Optional[int] = None,
|
||||
auto_model_cls=None,
|
||||
new_special_tokens: Optional[List[str]] = None,
|
||||
task_type: Literal['causal_lm', 'seq_cls', 'embedding', 'reranker', 'generative_reranker'] = None,
|
||||
num_labels: Optional[int] = None,
|
||||
problem_type: Literal['regression', 'single_label_classification', 'multi_label_classification'] = None,
|
||||
return_dummy_model: bool = False,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
) -> Tuple[Optional[PreTrainedModel], Processor]:
|
||||
"""Load a pretrained model and its processor from a model hub or local path.
|
||||
|
||||
Args:
|
||||
model_id_or_path: The model identifier from a hub (HuggingFace/ModelScope) or local path.
|
||||
torch_dtype: Data type for model parameters. If None, uses the dtype from config.json.
|
||||
device_map: Device mapping strategy for model loading. If None, uses default device map.
|
||||
Can be a string (e.g., 'auto', 'cuda:0') or a dictionary mapping layers to devices.
|
||||
load_model: Whether to load the model weights. If False, only returns the processor.
|
||||
|
||||
# Hub parameters
|
||||
use_hf: Force using HuggingFace Hub (True) or ModelScope (False). If None, it is controlled
|
||||
by the environment variable `USE_HF`, which defaults to '0'. Default: None.
|
||||
hub_token: Authentication token for accessing private models on the hub.
|
||||
revision: Specific model version to use.
|
||||
download_model: Whether to download model files. If None, determined by load_model value.
|
||||
|
||||
# Model configuration
|
||||
model_type: Explicit model type when it cannot be uniquely determined from model_id_or_path/config.json.
|
||||
quantization_config: Configuration for model quantization.
|
||||
max_memory: Maximum memory allocation per device.
|
||||
attn_impl: Attention implementation. 'flash_attn' for Flash Attention, None for auto-select (sdpa/eager).
|
||||
experts_impl: experts implementation. Options are 'grouped_mm', 'batched_mm', 'eager'. Defaults to None.
|
||||
This feature requires "transformers>=5.0.0".
|
||||
rope_scaling: RoPE (Rotary Position Embedding) scaling configuration dictionary.
|
||||
max_model_len: Maximum sequence length the model can handle.
|
||||
auto_model_cls: Custom AutoModel class to use for loading (e.g., AutoModelForCausalLM).
|
||||
new_special_tokens: List of new special tokens to add to the tokenizer.
|
||||
task_type: Task type for the model. Options: 'causal_lm', 'seq_cls', 'embedding', 'reranker',
|
||||
'generative_reranker'.
|
||||
num_labels: Number of labels for classification tasks.
|
||||
problem_type: Type of classification problem: 'regression', 'single_label_classification',
|
||||
or 'multi_label_classification'.
|
||||
return_dummy_model: If True, returns a dummy model (without loading weights).
|
||||
model_kwargs: Additional keyword arguments passed to the model's from_pretrained method.
|
||||
**kwargs: Additional keyword arguments passed to the loader.
|
||||
|
||||
Returns:
|
||||
A tuple of (model, processor) where:
|
||||
- model: The loaded PreTrainedModel instance, or None if load_model=False.
|
||||
- processor: The Processor (tokenizer, processor, etc.) for the model.
|
||||
|
||||
Examples:
|
||||
>>> # Load model and processor with default settings
|
||||
>>> model, processor = get_model_processor('Qwen/Qwen2.5-7B-Instruct')
|
||||
|
||||
>>> # Load only processor without model
|
||||
>>> _, processor = get_model_processor('Qwen/Qwen2.5-7B-Instruct', load_model=False)
|
||||
"""
|
||||
if load_model:
|
||||
patch_mp_ddp()
|
||||
if model_kwargs is None:
|
||||
model_kwargs = {}
|
||||
if download_model is None:
|
||||
download_model = load_model and not return_dummy_model
|
||||
model_info, model_meta = get_model_info_meta(
|
||||
model_id_or_path,
|
||||
torch_dtype=torch_dtype,
|
||||
use_hf=use_hf,
|
||||
hub_token=hub_token,
|
||||
revision=revision,
|
||||
download_model=download_model,
|
||||
model_type=model_type,
|
||||
quantization_config=quantization_config,
|
||||
task_type=task_type,
|
||||
num_labels=num_labels,
|
||||
problem_type=problem_type)
|
||||
if device_map is None:
|
||||
device_map = get_default_device_map()
|
||||
model_kwargs['device_map'] = device_map
|
||||
if quantization_config:
|
||||
model_kwargs['quantization_config'] = quantization_config
|
||||
if max_memory:
|
||||
model_kwargs['max_memory'] = max_memory
|
||||
loader = model_meta.loader(
|
||||
model_info,
|
||||
model_meta,
|
||||
load_model=load_model,
|
||||
attn_impl=attn_impl,
|
||||
experts_impl=experts_impl,
|
||||
rope_scaling=rope_scaling,
|
||||
max_model_len=max_model_len,
|
||||
auto_model_cls=auto_model_cls,
|
||||
return_dummy_model=return_dummy_model,
|
||||
new_special_tokens=new_special_tokens,
|
||||
model_kwargs=model_kwargs,
|
||||
**kwargs)
|
||||
return loader.load()
|
||||
|
||||
|
||||
def get_processor(
|
||||
model_id_or_path: str,
|
||||
*,
|
||||
# hub
|
||||
use_hf: Optional[bool] = None,
|
||||
hub_token: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
download_model: Optional[bool] = None,
|
||||
# model kwargs
|
||||
model_type: Optional[str] = None,
|
||||
task_type: Literal['causal_lm', 'seq_cls', 'embedding', 'reranker', 'generative_reranker'] = None,
|
||||
num_labels: Optional[int] = None,
|
||||
problem_type: Literal['regression', 'single_label_classification', 'multi_label_classification'] = None,
|
||||
**kwargs,
|
||||
) -> Processor:
|
||||
"""Load only the processor for a pretrained model.
|
||||
|
||||
This is a convenience function that wraps `get_model_processor` with `load_model=False`,
|
||||
returning only the processor without loading the model weights.
|
||||
"""
|
||||
return get_model_processor(
|
||||
model_id_or_path,
|
||||
use_hf=use_hf,
|
||||
hub_token=hub_token,
|
||||
revision=revision,
|
||||
download_model=download_model,
|
||||
model_type=model_type,
|
||||
task_type=task_type,
|
||||
num_labels=num_labels,
|
||||
problem_type=problem_type,
|
||||
load_model=False,
|
||||
**kwargs)[1]
|
||||
@@ -0,0 +1,338 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import os
|
||||
import shutil
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from accelerate.utils import find_device
|
||||
from functools import wraps
|
||||
from packaging import version
|
||||
from peft import PeftModel
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig, PreTrainedModel
|
||||
from transformers.integrations import is_deepspeed_zero3_enabled
|
||||
from transformers.utils import (is_torch_bf16_gpu_available, is_torch_cuda_available, is_torch_mps_available,
|
||||
is_torch_npu_available, strtobool)
|
||||
from types import MethodType
|
||||
from typing import List, Optional, TypeVar, Union
|
||||
|
||||
from swift.utils import (HfConfigFactory, Processor, deep_getattr, get_dist_setting, get_env_args, get_logger, is_mp,
|
||||
to_device)
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
_T = TypeVar('_T')
|
||||
|
||||
|
||||
class AttnImpl:
|
||||
attn_impl_keys = ['_attn_implementation', 'attn_implementation', 'llm_attn_implementation']
|
||||
use_flash_attn_keys = ['_flash_attn_2_enabled', 'use_flash_attn', '_use_flash_attention_2']
|
||||
|
||||
@staticmethod
|
||||
def to_use_flash_attn(attn_impl: Optional[str], auto_value: _T = None) -> Union[bool, _T]:
|
||||
if attn_impl is None:
|
||||
return auto_value
|
||||
return attn_impl in {'flash_attn', 'flash_attention_2'}
|
||||
|
||||
@staticmethod
|
||||
def update_attn_impl(config: PretrainedConfig,
|
||||
attn_impl: Optional[str],
|
||||
attn_impl_keys: Optional[List[str]] = None) -> None:
|
||||
if attn_impl is None:
|
||||
return
|
||||
logger.info(f'attn_impl: {attn_impl}')
|
||||
use_flash_attn = AttnImpl.to_use_flash_attn(attn_impl)
|
||||
if use_flash_attn:
|
||||
attn_impl = 'flash_attention_2'
|
||||
if isinstance(attn_impl_keys, str):
|
||||
attn_impl_keys = [attn_impl_keys]
|
||||
attn_impl_keys = attn_impl_keys or AttnImpl.attn_impl_keys
|
||||
for key in attn_impl_keys:
|
||||
HfConfigFactory.set_config_attr(config, key, attn_impl, include_vit=True, ensure_set=False)
|
||||
for key in AttnImpl.use_flash_attn_keys:
|
||||
HfConfigFactory.set_config_attr(config, key, use_flash_attn, include_vit=True, ensure_set=False)
|
||||
|
||||
|
||||
def get_llm_model(model: torch.nn.Module, model_meta=None, inner_backbone=True):
|
||||
"""Get LLM model, this function can be used to get the llm module from a multi-modal model.
|
||||
|
||||
Args:
|
||||
model: The model instance
|
||||
model_meta: The model_meta information
|
||||
inner_backbone: Get inner backbone model, like `QwenModel` or `LlamaModel`
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
from accelerate.utils import extract_model_from_parallel
|
||||
|
||||
from swift.tuners import SwiftModel
|
||||
model = extract_model_from_parallel(model)
|
||||
|
||||
if isinstance(model, (SwiftModel, PeftModel)):
|
||||
model = model.model
|
||||
if model_meta is None:
|
||||
model_meta = model.model_meta
|
||||
|
||||
llm_prefix = getattr(model_meta.model_arch, 'language_model', None)
|
||||
if llm_prefix:
|
||||
llm_model = deep_getattr(model, llm_prefix[0])
|
||||
else:
|
||||
llm_model = model
|
||||
|
||||
if inner_backbone:
|
||||
if hasattr(llm_model, 'thinker'):
|
||||
llm_model = llm_model.thinker.model
|
||||
elif hasattr(llm_model, 'model'):
|
||||
llm_model = llm_model.model
|
||||
return llm_model
|
||||
|
||||
|
||||
def use_submodel_func(model, submodel_name: str, func_list: Optional[List[str]] = None) -> None:
|
||||
if func_list is None:
|
||||
func_list = ['generate', 'get_input_embeddings', 'gradient_checkpointing_enable', 'forward']
|
||||
submodel = getattr(model, submodel_name)
|
||||
|
||||
def _get_new_func(func_name: str):
|
||||
# Please ensure the patch to submodel.forward is applied before this function.
|
||||
_old_func = getattr(submodel, func_name).__func__
|
||||
|
||||
@wraps(_old_func)
|
||||
def _new_func(self, *args, **kwargs):
|
||||
res = _old_func(submodel, *args, **kwargs)
|
||||
if func_name == 'forward':
|
||||
device = find_device(args)
|
||||
if device is None:
|
||||
device = find_device(kwargs)
|
||||
if hasattr(res, 'logits'):
|
||||
res.logits = to_device(res.logits, device)
|
||||
if hasattr(res, 'loss'):
|
||||
res.loss = to_device(res.loss, device)
|
||||
if isinstance(res, dict) and 'last_hidden_state' in res:
|
||||
res['last_hidden_state'] = to_device(res['last_hidden_state'], device)
|
||||
return res
|
||||
|
||||
return _new_func
|
||||
|
||||
for key in func_list:
|
||||
setattr(model, key, MethodType(_get_new_func(key), model))
|
||||
if key == 'generate' and model.device != submodel.device:
|
||||
submodel.__class__.device = model.device
|
||||
if key == 'forward' and 'generate' in func_list:
|
||||
setattr(submodel, key, MethodType(_get_new_func(key), submodel)) # fix device_map
|
||||
|
||||
|
||||
class InitModelStrategy:
|
||||
|
||||
@staticmethod
|
||||
def is_uninitialized(param: torch.Tensor) -> bool:
|
||||
"""
|
||||
Check if a parameter is uninitialized or has numerically unstable values.
|
||||
Criteria:
|
||||
- Tensor has NaN or Inf values
|
||||
- Tensor stats (mean or std) are outside reasonable range
|
||||
"""
|
||||
if param.numel() == 0:
|
||||
return False
|
||||
|
||||
with torch.no_grad():
|
||||
mean_abs = param.abs().mean()
|
||||
std = param.std()
|
||||
|
||||
# NaN or Inf
|
||||
if not torch.isfinite(mean_abs) or not torch.isfinite(std):
|
||||
return True
|
||||
|
||||
# Use empirically safe threshold
|
||||
MAX_THRESHOLD = 1e7
|
||||
if mean_abs > MAX_THRESHOLD or std > MAX_THRESHOLD:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def constant_init(param: torch.Tensor, c: float = 0) -> None:
|
||||
nn.init.constant_(param, c)
|
||||
|
||||
@staticmethod
|
||||
def uniform_init(param: torch.Tensor, a: float = -0.1, b: float = 0.1) -> None:
|
||||
nn.init.uniform_(param, a, b)
|
||||
|
||||
@staticmethod
|
||||
def normal_init(param: torch.Tensor, mean: float = 0.0, std: float = 0.01) -> None:
|
||||
nn.init.normal_(param, mean, std)
|
||||
|
||||
@staticmethod
|
||||
def _init_high_dim(param: torch.Tensor, init_func, *args, **kwargs) -> None:
|
||||
"""Helper for high-dimensional initialization methods."""
|
||||
if param.dim() > 1:
|
||||
init_func(param, *args, **kwargs)
|
||||
elif param.dim() == 1 and param.size(0) > 0:
|
||||
InitModelStrategy.constant_init(param)
|
||||
|
||||
@staticmethod
|
||||
def xavier_uniform_init(param: torch.Tensor) -> None:
|
||||
InitModelStrategy._init_high_dim(param, nn.init.xavier_uniform_)
|
||||
|
||||
@staticmethod
|
||||
def xavier_normal_init(param: torch.Tensor) -> None:
|
||||
InitModelStrategy._init_high_dim(param, nn.init.xavier_normal_)
|
||||
|
||||
@staticmethod
|
||||
def kaiming_uniform_init(param: torch.Tensor) -> None:
|
||||
InitModelStrategy._init_high_dim(
|
||||
param, nn.init.kaiming_uniform_, mode='fan_out', nonlinearity='leaky_relu', a=0.1)
|
||||
|
||||
@staticmethod
|
||||
def kaiming_normal_init(param: torch.Tensor) -> None:
|
||||
InitModelStrategy._init_high_dim(param, nn.init.kaiming_normal_, mode='fan_in', nonlinearity='relu')
|
||||
|
||||
@staticmethod
|
||||
def orthogonal_init(param: torch.Tensor) -> None:
|
||||
nn.init.orthogonal_(param, gain=1.0)
|
||||
|
||||
_INIT_STRATEGY_MAP = {
|
||||
'zero': constant_init,
|
||||
'uniform': uniform_init,
|
||||
'normal': normal_init,
|
||||
'xavier_uniform': xavier_uniform_init,
|
||||
'xavier_normal': xavier_normal_init,
|
||||
'kaiming_uniform': kaiming_uniform_init,
|
||||
'kaiming_normal': kaiming_normal_init,
|
||||
'orthogona': orthogonal_init,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def init_parameters(model: nn.Module, init_strategy: str) -> None:
|
||||
"""Initialize model parameters using the specified strategy.
|
||||
Args:
|
||||
model: The model whose parameters to initialize
|
||||
init_strategy: Name of initialization strategy
|
||||
"""
|
||||
if init_strategy not in InitModelStrategy._INIT_STRATEGY_MAP:
|
||||
raise ValueError(f'Unknown initialization strategy: {init_strategy}')
|
||||
|
||||
logger.info(f'initialization strategy: {init_strategy}')
|
||||
|
||||
init_func = InitModelStrategy._INIT_STRATEGY_MAP[init_strategy]
|
||||
|
||||
for name, param in model.named_parameters():
|
||||
if InitModelStrategy.is_uninitialized(param):
|
||||
logger.info(f'Initializing parameters: {name}.')
|
||||
init_func(param)
|
||||
|
||||
|
||||
def get_default_device_map():
|
||||
if is_deepspeed_zero3_enabled() or os.environ.get('ACCELERATE_USE_FSDP', 'False') == 'true':
|
||||
return None
|
||||
local_rank = get_dist_setting()[1]
|
||||
if local_rank == -1:
|
||||
local_rank = 0
|
||||
if is_torch_npu_available():
|
||||
return 'auto' if is_mp() else f'npu:{local_rank}'
|
||||
elif is_torch_mps_available():
|
||||
return f'mps:{local_rank}'
|
||||
elif is_torch_cuda_available():
|
||||
return 'auto' if is_mp() else f'cuda:{local_rank}'
|
||||
else:
|
||||
return 'cpu'
|
||||
|
||||
|
||||
def get_default_torch_dtype(torch_dtype: Optional[torch.dtype]):
|
||||
# torch_dtype: torch_dtype in config.json
|
||||
if torch_dtype is not None:
|
||||
return torch_dtype
|
||||
|
||||
try:
|
||||
is_bf16_available = is_torch_bf16_gpu_available() or (is_torch_npu_available()
|
||||
and torch.npu.is_bf16_supported())
|
||||
except Exception: # noqa
|
||||
is_bf16_available = False
|
||||
|
||||
if is_torch_cuda_available() or is_torch_npu_available():
|
||||
if is_bf16_available:
|
||||
return torch.bfloat16
|
||||
else:
|
||||
return torch.float16
|
||||
else:
|
||||
# cpu
|
||||
return torch.float32
|
||||
|
||||
|
||||
def _patch_conv3d():
|
||||
if hasattr(nn.Conv3d, '_original_forward'):
|
||||
return
|
||||
nn.Conv3d._original_forward = nn.Conv3d.forward
|
||||
|
||||
def forward(self, x):
|
||||
if any(s != k for s, k in zip(self.stride, self.kernel_size)) or any(p != 0 for p in self.padding) or any(
|
||||
d != 1 for d in self.dilation) or self.groups != 1:
|
||||
raise NotImplementedError(
|
||||
'Patched Conv3d only supports stride=kernel_size, padding=0, dilation=1, groups=1')
|
||||
N = x.shape[0]
|
||||
K = self.kernel_size
|
||||
x = x.unfold(2, K[0], K[0]).unfold(3, K[1], K[1]).unfold(4, K[2], K[2])
|
||||
D_out, H_out, W_out = x.shape[2:5]
|
||||
x = x.permute(0, 2, 3, 4, 1, 5, 6, 7).reshape(-1, self.in_channels * K[0] * K[1] * K[2])
|
||||
x = F.linear(x, self.weight.view(self.out_channels, -1), self.bias)
|
||||
x = x.view(N, D_out, H_out, W_out, self.out_channels).permute(0, 4, 1, 2, 3)
|
||||
return x
|
||||
|
||||
nn.Conv3d.forward = forward
|
||||
logger.info('Conv3d patched successfully')
|
||||
|
||||
|
||||
requires_patch = version.parse('2.9.0') <= version.parse(torch.__version__) < version.parse('2.10.0')
|
||||
if requires_patch:
|
||||
_patch_conv3d()
|
||||
|
||||
|
||||
def save_checkpoint(model: Optional[PreTrainedModel],
|
||||
processor: Processor,
|
||||
output_dir: str,
|
||||
*,
|
||||
safe_serialization: bool = True,
|
||||
max_shard_size: Union[int, str] = '5GB',
|
||||
model_dirs: List[str] = None,
|
||||
additional_saved_files: Optional[List[str]] = None) -> None:
|
||||
if model is not None:
|
||||
if model.__class__.__name__ != 'SentenceTransformer':
|
||||
model.save_pretrained(output_dir, safe_serialization=safe_serialization, max_shard_size=max_shard_size)
|
||||
else:
|
||||
model.save_pretrained(output_dir, safe_serialization=safe_serialization)
|
||||
# copy sentencetransformers files
|
||||
from swift.utils import copy_files_by_pattern
|
||||
copy_files_by_pattern(model.model_dir, output_dir, '*.py')
|
||||
copy_files_by_pattern(model.model_dir, output_dir, '*.json')
|
||||
processor.save_pretrained(output_dir)
|
||||
|
||||
if model_dirs is None:
|
||||
model_dirs = []
|
||||
else:
|
||||
model_dirs = model_dirs.copy()
|
||||
if model and model.model_dir and model.model_dir not in model_dirs:
|
||||
model_dirs.append(model.model_dir)
|
||||
for src_file in (additional_saved_files or []) + ['preprocessor_config.json', 'args.json']:
|
||||
tgt_path = os.path.join(output_dir, src_file)
|
||||
if os.path.exists(tgt_path) and src_file == 'args.json':
|
||||
continue
|
||||
for model_dir in model_dirs:
|
||||
src_path: str = os.path.join(model_dir, src_file)
|
||||
if os.path.isfile(src_path):
|
||||
shutil.copy(src_path, tgt_path)
|
||||
break
|
||||
elif os.path.isdir(src_path):
|
||||
shutil.copytree(src_path, tgt_path)
|
||||
break
|
||||
|
||||
|
||||
def get_ckpt_dir(model_dir: str, adapters_dir: Optional[List[str]]) -> str:
|
||||
model_dirs = (adapters_dir or []).copy()
|
||||
if model_dir:
|
||||
model_dirs.append(model_dir)
|
||||
# The adapter takes higher priority.
|
||||
ckpt_dir = None
|
||||
for model_dir in model_dirs:
|
||||
if os.path.exists(os.path.join(model_dir, 'args.json')):
|
||||
ckpt_dir = model_dir
|
||||
break
|
||||
return ckpt_dir
|
||||
Reference in New Issue
Block a user