This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,455 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import sys
|
||||
from functools import wraps
|
||||
from transformers import PretrainedConfig, PreTrainedModel
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from swift.template import TemplateType
|
||||
from swift.utils import git_clone_github, safe_snapshot_download
|
||||
from ..constant import MLLMModelType
|
||||
from ..model_arch import ModelArch
|
||||
from ..model_meta import Model, ModelGroup, ModelMeta
|
||||
from ..patcher import patch_get_input_embeddings
|
||||
from ..register import ModelLoader, register_model
|
||||
|
||||
|
||||
class LlavaLlamaHfLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
from transformers import LlavaConfig
|
||||
self.auto_config_cls = LlavaConfig
|
||||
return super().get_config(model_dir)
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_llama3_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-llama-3-8b-v1_1-transformers', 'xtuner/llava-llama-3-8b-v1_1-transformers'),
|
||||
]),
|
||||
],
|
||||
LlavaLlamaHfLoader,
|
||||
template=TemplateType.llava_llama3_hf,
|
||||
architectures=['LlavaForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.36'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
def _patch_llava(model):
|
||||
if hasattr(model, '__old_generate'):
|
||||
return
|
||||
generate = model.generate
|
||||
model.__old_generate = generate
|
||||
|
||||
@wraps(generate)
|
||||
def _new_generate(inputs=None, *args, **kwargs):
|
||||
input_ids = kwargs.pop('input_ids', None)
|
||||
if inputs is None and input_ids is not None:
|
||||
inputs = input_ids
|
||||
return generate(inputs, *args, **kwargs)
|
||||
|
||||
model.generate = _new_generate
|
||||
|
||||
|
||||
class LlavahfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_5_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-1.5-7b-hf', 'llava-hf/llava-1.5-7b-hf'),
|
||||
Model('llava-hf/llava-1.5-13b-hf', 'llava-hf/llava-1.5-13b-hf'),
|
||||
]),
|
||||
],
|
||||
LlavahfLoader,
|
||||
template=TemplateType.llava1_5_hf,
|
||||
architectures=['LlavaForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.36'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaOnevisionHfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaOnevisionForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaOnevisionForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_onevision_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-onevision-qwen2-0.5b-ov-hf', 'llava-hf/llava-onevision-qwen2-0.5b-ov-hf'),
|
||||
Model('llava-hf/llava-onevision-qwen2-7b-ov-hf', 'llava-hf/llava-onevision-qwen2-7b-ov-hf'),
|
||||
Model('llava-hf/llava-onevision-qwen2-72b-ov-hf', 'llava-hf/llava-onevision-qwen2-72b-ov-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaOnevisionHfLoader,
|
||||
template=TemplateType.llava_onevision_hf,
|
||||
architectures=['LlavaOnevisionForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.45'],
|
||||
tags=['vision', 'video'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextHfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaNextForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaNextForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_qwen_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-next-72b-hf', 'llava-hf/llava-next-72b-hf'),
|
||||
Model('llava-hf/llava-next-110b-hf', 'llava-hf/llava-next-110b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava_next_qwen_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama3_llava_next_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llama3-llava-next-8b-hf', 'llava-hf/llama3-llava-next-8b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llama3_llava_next_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_vicuna_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-v1.6-vicuna-7b-hf', 'llava-hf/llava-v1.6-vicuna-7b-hf'),
|
||||
Model('llava-hf/llava-v1.6-vicuna-13b-hf', 'llava-hf/llava-v1.6-vicuna-13b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava1_6_vicuna_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_mistral_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-v1.6-mistral-7b-hf', 'llava-hf/llava-v1.6-mistral-7b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava1_6_mistral_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_llama3_1_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('swift/llava-llama3.1-8b'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava_llama3_1_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.41'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextYiHfLoader(LlavaNextHfLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.image_token_index = 64003
|
||||
return config
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_yi_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/llava-v1.6-34b-hf', 'llava-hf/llava-v1.6-34b-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextHfLoader,
|
||||
template=TemplateType.llava1_6_yi_hf,
|
||||
architectures=['LlavaNextForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_hf,
|
||||
requires=['transformers>=4.39'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextVideoHfLoader(ModelLoader):
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
from transformers import LlavaNextVideoForConditionalGeneration
|
||||
self.auto_model_cls = self.auto_model_cls or LlavaNextVideoForConditionalGeneration
|
||||
return super().get_model(model_dir, *args, **kwargs)
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_video_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/LLaVA-NeXT-Video-7B-DPO-hf', 'llava-hf/LLaVA-NeXT-Video-7B-DPO-hf'),
|
||||
Model('llava-hf/LLaVA-NeXT-Video-7B-32K-hf', 'llava-hf/LLaVA-NeXT-Video-7B-32K-hf'),
|
||||
Model('llava-hf/LLaVA-NeXT-Video-7B-hf', 'llava-hf/LLaVA-NeXT-Video-7B-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextVideoHfLoader,
|
||||
template=TemplateType.llava_next_video_hf,
|
||||
architectures=['LlavaNextVideoForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_next_video_hf,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['video'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaNextVideoYiHfLoader(LlavaNextVideoHfLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.video_token_index = 64003
|
||||
config.image_token_index = 64004
|
||||
return config
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_video_yi_hf,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('llava-hf/LLaVA-NeXT-Video-34B-hf', 'llava-hf/LLaVA-NeXT-Video-34B-hf'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextVideoYiHfLoader,
|
||||
template=TemplateType.llava_next_video_hf,
|
||||
architectures=['LlavaNextVideoForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_next_video_hf,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['video'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaLoader(ModelLoader):
|
||||
llm_model_type = None
|
||||
|
||||
def get_config(self, model_dir: str):
|
||||
local_repo_path = self.local_repo_path
|
||||
if not local_repo_path:
|
||||
if 'next' in self.llm_model_type:
|
||||
repo_path = 'https://github.com/LLaVA-VL/LLaVA-NeXT'
|
||||
else:
|
||||
repo_path = 'https://github.com/haotian-liu/LLaVA'
|
||||
local_repo_path = git_clone_github(repo_path)
|
||||
sys.path.append(local_repo_path)
|
||||
if self.llm_model_type == 'mistral':
|
||||
from llava.model import LlavaMistralConfig
|
||||
self.auto_config_cls = LlavaMistralConfig
|
||||
elif 'llama' in self.llm_model_type: # llama
|
||||
from llava.model import LlavaConfig
|
||||
self.auto_config_cls = LlavaConfig
|
||||
config = super().get_config(model_dir)
|
||||
if not hasattr(config, 'max_sequence_length'):
|
||||
config.max_sequence_length = 2048
|
||||
return config
|
||||
|
||||
def get_model(self, model_dir: str, config, processor, model_kwargs) -> PreTrainedModel:
|
||||
if self.llm_model_type == 'mistral':
|
||||
from llava.model import LlavaMistralForCausalLM
|
||||
auto_model_cls = LlavaMistralForCausalLM
|
||||
elif 'llama' in self.llm_model_type: # llama
|
||||
from llava.model import LlavaLlamaForCausalLM
|
||||
if not hasattr(LlavaLlamaForCausalLM, '__old_forward'): # Avoid double patching
|
||||
forward = LlavaLlamaForCausalLM.forward
|
||||
LlavaLlamaForCausalLM.__old_forward = forward
|
||||
|
||||
@wraps(forward)
|
||||
def _new_forward(*args, **kwargs):
|
||||
kwargs.pop('cache_position', None)
|
||||
return forward(*args, **kwargs)
|
||||
|
||||
LlavaLlamaForCausalLM.forward = _new_forward
|
||||
auto_model_cls = LlavaLlamaForCausalLM
|
||||
else: # qwen
|
||||
from llava.model import LlavaQwenForCausalLM
|
||||
auto_model_cls = LlavaQwenForCausalLM
|
||||
|
||||
config.mm_vision_tower = safe_snapshot_download('AI-ModelScope/clip-vit-large-patch14-336', check_local=True)
|
||||
self.auto_model_cls = self.auto_model_cls or auto_model_cls
|
||||
model = super().get_model(model_dir, config, processor, model_kwargs)
|
||||
vision_tower = model.get_vision_tower()
|
||||
device_map = str(model_kwargs.get('device_map', str(model.device)))
|
||||
if not vision_tower.is_loaded:
|
||||
vision_tower.load_model(device_map=device_map)
|
||||
_patch_llava(model)
|
||||
model.resize_token_embeddings(len(processor))
|
||||
processor.image_processor = vision_tower.image_processor
|
||||
return model
|
||||
|
||||
|
||||
class Llama3LlavaNextLoader(LlavaLoader):
|
||||
llm_model_type = 'next_llama'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llama3_llava_next,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llama3-llava-next-8b', 'lmms-lab/llama3-llava-next-8b'),
|
||||
], ),
|
||||
],
|
||||
Llama3LlavaNextLoader,
|
||||
template=TemplateType.llama3_llava_next,
|
||||
architectures=['LlavaLlamaForCausalLM'],
|
||||
model_arch=ModelArch.llava_llama,
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaMistralLoader(LlavaLoader):
|
||||
llm_model_type = 'next_llama'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_mistral,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-v1.6-mistral-7b', 'liuhaotian/llava-v1.6-mistral-7b'),
|
||||
], ),
|
||||
],
|
||||
LlavaMistralLoader,
|
||||
template=TemplateType.llava1_6_mistral,
|
||||
requires=['transformers>=4.34'],
|
||||
architectures=['LlavaMistralForCausalLM'],
|
||||
model_arch=ModelArch.llava_mistral,
|
||||
tags=['vision'],
|
||||
))
|
||||
|
||||
|
||||
class LlavaLlamaLoader(LlavaLoader):
|
||||
llm_model_type = 'llama'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava1_6_yi, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-v1.6-34b', 'liuhaotian/llava-v1.6-34b'),
|
||||
], ),
|
||||
],
|
||||
LlavaLlamaLoader,
|
||||
template=TemplateType.llava1_6_yi,
|
||||
requires=['transformers>=4.34'],
|
||||
architectures=['LlavaLlamaForCausalLM'],
|
||||
tags=['vision'],
|
||||
model_arch=None))
|
||||
|
||||
|
||||
class LlavaNextQwenLoader(LlavaLoader):
|
||||
llm_model_type = 'next_qwen'
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_next_qwen, [
|
||||
ModelGroup([
|
||||
Model('AI-ModelScope/llava-next-72b', 'lmms-lab/llava-next-72b'),
|
||||
Model('AI-ModelScope/llava-next-110b', 'lmms-lab/llava-next-110b'),
|
||||
], ),
|
||||
],
|
||||
LlavaNextQwenLoader,
|
||||
template=TemplateType.llava_next_qwen,
|
||||
architectures=['LlavaQwenForCausalLM'],
|
||||
requires=['transformers>=4.42', 'av'],
|
||||
tags=['vision'],
|
||||
model_arch=None))
|
||||
|
||||
|
||||
class LlavaOnevisionLoader(ModelLoader):
|
||||
|
||||
def get_config(self, model_dir: str) -> PretrainedConfig:
|
||||
config = super().get_config(model_dir)
|
||||
config.vision_start_token_id = 151652
|
||||
return config
|
||||
|
||||
def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel:
|
||||
model_cls = get_class_from_dynamic_module(
|
||||
'modeling_llavaonevision1_5.LLaVAOneVision1_5_ForConditionalGeneration', model_dir)
|
||||
model_cls._no_split_modules = ['LLaVAOneVision1_5_DecoderLayer', 'RiceBlock']
|
||||
model = super().get_model(model_dir, *args, **kwargs)
|
||||
patch_get_input_embeddings(model.visual, 'patch_embed')
|
||||
return model
|
||||
|
||||
|
||||
register_model(
|
||||
ModelMeta(
|
||||
MLLMModelType.llava_onevision1_5,
|
||||
[
|
||||
ModelGroup([
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-4B-Instruct', 'lmms-lab/LLaVA-OneVision-1.5-4B-Instruct'),
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-8B-Instruct', 'lmms-lab/LLaVA-OneVision-1.5-8B-Instruct'),
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-4B-Base', 'lmms-lab/LLaVA-OneVision-1.5-4B-Base'),
|
||||
Model('lmms-lab/LLaVA-OneVision-1.5-8B-Base', 'lmms-lab/LLaVA-OneVision-1.5-8B-Base'),
|
||||
], ),
|
||||
],
|
||||
LlavaOnevisionLoader,
|
||||
template=TemplateType.llava_onevision1_5,
|
||||
architectures=['LLaVAOneVision1_5_ForConditionalGeneration'],
|
||||
model_arch=ModelArch.llava_onevision1_5,
|
||||
requires=['transformers>=4.53.0', 'qwen_vl_utils'],
|
||||
tags=['vision'],
|
||||
))
|
||||
Reference in New Issue
Block a user