This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,149 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Inference\n",
|
||||
"We have trained a well-trained checkpoint through the `self-cognition-sft.ipynb` tutorial, and here we use `TransformersEngine` to do the inference on it."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import some libraries\n",
|
||||
"import os\n",
|
||||
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
|
||||
"\n",
|
||||
"from swift.infer_engine import InferEngine, InferRequest, TransformersEngine, RequestConfig\n",
|
||||
"from swift import get_template"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Hyperparameters for inference\n",
|
||||
"last_model_checkpoint = 'output/checkpoint-xxx'\n",
|
||||
"\n",
|
||||
"# model\n",
|
||||
"model_id_or_path = 'Qwen/Qwen2.5-3B-Instruct' # model_id or model_path\n",
|
||||
"system = 'You are a helpful assistant.'\n",
|
||||
"infer_backend = 'transformers'\n",
|
||||
"\n",
|
||||
"# generation_config\n",
|
||||
"max_new_tokens = 512\n",
|
||||
"temperature = 0\n",
|
||||
"stream = True"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model and template, and load LoRA weights.\n",
|
||||
"engine = TransformersEngine(model_id_or_path, adapters=[last_model_checkpoint])\n",
|
||||
"template = get_template(engine.processor, default_system=system)\n",
|
||||
"# You can modify the `template` directly here, or pass it in during `engine.infer`.\n",
|
||||
"engine.template = template"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"query: who are you?\n",
|
||||
"response: I am an artificial intelligence language model named Xiao Huang, developed by ModelScope. I can answer various questions and engage in conversation with humans. If you have any questions or need help, feel free to ask me at any time.\n",
|
||||
"--------------------------------------------------\n",
|
||||
"query: What should I do if I can't sleep at night?\n",
|
||||
"response: If you're having trouble sleeping, there are several things you can try:\n",
|
||||
"\n",
|
||||
"1. Establish a regular sleep schedule: Try to go to bed and wake up at the same time every day, even on weekends.\n",
|
||||
"\n",
|
||||
"2. Create a relaxing bedtime routine: Engage in calming activities before bed, such as reading a book or taking a warm bath.\n",
|
||||
"\n",
|
||||
"3. Make your bedroom conducive to sleep: Keep your bedroom cool, dark, and quiet. Invest in comfortable bedding and pillows.\n",
|
||||
"\n",
|
||||
"4. Avoid stimulating activities before bed: Avoid using electronic devices, watching TV, or engaging in mentally stimulating activities before bed.\n",
|
||||
"\n",
|
||||
"5. Exercise regularly: Regular physical activity can help improve your sleep quality, but avoid exercising too close to bedtime.\n",
|
||||
"\n",
|
||||
"6. Manage stress: Practice relaxation techniques, such as deep breathing, meditation, or yoga, to help manage stress and promote better sleep.\n",
|
||||
"\n",
|
||||
"7. Limit caffeine and alcohol intake: Both caffeine and alcohol can disrupt sleep patterns, so it's best to limit their consumption, especially in the evening.\n",
|
||||
"\n",
|
||||
"8. Seek professional help: If you continue to have difficulty sleeping despite trying these strategies, consider seeking help from a healthcare provider or a sleep specialist.\n",
|
||||
"--------------------------------------------------\n",
|
||||
"query: 你是谁训练的？\n",
|
||||
"response: 我是由魔搭团队训练和开发的。\n",
|
||||
"--------------------------------------------------\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"query_list = [\n",
|
||||
" 'who are you?',\n",
|
||||
" \"What should I do if I can't sleep at night?\",\n",
|
||||
" '你是谁训练的？',\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"def infer_stream(engine: InferEngine, infer_request: InferRequest):\n",
|
||||
" request_config = RequestConfig(max_tokens=max_new_tokens, temperature=temperature, stream=True)\n",
|
||||
" gen_list = engine.infer([infer_request], request_config)\n",
|
||||
" query = infer_request.messages[0]['content']\n",
|
||||
" print(f'query: {query}\\nresponse: ', end='')\n",
|
||||
" for resp in gen_list[0]:\n",
|
||||
" if resp is None:\n",
|
||||
" continue\n",
|
||||
" print(resp.choices[0].delta.content, end='', flush=True)\n",
|
||||
" print()\n",
|
||||
"\n",
|
||||
"def infer(engine: InferEngine, infer_request: InferRequest):\n",
|
||||
" request_config = RequestConfig(max_tokens=max_new_tokens, temperature=temperature)\n",
|
||||
" resp_list = engine.infer([infer_request], request_config)\n",
|
||||
" query = infer_request.messages[0]['content']\n",
|
||||
" response = resp_list[0].choices[0].message.content\n",
|
||||
" print(f'query: {query}')\n",
|
||||
" print(f'response: {response}')\n",
|
||||
"\n",
|
||||
"infer_func = infer_stream if stream else infer\n",
|
||||
"for query in query_list:\n",
|
||||
" infer_func(engine, InferRequest(messages=[{'role': 'user', 'content': query}]))\n",
|
||||
" print('-' * 50)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "test_py310",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# Here is the command-line style inference code.
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
swift infer \
|
||||
--adapters output/vx-xxx/checkpoint-xxx \
|
||||
--stream true \
|
||||
--temperature 0 \
|
||||
--max_new_tokens 2048
|
||||
@@ -0,0 +1,218 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 10-minute self-cognition SFT\n",
|
||||
"\n",
|
||||
"Here is a demonstration of using python to perform self-cognition SFT of Qwen2.5-3B-Instruct. Through this tutorial, you can quickly understand some details of swift sft, which will be of great help in customizing ms-swift for you~\n",
|
||||
"\n",
|
||||
"Are you ready? Let's begin the journey...\n",
|
||||
"\n",
|
||||
"中文版：[魔搭教程](https://github.com/modelscope/modelscope-classroom/blob/main/LLM-tutorial/R.10%E5%88%86%E9%92%9F%E6%94%B9%E5%8F%98%E5%A4%A7%E6%A8%A1%E5%9E%8B%E8%87%AA%E6%88%91%E8%AE%A4%E7%9F%A5.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "shellscript"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# # install ms-swift\n",
|
||||
"# pip install ms-swift -U"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import some libraries\n",
|
||||
"import os\n",
|
||||
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
|
||||
"\n",
|
||||
"from swift import get_model_processor, load_dataset, get_template\n",
|
||||
"from swift.dataset import EncodePreprocessor\n",
|
||||
"from swift.utils import get_logger, find_all_linears, get_model_parameter_info, plot_images, seed_everything\n",
|
||||
"from peft import get_peft_model, LoraConfig\n",
|
||||
"from swift.trainers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n",
|
||||
"\n",
|
||||
"logger = get_logger()\n",
|
||||
"seed_everything(42)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Hyperparameters for training\n",
|
||||
"# model\n",
|
||||
"model_id_or_path = 'Qwen/Qwen2.5-3B-Instruct' # model_id or model_path\n",
|
||||
"system = 'You are a helpful assistant.'\n",
|
||||
"output_dir = 'output'\n",
|
||||
"\n",
|
||||
"# dataset\n",
|
||||
"dataset = ['AI-ModelScope/alpaca-gpt4-data-zh#500', 'AI-ModelScope/alpaca-gpt4-data-en#500',\n",
|
||||
" 'swift/self-cognition#500'] # dataset_id or dataset_path\n",
|
||||
"data_seed = 42\n",
|
||||
"max_length = 2048\n",
|
||||
"split_dataset_ratio = 0.01 # Split validation set\n",
|
||||
"num_proc = 4 # The number of processes for data loading.\n",
|
||||
"# The following two parameters are used to override the placeholders in the self-cognition dataset.\n",
|
||||
"model_name = ['小黄', 'Xiao Huang'] # The Chinese name and English name of the model\n",
|
||||
"model_author = ['魔搭', 'ModelScope'] # The Chinese name and English name of the model author\n",
|
||||
"\n",
|
||||
"# lora\n",
|
||||
"lora_rank = 8\n",
|
||||
"lora_alpha = 32\n",
|
||||
"\n",
|
||||
"# training_args\n",
|
||||
"training_args = Seq2SeqTrainingArguments(\n",
|
||||
" output_dir=output_dir,\n",
|
||||
" learning_rate=1e-4,\n",
|
||||
" per_device_train_batch_size=1,\n",
|
||||
" per_device_eval_batch_size=1,\n",
|
||||
" gradient_checkpointing=True,\n",
|
||||
" weight_decay=0.1,\n",
|
||||
" lr_scheduler_type='cosine',\n",
|
||||
" warmup_ratio=0.05,\n",
|
||||
" report_to=['tensorboard'],\n",
|
||||
" logging_first_step=True,\n",
|
||||
" save_strategy='steps',\n",
|
||||
" save_steps=50,\n",
|
||||
" eval_strategy='steps',\n",
|
||||
" eval_steps=50,\n",
|
||||
" gradient_accumulation_steps=16,\n",
|
||||
" num_train_epochs=1,\n",
|
||||
" metric_for_best_model='loss',\n",
|
||||
" save_total_limit=2,\n",
|
||||
" logging_steps=5,\n",
|
||||
" dataloader_num_workers=1,\n",
|
||||
" data_seed=data_seed,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"output_dir = os.path.abspath(os.path.expanduser(output_dir))\n",
|
||||
"logger.info(f'output_dir: {output_dir}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Obtain the model and template, and add a trainable Lora layer on the model.\n",
|
||||
"model, tokenizer = get_model_processor(model_id_or_path)\n",
|
||||
"logger.info(f'model_info: {model.model_info}')\n",
|
||||
"template = get_template(tokenizer, default_system=system, max_length=max_length)\n",
|
||||
"template.set_mode('train')\n",
|
||||
"\n",
|
||||
"target_modules = find_all_linears(model)\n",
|
||||
"lora_config = LoraConfig(task_type='CAUSAL_LM', r=lora_rank, lora_alpha=lora_alpha,\n",
|
||||
" target_modules=target_modules)\n",
|
||||
"model = get_peft_model(model, lora_config)\n",
|
||||
"logger.info(f'lora_config: {lora_config}')\n",
|
||||
"\n",
|
||||
"# Print model structure and trainable parameters.\n",
|
||||
"logger.info(f'model: {model}')\n",
|
||||
"model_parameter_info = get_model_parameter_info(model)\n",
|
||||
"logger.info(f'model_parameter_info: {model_parameter_info}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Download and load the dataset, split it into a training set and a validation set,\n",
|
||||
"# and encode the text data into tokens.\n",
|
||||
"train_dataset, val_dataset = load_dataset(dataset, split_dataset_ratio=split_dataset_ratio, num_proc=num_proc,\n",
|
||||
" model_name=model_name, model_author=model_author, seed=data_seed)\n",
|
||||
"\n",
|
||||
"logger.info(f'train_dataset: {train_dataset}')\n",
|
||||
"logger.info(f'val_dataset: {val_dataset}')\n",
|
||||
"logger.info(f'train_dataset[0]: {train_dataset[0]}')\n",
|
||||
"\n",
|
||||
"train_dataset = EncodePreprocessor(template=template)(train_dataset, num_proc=num_proc)\n",
|
||||
"val_dataset = EncodePreprocessor(template=template)(val_dataset, num_proc=num_proc)\n",
|
||||
"logger.info(f'encoded_train_dataset[0]: {train_dataset[0]}')\n",
|
||||
"\n",
|
||||
"# Print a sample\n",
|
||||
"template.print_inputs(train_dataset[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get the trainer and start the training.\n",
|
||||
"model.enable_input_require_grads() # Compatible with gradient checkpointing\n",
|
||||
"trainer = Seq2SeqTrainer(\n",
|
||||
" model=model,\n",
|
||||
" args=training_args,\n",
|
||||
" template=template,\n",
|
||||
" train_dataset=train_dataset,\n",
|
||||
" eval_dataset=val_dataset,\n",
|
||||
")\n",
|
||||
"trainer.train()\n",
|
||||
"\n",
|
||||
"last_model_checkpoint = trainer.state.last_model_checkpoint\n",
|
||||
"logger.info(f'last_model_checkpoint: {last_model_checkpoint}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Visualize the training loss.\n",
|
||||
"# You can also use the TensorBoard visualization interface during training by entering\n",
|
||||
"# `tensorboard --logdir '{output_dir}/runs'` at the command line.\n",
|
||||
"images_dir = os.path.join(output_dir, 'images')\n",
|
||||
"logger.info(f'images_dir: {images_dir}')\n",
|
||||
"plot_images(images_dir, training_args.logging_dir, ['train/loss'], 0.9) # save images\n",
|
||||
"\n",
|
||||
"# Read and display the image.\n",
|
||||
"# The light yellow line represents the actual loss value,\n",
|
||||
"# while the yellow line represents the loss value smoothed with a smoothing factor of 0.9.\n",
|
||||
"from IPython.display import display\n",
|
||||
"from PIL import Image\n",
|
||||
"image = Image.open(os.path.join(images_dir, 'train_loss.png'))\n",
|
||||
"display(image)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "hjt",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.14"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# Here is the command-line style training code.
|
||||
# 22GB
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
swift sft \
|
||||
--model Qwen/Qwen2.5-3B-Instruct \
|
||||
--tuner_type lora \
|
||||
--dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
|
||||
'AI-ModelScope/alpaca-gpt4-data-en#500' \
|
||||
'swift/self-cognition#500' \
|
||||
--torch_dtype bfloat16 \
|
||||
--num_train_epochs 1 \
|
||||
--per_device_train_batch_size 1 \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--learning_rate 1e-4 \
|
||||
--lora_rank 8 \
|
||||
--lora_alpha 32 \
|
||||
--target_modules all-linear \
|
||||
--gradient_accumulation_steps 16 \
|
||||
--eval_steps 50 \
|
||||
--save_steps 50 \
|
||||
--save_total_limit 2 \
|
||||
--logging_steps 5 \
|
||||
--max_length 2048 \
|
||||
--output_dir output \
|
||||
--system 'You are a helpful assistant.' \
|
||||
--warmup_ratio 0.05 \
|
||||
--dataloader_num_workers 4 \
|
||||
--dataset_num_proc 4 \
|
||||
--model_name 小黄 'Xiao Huang' \
|
||||
--model_author '魔搭' 'ModelScope'
|
||||
@@ -0,0 +1,263 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Qwen2.5-VL Grounding任务\n",
|
||||
"\n",
|
||||
"这里介绍使用qwen2.5-vl进行grounding任务的全流程介绍。当然，你也可以使用internvl2.5或者qwen2-vl等多模态模型。\n",
|
||||
"\n",
|
||||
"我们使用[AI-ModelScope/coco](https://modelscope.cn/datasets/AI-ModelScope/coco)数据集来展示整个流程。\n",
|
||||
"\n",
|
||||
"如果需要使用自定义数据集，需要符合以下格式："
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"<image>描述图像\"}, {\"role\": \"assistant\", \"content\": \"<ref-object><bbox>和<ref-object><bbox>正在沙滩上玩耍\"}], \"images\": [\"/xxx/x.jpg\"], \"objects\": {\"ref\": [\"一只狗\", \"一个女人\"], \"bbox\": [[331.5, 761.4, 853.5, 1594.8], [676.5, 685.8, 1099.5, 1427.4]]}}\n",
|
||||
"{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"<image>找到图像中的<ref-object>\"}, {\"role\": \"assistant\", \"content\": \"<bbox><bbox>\"}], \"images\": [\"/xxx/x.jpg\"], \"objects\": {\"ref\": [\"羊\"], \"bbox\": [[90.9, 160.8, 135, 212.8], [360.9, 480.8, 495, 532.8]]}}\n",
|
||||
"{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"<image>帮我打开谷歌浏览器\"}, {\"role\": \"assistant\", \"content\": \"Action: click(start_box='<bbox>')\"}], \"images\": [\"/xxx/x.jpg\"], \"objects\": {\"ref\": [], \"bbox\": [[615, 226]]}}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"ms-swift在预处理数据集时，会使用模型特有的grounding任务格式，将objects中的ref填充`<ref-object>`，bbox会根据模型类型选择是否进行0-1000的归一化，并填充`<bbox>`。例如：qwen2-vl为`f'<|object_ref_start|>羊<|object_ref_end|>'`和`f'<|box_start|>(101,201),(150,266)<|box_end|>'`（qwen2.5-vl不进行归一化，只将float型转成int型），internvl2.5则为`f'<ref>羊</ref>'`和`f'<box>[[101, 201, 150, 266]]</box>'`等。\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"训练之前，你需要从main分支安装ms-swift："
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "shellscript"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# pip install git+https://github.com/modelscope/ms-swift.git\n",
|
||||
"\n",
|
||||
"git clone https://github.com/modelscope/ms-swift.git\n",
|
||||
"cd ms-swift\n",
|
||||
"pip install -e ."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"然后，使用以下shell进行训练。MAX_PIXELS的参数含义可以查看[这里](https://swift.readthedocs.io/en/latest/Instruction/Command-line-parameters.html#specific-model-arguments)\n",
|
||||
"\n",
|
||||
"### 训练\n",
|
||||
"\n",
|
||||
"单卡训练："
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "shellscript"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 显存资源：24GiB\n",
|
||||
"CUDA_VISIBLE_DEVICES=0 \\\n",
|
||||
"MAX_PIXELS=1003520 \\\n",
|
||||
"swift sft \\\n",
|
||||
" --model Qwen/Qwen2.5-VL-7B-Instruct \\\n",
|
||||
" --dataset 'AI-ModelScope/coco#2000' \\\n",
|
||||
" --load_from_cache_file true \\\n",
|
||||
" --split_dataset_ratio 0.01 \\\n",
|
||||
" --tuner_type lora \\\n",
|
||||
" --torch_dtype bfloat16 \\\n",
|
||||
" --num_train_epochs 1 \\\n",
|
||||
" --per_device_train_batch_size 1 \\\n",
|
||||
" --per_device_eval_batch_size 1 \\\n",
|
||||
" --learning_rate 1e-4 \\\n",
|
||||
" --lora_rank 8 \\\n",
|
||||
" --lora_alpha 32 \\\n",
|
||||
" --target_modules all-linear \\\n",
|
||||
" --freeze_vit true \\\n",
|
||||
" --freeze_aligner true \\\n",
|
||||
" --gradient_accumulation_steps 16 \\\n",
|
||||
" --eval_steps 100 \\\n",
|
||||
" --save_steps 100 \\\n",
|
||||
" --save_total_limit 5 \\\n",
|
||||
" --logging_steps 5 \\\n",
|
||||
" --max_length 2048 \\\n",
|
||||
" --output_dir output \\\n",
|
||||
" --warmup_ratio 0.05 \\\n",
|
||||
" --dataloader_num_workers 4 \\\n",
|
||||
" --dataset_num_proc 4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"然后我们将训练的模型推送到ModelScope："
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "shellscript"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"swift export \\\n",
|
||||
" --adapters output/vx-xxx/checkpoint-xxx \\\n",
|
||||
" --push_to_hub true \\\n",
|
||||
" --hub_model_id '<model-id>' \\\n",
|
||||
" --hub_token '<sdk-token>' \\\n",
|
||||
" --use_hf false"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"我们将训练的checkpoint推送到[swift/test_grounding](https://modelscope.cn/models/swift/test_grounding)。\n",
|
||||
"\n",
|
||||
"### 推理\n",
|
||||
"\n",
|
||||
"训练完成后，我们使用以下命令对训练时的验证集进行推理。这里`--adapters`需要替换成训练生成的last checkpoint文件夹。由于adapters文件夹中包含了训练的参数文件，因此不需要额外指定`--model`。\n",
|
||||
"\n",
|
||||
"若模型采用的是绝对坐标的方式进行输出，推理时请提前对图像进行缩放而不使用`MAX_PIXELS`或者`--max_pixels`。若是千分位坐标，则没有此约束。\n",
|
||||
"\n",
|
||||
"由于我们已经将训练后的checkpoint推送到了ModelScope上，以下推理脚本可以直接运行："
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "shellscript"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"CUDA_VISIBLE_DEVICES=0 \\\n",
|
||||
"swift infer \\\n",
|
||||
" --adapters swift/test_grounding \\\n",
|
||||
" --stream true \\\n",
|
||||
" --load_data_args true \\\n",
|
||||
" --max_new_tokens 512 \\\n",
|
||||
" --dataset_num_proc 4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"我们也可以使用代码的方式进行推理：\n",
|
||||
"\n",
|
||||
"单样本推理的例子可以查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_grounding.py)。"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
|
||||
"\n",
|
||||
"import re\n",
|
||||
"from typing import Literal\n",
|
||||
"from swift.arguments import BaseArguments\n",
|
||||
"from swift.template import load_image, draw_bbox\n",
|
||||
"from swift.dataset import load_dataset\n",
|
||||
"from swift.infer_engine import TransformersEngine, RequestConfig, InferRequest, InferEngine\n",
|
||||
"from swift.utils import safe_snapshot_download\n",
|
||||
"from IPython.display import display\n",
|
||||
"\n",
|
||||
"def infer_stream(engine: InferEngine, infer_request: InferRequest):\n",
|
||||
" request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)\n",
|
||||
" gen_list = engine.infer([infer_request], request_config)\n",
|
||||
" query = infer_request.messages[0]['content']\n",
|
||||
" print(f'query: {query}\\nresponse: ', end='')\n",
|
||||
" response = ''\n",
|
||||
" for resp in gen_list[0]:\n",
|
||||
" if resp is None:\n",
|
||||
" continue\n",
|
||||
" delta = resp.choices[0].delta.content\n",
|
||||
" response += delta\n",
|
||||
" print(delta, end='', flush=True)\n",
|
||||
" print()\n",
|
||||
" return response\n",
|
||||
"\n",
|
||||
"def draw_bbox_qwen2_vl(image, response, norm_bbox: Literal['norm1000', 'none']):\n",
|
||||
" matches = re.findall(\n",
|
||||
" r'<\\|object_ref_start\\|>(.*?)<\\|object_ref_end\\|><\\|box_start\\|>\\((\\d+),(\\d+)\\),\\((\\d+),(\\d+)\\)<\\|box_end\\|>',\n",
|
||||
" response)\n",
|
||||
" ref = []\n",
|
||||
" bbox = []\n",
|
||||
" for match_ in matches:\n",
|
||||
" ref.append(match_[0])\n",
|
||||
" bbox.append(list(match_[1:]))\n",
|
||||
" draw_bbox(image, ref, bbox, norm_bbox=norm_bbox)\n",
|
||||
"\n",
|
||||
"# 下载权重，并加载模型\n",
|
||||
"output_dir = 'images_bbox'\n",
|
||||
"model_id_or_path = 'swift/test_grounding'\n",
|
||||
"output_dir = os.path.abspath(os.path.expanduser(output_dir))\n",
|
||||
"adapter_path = safe_snapshot_download(model_id_or_path)\n",
|
||||
"args = BaseArguments.from_pretrained(adapter_path)\n",
|
||||
"engine = TransformersEngine(args.model, adapters=[adapter_path])\n",
|
||||
"\n",
|
||||
"# 获取验证集并推理\n",
|
||||
"_, val_dataset = load_dataset(args.dataset, split_dataset_ratio=args.split_dataset_ratio, num_proc=4, seed=args.seed)\n",
|
||||
"print(f'output_dir: {output_dir}')\n",
|
||||
"os.makedirs(output_dir, exist_ok=True)\n",
|
||||
"for i, data in enumerate(val_dataset):\n",
|
||||
" image = data['images'][0]\n",
|
||||
" image = load_image(image['bytes'] or image['path'])\n",
|
||||
" display(image)\n",
|
||||
" response = infer_stream(engine, InferRequest(**data))\n",
|
||||
" draw_bbox_qwen2_vl(image, response, norm_bbox=args.norm_bbox)\n",
|
||||
" print('-' * 50)\n",
|
||||
" image.save(os.path.join(output_dir, f'{i}.png'))\n",
|
||||
" display(image)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "test_py310",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Inference\n",
|
||||
"We have trained a well-trained checkpoint through the `ocr-sft.ipynb` tutorial, and here we use `TransformersEngine` to do the inference on it."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import some libraries\n",
|
||||
"import os\n",
|
||||
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
|
||||
"\n",
|
||||
"from swift.infer_engine import (\n",
|
||||
" InferEngine, InferRequest, TransformersEngine, RequestConfig, get_template, load_dataset, load_image\n",
|
||||
")\n",
|
||||
"from swift.utils import get_model_parameter_info, get_logger, seed_everything\n",
|
||||
"logger = get_logger()\n",
|
||||
"seed_everything(42)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Hyperparameters for inference\n",
|
||||
"last_model_checkpoint = 'output/checkpoint-xxx'\n",
|
||||
"\n",
|
||||
"# model\n",
|
||||
"model_id_or_path = 'Qwen/Qwen2-VL-2B-Instruct' # model_id or model_path\n",
|
||||
"system = None\n",
|
||||
"infer_backend = 'transformers'\n",
|
||||
"\n",
|
||||
"# dataset\n",
|
||||
"dataset = ['AI-ModelScope/LaTeX_OCR#20000']\n",
|
||||
"data_seed = 42\n",
|
||||
"split_dataset_ratio = 0.01\n",
|
||||
"num_proc = 4\n",
|
||||
"strict = False\n",
|
||||
"\n",
|
||||
"# generation_config\n",
|
||||
"max_new_tokens = 512\n",
|
||||
"temperature = 0\n",
|
||||
"stream = True"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model and template, and load LoRA weights.\n",
|
||||
"engine = TransformersEngine(model_id_or_path, adapters=[last_model_checkpoint])\n",
|
||||
"template = get_template(engine.model_meta.template, engine.processor, default_system=system)\n",
|
||||
"# The default mode of the template is 'transformers', so there is no need to make any changes.\n",
|
||||
"# template.set_mode('transformers')\n",
|
||||
"\n",
|
||||
"model_parameter_info = get_model_parameter_info(engine.model)\n",
|
||||
"logger.info(f'model_parameter_info: {model_parameter_info}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Due to the data_seed setting, the validation set here is the same as the validation set used during training.\n",
|
||||
"_, val_dataset = load_dataset(dataset, split_dataset_ratio=split_dataset_ratio, num_proc=num_proc,\n",
|
||||
" strict=strict, seed=data_seed)\n",
|
||||
"val_dataset = val_dataset.select(range(10)) # Take the first 10 items"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Streaming inference and save images from the validation set.\n",
|
||||
"# The batch processing code can be found here: https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_mllm.py\n",
|
||||
"def infer_stream(engine: InferEngine, infer_request: InferRequest):\n",
|
||||
" request_config = RequestConfig(max_tokens=max_new_tokens, temperature=temperature, stream=True)\n",
|
||||
" gen_list = engine.infer([infer_request], request_config)\n",
|
||||
" query = infer_request.messages[0]['content']\n",
|
||||
" print(f'query: {query}\\nresponse: ', end='')\n",
|
||||
" for resp in gen_list[0]:\n",
|
||||
" if resp is None:\n",
|
||||
" continue\n",
|
||||
" print(resp.choices[0].delta.content, end='', flush=True)\n",
|
||||
" print()\n",
|
||||
"\n",
|
||||
"from IPython.display import display\n",
|
||||
"os.makedirs('images', exist_ok=True)\n",
|
||||
"for i, data in enumerate(val_dataset):\n",
|
||||
" image = data['images'][0]\n",
|
||||
" image = load_image(image['bytes'] or image['path'])\n",
|
||||
" image.save(f'images/{i}.png')\n",
|
||||
" display(image)\n",
|
||||
" infer_stream(engine, InferRequest(**data))\n",
|
||||
" print('-' * 50)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "test_py310",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Latex-OCR SFT\n",
|
||||
"\n",
|
||||
"Here is a demonstration of using python to perform Latex-OCR SFT of Qwen2-VL-2B-Instruct. Through this tutorial, you can quickly understand some details of swift sft, which will be of great help in customizing ms-swift for you~\n",
|
||||
"\n",
|
||||
"Are you ready? Let's begin the journey..."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "shellscript"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# # install ms-swift\n",
|
||||
"# pip install ms-swift -U"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import some libraries\n",
|
||||
"import os\n",
|
||||
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
|
||||
"\n",
|
||||
"from swift import get_model_processor, load_dataset, get_template\n",
|
||||
"from peft import get_peft_model, LoraConfig\n",
|
||||
"from swift.dataset import EncodePreprocessor, LazyLLMDataset\n",
|
||||
"from swift.utils import get_logger, get_model_parameter_info, plot_images, seed_everything, get_multimodal_target_regex\n",
|
||||
"from swift.trainers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n",
|
||||
"\n",
|
||||
"logger = get_logger()\n",
|
||||
"seed_everything(42)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Hyperparameters for training\n",
|
||||
"# model\n",
|
||||
"model_id_or_path = 'Qwen/Qwen2-VL-2B-Instruct'\n",
|
||||
"system = None # Using the default system defined in the template.\n",
|
||||
"output_dir = 'output'\n",
|
||||
"\n",
|
||||
"# dataset\n",
|
||||
"dataset = ['AI-ModelScope/LaTeX_OCR#20000'] # dataset_id or dataset_path. Sampling 20000 data points\n",
|
||||
"data_seed = 42\n",
|
||||
"max_length = 2048\n",
|
||||
"split_dataset_ratio = 0.01 # Split validation set\n",
|
||||
"num_proc = 4 # The number of processes for data loading.\n",
|
||||
"\n",
|
||||
"# lora\n",
|
||||
"lora_rank = 8\n",
|
||||
"lora_alpha = 32\n",
|
||||
"freeze_llm = False\n",
|
||||
"freeze_vit = True\n",
|
||||
"freeze_aligner = True\n",
|
||||
"\n",
|
||||
"# training_args\n",
|
||||
"training_args = Seq2SeqTrainingArguments(\n",
|
||||
" output_dir=output_dir,\n",
|
||||
" learning_rate=1e-4,\n",
|
||||
" per_device_train_batch_size=1,\n",
|
||||
" per_device_eval_batch_size=1,\n",
|
||||
" gradient_checkpointing=True,\n",
|
||||
" weight_decay=0.1,\n",
|
||||
" lr_scheduler_type='cosine',\n",
|
||||
" warmup_ratio=0.05,\n",
|
||||
" report_to=['tensorboard'],\n",
|
||||
" logging_first_step=True,\n",
|
||||
" save_strategy='steps',\n",
|
||||
" save_steps=50,\n",
|
||||
" eval_strategy='steps',\n",
|
||||
" eval_steps=50,\n",
|
||||
" gradient_accumulation_steps=16,\n",
|
||||
" # To observe the training results more quickly, this is set to 1 here. \n",
|
||||
" # Under normal circumstances, a larger number should be used.\n",
|
||||
" num_train_epochs=1,\n",
|
||||
" metric_for_best_model='loss',\n",
|
||||
" save_total_limit=5,\n",
|
||||
" logging_steps=5,\n",
|
||||
" dataloader_num_workers=4,\n",
|
||||
" data_seed=data_seed,\n",
|
||||
" remove_unused_columns=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"output_dir = os.path.abspath(os.path.expanduser(output_dir))\n",
|
||||
"logger.info(f'output_dir: {output_dir}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Obtain the model and template\n",
|
||||
"model, processor = get_model_processor(model_id_or_path)\n",
|
||||
"logger.info(f'model_info: {model.model_info}')\n",
|
||||
"template = get_template(processor, default_system=system, max_length=max_length)\n",
|
||||
"template.set_mode('train')\n",
|
||||
"if template.use_model:\n",
|
||||
" template.model = model\n",
|
||||
"\n",
|
||||
"# Get target_modules and add trainable LoRA modules to the model.\n",
|
||||
"target_modules = get_multimodal_target_regex(model, freeze_llm=freeze_llm, freeze_vit=freeze_vit,\n",
|
||||
" freeze_aligner=freeze_aligner)\n",
|
||||
"lora_config = LoraConfig(task_type='CAUSAL_LM', r=lora_rank, lora_alpha=lora_alpha,\n",
|
||||
" target_modules=target_modules)\n",
|
||||
"model = get_peft_model(model, lora_config)\n",
|
||||
"logger.info(f'lora_config: {lora_config}')\n",
|
||||
"\n",
|
||||
"# Print model structure and trainable parameters.\n",
|
||||
"logger.info(f'model: {model}')\n",
|
||||
"model_parameter_info = get_model_parameter_info(model)\n",
|
||||
"logger.info(f'model_parameter_info: {model_parameter_info}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Download and load the dataset, split it into a training set and a validation set,\n",
|
||||
"# and encode the text data into tokens.\n",
|
||||
"train_dataset, val_dataset = load_dataset(dataset, split_dataset_ratio=split_dataset_ratio, num_proc=num_proc,\n",
|
||||
" seed=data_seed)\n",
|
||||
"\n",
|
||||
"logger.info(f'train_dataset: {train_dataset}')\n",
|
||||
"logger.info(f'val_dataset: {val_dataset}')\n",
|
||||
"logger.info(f'train_dataset[0]: {train_dataset[0]}')\n",
|
||||
"\n",
|
||||
"train_dataset = LazyLLMDataset(train_dataset, template.encode, random_state=data_seed)\n",
|
||||
"val_dataset = LazyLLMDataset(val_dataset, template.encode, random_state=data_seed)\n",
|
||||
"data = train_dataset[0]\n",
|
||||
"logger.info(f'encoded_train_dataset[0]: {data}')\n",
|
||||
"\n",
|
||||
"template.print_inputs(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get the trainer and start the training.\n",
|
||||
"model.enable_input_require_grads() # Compatible with gradient checkpointing\n",
|
||||
"trainer = Seq2SeqTrainer(\n",
|
||||
" model=model,\n",
|
||||
" args=training_args,\n",
|
||||
" template=template,\n",
|
||||
" train_dataset=train_dataset,\n",
|
||||
" eval_dataset=val_dataset,\n",
|
||||
")\n",
|
||||
"trainer.train()\n",
|
||||
"\n",
|
||||
"last_model_checkpoint = trainer.state.last_model_checkpoint\n",
|
||||
"logger.info(f'last_model_checkpoint: {last_model_checkpoint}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Visualize the training loss.\n",
|
||||
"# You can also use the TensorBoard visualization interface during training by entering\n",
|
||||
"# `tensorboard --logdir '{output_dir}/runs'` at the command line.\n",
|
||||
"images_dir = os.path.join(output_dir, 'images')\n",
|
||||
"logger.info(f'images_dir: {images_dir}')\n",
|
||||
"plot_images(images_dir, training_args.logging_dir, ['train/loss'], 0.9) # save images\n",
|
||||
"\n",
|
||||
"# Read and display the image.\n",
|
||||
"# The light yellow line represents the actual loss value,\n",
|
||||
"# while the yellow line represents the loss value smoothed with a smoothing factor of 0.9.\n",
|
||||
"from IPython.display import display\n",
|
||||
"from PIL import Image\n",
|
||||
"image = Image.open(os.path.join(images_dir, 'train_loss.png'))\n",
|
||||
"display(image)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "hjt",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.14"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user