chore: import upstream snapshot with attribution

2026-07-13 13:34:58 +08:00
commit a203934033
1368 changed files with 175001 additions and 0 deletions
@@ -0,0 +1,13 @@
+# Instructions
+
+The example provides instructions for using SWIFT for training, inference, deployment, evaluation, and quantization. By default, the model will be downloaded from the ModelScope community.
+
+If you want to use the Huggingface community, you can change the command line like this:
+
+```shell
+...
+swift sft \
+    --model <model_id_or_path> \
+    --use_hf 1 \
+    ...
+```
@@ -0,0 +1,14 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+if __name__ == '__main__':
+    from swift import AppArguments, DeployArguments, app_main, run_deploy
+
+    # Here's a runnable demo provided.
+    # In a real scenario, you can simply remove the deployed context.
+    with run_deploy(
+            DeployArguments(model='Qwen/Qwen2.5-1.5B-Instruct', verbose=False, log_interval=-1, infer_backend='vllm'),
+            return_url=True) as url:
+        app_main(AppArguments(model='Qwen2.5-1.5B-Instruct', base_url=url, stream=True, max_new_tokens=2048))
@@ -0,0 +1,7 @@
+# You need to have a deployed model or api service first
+CUDA_VISIBLE_DEVICES=0 swift app \
+    --model '<model_name>' \
+    --base_url http://127.0.0.1:8000/v1 \
+    --stream true \
+    --max_new_tokens 2048 \
+    --lang zh
@@ -0,0 +1,7 @@
+# test_env: pip install "sglang[all]==0.4.6.*" -U
+CUDA_VISIBLE_DEVICES=0 swift app \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --stream true \
+    --infer_backend sglang \
+    --max_new_tokens 2048 \
+    --lang zh
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0 swift app \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --stream true \
+    --infer_backend vllm \
+    --max_new_tokens 2048 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --lang zh
@@ -0,0 +1,13 @@
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+swift app \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --stream true \
+    --infer_backend vllm \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --vllm_limit_mm_per_prompt '{"image": 5, "video": 2}' \
+    --lang zh
@@ -0,0 +1,26 @@
+{
+    "_description": "FSDP2 configuration for distributed training (PyTorch native FSDP v2)",
+    "_requires": "torch>=2.4.0",
+    "_note": "This is the recommended configuration for multi-GPU training without CPU offloading. NOTE: When using FSDP2, do NOT use --gradient_checkpointing, use activation_checkpointing in fsdp_config instead.",
+
+    "_param_docs": {
+        "fsdp": "FSDP strategy string. Options: 'full_shard' (ZeRO-3 style, shards params+grads+optimizer), 'shard_grad_op' (ZeRO-2 style, shards grads+optimizer only). Add 'auto_wrap' to enable automatic layer wrapping. Add 'offload' to enable CPU offloading.",
+        "fsdp_version": "FSDP version. Use 2 for PyTorch native FSDP2 (recommended). FSDP2 uses DTensor for per-parameter sharding, supports LoRA/QLoRA natively.",
+        "auto_wrap_policy": "How to wrap model layers. 'TRANSFORMER_BASED_WRAP' wraps transformer decoder layers (from model._no_split_modules). 'SIZE_BASED_WRAP' wraps modules exceeding min_num_params.",
+        "cpu_ram_efficient_loading": "If true, only rank 0 loads full model weights, then broadcasts to other ranks. Reduces CPU RAM usage during initialization.",
+        "state_dict_type": "'SHARDED_STATE_DICT' (recommended): each rank saves its own shard without extra communication. 'FULL_STATE_DICT': gathers full model on rank 0 (higher memory, slower).",
+        "reshard_after_forward": "true = FULL_SHARD (ZeRO-3), reshards params after forward pass. false = SHARD_GRAD_OP (ZeRO-2), keeps params gathered during forward/backward.",
+        "activation_checkpointing": "Use FSDP's native activation checkpointing instead of gradient_checkpointing. This is the correct way to save memory with FSDP.",
+        "activation_cpu_offload": "true = offload activations to CPU. false = keep activations on GPU,can enable when using activation_checkpointing."
+    },
+    "fsdp": "full_shard auto_wrap",
+    "fsdp_config": {
+        "fsdp_version": 2,
+        "reshard_after_forward": true,
+        "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+        "cpu_ram_efficient_loading": true,
+        "state_dict_type": "SHARDED_STATE_DICT",
+        "activation_checkpointing": false,
+        "activation_cpu_offload": true
+    }
+}
@@ -0,0 +1,57 @@
+#!/bin/bash
+ASCEND_RT_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=2 \
+swift sft \
+    --model 'Qwen/Qwen3-8B' \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_checkpointing false \
+    --weight_decay 0.1 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 5 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --system You\ are\ a\ helpful\ assistant. \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --fsdp './examples/ascend/activation_cpu_offload/fsdp2.json'
+
+#  --dataset AI-ModelScope/LongAlpaca-12k
+# activation_cpu_offload=false
+
+# {'loss': 2.93329144, 'grad_norm': 2.44835496, 'learning_rate': 0.0001, 'token_acc': 0.56405613, 'epoch': 0.06, 'global_step/max_steps': '1/16', 'percentage': '6.25%', 'elapsed_time': '8s', 'remaining_time': '2m 6s', 'memory(GiB)': 24.8, 'train_speed(iter/s)': 0.118837}
+# {'loss': 2.93490505, 'grad_norm': 2.63550186, 'learning_rate': 8.346e-05, 'token_acc': 0.58979954, 'epoch': 0.32, 'global_step/max_steps': '5/16', 'percentage': '31.25%', 'elapsed_time': '28s', 'remaining_time': '1m 2s', 'memory(GiB)': 57.91, 'train_speed(iter/s)': 0.175644}
+# Train:  31%|███████████████████████████████████                                                                             | 5/16 [00:28<00:57,  5.22s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-5
+# {'loss': 1.61339226, 'grad_norm': 1.05343676, 'learning_rate': 3.455e-05, 'token_acc': 0.63342983, 'epoch': 0.64, 'global_step/max_steps': '10/16', 'percentage': '62.50%', 'elapsed_time': '51s', 'remaining_time': '31s', 'memory(GiB)': 58.02, 'train_speed(iter/s)': 0.192856}
+# Train:  62%|█████████████████████████████████████████████████████████████████████▍                                         | 10/16 [00:51<00:27,  4.66s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-10
+# {'loss': 1.32472887, 'grad_norm': 0.60581738, 'learning_rate': 1.09e-06, 'token_acc': 0.64779323, 'epoch': 0.96, 'global_step/max_steps': '15/16', 'percentage': '93.75%', 'elapsed_time': '1m 13s', 'remaining_time': '4s', 'memory(GiB)': 58.02, 'train_speed(iter/s)': 0.204973}
+# Train:  94%|████████████████████████████████████████████████████████████████████████████████████████████████████████       | 15/16 [01:13<00:04,  4.12s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-15
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:17<00:00,  4.25s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-16
+# {'train_runtime': 79.7064, 'train_samples_per_second': 6.311, 'train_steps_per_second': 0.201, 'train_loss': 1.91648413, 'token_acc': 0.68027888, 'epoch': 1.0, 'global_step/max_steps': '16/16', 'percentage': '100.00%', 'elapsed_time': '1m 19s', 'remaining_time': '0s', 'memory(GiB)': 58.02, 'train_speed(iter/s)': 0.200728}
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:19<00:00,  4.98s/it]
+
+
+#  --dataset AI-ModelScope/LongAlpaca-12k
+# "activation_cpu_offload": true
+
+# {'loss': 2.93329144, 'grad_norm': 2.44853568, 'learning_rate': 0.0001, 'token_acc': 0.56405613, 'epoch': 0.06, 'global_step/max_steps': '1/16', 'percentage': '6.25%', 'elapsed_time': '26s', 'remaining_time': '6m 43s', 'memory(GiB)': 24.62, 'train_speed(iter/s)': 0.037168}
+# {'loss': 2.93512678, 'grad_norm': 2.6212213, 'learning_rate': 8.346e-05, 'token_acc': 0.5895268, 'epoch': 0.32, 'global_step/max_steps': '5/16', 'percentage': '31.25%', 'elapsed_time': '1m 21s', 'remaining_time': '2m 58s', 'memory(GiB)': 26.93, 'train_speed(iter/s)': 0.061631}
+# Train:  31%|███████████████████████████████████                                                                             | 5/16 [01:21<02:30, 13.67s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-5
+# {'loss': 1.61200867, 'grad_norm': 1.05091298, 'learning_rate': 3.455e-05, 'token_acc': 0.63310818, 'epoch': 0.64, 'global_step/max_steps': '10/16', 'percentage': '62.50%', 'elapsed_time': '2m 20s', 'remaining_time': '1m 24s', 'memory(GiB)': 26.93, 'train_speed(iter/s)': 0.0712}
+# Train:  62%|█████████████████████████████████████████████████████████████████████▍                                         | 10/16 [02:20<01:11, 11.97s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-10
+# {'loss': 1.32489185, 'grad_norm': 0.60476321, 'learning_rate': 1.09e-06, 'token_acc': 0.64746468, 'epoch': 0.96, 'global_step/max_steps': '15/16', 'percentage': '93.75%', 'elapsed_time': '3m 11s', 'remaining_time': '12s', 'memory(GiB)': 26.94, 'train_speed(iter/s)': 0.078265}
+# Train:  94%|████████████████████████████████████████████████████████████████████████████████████████████████████████       | 15/16 [03:11<00:10, 10.03s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-15
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [03:20<00:00,  9.65s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-16
+# {'train_runtime': 202.2537, 'train_samples_per_second': 2.487, 'train_steps_per_second': 0.079, 'train_loss': 1.91632293, 'token_acc': 0.67729084, 'epoch': 1.0, 'global_step/max_steps': '16/16', 'percentage': '100.00%', 'elapsed_time': '3m 22s', 'remaining_time': '0s', 'memory(GiB)': 26.94, 'train_speed(iter/s)': 0.078996}
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [03:22<00:00, 12.66s/it]
@@ -0,0 +1,4 @@
+ASCEND_RT_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --served_model_name Qwen2.5-7B-Instruct
@@ -0,0 +1,11 @@
+NPROC_PER_NODE=4 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen3-8B \
+    --infer_backend vllm \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#2000 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --vllm_tensor_parallel_size 2 \
+    --max_new_tokens 2048 \
+    --write_batch_size 1000
@@ -0,0 +1,35 @@
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+megatron sft \
+    --model Qwen/Qwen3-4B \
+    --save_safetensors true \
+    --dataset 'llm-wizard/alpaca-gpt4-data-zh' \
+    --use_hf true \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --packing True \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 5 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-4B \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 8192 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --cross_entropy_loss_fusion true \
+    --cross_entropy_fusion_impl native \
+    --attention_backend flash
@@ -0,0 +1,33 @@
+# Atlas A2 * 2 nodes * 8 cards per node
+
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NNODES=2 \
+NODE_RANK=0 \
+MASTER_ADDR=127.0.0.1 \
+MASTER_PORT=29500 \
+NPROC_PER_NODE=8 \
+HCCL_SOCKET_IFNAME=xxx \
+megatron sft \
+    --model 'Qwen/Qwen3-8B' \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+    --output_dir './SAVE' \
+    --tuner_type 'lora' \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules 'all-linear' \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 1 \
+    --context_parallel_size 1 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 64 \
+    --recompute_granularity selective \
+    --recompute_modules core_attn \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --logging_steps 5 \
+    --dataloader_num_workers 4
@@ -0,0 +1,33 @@
+# Atlas A2 * 2 nodes * 8 cards per node
+
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NNODES=2 \
+NODE_RANK=1 \
+MASTER_ADDR=xxx.xxx.xxx.xxx \
+MASTER_PORT=29500 \
+NPROC_PER_NODE=8 \
+HCCL_SOCKET_IFNAME=xxx \
+megatron sft \
+    --model 'Qwen/Qwen3-8B' \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+    --output_dir './SAVE' \
+    --tuner_type 'lora' \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules 'all-linear' \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 1 \
+    --context_parallel_size 1 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 64 \
+    --recompute_granularity selective \
+    --recompute_modules core_attn \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --logging_steps 5 \
+    --dataloader_num_workers 4
@@ -0,0 +1,31 @@
+# hardware: Atlas 900 A2
+export TASK_QUEUE_ENABLE=2
+export CPU_AFFINITY_CONF=2
+nproc_per_node=8
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model 'Qwen/Qwen3-32B' \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero3
@@ -0,0 +1,26 @@
+{
+  "compute_environment": "LOCAL_MACHINE",
+  "debug": false,
+  "distributed_type": "FSDP",
+  "downcast_bf16": "no",
+  "mixed_precision": "bf16",
+  "num_machines": 1,
+  "num_processes": 8,
+  "machine_rank": 0,
+  "rdzv_backend": "static",
+  "same_network": true,
+  "use_cpu": false,
+  "fsdp_config": {
+    "fsdp_auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+    "fsdp_transformer_cls_names_to_wrap": "Qwen3DecoderLayer",
+    "fsdp_sharding_strategy": "FULL_SHARD",
+    "fsdp_backward_prefetch": "BACKWARD_PRE",
+    "fsdp_forward_prefetch": true,
+    "fsdp_limit_all_gathers": true,
+    "fsdp_state_dict_type": "FULL_STATE_DICT",
+    "fsdp_sync_module_states": true,
+    "fsdp_cpu_ram_efficient_loading": false,
+    "fsdp_use_orig_params": true,
+    "fsdp_offload_params": false
+  }
+}
@@ -0,0 +1,29 @@
+# hardware: Atlas 900 A2
+# For NPU, in Transformers versions 5.0 and above, it is recommended to disable
+# cpu_ram_efficient_loading in fsdp.json to avoid timeout issues at the first
+# synchronization point caused by inter-card desynchronization when loading the model.
+export TASK_QUEUE_ENABLE=2
+export CPU_AFFINITY_CONF=2
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+accelerate launch --config_file "./examples/ascend/train/qwen3/qwen3_lora_fsdp/fsdp.json" \
+    swift/cli/sft.py \
+    --model 'Qwen/Qwen3-32B' \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 10 \
+    --gradient_accumulation_steps 2 \
+    --gradient_checkpointing true \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}' \
+    --max_length 1200 \
+    --num_train_epochs 2 \
+    --eval_strategy no \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output \
+    --attn_impl 'flash_attention_2' \
+    --packing true
@@ -0,0 +1,37 @@
+NPROC_PER_NODE=2 \
+ASCEND_RT_VISIBLE_DEVICES=0,1 \
+megatron sft \
+    --model Qwen/Qwen3-8B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 2 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-8B \
+    --save_steps 100 \
+    --max_length 2048 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --model_author swift \
+    --model_name swift-robot
@@ -0,0 +1,58 @@
+# 16 * 64GiB Ascend A3
+
+# NPU stability environment variables
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export MULTI_STREAM_MEMORY_REUSE=1
+# NPU memory management environment variables
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+# NPU performance environment variables
+export TASK_QUEUE_ENABLE=2
+export USE_MCORE_GDN=0
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 16 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    \
+    --micro_batch_size 2 \
+    --global_batch_size 32 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 100 \
+    --max_length 1024 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot \
+    --save_total_limit 3
@@ -0,0 +1,56 @@
+# 16 * 64GiB Ascend A3
+export USE_MCORE_GDN=0
+
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export MULTI_STREAM_MEMORY_REUSE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export TASK_QUEUE_ENABLE=2
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+MASTER_PORT=29609 \
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors false \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    \
+    --micro_batch_size 1 \
+    --global_batch_size 16 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 32 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot
@@ -0,0 +1,52 @@
+# 8 * 96GiB Ascend A5
+
+export TASK_QUEUE_ENABLE=2
+export USE_MCORE_GDN=0
+
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    \
+    --micro_batch_size 2 \
+    --global_batch_size 16 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 32 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot
@@ -0,0 +1,39 @@
+export TASK_QUEUE_ENABLE=2
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --export_model_parallel_size 4 \
+    --model_type qwen3_next \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 4 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-Next-Instruct \
+    --save_steps 100 \
+    --max_length 1024 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --model_author swift \
+    --model_name swift-robot
@@ -0,0 +1,50 @@
+# 16 * 64GiB Ascend A3
+# Modified from https://github.com/modelscope/ms-swift/blob/main/examples/megatron/multimodal/omni/moe.sh
+export TASK_QUEUE_ENABLE=2
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=16 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+megatron sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite' \
+    --load_from_cache_file true \
+    --sequence_parallel true \
+    --packing true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --split_dataset_ratio 0.01 \
+    --expert_model_parallel_size 8 \
+    --expert_tensor_parallel_size 1 \
+    --tensor_model_parallel_size 1 \
+    --pipeline_model_parallel_size 2 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-3 \
+    --micro_batch_size 1 \
+    --global_batch_size 8 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --cross_entropy_fusion_impl native \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --num_train_epochs 3 \
+    --output_dir megatron_output/Qwen3-Omni-30B-A3B-Instruct \
+    --eval_steps 1000 \
+    --save_steps 10000 \
+    --max_length 1024 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --attention_backend flash \
+    --gradient_accumulation_fusion False \
+    --masked_softmax_fusion False
@@ -0,0 +1,54 @@
+# 16 * 64GiB Ascend A3
+
+export USE_MCORE_GDN=0
+
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export HCCL_CONNECT_TIMEOUT=600
+export MULTI_STREAM_MEMORY_REUSE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+
+export TASK_QUEUE_ENABLE=2
+
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --save_safetensors false \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --micro_batch_size 2 \
+    --global_batch_size 32 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-omni \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot
@@ -0,0 +1,52 @@
+# 8 * 96GiB Ascend A5
+
+export USE_MCORE_GDN=0
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export TASK_QUEUE_ENABLE=2
+export HCCL_CONNECT_TIMEOUT=600
+
+MODEL_PATH=Qwen/Qwen3-Omni-30B-A3B-Instruct
+DATASET_PATH='AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000'
+
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model ${MODEL_PATH} \
+    --save_safetensors false \
+    --dataset ${DATASET_PATH} \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --micro_batch_size 4 \
+    --global_batch_size 32 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-omni \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot
@@ -0,0 +1,55 @@
+# 16 * 64GiB Ascend A3
+# Modified from https://github.com/modelscope/ms-swift/blob/main/examples/models/qwen3_vl/mcore_full.sh
+export TASK_QUEUE_ENABLE=2
+export COMBINED_ENABLE=1
+export CPU_AFFINITY_CONF=1
+export TORCH_HCCL_ZERO_COPY=1
+
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+MULTI_STREAM_MEMORY_REUSE=2 \
+OMP_NUM_THREADS=14 \
+NPROC_PER_NODE=16 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+megatron sft \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --expert_model_parallel_size 8 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-VL-30B-A3B-Instruct \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --packing true \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --moe_expert_capacity_factor 2 \
+    --attention_backend flash
+    # --moe_permute_fusion true
+    # --optimizer_cpu_offload true
+    # --use_precision_aware_optimizer true
+    # --optimizer_offload_fraction 0.2
@@ -0,0 +1,30 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from typing import Any, Dict, Optional
+
+from swift.dataset import DatasetMeta, ResponsePreprocessor, load_dataset, register_dataset
+
+
+class CustomPreprocessor(ResponsePreprocessor):
+    prompt = """Task: Based on the given two sentences, provide a similarity score between 0.0 and 5.0.
+Sentence 1: {text1}
+Sentence 2: {text2}
+Similarity score: """
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        return super().preprocess({
+            'query': self.prompt.format(text1=row['text1'], text2=row['text2']),
+            'response': f"{row['label']:.1f}"
+        })
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='swift/stsb',
+        hf_dataset_id='SetFit/stsb',
+        preprocess_func=CustomPreprocessor(),
+    ))
+
+if __name__ == '__main__':
+    dataset = load_dataset(['swift/stsb'])[0]
+    print(f'dataset: {dataset}')
+    print(f'dataset[0]: {dataset[0]}')
@@ -0,0 +1,9 @@
+# sh examples/custom/infer.sh
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --load_data_args true \
+    --infer_backend transformers \
+    --max_batch_size 16 \
+    --max_new_tokens 256 \
+    --temperature 0
@@ -0,0 +1,35 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+from swift.model import Model, ModelGroup, ModelMeta, register_model
+from swift.template import TemplateMeta, register_template
+
+register_template(
+    TemplateMeta(
+        template_type='custom',
+        prefix=['<extra_id_0>System\n{{SYSTEM}}\n'],
+        prompt=['<extra_id_1>User\n{{QUERY}}\n<extra_id_1>Assistant\n'],
+        chat_sep=['\n']))
+
+register_model(
+    ModelMeta(
+        model_type='custom',
+        model_groups=[
+            ModelGroup([Model('AI-ModelScope/Nemotron-Mini-4B-Instruct', 'nvidia/Nemotron-Mini-4B-Instruct')])
+        ],
+        template='custom',
+        ignore_patterns=['nemo'],
+        is_multimodal=False,
+    ))
+
+if __name__ == '__main__':
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    engine = TransformersEngine('AI-ModelScope/Nemotron-Mini-4B-Instruct')
+    response = engine.infer([infer_request], request_config)
+    swift_response = response[0].choices[0].message.content
+
+    engine.template.template_backend = 'jinja'
+    response = engine.infer([infer_request], request_config)
+    jinja_response = response[0].choices[0].message.content
+    assert swift_response == jinja_response, f'swift_response: {swift_response}\njinja_response: {jinja_response}'
+    print(f'response: {swift_response}')
@@ -0,0 +1,59 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""
+Here is another way to register the model, by customizing the get_function.
+
+The get_function just needs to return the model + tokenizer/processor.
+"""
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, PretrainedConfig, PreTrainedModel
+
+from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+from swift.model import Model, ModelGroup, ModelLoader, ModelMeta, register_model
+from swift.template import TemplateMeta, register_template
+from swift.utils import Processor
+
+register_template(
+    TemplateMeta(
+        template_type='custom',
+        prefix=['<extra_id_0>System\n{{SYSTEM}}\n'],
+        prompt=['<extra_id_1>User\n{{QUERY}}\n<extra_id_1>Assistant\n'],
+        chat_sep=['\n']))
+
+
+class MyModelLoader(ModelLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        return AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
+
+    def get_model(self, model_dir: str, config: PretrainedConfig, processor: Processor,
+                  model_kwargs) -> PreTrainedModel:
+        return AutoModelForCausalLM.from_pretrained(
+            model_dir, config=config, torch_dtype=self.torch_dtype, trust_remote_code=True, **model_kwargs)
+
+
+register_model(
+    ModelMeta(
+        model_type='custom',
+        model_groups=[
+            ModelGroup([Model('AI-ModelScope/Nemotron-Mini-4B-Instruct', 'nvidia/Nemotron-Mini-4B-Instruct')])
+        ],
+        loader=MyModelLoader,
+        template='custom',
+        ignore_patterns=['nemo'],
+        is_multimodal=False,
+    ))
+
+if __name__ == '__main__':
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    engine = TransformersEngine('AI-ModelScope/Nemotron-Mini-4B-Instruct')
+    response = engine.infer([infer_request], request_config)
+    swift_response = response[0].choices[0].message.content
+
+    engine.template.template_backend = 'jinja'
+    response = engine.infer([infer_request], request_config)
+    jinja_response = response[0].choices[0].message.content
+    assert swift_response == jinja_response, f'swift_response: {swift_response}\njinja_response: {jinja_response}'
+    print(f'response: {swift_response}')
@@ -0,0 +1,454 @@
+import torch
+from functools import partial
+from transformers import AutoConfig, PretrainedConfig, PreTrainedModel
+from transformers.integrations import is_deepspeed_zero3_enabled
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.model import (Model, ModelGroup, ModelLoader, ModelMeta, MultiModelKeys, get_model_processor, register_model,
+                         register_model_arch)
+from swift.model.models.qwen import patch_qwen_vl_utils
+from swift.model.patcher import patch_get_input_embeddings
+from swift.model.utils import use_submodel_func
+from swift.template import StdTemplateInputs, Template, TemplateMeta, get_template, register_template
+from swift.template.utils import Context, findall
+from swift.template.vision_utils import load_audio
+from swift.utils import Processor, get_env_args, get_logger, get_packed_seq_params, is_deepspeed_enabled, to_float_dtype
+
+register_model_arch(
+    MultiModelKeys(
+        'my_qwen2_5_omni',
+        # `freeze_llm`, `freeze_vit`, `freeze_aligner` behavior is determined by the values below.
+        # For example: full parameter training, if `freeze_vit=True`, it will freeze parameters of
+        # model layers prefixed with `thinker.audio_tower` and `thinker.visual`.
+        # LoRA training, if `freeze_vit=False`, it will additionally add LoRA to Linear layers
+        # prefixed with `thinker.audio_tower` and `thinker.visual`.
+        language_model=['thinker.model', 'thinker.lm_head'],
+        vision_tower=['thinker.audio_tower', 'thinker.visual'],
+        aligner=['thinker.audio_tower.proj', 'thinker.visual.merger'],
+        # Generator parts will never be trained or remain frozen.
+        # If you want `thinker.audio_tower` and `thinker.audio_tower.proj` to never be trained,
+        # you can place them in the generator and remove them from vision_tower and aligner.
+        generator=['talker', 'token2wav'],
+    ))
+
+
+class Qwen2_5OmniLoader(ModelLoader):
+
+    def get_config(self, model_dir: str) -> PretrainedConfig:
+        config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+        enable_audio_output = get_env_args('ENABLE_AUDIO_OUTPUT', bool, None)
+        if enable_audio_output is not None:
+            config.enable_audio_output = enable_audio_output
+        return config
+
+    def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:
+        from qwen_omni_utils import vision_process
+        from transformers import Qwen2_5OmniProcessor
+        processor = Qwen2_5OmniProcessor.from_pretrained(model_dir, trust_remote_code=True)
+        # Control constants in qwen_omni_utils library via environment variables,
+        # e.g., `MAX_PIXELS`, etc.
+        patch_qwen_vl_utils(vision_process)
+        return processor
+
+    def get_model(self, model_dir: str, config: PretrainedConfig, processor: Processor,
+                  model_kwargs) -> PreTrainedModel:
+        from transformers import Qwen2_5OmniForConditionalGeneration
+        print('Run my_qwen2_5_omni...')
+        self.auto_model_cls = self.auto_model_cls or Qwen2_5OmniForConditionalGeneration
+        model = super().get_model(model_dir, config, processor, model_kwargs)
+        # For multimodal model consistency, we replace the model's forward/generate functions
+        # with those of its language_model.
+        # Handle additional parts separately.
+        use_submodel_func(model, 'thinker')
+        # Avoid inplace operations on leaf_variable during training
+        # (replacing parts of input_embeds with images_embeds)
+        patch_get_input_embeddings(model.thinker.visual, 'patch_embed')
+        # Some custom settings for model/config (usually not needed; configure based on
+        # specific model if errors occur during training/inference)
+        model.config.keys_to_ignore_at_inference += ['hidden_states', 'attention_mask']
+        model.config.talker_config.pad_token_id = None
+        return model
+
+
+register_model(
+    ModelMeta(
+        'my_qwen2_5_omni',
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Omni-3B', 'Qwen/Qwen2.5-Omni-3B'),
+                Model('Qwen/Qwen2.5-Omni-7B', 'Qwen/Qwen2.5-Omni-7B'),
+            ]),
+        ],
+        # Function to get model and processor.
+        Qwen2_5OmniLoader,
+        template='my_qwen2_5_omni',
+        is_multimodal=True,  # Whether it's a multimodal model
+        model_arch='my_qwen2_5_omni',  # Usually set only for multimodal models
+        # Used for automatic model_type matching
+        architectures=['Qwen2_5OmniModel', 'Qwen2_5OmniForConditionalGeneration'],
+        # Used to prompt users about dependency versions (can be removed)
+        requires=['transformers>=4.50', 'soundfile', 'qwen_omni_utils', 'decord'],
+        # Used to prompt users (can be removed)
+        tags=['vision', 'video', 'audio'],
+        # Additional files to save during full parameter training/merge-lora
+        additional_saved_files=['spk_dict.pt'],
+    ))
+
+if __name__ == '__main__':
+    # Test and debug
+    model, processor = get_model_processor('Qwen/Qwen2.5-Omni-7B', model_type='my_qwen2_5_omni')
+
+logger = get_logger()
+
+
+class Qwen2_5OmniTemplate(Template):
+    use_model = True  # Whether model participation is required during preprocessing
+    # Note: Not all multimodal models support padding_free/packing. Models in `transformers` library usually support it
+    support_padding_free = True  # Whether padding_free and packing are supported (multimodal models)
+    norm_bbox = 'none'  # Whether grounding tasks use absolute or norm1000 coordinates
+
+    # These tokens will not be truncated (e.g., when setting `--truncation_strategy left/right`)
+    # and will be printed in abbreviated form (calling `template.safe_decode`)
+    placeholder_tokens = ['<|IMAGE|>', '<|AUDIO|>', '<|VIDEO|>']
+
+    def init_processor(self, processor) -> None:
+        """Initialize some required constants when initializing the processor"""
+        if processor is None:
+            return
+        super().init_processor(processor)
+        from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs
+        default = Qwen2_5OmniProcessorKwargs._defaults
+        self.seconds_per_chunk = default['videos_kwargs']['seconds_per_chunk']
+        self.position_id_per_seconds = default['videos_kwargs']['position_id_per_seconds']
+        self.use_audio_in_video = get_env_args('use_audio_in_video', bool, False)
+        self.sampling_rate = get_env_args('sampling_rate', int, self.processor.feature_extractor.sampling_rate)
+        # See grounding dataset customization documentation for `QWENVL_BBOX_FORMAT` meaning
+        self.bbox_format = get_env_args('QWENVL_BBOX_FORMAT', str, 'legacy')
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        """Load multimodal data and replace generic multimodal tags.
+        For example: image tag from `<image>` -> `<|vision_bos|><|IMAGE|><|vision_eos|>`"""
+        # Loading multimodal data can also be done in the `_encode` function, whichever is more convenient.
+        from qwen_omni_utils import fetch_image, fetch_video
+        if media_type == 'image':
+            inputs.images[index] = fetch_image({'image': inputs.images[index]})
+            return ['<|vision_bos|><|IMAGE|><|vision_eos|>']
+        elif media_type == 'audio':
+            if self.mode != 'vllm':  # No processing needed in 'vllm' inference scenario
+                inputs.audios[index] = load_audio(inputs.audios[index], self.sampling_rate)
+            return ['<|audio_bos|><|AUDIO|><|audio_eos|>']
+        elif media_type == 'video':
+            video = inputs.videos[index]
+            _video = fetch_video({'video': video})
+            if isinstance(_video, torch.Tensor):
+                _video = _video.to(torch.uint8)
+            inputs.videos[index] = _video
+            if self.use_audio_in_video:
+                import librosa
+                if video.startswith('http://') or video.startswith('https://'):
+                    import audioread
+                    video = audioread.ffdec.FFmpegAudioFile(video)
+                video = librosa.load(video, sr=self.sampling_rate)[0]
+                inputs.audios.insert(inputs.audio_idx, (video, 'video'))
+                inputs.audio_idx += 1
+                return ['<|vision_bos|><|audio_bos|><|VIDEO|><|audio_eos|><|vision_eos|>']
+            else:
+                return ['<|vision_bos|><|VIDEO|><|vision_eos|>']
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace generic tag for grounding tasks: `<ref-object>`"""
+        if self.bbox_format == 'legacy':
+            return [f'<|object_ref_start|>{ref}<|object_ref_end|>']
+        else:
+            return [ref]
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace generic tag for grounding tasks: `<bbox>`"""
+        if self.bbox_format == 'legacy':
+            return [f'<|box_start|>{self._get_bbox_str(bbox)}<|box_end|>']
+        else:
+            return [str(bbox)]
+
+    def packing_row(self, row: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Support packing & mrope.
+
+        Usually no need to inherit this function; here for customizing mrope's position_ids."""
+        position_ids = []
+        for r in row:
+            r = r.copy()
+            r['input_ids'] = torch.tensor(r['input_ids'])[None]
+            position_ids.append(self._get_position_ids(r))
+        packed = super().packing_row(row)
+        packed['position_ids'] = torch.concat(position_ids, dim=-1)
+        return packed
+
+    def _get_new_tokens_use_audio_in_video(self, i, *, video_grid_thw, video_second_per_grid, audio_lengths,
+                                           video_token_id, audio_token_id):
+        """Helper function to support `use_audio_in_video` being True"""
+        merge_size = self.processor.image_processor.merge_size
+        grid_thw = video_grid_thw[i]
+        height = grid_thw[1] // merge_size
+        width = grid_thw[2] // merge_size
+        audio_token_indices = torch.arange(audio_lengths[i])
+        video_token_indices = torch.arange(grid_thw[0]).reshape(-1, 1, 1)
+
+        video_token_indices = torch.broadcast_to(video_token_indices,
+                                                 (video_token_indices.shape[0], height, width)).reshape(-1)
+        video_token_indices = (video_token_indices * video_second_per_grid[i] * self.position_id_per_seconds)
+        tokens_per_chunk = int(self.position_id_per_seconds * self.seconds_per_chunk)
+        video_chunk_indexes = self.processor.get_chunked_index(video_token_indices, tokens_per_chunk)
+        audio_chunk_indexes = self.processor.get_chunked_index(audio_token_indices, tokens_per_chunk)
+
+        res = []
+        for j in range(max(len(video_chunk_indexes), len(audio_chunk_indexes))):
+            if j < len(video_chunk_indexes):
+                video_seq_length = video_chunk_indexes[j][1] - video_chunk_indexes[j][0]
+                res += video_token_id * video_seq_length
+            if j < len(audio_chunk_indexes):
+                audio_seq_length = audio_chunk_indexes[j][1] - audio_chunk_indexes[j][0]
+                res += audio_token_id * audio_seq_length
+        return res
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        """This determines how to convert text/images/audios/videos ->
+        input_ids, labels, loss_scale, and multimodal content like pixel_values.
+
+        Processing logic can usually be borrowed from the corresponding model's preprocessing code implementation.
+        Recommended: Perform inference alignment first, then training."""
+        encoded = Template._encode(self, inputs)  # Process text-only part; see custom model documentation for details
+        logger.info_once('Run qwen2_5_omni template')
+        processor = self.processor
+        # Get multimodal content
+        media_inputs = processor(
+            text='',
+            audio=inputs.audios or None,
+            images=inputs.images or None,
+            videos=inputs.videos or None,
+            do_resize=False,
+            return_tensors='pt')
+        # We don't use input_ids and attention_mask produced by `processor` because it doesn't produce `labels`.
+        media_inputs.pop('input_ids')
+        media_inputs.pop('attention_mask')
+        media_inputs = to_float_dtype(media_inputs, self.model_info.torch_dtype)
+
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        loss_scale = encoded.get('loss_scale', None)
+        # audio modality
+        audio_token_id = self._tokenize('<|AUDIO|>')
+        idx_list = findall(input_ids, audio_token_id)  # Find all audio_tokens
+        feature_attention_mask = media_inputs.get('feature_attention_mask')
+        if feature_attention_mask is not None:
+            audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
+            audio_lengths = ((audio_feature_lengths - 1) // 2 + 1 - 2) // 2 + 1
+        else:
+            audio_lengths = None
+        audio_lengths_origin = audio_lengths
+        # video_audios_mask is used to handle `use_audio_in_video`, distinguishing pure audio(0) from audio in video(1)
+        video_audios_mask = []
+        for i, audio in enumerate(inputs.audios):
+            if isinstance(audio, tuple) and audio[1] == 'video':
+                inputs.audios[i] = audio[0]
+                video_audios_mask.append(True)
+            else:
+                video_audios_mask.append(False)
+        video_audios_mask = torch.tensor(video_audios_mask)
+        if idx_list:
+            # Filter out audio content in videos (will be handled in video section)
+            if self.use_audio_in_video:
+                audio_lengths = audio_lengths[~video_audios_mask]
+
+            def _get_new_audio_tokens(i):
+                return audio_token_id * audio_lengths[i]
+
+            # Expand multimodal tokens in input_ids
+            input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                _get_new_audio_tokens)
+
+        # image and video modalities
+        for media_type in ['image', 'video']:
+            token = f'<|{media_type.upper()}|>'
+            token_id = self._tokenize(token)
+            idx_list = findall(input_ids, token_id)
+            if idx_list:
+                merge_size = processor.image_processor.merge_size
+                media_grid_thw = media_inputs.get(f'{media_type}_grid_thw')
+                if media_type == 'video' and self.use_audio_in_video:
+                    audio_lengths = audio_lengths_origin[video_audios_mask]
+                    video_second_per_grid = media_inputs['video_second_per_grid']
+                    _get_new_tokens_use_audio_in_video = partial(
+                        self._get_new_tokens_use_audio_in_video,
+                        video_grid_thw=media_grid_thw,
+                        video_second_per_grid=video_second_per_grid,
+                        audio_lengths=audio_lengths,
+                        video_token_id=token_id,
+                        audio_token_id=audio_token_id)
+                    input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                        _get_new_tokens_use_audio_in_video)
+
+                else:
+
+                    def _get_new_tokens(i):
+                        token_len = (media_grid_thw[i].prod() // (merge_size**2))
+                        return token_id * token_len
+
+                    input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,
+                                                                        _get_new_tokens)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        encoded.update(media_inputs)  # Add multimodal content
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        """This function is typically used to solve the zero2/zero3 hanging issue in mixed model training,
+        i.e., some processes have pure text data without passing through vit,
+        while others have image data that passed through vit.
+        Here we create dummy_image to solve this.
+
+        This function will be registered in the pre_forward_hook before `model.forward`.
+        This function should return input_embeds containing multimodal information.
+        """
+        if not self.is_training:
+            return inputs
+
+        input_ids = inputs['input_ids']
+        input_features = inputs.get('input_features')
+        feature_attention_mask = inputs.get('feature_attention_mask')
+
+        base_model = self.get_base_model(model)
+        inputs_embeds = base_model.thinker.model.embed_tokens(input_ids)
+        thinker_config = model.config.thinker_config
+        # Helper function for handling text/image/video mixed modality data scenarios. (internally creates dummy_image)
+        inputs_embeds = self._get_inputs_embeds_hf(inputs_embeds, inputs, model.thinker.visual, self.processor,
+                                                   thinker_config)
+        # Mixed modality data scenarios containing audio
+        if input_features is None:
+            if is_deepspeed_enabled() and not is_deepspeed_zero3_enabled():
+                # Note: Due to transformers implementation,
+                # the number of passes through audio model layers is related to the number of audios
+                # Therefore, zero3 will hang in scenarios where different processes have different numbers of audios
+                # (requires modification of transformers code to fix).
+                # Use zero2 in this scenario.
+                input_features = input_ids.new_zeros([1, 128, 128], dtype=model.thinker.audio_tower.dtype)
+                feature_attention_mask = input_ids.new_ones([1, 128], dtype=torch.bool)
+                audio_res = model.thinker.get_audio_features(input_features, feature_attention_mask)
+                # Compatible with transformers 5.0
+                if hasattr(audio_res, 'last_hidden_state'):
+                    audio_embeds = audio_res.last_hidden_state
+                else:
+                    audio_embeds = audio_res
+                inputs_embeds = inputs_embeds + audio_embeds.mean() * 0.
+        else:
+            audio_res = model.thinker.get_audio_features(input_features, feature_attention_mask)
+            # Compatible with transformers 5.0
+            if hasattr(audio_res, 'last_hidden_state'):
+                audio_embeds = audio_res.last_hidden_state
+            else:
+                audio_embeds = audio_res
+            audio_mask = (input_ids == thinker_config.audio_token_index).unsqueeze(-1).expand_as(inputs_embeds)
+            audio_embeds = audio_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+            inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_embeds)
+
+        return {'inputs_embeds': inputs_embeds}
+
+    def _get_position_ids(self, inputs: Dict[str, Any]):
+        """Helper function to get mrope's position_ids"""
+        feature_attention_mask = inputs.get('feature_attention_mask')
+        if feature_attention_mask is not None:
+            audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
+        else:
+            audio_feature_lengths = None
+        video_second_per_grid = inputs.pop('video_second_per_grid', None)
+        input_ids = inputs['input_ids']
+        attention_mask = inputs.get('attention_mask')
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+        position_ids, _ = self.model.thinker.get_rope_index(
+            input_ids,
+            inputs.get('image_grid_thw'),
+            inputs.get('video_grid_thw'),
+            attention_mask,
+            self.use_audio_in_video,
+            audio_feature_lengths,
+            video_second_per_grid,
+        )
+        return self._concat_text_position_ids(position_ids)  # First dimension is text_position_ids
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        """Passed to dataloader's `collate_fn`"""
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if not self.padding_free and self.is_training:
+            # padding_free/packing scenarios will handle position_ids in packing_row.
+            res['position_ids'] = self._get_position_ids(res)
+        if 'position_ids' in res:
+            # Create `packed_seq_params` to support padding_free/packing & flash-attn
+            position_ids = res['position_ids']
+            res['position_ids'] = position_ids[1:]
+            res['text_position_ids'] = text_position_ids = position_ids[0]
+            # https://github.com/huggingface/transformers/pull/40194
+            res.update(get_packed_seq_params(text_position_ids))
+        return res
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Handle multimodal part in `_data_collator` function.
+        (This function is compatible with padding_free/packing)"""
+        res = super()._data_collator_mm_data(batch)
+        video_second_per_grid = self.gather_list(batch, 'video_second_per_grid')
+        if video_second_per_grid:
+            res['video_second_per_grid'] = video_second_per_grid
+        input_features = [b['input_features'] for b in batch if b.get('input_features') is not None]
+        feature_attention_mask = [
+            b['feature_attention_mask'] for b in batch if b.get('feature_attention_mask') is not None
+        ]
+        if input_features:
+            res['input_features'] = torch.concat(input_features)
+            res['feature_attention_mask'] = torch.concat(feature_attention_mask)
+        return res
+
+    def generate(self, model, *args, **kwargs):
+        """`TransformersEngine` will call template.generate method for text generation;
+        inherit here for customization."""
+        if kwargs.get('video_grid_thw') is not None:
+            kwargs['use_audio_in_video'] = self.use_audio_in_video
+        return super().generate(model, *args, **kwargs)
+
+
+register_template(
+    TemplateMeta(
+        'my_qwen2_5_omni',
+        prefix=[],
+        prompt=['<|im_start|>user\n{{QUERY}}<|im_end|>\n<|im_start|>assistant\n'],
+        chat_sep=['<|im_end|>\n'],
+        suffix=['<|im_end|>'],
+        system_prefix=['<|im_start|>system\n{{SYSTEM}}<|im_end|>\n'],
+        default_system='You are a helpful assistant.',
+        stop_words=['<|endoftext|>'],
+        agent_template='hermes',
+        template_cls=Qwen2_5OmniTemplate))
+
+if __name__ == '__main__':
+    # Test and debug
+    model, processor = get_model_processor('Qwen/Qwen2.5-Omni-7B', model_type='my_qwen2_5_omni')
+    template = get_template(processor, template_type='my_qwen2_5_omni')
+    data = {
+        'messages': [
+            {
+                'role': 'user',
+                'content': 'Describe the video<video> and image<image> content.'
+            },
+            {
+                'role': 'assistant',
+                'content': 'A child and a cat.'
+            },
+        ],
+        'videos': ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'],
+        'images': ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png'],
+    }
+    template.set_mode('train')
+    encoded = template.encode(data)
+    print('input_ids: ' + template.safe_decode(encoded['input_ids']))
+    print('labels: ' + template.safe_decode(encoded['labels']))
+    print('keys: ' + str(encoded.keys()))
@@ -0,0 +1,89 @@
+import os
+import requests
+import sys
+from modelscope import snapshot_download
+from qwen_omni_utils import process_mm_info
+from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
+
+from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+
+sys.path.append('examples/custom/my_qwen2_5_omni')
+
+
+def infer_hf():
+    model_dir = snapshot_download('Qwen/Qwen2.5-Omni-7B')
+    model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
+        model_dir, torch_dtype='auto', device_map='auto', attn_implementation='flash_attention_2')
+    processor = Qwen2_5OmniProcessor.from_pretrained(model_dir)
+    # Use decord to read video (url not yet supported)
+    resp = requests.get('https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4')
+    with open('_baby.mp4', 'wb') as f:
+        f.write(resp.content)
+
+    conversation = [
+        {
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'video',
+                    'video': '_baby.mp4'
+                },
+                {
+                    'type': 'image',
+                    'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png'
+                },
+                {
+                    'type': 'text',
+                    'text': 'Describe the video and image.'
+                },
+            ],
+        },
+    ]
+
+    USE_AUDIO_IN_VIDEO = False
+    text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
+    audios, images, videos = process_mm_info(conversation, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+    inputs = processor(
+        text=text,
+        audio=audios,
+        images=images,
+        videos=videos,
+        return_tensors='pt',
+        padding=True,
+        use_audio_in_video=USE_AUDIO_IN_VIDEO)
+    inputs = inputs.to(model.device).to(model.dtype)
+    text_ids = model.generate(
+        **inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO, thinker_do_sample=False, return_audio=False)
+    text = processor.batch_decode(
+        text_ids[:, inputs['input_ids'].shape[1]:], skip_special_tokens=True, clean_up_tokenization_spaces=False)
+    return inputs['input_ids'][0].tolist(), text[0]
+
+
+def test_my_qwen2_5_omni():
+    engine = TransformersEngine('Qwen/Qwen2.5-Omni-7B', model_type='my_qwen2_5_omni', attn_impl='flash_attention_2')
+    infer_request = InferRequest(
+        messages=[{
+            'role': 'user',
+            'content': '<video><image>Describe the video and image.',
+        }],
+        videos=['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'],
+        images=['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png'],
+    )
+    request_config = RequestConfig(temperature=0, max_tokens=512)
+    input_ids = engine.template.encode(infer_request)['input_ids']
+    resp_list = engine.infer([infer_request], request_config)
+    resp = resp_list[0].choices[0].message.content
+    return input_ids, resp
+
+
+if __name__ == '__main__':
+    import my_register
+
+    # Enable debug mode, will print input_ids and generate_ids from `TransformersEngine.infer`
+    os.environ['SWIFT_DEBUG'] = '1'
+    input_ids_hf, response_hf = infer_hf()
+    input_ids_swift, response_swift = test_my_qwen2_5_omni()
+    # Test input_ids and response alignment
+    assert input_ids_hf == input_ids_swift
+    assert response_hf == response_swift
@@ -0,0 +1,42 @@
+import os
+import sys
+
+from swift import SftArguments, sft_main
+
+sys.path.append('examples/custom/my_qwen2_5_omni')
+
+if __name__ == '__main__':
+    import my_register
+    os.environ['MAX_PIXELS'] = '1003520'
+    sft_main(
+        SftArguments(
+            model='Qwen/Qwen2.5-Omni-7B',
+            dataset=['AI-ModelScope/LaTeX_OCR#5000'],
+            model_type='my_qwen2_5_omni',
+            template='my_qwen2_5_omni',
+            load_from_cache_file=True,
+            split_dataset_ratio=0.01,
+            tuner_type='lora',
+            torch_dtype='bfloat16',
+            attn_impl='flash_attn',
+            padding_free=True,
+            num_train_epochs=1,
+            per_device_train_batch_size=16,
+            per_device_eval_batch_size=16,
+            learning_rate=1e-4,
+            lora_rank=8,
+            lora_alpha=32,
+            target_modules=['all-linear'],
+            freeze_vit=True,
+            freeze_aligner=True,
+            gradient_accumulation_steps=1,
+            eval_steps=50,
+            save_steps=50,
+            save_total_limit=2,
+            logging_steps=5,
+            max_length=2048,
+            output_dir='output',
+            warmup_ratio=0.05,
+            dataloader_num_workers=4,
+            dataset_num_proc=1,
+        ))
@@ -0,0 +1,26 @@
+# sh examples/custom/sft.sh
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --external_plugins examples/custom/dataset.py \
+                       examples/custom/model.py \
+    --model AI-ModelScope/Nemotron-Mini-4B-Instruct \
+    --tuner_type lora \
+    --dataset swift/stsb \
+    --split_dataset_ratio 0.01 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_length 2048 \
+    --output_dir output \
+    --dataset_num_proc 4
@@ -0,0 +1,9 @@
+Please refer to the examples in [examples/infer](../../infer/) and change `swift infer` to `swift deploy` to start the service. (You need to additionally remove `--val_dataset`)
+
+e.g.
+```shell
+CUDA_VISIBLE_DEVICES=0 \
+swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm
+```
@@ -0,0 +1,90 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def get_infer_request():
+    messages = [{'role': 'user', 'content': "How's the weather in Beijing today?"}]
+    tools = [{
+        'name': 'get_current_weather',
+        'description': 'Get the current weather in a given location',
+        'parameters': {
+            'type': 'object',
+            'properties': {
+                'location': {
+                    'type': 'string',
+                    'description': 'The city and state, e.g. San Francisco, CA'
+                },
+                'unit': {
+                    'type': 'string',
+                    'enum': ['celsius', 'fahrenheit']
+                }
+            },
+            'required': ['location']
+        }
+    }]
+    return messages, tools
+
+
+def infer(client, model: str, messages, tools):
+    messages = messages.copy()
+    query = messages[0]['content']
+    resp = client.chat.completions.create(model=model, messages=messages, tools=tools, max_tokens=512, temperature=0)
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    print(f'tool_calls: {resp.choices[0].message.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    resp = client.chat.completions.create(model=model, messages=messages, tools=tools, max_tokens=512, temperature=0)
+    response2 = resp.choices[0].message.content
+    print(f'response2: {response2}')
+
+
+# streaming
+def infer_stream(client, model: str, messages, tools):
+    messages = messages.copy()
+    query = messages[0]['content']
+    gen = client.chat.completions.create(
+        model=model, messages=messages, tools=tools, max_tokens=512, temperature=0, stream=True)
+    response = ''
+    print(f'query: {query}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        delta = chunk.choices[0].delta.content
+        response += delta
+        print(delta, end='', flush=True)
+    print()
+    print(f'tool_calls: {chunk.choices[0].delta.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    gen = client.chat.completions.create(
+        model=model, messages=messages, tools=tools, max_tokens=512, temperature=0, stream=True)
+    print(f'query: {query}\nresponse2: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+if __name__ == '__main__':
+    host: str = '127.0.0.1'
+    port: int = 8000
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages, tools = get_infer_request()
+    infer(client, model, messages, tools)
+    infer_stream(client, model, messages, tools)
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --agent_template hermes \
+    --served_model_name Qwen2.5-7B-Instruct
@@ -0,0 +1,29 @@
+from typing import List
+
+from swift.infer_engine import InferClient, InferRequest
+
+
+def infer_batch(engine: InferClient, infer_requests: List[InferRequest]):
+    resp_list = engine.infer(infer_requests)
+    query0 = infer_requests[0].messages[0]['content']
+    query1 = infer_requests[1].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'query1: {query1}')
+    print(f'response1: {resp_list[1].choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    engine = InferClient(host='127.0.0.1', port=8000)
+    models = engine.models
+    print(f'models: {models}')
+    infer_batch(engine, [
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '今天天气真好呀'
+        }]),
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '真倒霉'
+        }])
+    ])
@@ -0,0 +1,10 @@
+# Since `swift/test_lora` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --adapters swift/test_bert \
+    --served_model_name bert-base-chinese \
+    --infer_backend transformers \
+    --truncation_strategy right \
+    --max_length 512
@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    query = messages[0]['content']
+    print(f'query: {query}')
+    resp = client.completions.create(model=model, prompt=query, max_tokens=64, temperature=0)
+    response = resp.choices[0].text
+    print(f'response: {response}')
+    # or (The two calling methods are equivalent.)
+    resp = client.chat.completions.create(model=model, messages=messages, max_tokens=64, temperature=0)
+    response = resp.choices[0].message.content
+    print(f'response: {response}')
+    return response
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{'role': 'user', 'content': '浙江 -> 杭州\n安徽 -> 合肥\n四川 ->'}]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen2.5-1.5B',
+                verbose=False,
+                log_interval=-1,
+                infer_backend='transformers',
+                use_chat_template=False)) as port:
+        run_client(port=port)
@@ -0,0 +1,37 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=64, temperature=0)
+
+    resp_list = engine.infer(infer_requests, request_config)
+
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+
+    infer_requests = [InferRequest(messages=[{'role': 'user', 'content': '浙江 -> 杭州\n安徽 -> 合肥\n四川 ->'}])]
+    infer_batch(engine, infer_requests)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, InferClient, InferEngine, InferRequest, RequestConfig, run_deploy
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen2.5-1.5B',
+                verbose=False,
+                log_interval=-1,
+                infer_backend='transformers',
+                use_chat_template=False)) as port:
+        run_client(port=port)
@@ -0,0 +1,65 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        max_tokens=512,
+        temperature=0,
+        extra_body={
+            'chat_template_kwargs': {
+                'enable_thinking': False
+            },
+        })
+    query = messages[0]['content']
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    return response
+
+
+# streaming
+def infer_stream(client, model: str, messages):
+    gen = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        stream=True,
+        temperature=0,
+        extra_body={
+            'chat_template_kwargs': {
+                'enable_thinking': False
+            },
+        })
+    print(f'messages: {messages}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    query = 'Where is the capital of Zhejiang?'
+    messages = [{'role': 'user', 'content': query}]
+    response = infer(client, model, messages)
+    messages.append({'role': 'assistant', 'content': response})
+    messages.append({'role': 'user', 'content': 'What delicious food is there?'})
+    infer_stream(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(DeployArguments(model='Qwen/Qwen3.5-4B', verbose=False, log_interval=-1)) as port:
+        run_client(port=port)
@@ -0,0 +1,59 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    # # The asynchronous interface below is equivalent to the synchronous interface above.
+    # async def _run():
+    #     tasks = [engine.infer_async(infer_request, request_config) for infer_request in infer_requests]
+    #     return await asyncio.gather(*tasks)
+    # resp_list = asyncio.run(_run())
+
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{'role': 'user', 'content': 'who are you?'}]
+    infer_stream(engine, InferRequest(messages=messages, chat_template_kwargs={'enable_thinking': False}))
+
+
+if __name__ == '__main__':
+    from swift import (DeployArguments, InferClient, InferEngine, InferRequest, InferStats, RequestConfig, load_dataset,
+                       run_deploy)
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(DeployArguments(model='Qwen/Qwen3.5-4B', verbose=False, log_interval=-1,
+                                    infer_backend='vllm')) as port:
+        run_client(port=port)
@@ -0,0 +1,98 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages, max_tokens=512, temperature=0)
+    query = messages[0]['content']
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    return response
+
+
+# streaming
+def infer_stream(client, model: str, messages):
+    gen = client.chat.completions.create(model=model, messages=messages, stream=True, temperature=0)
+    print(f'messages: {messages}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'image',
+                'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+            }, {
+                'type': 'text',
+                'text': 'How many sheep are there in the picture?'
+            }]
+        }
+
+    elif mm_type == 'video':
+        # # use base64
+        # import base64
+        # with open('baby.mp4', 'rb') as f:
+        #     vid_base64 = base64.b64encode(f.read()).decode('utf-8')
+        # video = f'data:video/mp4;base64,{vid_base64}'
+
+        # use url
+        video = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+        message = {
+            'role': 'user',
+            'content': [{
+                'type': 'video',
+                'video': video
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    query = 'who are you?'
+    messages = [{'role': 'user', 'content': query}]
+    response = infer(client, model, messages)
+    messages.append({'role': 'assistant', 'content': response})
+    messages.append(get_message(mm_type='video'))
+    infer_stream(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(DeployArguments(model='Qwen/Qwen2.5-VL-3B-Instruct', verbose=False, log_interval=-1)) as port:
+        run_client(port=port)
@@ -0,0 +1,127 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List, Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'image',
+                    # url or local_path or PIL.Image or base64
+                    'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+                },
+                {
+                    'type': 'text',
+                    'text': 'How many sheep are there in the picture?'
+                }
+            ]
+        }
+
+    elif mm_type == 'video':
+        # # use base64
+        # import base64
+        # with open('baby.mp4', 'rb') as f:
+        #     vid_base64 = base64.b64encode(f.read()).decode('utf-8')
+        # video = f'data:video/mp4;base64,{vid_base64}'
+
+        # use url
+        video = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+        message = {
+            'role': 'user',
+            'content': [{
+                'type': 'video',
+                'video': video
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def get_data(mm_type: Literal['text', 'image', 'video', 'audio']):
+    data = {}
+    if mm_type == 'text':
+        messages = [{'role': 'user', 'content': 'who are you?'}]
+    elif mm_type == 'image':
+        # The number of <image> tags must be the same as len(images).
+        messages = [{'role': 'user', 'content': '<image>How many sheep are there in the picture?'}]
+        # Support URL/Path/base64/PIL.Image
+        data['images'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']
+    elif mm_type == 'video':
+        messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+        data['videos'] = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    elif mm_type == 'audio':
+        messages = [{'role': 'user', 'content': '<audio>What does this audio say?'}]
+        data['audios'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    data['messages'] = messages
+    return data
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/LaTeX_OCR:small#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_stream(engine, InferRequest(messages=[get_message(mm_type='video')]))
+    # This writing is equivalent to the above writing.
+    infer_stream(engine, InferRequest(**get_data(mm_type='video')))
+
+
+if __name__ == '__main__':
+    from swift import (DeployArguments, InferClient, InferEngine, InferRequest, InferStats, RequestConfig, load_dataset,
+                       run_deploy)
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(model='Qwen/Qwen2.5-VL-3B-Instruct', verbose=False, log_interval=-1,
+                            infer_backend='vllm')) as port:
+        run_client(port=port)
@@ -0,0 +1,56 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    # You can also use client.embeddings.create
+    # But this interface does not support multi-modal medias
+    resp = client.chat.completions.create(model=model, messages=messages)
+    emb = resp.data[0]['embedding']
+    shape = len(emb)
+    sample = str(emb)
+    if len(emb) > 6:
+        sample = str(emb[:3])[:-1] + ', ..., ' + str(emb[-3:])[1:]
+    print(f'messages: {messages}')
+    print(f'Embedding(shape: [1, {shape}]): {sample}')
+    return emb
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role':
+        'user',
+        'content': [
+            # {
+            #   'type': 'image',
+            #   'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+            # },
+            {
+                'type': 'text',
+                'text': 'What is the capital of China?'
+            },
+        ]
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen3-Embedding-0.6B',  # GME/GTE models or your checkpoints are also supported
+                task_type='embedding',
+                infer_backend='vllm',
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)
@@ -0,0 +1,8 @@
+# GME/GTE models or your checkpoints are also supported
+# transformers/vllm/sglang supported
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --task_type embedding \
+    --model Qwen/Qwen3-Embedding-0.6B \
+    --infer_backend sglang
@@ -0,0 +1,27 @@
+from swift.infer_engine import InferClient, InferRequest, RequestConfig
+
+
+def infer_multilora(engine: InferClient, infer_request: InferRequest):
+    # Dynamic LoRA
+    models = engine.models
+    print(f'models: {models}')
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+
+    # use lora1
+    resp_list = engine.infer([infer_request], request_config, model=models[1])
+    response = resp_list[0].choices[0].message.content
+    print(f'lora1-response: {response}')
+    # origin model
+    resp_list = engine.infer([infer_request], request_config, model=models[0])
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+    # use lora2
+    resp_list = engine.infer([infer_request], request_config, model=models[2])
+    response = resp_list[0].choices[0].message.content
+    print(f'lora2-response: {response}')
+
+
+if __name__ == '__main__':
+    engine = InferClient(host='127.0.0.1', port=8000)
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+    infer_multilora(engine, infer_request)
@@ -0,0 +1,7 @@
+# Since `swift/test_lora` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --adapters lora1=swift/test_lora lora2=swift/test_lora2 \
+    --infer_backend vllm
@@ -0,0 +1,46 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages)
+    scores = resp.choices[0].message.content
+    print(f'messages: {messages}')
+    print(f'scores: {scores}')
+    return scores
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role': 'user',
+        'content': 'what is the capital of China?',
+    }, {
+        'role': 'assistant',
+        'content': 'Beijing',
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='BAAI/bge-reranker-v2-m3',
+                task_type='reranker',
+                infer_backend='vllm',
+                gpu_memory_utilization=0.7,
+                vllm_enforce_eager=True,
+                reranker_use_activation=True,
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)
@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages)
+    scores = resp.choices[0].message.content
+    print(f'messages: {messages}')
+    print(f'scores: {scores}')
+    return scores
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role': 'user',
+        'content': 'what is the capital of China?',
+    }, {
+        'role': 'assistant',
+        'content': 'Beijing.',
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen3-Reranker-0.6B',
+                task_type='generative_reranker',
+                infer_backend='vllm',
+                gpu_memory_utilization=0.7,
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)
@@ -0,0 +1,9 @@
+# GME/GTE models or your checkpoints are also supported
+# transformers/vllm/sglang supported
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --model BAAI/bge-reranker-v2-m3 \
+    --infer_backend vllm \
+    --task_type reranker \
+    --vllm_enforce_eager true \
@@ -0,0 +1,17 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.infer_engine import InferClient, InferRequest
+
+if __name__ == '__main__':
+    engine = InferClient(host='127.0.0.1', port=8000)
+    models = engine.models
+    print(f'models: {models}')
+    messages = [{
+        'role': 'user',
+        'content': "Hello! What's your name?"
+    }, {
+        'role': 'assistant',
+        'content': 'My name is InternLM2! A helpful AI assistant. What can I do for you?'
+    }]
+    resp_list = engine.infer([InferRequest(messages=messages)])
+    print(f'messages: {messages}')
+    print(f'response: {resp_list[0].choices[0].message.content}')
@@ -0,0 +1,5 @@
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
+    --infer_backend transformers
@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages)
+    classify = resp.choices[0].message.content
+    print(f'messages: {messages}')
+    print(f'classify: {classify}')
+    return classify
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role': 'user',
+        'content': 'What is the capital of China?',
+    }, {
+        'role': 'assistant',
+        'content': 'Beijing',
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='/your/seq_cls/checkpoint-xxx',
+                task_type='seq_cls',
+                infer_backend='vllm',
+                num_labels=2,
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)
@@ -0,0 +1,9 @@
+# GME/GTE models or your checkpoints are also supported
+# transformers/vllm/sglang supported
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --model /your/seq_cls/checkpoint-xxx \
+    --infer_backend vllm \
+    --task_type seq_cls \
+    --num_labels 2 \
@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+swift deploy \
+    --model Qwen/Qwen3-8B \
+    --infer_backend sglang \
+    --max_new_tokens 2048 \
+    --sglang_context_length 8192 \
+    --sglang_tp_size 2 \
+    --served_model_name Qwen3-8B
+
+# After the server-side deployment above is successful, use the command below to perform a client call test.
+
+# curl http://localhost:8000/v1/chat/completions \
+# -H "Content-Type: application/json" \
+# -d '{
+# "model": "Qwen3-8B",
+# "messages": [{"role": "user", "content": "What is your name?"}],
+# "temperature": 0
+# }'
@@ -0,0 +1,14 @@
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --served_model_name Qwen2.5-7B-Instruct
+
+# After the server-side deployment above is successful, use the command below to perform a client call test.
+
+# curl http://localhost:8000/v1/chat/completions \
+# -H "Content-Type: application/json" \
+# -d '{
+# "model": "Qwen2.5-7B-Instruct",
+# "messages": [{"role": "user", "content": "What is your name?"}],
+# "temperature": 0
+# }'
@@ -0,0 +1,22 @@
+CUDA_VISIBLE_DEVICES=0,1 swift deploy \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --infer_backend vllm \
+    --served_model_name Qwen2.5-VL-7B-Instruct \
+    --vllm_max_model_len 8192 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_data_parallel_size 2
+
+# After the server-side deployment above is successful, use the command below to perform a client call test.
+
+# curl http://localhost:8000/v1/chat/completions \
+# -H "Content-Type: application/json" \
+# -d '{
+# "model": "Qwen2.5-VL-7B-Instruct",
+# "messages": [{"role": "user", "content": [
+#     {"type": "image", "image": "http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png"},
+#     {"type": "image", "image": "http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png"},
+#     {"type": "text", "text": "What is the difference between the two images?"}
+# ]}],
+# "max_tokens": 256,
+# "temperature": 0
+# }'
@@ -0,0 +1,15 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+if __name__ == '__main__':
+    from swift import DeployArguments, EvalArguments, eval_main, run_deploy
+
+    # Here's a runnable demo provided. Use the eval_url method for evaluation.
+    # In a real scenario, you can simply remove the deployed context.
+    print(EvalArguments.list_eval_dataset())
+    with run_deploy(
+            DeployArguments(model='Qwen/Qwen2.5-0.5B-Instruct', verbose=False, log_interval=-1, infer_backend='vllm'),
+            return_url=True) as url:
+        eval_main(EvalArguments(model='Qwen2.5-0.5B-Instruct', eval_url=url, eval_dataset=['arc']))
@@ -0,0 +1,7 @@
+# You need to have a deployed model or api service first
+swift eval \
+    --model '<model_name>' \
+    --eval_backend OpenCompass \
+    --eval_url http://127.0.0.1:8000/v1 \
+    --eval_limit 100 \
+    --eval_dataset gsm8k
@@ -0,0 +1,7 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift eval \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --eval_backend OpenCompass \
+    --infer_backend sglang \
+    --eval_limit 100 \
+    --eval_dataset gsm8k
@@ -0,0 +1,7 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift eval \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --eval_backend OpenCompass \
+    --infer_backend vllm \
+    --eval_limit 100 \
+    --eval_dataset gsm8k
@@ -0,0 +1,24 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+  --model "Qwen/Qwen2.5-0.5B-Instruct" \
+  --tuner_type "lora" \
+  --dataset "AI-ModelScope/alpaca-gpt4-data-zh#100" \
+  --torch_dtype "bfloat16" \
+  --num_train_epochs "1" \
+  --per_device_train_batch_size "1" \
+  --learning_rate "1e-4" \
+  --lora_rank "8" \
+  --lora_alpha "32" \
+  --target_modules "all-linear" \
+  --gradient_accumulation_steps "16" \
+  --save_steps "50" \
+  --save_total_limit "5" \
+  --logging_steps "5" \
+  --max_length "2048" \
+  --eval_strategy "steps" \
+  --eval_steps "5" \
+  --per_device_eval_batch_size "5" \
+  --eval_use_evalscope \
+  --eval_dataset "gsm8k" \
+  --eval_dataset_args '{"gsm8k": {"few_shot_num": 0}}' \
+  --eval_limit "10"
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+swift eval \
+  --model Qwen/Qwen2.5-VL-3B-Instruct \
+  --infer_backend vllm \
+  --eval_limit 100 \
+  --eval_dataset realWorldQA \
+  --eval_backend VLMEvalKit
@@ -0,0 +1,5 @@
+# Since `output/vx-xxx/checkpoint-xxx` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+swift export \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --merge_lora true
@@ -0,0 +1,4 @@
+swift export \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --to_ollama true \
+    --output_dir Qwen2.5-1.5B-Instruct-ollama
@@ -0,0 +1,6 @@
+swift export \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --push_to_hub true \
+    --hub_model_id '<model-id>' \
+    --hub_token '<sdk-token>' \
+    --use_hf false
@@ -0,0 +1,14 @@
+pip install "transformers<4.52"
+
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen2.5-72B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+    --device_map cpu \
+    --quant_n_samples 256 \
+    --quant_batch_size 1 \
+    --max_length 2048 \
+    --quant_method awq \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-72B-Instruct-AWQ
@@ -0,0 +1,16 @@
+# merge-lora
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --adapters swift/test_bert \
+    --output_dir output/swift_test_bert_merged \
+    --merge_lora true
+
+# bnb quantize
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --model output/swift_test_bert_merged \
+    --output_dir output/swift_test_bert_bnb_int4 \
+    --quant_bits 4 \
+    --quant_method bnb
+
+# infer
+CUDA_VISIBLE_DEVICES=0 swift infer \
+    --model output/swift_test_bert_bnb_int4
@@ -0,0 +1,33 @@
+# merge-lora
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --adapters swift/test_bert \
+    --output_dir output/swift_test_bert_merged \
+    --merge_lora true
+
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "Error: LoRA merge failed with exit code $EXIT_CODE"
+    exit $EXIT_CODE
+fi
+
+# gptq quantize
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --model output/swift_test_bert_merged \
+    --load_data_args true \
+    --output_dir output/swift_test_bert_gptq_int4 \
+    --quant_bits 4 \
+    --quant_method gptq \
+    --max_length 512
+
+
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "Error: GPTQ quantization failed with exit code $EXIT_CODE"
+    exit $EXIT_CODE
+fi
+
+# infer
+CUDA_VISIBLE_DEVICES=0 swift infer \
+    --model output/swift_test_bert_gptq_int4
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --quant_method bnb \
+    --quant_bits 4 \
+    --bnb_4bit_quant_type nf4 \
+    --bnb_4bit_use_double_quant true \
+    --output_dir Qwen2.5-1.5B-Instruct-BNB-NF4
@@ -0,0 +1,9 @@
+# Due to the structural changes made to MoE architecture in `transformers>=5.0`,
+# if you need to apply FP8 quantization to MoE models, please use `megatron export`
+# (compatible with vLLM inference).
+# Reference: https://github.com/modelscope/ms-swift/blob/main/examples/megatron/fp8/quant.sh
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --quant_method fp8 \
+    --output_dir Qwen2.5-3B-Instruct-FP8
@@ -0,0 +1,13 @@
+# OMP_NUM_THREADS=14 please Check issue: https://github.com/AutoGPTQ/AutoGPTQ/issues/439
+OMP_NUM_THREADS=14 \
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+    --quant_n_samples 256 \
+    --quant_batch_size 1 \
+    --max_length 2048 \
+    --quant_method gptq \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-1.5B-Instruct-GPTQ-Int4
@@ -0,0 +1,14 @@
+# You need to install gptqmodel.
+# OMP_NUM_THREADS=14 please Check issue: https://github.com/AutoGPTQ/AutoGPTQ/issues/439
+OMP_NUM_THREADS=14 \
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+    --quant_n_samples 256 \
+    --quant_batch_size 1 \
+    --max_length 2048 \
+    --quant_method gptq_v2 \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-1.5B-Instruct-GPTQ-V2-Int4
@@ -0,0 +1,18 @@
+pip install "transformers==4.51.*"
+
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+swift export \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'modelscope/coco_2014_caption:validation#500' \
+              'swift/VideoChatGPT:Generic#500' \
+    --quant_n_samples 256 \
+    --quant_batch_size -1 \
+    --max_length 2048 \
+    --quant_method awq \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-VL-3B-Instruct-AWQ
@@ -0,0 +1,6 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --quant_method bnb \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-VL-3B-Instruct-BNB-Int4
@@ -0,0 +1,10 @@
+# use transformers==5.2.0
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen3.5-4B \
+    --quant_method fp8 \
+    --output_dir Qwen3.5-4B-FP8
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --model Qwen3.5-4B-FP8
@@ -0,0 +1,18 @@
+# OMP_NUM_THREADS=14 please Check issue: https://github.com/AutoGPTQ/AutoGPTQ/issues/439
+OMP_NUM_THREADS=14 \
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+swift export \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'modelscope/coco_2014_caption:validation#500' \
+              'swift/VideoChatGPT:Generic#500' \
+    --quant_n_samples 256 \
+    --quant_batch_size 1 \
+    --max_length 2048 \
+    --quant_method gptq \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-VL-3B-Instruct-GPTQ-Int4
@@ -0,0 +1,13 @@
+pip install "transformers<4.52"
+
+CUDA_VISIBLE_DEVICES=0,1 \
+swift export \
+    --model Qwen/Qwen3-30B-A3B \
+    --dataset 'swift/Qwen3-SFT-Mixin' \
+    --device_map auto \
+    --quant_n_samples 64 \
+    --quant_batch_size -1 \
+    --max_length 8192 \
+    --quant_method awq \
+    --quant_bits 4 \
+    --output_dir Qwen3-30B-A3B-AWQ
@@ -0,0 +1,6 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen3-30B-A3B \
+    --quant_method bnb \
+    --quant_bits 4 \
+    --output_dir Qwen3-30B-A3B-BNB-Int4
@@ -0,0 +1,11 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --model Qwen/Qwen3-30B-A3B \
+    --quant_method fp8 \
+    --output_dir Qwen3-30B-A3B-FP8
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --model Qwen3-30B-A3B-FP8 \
+#     --infer_backend vllm \
+#     --stream true
@@ -0,0 +1,13 @@
+# 2 * 80GB
+OMP_NUM_THREADS=14 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift export \
+    --model Qwen/Qwen2-57B-A14B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#1000' \
+    --quant_n_samples 512 \
+    --quant_batch_size 1 \
+    --max_length 4096 \
+    --quant_method gptq \
+    --quant_bits 4 \
+    --output_dir Qwen2-57B-A14B-Instruct-GPTQ-Int4
@@ -0,0 +1,18 @@
+# OMP_NUM_THREADS=14 please Check issue: https://github.com/AutoGPTQ/AutoGPTQ/issues/439
+OMP_NUM_THREADS=14 \
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+swift export \
+    --model Qwen/Qwen2.5-Omni-7B \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'modelscope/coco_2014_caption:validation#500' \
+              'swift/VideoChatGPT:Generic#500' \
+    --quant_n_samples 256 \
+    --quant_batch_size 1 \
+    --max_length 2048 \
+    --quant_method gptq \
+    --quant_bits 4 \
+    --output_dir Qwen2.5-Omni-7B-GPTQ-Int4
@@ -0,0 +1,12 @@
+# bnb quantize
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
+    --output_dir output/internlm2-1_8b-reward-bnb-int4 \
+    --quant_bits 4 \
+    --quant_method bnb
+
+# infer
+CUDA_VISIBLE_DEVICES=0 swift infer \
+    --model output/internlm2-1_8b-reward-bnb-int4 \
+    --val_dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+    --max_batch_size 16
@@ -0,0 +1,14 @@
+# gptq quantize
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
+    --output_dir output/internlm2-1_8b-reward-gptq-int4 \
+    --quant_bits 4 \
+    --max_length 2048 \
+    --quant_method gptq \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' 'AI-ModelScope/alpaca-gpt4-data-en#1000'
+
+# infer
+CUDA_VISIBLE_DEVICES=0 swift infer \
+    --model output/internlm2-1_8b-reward-gptq-int4 \
+    --val_dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+    --max_batch_size 16
@@ -0,0 +1,6 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend transformers \
+    --stream true \
+    --max_new_tokens 2048
@@ -0,0 +1,57 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+    # metric.reset()  # reuse
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+if __name__ == '__main__':
+    from swift import InferEngine, InferRequest, InferStats, RequestConfig, TransformersEngine, load_dataset
+    model = 'Qwen/Qwen2.5-1.5B-Instruct'
+    infer_backend = 'transformers'
+
+    if infer_backend == 'transformers':
+        engine = TransformersEngine(model, max_batch_size=64)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine(model, max_model_len=8192)
+    elif infer_backend == 'sglang':
+        from swift.infer_engine import SglangEngine
+        engine = SglangEngine(model)
+    elif infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine(model)
+
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{'role': 'user', 'content': 'who are you?'}]
+    infer_stream(engine, InferRequest(messages=messages))
@@ -0,0 +1,114 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+# os.environ['SWIFT_DEBUG'] = '1'
+
+
+def infer(engine: 'InferEngine', infer_request: 'InferRequest'):
+    stop = [engine.template.agent_template.keyword.observation]  # compat react_en
+    request_config = RequestConfig(max_tokens=512, temperature=0, stop=stop)
+    resp_list = engine.infer([infer_request], request_config)
+    query = infer_request.messages[0]['content']
+    response = resp_list[0].choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    print(f'tool_calls: {resp_list[0].choices[0].message.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    infer_request.messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    resp_list = engine.infer([infer_request], request_config)
+    response2 = resp_list[0].choices[0].message.content
+    print(f'response2: {response2}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    stop = [engine.template.agent_template.keyword.observation]
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True, stop=stop)
+    gen_list = engine.infer([infer_request], request_config)
+    query = infer_request.messages[0]['content']
+    response = ''
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        delta = resp.choices[0].delta.content
+        response += delta
+        print(delta, end='', flush=True)
+    print()
+    print(f'tool_calls: {resp.choices[0].delta.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}\nresponse2: ', end='')
+    infer_request.messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    gen_list = engine.infer([infer_request], request_config)
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def get_infer_request():
+    return InferRequest(
+        messages=[{
+            'role': 'user',
+            'content': "How's the weather in Beijing today?"
+        }],
+        tools=[{
+            'name': 'get_current_weather',
+            'description': 'Get the current weather in a given location',
+            'parameters': {
+                'type': 'object',
+                'properties': {
+                    'location': {
+                        'type': 'string',
+                        'description': 'The city and state, e.g. San Francisco, CA'
+                    },
+                    'unit': {
+                        'type': 'string',
+                        'enum': ['celsius', 'fahrenheit']
+                    }
+                },
+                'required': ['location']
+            }
+        }])
+
+
+def infer_continue_generate(engine):
+    # Continue generating after the assistant message.
+    infer_request = InferRequest(messages=[{
+        'role': 'user',
+        'content': 'How is the weather today?'
+    }, {
+        'role': 'assistant',
+        'content': 'It is sunny today, '
+    }])
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    resp_list = engine.infer([infer_request], request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+
+
+if __name__ == '__main__':
+    from swift.agent_template import agent_template_map
+    from swift.infer_engine import InferEngine, InferRequest, RequestConfig, TransformersEngine
+    model = 'Qwen/Qwen2.5-1.5B-Instruct'
+    infer_backend = 'transformers'
+
+    if infer_backend == 'transformers':
+        engine = TransformersEngine(model, max_batch_size=64)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine(model, max_model_len=8192)
+    elif infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine(model)
+
+    # engine.template._agent_template = 'hermes'  # react_en/qwen_en/qwen_en_parallel
+
+    infer(engine, get_infer_request())
+    infer_stream(engine, get_infer_request())
+
+    # infer_continue_generate(engine)
@@ -0,0 +1,55 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+# demo_seq_cls: https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls/qwen2_5_omni/infer.py
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    resp_list = engine.infer(infer_requests)
+    query0 = infer_requests[0].messages[0]['content']
+    query1 = infer_requests[1].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'query1: {query1}')
+    print(f'response1: {resp_list[1].choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    # This is an example of BERT with LoRA.
+    from peft import PeftModel
+
+    from swift import BaseArguments, InferEngine, InferRequest, TransformersEngine, load_dataset, safe_snapshot_download
+    adapter_path = safe_snapshot_download('swift/test_bert')
+    args = BaseArguments.from_pretrained(adapter_path)
+    args.max_length = 512
+    args.truncation_strategy = 'right'
+    # method1
+    model, processor = args.get_model_processor()
+    model = PeftModel.from_pretrained(model, adapter_path)
+    template = args.get_template(processor)
+    engine = TransformersEngine(model, template=template, max_batch_size=64)
+
+    # method2
+    # engine = TransformersEngine(args.model, adapters=[adapter_path], max_batch_size=64,
+    #                   task_type=args.task_type, num_labels=args.num_labels)
+    # template = args.get_template(engine.processor)
+    # engine.template = template
+
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['DAMO_NLP/jd:cls#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(messages=data['messages']) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_batch(engine, [
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '今天天气真好呀'
+        }]),
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '真倒霉'
+        }])
+    ])
@@ -0,0 +1,64 @@
+import torch
+
+from swift.infer_engine import InferRequest, TransformersEngine
+
+
+def run_qwen3_emb():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-Embedding-4B', task_type='embedding', torch_dtype=torch.float16, attn_impl='flash_attention_2')
+
+    infer_requests = [
+        InferRequest(messages=[
+            {
+                'role':
+                'user',
+                'content':
+                'Instruct: Given a web search query, retrieve relevant passages that answer the query\n'
+                'Query:What is the capital of China?'
+            },
+        ]),
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'The capital of China is Beijing.'
+            },
+        ])
+    ]
+    resp_list = engine.infer(infer_requests)
+    embedding0 = torch.tensor(resp_list[0].data[0].embedding)
+    embedding1 = torch.tensor(resp_list[1].data[0].embedding)
+    print(f'scores: {(embedding0 * embedding1).sum()}')
+
+
+def run_qwen3_vl_emb():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-VL-Embedding-2B', task_type='embedding', max_batch_size=2, attn_impl='flash_attention_2')
+
+    infer_requests = [
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'A woman playing with her dog on a beach at sunset.'
+            },
+        ]),
+        InferRequest(
+            messages=[
+                {
+                    'role':
+                    'user',
+                    'content':
+                    '<image>A woman shares a joyful moment with her golden retriever on a sun-drenched beach at '
+                    'sunset, as the dog offers its paw in a heartwarming display of companionship and trust.'
+                },
+            ],
+            images=['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg'])
+    ]
+    resp_list = engine.infer(infer_requests)
+    embedding0 = torch.tensor(resp_list[0].data[0].embedding)
+    embedding1 = torch.tensor(resp_list[1].data[0].embedding)
+    print(f'scores: {(embedding0 * embedding1).sum()}')
+
+
+if __name__ == '__main__':
+    # run_qwen3_emb()
+    run_qwen3_vl_emb()
@@ -0,0 +1,45 @@
+import os
+import re
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['MAX_PIXELS'] = '1003520'
+
+
+def draw_bbox_qwen2_vl(image, response, norm_bbox: Literal['norm1000', 'none']):
+    matches = re.findall(
+        r'<\|object_ref_start\|>(.*?)<\|object_ref_end\|><\|box_start\|>\((\d+),(\d+)\),\((\d+),(\d+)\)<\|box_end\|>',
+        response)
+    ref = []
+    bbox = []
+    for match_ in matches:
+        ref.append(match_[0])
+        bbox.append(list(match_[1:]))
+    draw_bbox(image, ref, bbox, norm_bbox=norm_bbox)
+
+
+def infer_grounding():
+    # use transformers==4.51.3
+    from swift import BaseArguments, InferRequest, RequestConfig, TransformersEngine, safe_snapshot_download
+    output_path = 'bbox.png'
+    image = load_image('http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png')
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'Task: Object Detection'}], images=[image])
+
+    request_config = RequestConfig(max_tokens=512, temperature=0, return_details=True)
+    adapter_path = safe_snapshot_download('swift/test_grounding')
+    args = BaseArguments.from_pretrained(adapter_path)
+
+    engine = TransformersEngine(args.model, adapters=[adapter_path])
+    resp_list = engine.infer([infer_request], request_config)
+    image = image.resize(resp_list[0].images_size[0])
+    response = resp_list[0].choices[0].message.content
+    print(f'lora-response: {response}')
+
+    draw_bbox_qwen2_vl(image, response, norm_bbox=args.norm_bbox)
+    print(f'output_path: {output_path}')
+    image.save(output_path)
+
+
+if __name__ == '__main__':
+    from swift.template import draw_bbox, load_image
+    infer_grounding()
@@ -0,0 +1,66 @@
+def infer_hf():
+    from modelscope import snapshot_download
+    from peft import PeftModel
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    model_dir = snapshot_download('Qwen/Qwen2.5-7B-Instruct')
+    adapter_dir = snapshot_download('swift/test_lora')
+    model = AutoModelForCausalLM.from_pretrained(
+        model_dir, torch_dtype='auto', device_map='auto', trust_remote_code=True)
+    model = PeftModel.from_pretrained(model, adapter_dir)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
+
+    messages = [{
+        'role': 'system',
+        'content': 'You are a helpful assistant.'
+    }, {
+        'role': 'user',
+        'content': 'who are you?'
+    }]
+    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    model_inputs = tokenizer([text], return_tensors='pt', add_special_tokens=False).to(model.device)
+
+    generated_ids = model.generate(**model_inputs, max_new_tokens=512, do_sample=False)
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    print(f'response: {response}')
+    return response
+
+
+def infer_swift():
+    from modelscope import snapshot_download
+    from peft import PeftModel
+
+    from swift import get_model_processor, get_template
+    from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+    from swift.tuners import Swift
+    model_dir = snapshot_download('Qwen/Qwen2.5-7B-Instruct')
+    adapter_dir = snapshot_download('swift/test_lora')
+    model, tokenizer = get_model_processor(model_dir, device_map='auto')
+    model = Swift.from_pretrained(model, adapter_dir)
+    # You can also write it as:
+    # model = PeftModel.from_pretrained(model, adapter_dir)
+    template = get_template(tokenizer)
+    engine = TransformersEngine(model, template=template)
+
+    messages = [{
+        'role': 'system',
+        'content': 'You are a helpful assistant.'
+    }, {
+        'role': 'user',
+        'content': 'who are you?'
+    }]
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    resp_list = engine.infer([InferRequest(messages=messages)], request_config=request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+    return response
+
+
+if __name__ == '__main__':
+    response = infer_hf()
+    response2 = infer_swift()
+    assert response == response2
@@ -0,0 +1,70 @@
+import os
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_multilora(infer_request: 'InferRequest', infer_backend: Literal['vllm', 'transformers']):
+    # Dynamic LoRA
+    adapter_path = safe_snapshot_download('swift/test_lora')
+    adapter_path2 = safe_snapshot_download('swift/test_lora2')
+    args = BaseArguments.from_pretrained(adapter_path)
+    if infer_backend == 'transformers':
+        engine = TransformersEngine(args.model)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine(args.model, enable_lora=True, max_loras=1, max_lora_rank=16)
+    template = get_template(engine.processor, template_type=args.template, default_system=args.system)
+    engine.template = template
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    adapter_request = AdapterRequest('lora1', adapter_path)
+    adapter_request2 = AdapterRequest('lora2', adapter_path2)
+
+    # use lora
+    resp_list = engine.infer([infer_request], request_config, adapter_request=adapter_request)
+    response = resp_list[0].choices[0].message.content
+    print(f'lora1-response: {response}')
+    # origin model
+    resp_list = engine.infer([infer_request], request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+    # use lora
+    resp_list = engine.infer([infer_request], request_config, adapter_request=adapter_request2)
+    response = resp_list[0].choices[0].message.content
+    print(f'lora2-response: {response}')
+
+
+def infer_lora(infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    adapter_path = safe_snapshot_download('swift/test_lora')
+    args = BaseArguments.from_pretrained(adapter_path)
+    # method1
+    # engine = TransformersEngine(args.model, adapters=[adapter_path])
+    # template = get_template(engine.processor, args.system, template_type=args.template)
+    # engine.template = template
+
+    # method2
+    # model, processor = args.get_model_processor()
+    # model = PeftModel.from_pretrained(model, adapter_path)
+    # template = args.get_template(processor)
+    # engine = TransformersEngine(model, template=template)
+
+    # method3
+    model, tokenizer = get_model_processor(args.model)
+    model = PeftModel.from_pretrained(model, adapter_path)
+    template = get_template(tokenizer, args.system, template_type=args.template)
+    engine = TransformersEngine(model, template=template)
+
+    resp_list = engine.infer([infer_request], request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'lora-response: {response}')
+
+
+if __name__ == '__main__':
+    from peft import PeftModel
+
+    from swift import (AdapterRequest, BaseArguments, InferRequest, RequestConfig, TransformersEngine,
+                       get_model_processor, get_template, safe_snapshot_download)
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+    # infer_lora(infer_request)
+    infer_multilora(infer_request, 'transformers')
@@ -0,0 +1,145 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List, Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+    # metric.reset()  # reuse
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'image',
+                    # url or local_path or PIL.Image or base64
+                    'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+                },
+                {
+                    'type': 'text',
+                    'text': 'How many sheep are there in the picture?'
+                }
+            ]
+        }
+
+    elif mm_type == 'video':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'video',
+                'video': 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def get_data(mm_type: Literal['text', 'image', 'video', 'audio']):
+    data = {}
+    if mm_type == 'text':
+        messages = [{'role': 'user', 'content': 'who are you?'}]
+    elif mm_type == 'image':
+        # The number of <image> tags must be the same as len(images).
+        messages = [{'role': 'user', 'content': '<image>How many sheep are there in the picture?'}]
+        # Support URL/Path/base64/PIL.Image
+        data['images'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']
+    elif mm_type == 'video':
+        messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+        data['videos'] = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    elif mm_type == 'audio':
+        messages = [{'role': 'user', 'content': '<audio>What does this audio say?'}]
+        data['audios'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    data['messages'] = messages
+    return data
+
+
+if __name__ == '__main__':
+    # The inference of the trained model can be referred to as:
+    # https://github.com/modelscope/ms-swift/tree/main/examples/notebook
+    from swift import InferEngine, InferRequest, InferStats, RequestConfig, TransformersEngine, load_dataset
+    infer_backend = 'transformers'
+
+    if infer_backend == 'transformers':
+        # test env: transformers==4.55.2
+        model = 'Qwen/Qwen2.5-Omni-7B'
+        mm_type = 'audio'
+        engine = TransformersEngine(model, max_batch_size=64, attn_impl='flash_attention_2')
+    elif infer_backend == 'vllm':
+        # test env: vllm==0.8.5.post1, transformers==4.51.3
+        # The meaning of environment variables can be found at:
+        # https://swift.readthedocs.io/zh-cn/latest/Instruction/%E5%91%BD%E4%BB%A4%E8%A1%8C%E5%8F%82%E6%95%B0.html#id17
+        from swift.infer_engine import VllmEngine
+        os.environ['MAX_PIXELS'] = '1003520'
+        os.environ['VIDEO_MAX_PIXELS'] = '50176'
+        os.environ['FPS_MAX_FRAMES'] = '12'
+        model = 'Qwen/Qwen2.5-VL-3B-Instruct'
+        # If you encounter insufficient GPU memory, please reduce `max_model_len` and set `max_num_seqs=5`.
+        engine = VllmEngine(model, max_model_len=8192, limit_mm_per_prompt={'image': 5, 'video': 2})
+        mm_type = 'image'  # or 'video'
+    elif infer_backend == 'lmdeploy':
+        # test env: lmdeploy==0.7.1
+        from swift.infer_engine import LmdeployEngine
+        model = 'OpenGVLab/InternVL2_5-1B'
+        engine = LmdeployEngine(model, vision_batch_size=8)
+        mm_type = 'image'  # or 'video'
+
+    # infer dataset
+    if mm_type == 'audio':
+        dataset = 'speech_asr/speech_asr_aishell1_trainsets:validation#1000'
+    elif mm_type == 'image':
+        dataset = 'AI-ModelScope/LaTeX_OCR:small#1000'
+    elif mm_type == 'video':
+        dataset = 'swift/VideoChatGPT:Generic#100'
+
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset([dataset], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_stream(engine, InferRequest(messages=[get_message(mm_type)]))
+    # This writing is equivalent to the above writing.
+    infer_stream(engine, InferRequest(**get_data(mm_type)))
@@ -0,0 +1,55 @@
+import torch
+
+from swift.infer_engine import InferRequest, TransformersEngine
+
+
+def run_qwen3_reranker():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-Reranker-4B',
+        task_type='generative_reranker',
+        torch_dtype=torch.float16,
+        attn_impl='flash_attention_2')
+
+    infer_request = InferRequest(
+        messages=[{
+            'role': 'system',
+            'content': 'Given a web search query, retrieve relevant passages that answer the query'
+        }, {
+            'role': 'user',
+            'content': 'What is the capital of China?'
+        }, {
+            'role': 'assistant',
+            'content': 'The capital of China is Beijing.'
+        }])
+
+    response = engine.infer([infer_request])[0]
+    print(f'scores: {response.choices[0].message.content}')
+
+
+def run_qwen3_vl_reranker():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-VL-Reranker-2B', task_type='generative_reranker', attn_impl='flash_attention_2')
+
+    infer_request = InferRequest(
+        messages=[{
+            'role': 'system',
+            'content': "Retrieval relevant image or text with user's query"
+        }, {
+            'role': 'user',
+            'content': 'A woman playing with her dog on a beach at sunset.'
+        }, {
+            'role':
+            'assistant',
+            'content':
+            '<image>A woman shares a joyful moment with her golden retriever on a sun-drenched beach '
+            'at sunset, as the dog offers its paw in a heartwarming display of companionship and trust.'
+        }],
+        images=['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg'])
+
+    response = engine.infer([infer_request])[0]
+    print(f'scores: {response.choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    # run_qwen3_reranker()
+    run_qwen3_vl_reranker()
@@ -0,0 +1,31 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    resp_list = engine.infer(infer_requests)
+    print(f'messages0: {infer_requests[0].messages}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    from swift import InferEngine, InferRequest, TransformersEngine, load_dataset
+    model = 'Shanghai_AI_Laboratory/internlm2-1_8b-reward'
+    engine = TransformersEngine(model, max_batch_size=64)
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{
+        'role': 'user',
+        'content': "Hello! What's your name?"
+    }, {
+        'role': 'assistant',
+        'content': 'My name is InternLM2! A helpful AI assistant. What can I do for you?'
+    }]
+    infer_batch(engine, [InferRequest(messages=messages)])
@@ -0,0 +1,85 @@
+"""
+Example of using reasoning_parser
+
+This example demonstrates how to use reasoning_parser in Swift's VllmEngine to support reasoning models.
+"""
+
+from swift.infer_engine import InferRequest, RequestConfig, VllmEngine
+
+
+def main(engine: VllmEngine):
+    # Create inference request
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': '9.11 and 9.8, which is greater?'}])
+
+    # Configure request parameters
+    request_config = RequestConfig(
+        max_tokens=8192,
+        temperature=0.7,
+        stream=False  # Non-streaming inference
+    )
+
+    # Execute inference
+    responses = engine.infer(infer_requests=[infer_request], request_config=request_config)
+
+    # Process responses
+    for response in responses:
+        if hasattr(response, 'choices') and response.choices:
+            choice = response.choices[0]
+            message = choice.message
+
+            print('=== Reasoning Content ===')
+            if message.reasoning_content:
+                print(f'Reasoning steps: {message.reasoning_content}')
+            else:
+                print('No reasoning content detected')
+
+            print('\n=== Final Answer ===')
+            print(f'Answer: {message.content}')
+
+            print('\n=== Finish Reason ===')
+            print(f'Reason: {choice.finish_reason}')
+
+
+def streaming_example(engine: VllmEngine):
+    """Streaming inference example"""
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'Calculate the result of 15 + 27'}])
+
+    request_config = RequestConfig(
+        max_tokens=8192,
+        temperature=0.7,
+        stream=True  # Enable streaming inference
+    )
+
+    # Streaming inference
+    responses = engine.infer(infer_requests=[infer_request], request_config=request_config)
+
+    print('=== Streaming Inference Results ===')
+    for chunk in responses[0]:  # responses[0] is the streaming generator
+        if chunk and chunk.choices:
+            choice = chunk.choices[0]
+            delta = choice.delta
+
+            if delta.reasoning_content:
+                print(f'Reasoning: {delta.reasoning_content}', end='', flush=True)
+
+            if delta.content:
+                print(f'Content: {delta.content}', end='', flush=True)
+
+    print('\n=== Inference Complete ===')
+
+
+if __name__ == '__main__':
+    # Initialize VllmEngine with reasoning_parser enabled
+    engine = VllmEngine(
+        model_id_or_path='Qwen/Qwen3-8B',
+        reasoning_parser='qwen3',  # Specify reasoning parser
+        gpu_memory_utilization=0.9,
+    )
+
+    print('=== Non-streaming Inference Example ===')
+    main(engine)
+
+    print('\n' + '=' * 50 + '\n')
+
+    print('=== Streaming Inference Example ===')
+    streaming_example(engine)
@@ -0,0 +1,8 @@
+# test env: lmdeploy 0.9.2.post1
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend lmdeploy \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#1000 \
+    --max_new_tokens 512
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+swift infer \
+    --model OpenGVLab/InternVL2_5-1B \
+    --infer_backend lmdeploy \
+    --val_dataset AI-ModelScope/captcha-images#1000 \
+    --lmdeploy_tp 2 \
+    --lmdeploy_vision_batch_size 8 \
+    --max_new_tokens 2048
@@ -0,0 +1,7 @@
+# test_env: pip install "sglang[all]==0.4.6.*" -U
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend sglang \
+    --stream true \
+    --max_new_tokens 2048
--- a/Show More
+++ b/Show More