chore: import upstream snapshot with attribution

examples/models/deepseek_ocr/infer.py

@@ -0,0 +1,29 @@
+# pip install "transformers==4.46.3" easydict
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+# os.environ['SWIFT_DEBUG'] = '1'
+
+if __name__ == '__main__':
+    from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+    engine = TransformersEngine('deepseek-ai/DeepSeek-OCR')
+    infer_request = InferRequest(
+        messages=[{
+            'role': 'user',
+            # or
+            'content': '<image>Free OCR.',
+            # "content": '<image><|grounding|>Convert the document to markdown.',
+        }],
+        images=['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/ocr.png'])
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    resp_list = engine.infer([infer_request], request_config=request_config)
+    response = resp_list[0].choices[0].message.content
+
+    # use stream
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    gen_list = engine.infer([infer_request], request_config=request_config)
+    for chunk in gen_list[0]:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()

examples/models/deepseek_ocr/train.sh

@@ -0,0 +1,30 @@
+# 24GiB
+pip install "transformers==4.46.3" easydict "peft==0.15.2"
+
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model deepseek-ai/DeepSeek-OCR \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/deepseek_vl2/train.sh

@@ -0,0 +1,31 @@
+# 9GiB
+pip install "transformers==4.41.*" "peft==0.11.*"
+# pip uninstall autoawq
+
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model deepseek-ai/deepseek-vl2-tiny \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/gemma4/diffusion_gemma.sh

@@ -0,0 +1,47 @@
+# 2 * 60GiB
+# This is just a demo for DiffusionGemma training.
+# Notes:
+# 1. Currently only --per_device_train_batch_size 1 is supported,
+# and the response length of a single sample must be less than config.canvas_length.
+# 2. --gradient_checkpointing false must be set. DiffusionGemma's encoder passes
+# KV to the decoder via DynamicCache, and gradient checkpointing causes errors
+# when recomputing the forward pass during backward.
+# 3. For customizing the specific training loss, refer to:
+# https://github.com/modelscope/ms-swift/blob/104048e374b954b4df6961f83f77392031f38fb0/swift/template/templates/gemma.py#L386-L428
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=2 \
+swift sft \
+    --model google/diffusiongemma-26B-A4B-it \
+    --dataset 'sapientinc/sudoku-extreme-1k' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --num_train_epochs 3 \
+    --loss_scale ignore_empty_think \
+    --gradient_checkpointing false \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --deepspeed zero2 \
+    --dataloader_num_workers 4
+
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --load_data_args true \
+    --enable_thinking false

examples/models/gemma4/gemma4_unified.sh

@@ -0,0 +1,49 @@
+# 4 * 65GiB
+# Due to the use of group_by_length, the data is not sufficiently shuffled,
+# which may cause fluctuations in the loss curve. Please adjust the parameters accordingly.
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+megatron sft \
+    --model google/gemma-4-12B-it \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --load_from_cache_file true \
+    --add_non_thinking_prefix true \
+    --loss_scale ignore_empty_think \
+    --split_dataset_ratio 0.01 \
+    --tuner_type full \
+    --tensor_model_parallel_size 4 \
+    --micro_batch_size 16 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/gemma-4-12B-it \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend unfused \
+    --group_by_length true \
+    --padding_free false
+
+# CUDA_VISIBLE_DEVICES=0 swift infer \
+#     --model megatron_output/gemma-4-12B-it/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --enable_thinking false \
+#     --load_data_args true \
+#     --max_new_tokens 2048

examples/models/gemma4/mcore.sh

@@ -0,0 +1,60 @@
+# 8 * 80GiB
+# Due to the use of group_by_length, the data is not sufficiently shuffled,
+# which may cause fluctuations in the loss curve. Please adjust the parameters accordingly.
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model google/gemma-4-26B-A4B-it \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --load_from_cache_file true \
+    --add_non_thinking_prefix true \
+    --loss_scale ignore_empty_think \
+    --split_dataset_ratio 0.01 \
+    --tuner_type full \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --pipeline_model_parallel_size 2 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 8 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/gemma-4-26B-A4B-it \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend unfused \
+    --group_by_length true \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot
+
+# CUDA_VISIBLE_DEVICES=0 swift infer \
+#     --model megatron_output/gemma-4-26B-A4B-it/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --enable_thinking false \
+#     --load_data_args true \
+#     --max_new_tokens 2048

examples/models/gemma4/train.sh

@@ -0,0 +1,35 @@
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift sft \
+    --model google/gemma-4-E2B-it \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --adapters output/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --load_data_args true

examples/models/glm-4.6v/flash.sh

@@ -0,0 +1,41 @@
+# 2 * 90GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+megatron sft \
+    --model ZhipuAI/GLM-4.6V-Flash \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+    --load_from_cache_file true \
+    --tensor_model_parallel_size 2 \
+    --sequence_parallel true \
+    --packing true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --split_dataset_ratio 0.01 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/GLM-4.6V-Flash \
+    --save_steps 200 \
+    --max_length 2048 \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 8
+
+# PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --model megatron_output/GLM-4.6V-Flash/vx-xxx/checkpoint-xxx \
+#     --load_data_args true \
+#     --stream true

examples/models/glm-4.6v/mcore.sh

@@ -0,0 +1,48 @@
+# 4 * 66GiB, 5s/it
+# Fine-tuning GLM-4.6V requires Transformers 5.0, or you can switch the model to 'ZhipuAI/GLM-4.5V'.
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+megatron sft \
+    --model ZhipuAI/GLM-4.6V \
+    --save_safetensors true \
+    --merge_lora true \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --sequence_parallel true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --packing true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 4 \
+    --expert_tensor_parallel_size 1 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-3 \
+    --micro_batch_size 1 \
+    --global_batch_size 2 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/GLM-4.6V \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 2048 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --attention_backend flash

examples/models/gpt_oss/internvl3_5_gpt.sh

@@ -0,0 +1,49 @@
+# 8 * 80GiB 4s/it
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+OMP_NUM_THREADS=14 \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --moe_permute_fusion true \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 8 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 2 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/InternVL3_5-GPT-OSS-20B-A4B-Preview \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --moe_expert_capacity_factor 2 \
+    --padding_free false \
+    --attention_backend unfused
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --model megatron_output/InternVL3_5-GPT-OSS-20B-A4B-Preview/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --load_data_args true \
+#     --max_new_tokens 512

examples/models/gpt_oss/mcore.sh

@@ -0,0 +1,50 @@
+# 2 * 40GiB
+# dataset format: https://github.com/modelscope/ms-swift/pull/5277
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+megatron sft \
+    --model openai-mirror/gpt-oss-20b \
+    --save_safetensors true \
+    --merge_lora true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --expert_model_parallel_size 2 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 8 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/gpt-oss-20b \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --max_length 2048 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --padding_free false \
+    --attention_backend unfused \
+    --model_author swift \
+    --model_name swift-robot
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --model megatron_output/gpt-oss-20b/vx-xxx/checkpoint-xxx \
+#     --stream true

examples/models/gpt_oss/train.sh

@@ -0,0 +1,29 @@
+# 42GB
+# https://github.com/modelscope/ms-swift/pull/5277
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model openai-mirror/gpt-oss-20b \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --router_aux_loss_coef 1e-3 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/models/hunyuan_ocr/train.sh

@@ -0,0 +1,40 @@
+# Custom dataset format
+# {"messages": [{"role": "user", "content": "Extract all information from the main body of the document image and represent it in markdown format, ignoring headers and footers. Tables should be expressed in HTML format, formulas in the document should be represented using LaTeX format, and the parsing should be organized according to the reading order."}, {"role": "assistant", "content": "<response1>"}], "images": ["ocr.png"]}
+
+pip install git+https://github.com/huggingface/transformers@82a06db03535c49aa987719ed0746a76093b1ec4
+
+
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Tencent-Hunyuan/HunyuanOCR \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4
+
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#     --adapters output/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --load_data_args true

examples/models/internvl3/train.sh

@@ -0,0 +1,30 @@
+# 24GiB
+pip install "transformers==4.51.*"
+
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model OpenGVLab/InternVL3-8B \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/keye/train.sh

@@ -0,0 +1,28 @@
+# 24GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Kwai-Keye/Keye-VL-1_5-8B \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/llama4/mcore.sh

@@ -0,0 +1,56 @@
+# transformers: https://github.com/modelscope/ms-swift/blob/main/examples/train/moe/llama4.sh
+# 4 * 80GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+megatron sft \
+    --model LLM-Research/Llama-4-Scout-17B-16E-Instruct \
+    --save_safetensors true \
+    --merge_lora true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --sequence_parallel true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-3 \
+    --micro_batch_size 4 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Llama-4-Scout-17B-16E-Instruct \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --max_length 4096 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --padding_free false \
+    --attention_backend unfused
+
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model megatron_output/Llama-4-Scout-17B-16E-Instruct/vx-xxx/checkpoint-xxx \
+    --stream true \
+    --load_data_args true \
+    --max_new_tokens 2048

examples/models/minicpmv/train.sh

@@ -0,0 +1,28 @@
+# 10.5GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model OpenBMB/MiniCPM-V-4.6 \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/ovis2/train.sh

@@ -0,0 +1,33 @@
+# 17GiB
+
+pip install "transformers==4.51.*"
+
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model AIDC-AI/Ovis2.5-2B \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 16 \
+    --attn_impl flash_attn \
+    --padding_free true \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/paddle_ocr/train.sh

@@ -0,0 +1,35 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model PaddlePaddle/PaddleOCR-VL-1.6 \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --padding_free true \
+    --attn_impl flash_attention_2 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 16 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift infer \
+#   --adapters output/vx-xxx/checkpoint-xxx \
+#   --stream true \
+#   --load_data_args true

examples/models/qwen3_5/fp8.sh

@@ -0,0 +1,81 @@
+# 2 * 50GiB
+# `--model` can directly accept a model with FP8 weights,
+# and the FP8 weight model will be saved directly upon training completion.
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+megatron sft \
+    --model Qwen/Qwen3.5-4B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --model_author swift \
+    --model_name swift-robot \
+    --linear_decoupled_in_proj true \
+    --load_from_cache_file true \
+    --add_non_thinking_prefix true \
+    --loss_scale ignore_empty_think \
+    --fp8_recipe blockwise \
+    --fp8_format e4m3 \
+    --fp8_param_gather true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type full \
+    --tensor_model_parallel_size 2 \
+    --micro_batch_size 1 \
+    --global_batch_size 2 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --packing true \
+    --finetune true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3.5-4B-FP8 \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 4096 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --mtp_num_layers 1 \
+    --attention_backend flash
+
+# PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+# CUDA_VISIBLE_DEVICES=0 \
+# IMAGE_MAX_TOKEN_NUM=1024 \
+# VIDEO_MAX_TOKEN_NUM=128 \
+# FPS_MAX_FRAMES=12 \
+# swift infer \
+#     --model megatron_output/Qwen3.5-4B-FP8/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --enable_thinking false \
+#     --load_data_args true
+
+# vllm
+# PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+# CUDA_VISIBLE_DEVICES=0 \
+# IMAGE_MAX_TOKEN_NUM=1024 \
+# VIDEO_MAX_TOKEN_NUM=128 \
+# FPS_MAX_FRAMES=12 \
+# swift infer \
+#     --model megatron_output/Qwen3.5-4B-FP8/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --enable_thinking false \
+#     --load_data_args true \
+#     --infer_backend vllm \
+#     --vllm_tensor_parallel_size 1 \
+#     --vllm_max_model_len 8192 \
+#     --vllm_speculative_config '{"method":"qwen3_next_mtp","num_speculative_tokens":2}'

examples/models/qwen3_5/mcore.sh

@@ -0,0 +1,70 @@
+# 4 * 40GiB
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --merge_lora true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --load_from_cache_file true \
+    --add_non_thinking_prefix true \
+    --loss_scale ignore_empty_think \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 4 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --group_by_length true \
+    --finetune true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/Qwen3.5-35B-A3B \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 2048 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot
+
+
+# PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+# CUDA_VISIBLE_DEVICES=0,1,2,3 \
+# IMAGE_MAX_TOKEN_NUM=1024 \
+# VIDEO_MAX_TOKEN_NUM=128 \
+# FPS_MAX_FRAMES=12 \
+# swift infer \
+#     --model megatron_output/Qwen3.5-35B-A3B/vx-xxx/checkpoint-xxx-merged \
+#     --stream true \
+#     --experts_impl grouped_mm \
+#     --enable_thinking false \
+#     --load_data_args true

examples/models/qwen3_5/mcore_grpo_moe.sh

@@ -0,0 +1,64 @@
+# Qwen3.5-35B-A3B (MoE) Megatron GRPO LoRA Training Example
+
+SYSTEM_PROMPT="""You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}."""
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+megatron rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --enable_thinking false \
+    --merge_lora true \
+    --context_parallel_size 1 \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --pipeline_model_parallel_size 1 \
+    --moe_permute_fusion true \
+    --dataset open-r1/DAPO-Math-17k-Processed \
+    --system "$SYSTEM_PROMPT" \
+    --num_train_epochs 1 \
+    --global_batch_size 64 \
+    --micro_batch_size 1 \
+    --steps_per_generation 2 \
+    --num_generations 8 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_max_model_len 9192 \
+    --max_length 1000 \
+    --max_completion_length 8192 \
+    --tuner_type lora \
+    --target_modules all-linear \
+    --lr 5e-5 \
+    --bf16 true \
+    --beta 0.00 \
+    --epsilon 0.2 \
+    --epsilon_high 0.28 \
+    --dynamic_sample false \
+    --overlong_filter true \
+    --loss_type grpo \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_bridge false \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim \
+    --no_save_rng \
+    --save_steps 20 \
+    --attention_backend flash \
+    --moe_expert_capacity_factor 2 \
+    --temperature 1.0 \
+    --padding_free false \
+    --sequence_parallel true \
+    --log_completions true \
+    --report_to tensorboard swanlab

examples/models/qwen3_5/packing.sh

@@ -0,0 +1,51 @@
+# 8 * 80GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/LongAlpaca-12k' \
+    --load_from_cache_file true \
+    --add_non_thinking_prefix true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type full \
+    --tensor_model_parallel_size 4 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --packing true \
+    --finetune true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3.5-35B-A3B \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 32768 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --moe_expert_capacity_factor 2 \
+    --mtp_num_layers 1 \
+    --optimizer_cpu_offload true \
+    --use_precision_aware_optimizer true \
+    --optimizer_offload_fraction 0.64 \
+    --attention_backend flash

examples/models/qwen3_5/transformers.sh

@@ -0,0 +1,49 @@
+# 4 * 30GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \
+    --load_from_cache_file true \
+    --add_non_thinking_prefix true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --experts_impl grouped_mm \
+    --router_aux_loss_coef 1e-3 \
+    --gradient_accumulation_steps 1 \
+    --group_by_length true \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero3
+
+
+# PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+# CUDA_VISIBLE_DEVICES=0,1,2,3 \
+# IMAGE_MAX_TOKEN_NUM=1024 \
+# VIDEO_MAX_TOKEN_NUM=128 \
+# FPS_MAX_FRAMES=12 \
+# swift infer \
+#     --adapters output/Qwen3.5-35B-A3B/vx-xxx/checkpoint-xxx \
+#     --stream true \
+#     --experts_impl grouped_mm \
+#     --enable_thinking false \
+#     --load_data_args true

examples/models/qwen3_next/mcore.sh

@@ -0,0 +1,58 @@
+# 8 * 56GiB
+# padding_free/packing support requires "mcore-bridge>=1.4.0":
+# https://github.com/modelscope/mcore-bridge/pull/76
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --save_safetensors true \
+    --merge_lora false \
+    --dataset 'swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT#2000' \
+              'swift/self-cognition#1000' \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --packing true \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/Qwen3-Next-80B-A3B-Instruct \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 4096 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot
+
+
+# CUDA_VISIBLE_DEVICES=0,1,2,3 \
+# swift infer \
+#     --adapters megatron_output/Qwen3-Next-80B-A3B-Instruct/vx-xxx/checkpoint-xxx \
+#     --vllm_tensor_parallel_size 4 \
+#     --infer_backend vllm \
+#     --vllm_max_model_len 8192 \
+#     --vllm_gpu_memory_utilization 0.9 \
+#     --stream true \
+#     --max_new_tokens 2048

examples/models/qwen3_next/mtp.sh

@@ -0,0 +1,55 @@
+# 8 * 60GiB
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --save_safetensors true \
+    --mtp_num_layers 1 \
+    --dataset 'swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT#2000' \
+              'swift/self-cognition#1000' \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 2 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/Qwen3-Next-80B-A3B-Instruct \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 2048 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot
+
+
+# CUDA_VISIBLE_DEVICES=0,1,2,3 \
+# swift infer \
+#     --model megatron_output/Qwen3-Next-80B-A3B-Instruct/vx-xxx/checkpoint-xxx-merged \
+#     --vllm_tensor_parallel_size 4 \
+#     --infer_backend vllm \
+#     --vllm_max_model_len 8192 \
+#     --vllm_gpu_memory_utilization 0.9 \
+#     --vllm_speculative_config '{"method":"qwen3_next_mtp","num_speculative_tokens":2}' \
+#     --max_new_tokens 2048

examples/models/qwen3_next/transformers.sh

@@ -0,0 +1,28 @@
+# 4 * 60GiB
+# Note: Due to linear attention, this model currently does not support padding_free and packing.
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --experts_impl grouped_mm \
+    --router_aux_loss_coef 1e-3 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/models/qwen3_omni/transformers.sh

@@ -0,0 +1,37 @@
+# 2 * 48GiB
+# mcore shell: https://github.com/modelscope/ms-swift/blob/main/examples/megatron/multimodal/omni/moe.sh
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'speech_asr/speech_asr_aishell1_trainsets:validation#5000' \
+    --split_dataset_ratio 0.01 \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --attn_impl flash_attn \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --packing true \
+    --gradient_accumulation_steps 4 \
+    --gradient_checkpointing true \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 1 \
+    --dataloader_num_workers 4

examples/models/qwen3_omni/zero3.sh

@@ -0,0 +1,41 @@
+# 2 * 60GiB
+# mcore shell: https://github.com/modelscope/ms-swift/blob/main/examples/megatron/multimodal/omni/moe.sh
+NPROC_PER_NODE=2 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+              'speech_asr/speech_asr_aishell1_trainsets:validation#5000' \
+    --split_dataset_ratio 0.01 \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 8 \
+    --attn_impl flash_attn \
+    --experts_impl grouped_mm \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --padding_free true \
+    --gradient_accumulation_steps 1 \
+    --gradient_checkpointing true \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --dataloader_num_workers 4

examples/models/qwen3_tts/infer.py

@@ -0,0 +1,15 @@
+import soundfile as sf
+import torch
+from qwen_tts import Qwen3TTSModel
+
+tts = Qwen3TTSModel.from_pretrained(
+    'output/Qwen3-TTS-12Hz-1.7B-Base/vx-xxx/checkpoint-xxx',
+    device_map='cuda:0',
+    dtype=torch.bfloat16,
+)
+
+wavs, sr = tts.generate_custom_voice(
+    text='我不是俱乐部的会员。只是一大早被克洛琳德敲门叫醒，说有个不错的剧本可以体验，这才决定加入的。',
+    speaker='speaker_test',
+)
+sf.write('output.wav', wavs[0], sr)

examples/models/qwen3_tts/prepare_data.py

@@ -0,0 +1,23 @@
+# This file is used to generate `audio_codes` in the dataset.
+from modelscope import snapshot_download
+from qwen_tts import Qwen3TTSTokenizer
+
+from swift import load_dataset
+
+BATCH_INFER_NUM = 32
+dataset = load_dataset('qsdong/Qwen3-1.7-TTS-SFT-Furina')[0]
+tokenizer_model_path = snapshot_download('Qwen/Qwen3-TTS-Tokenizer-12Hz')
+tokenizer = Qwen3TTSTokenizer.from_pretrained(
+    tokenizer_model_path,
+    device_map='cuda:0',
+)
+audio_codes = []
+
+for i in range(0, len(dataset), BATCH_INFER_NUM):
+    batch_lines = dataset[i:i + BATCH_INFER_NUM]
+    batch_audios = [audios[0] for audios in batch_lines['audios']]
+    enc_res = tokenizer.encode(batch_audios)
+    audio_codes += [code.cpu().tolist() for code in enc_res.audio_codes]
+
+dataset = dataset.add_column('audio_codes', audio_codes)
+dataset.to_parquet('tts_data.parquet')

examples/models/qwen3_tts/train.sh

@@ -0,0 +1,30 @@
+# Execution order: prepare_data.py -> train.sh -> infer.py
+# The dataset can also be set to `--dataset qsdong/Qwen3-1.7-TTS-SFT-Furina`,
+# but the preprocessing will affect training speed.
+
+SPEAKER_NAME='speaker_test' \
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift sft \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --dataset 'tts_data.parquet' \
+    --split_dataset_ratio 0.01 \
+    --load_from_cache_file true \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --gradient_checkpointing true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --ddp_find_unused_parameters true \
+    --dataset_num_proc 1 \
+    --dataloader_num_workers 4

examples/models/qwen3_vl/mcore.sh

@@ -0,0 +1,46 @@
+# 8 * 80GiB; 45min
+# If you're doing full-parameter training, you'll need 64 × 80 GiB of GPU memory
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3-VL-235B-A22B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --moe_permute_fusion true \
+    --tensor_model_parallel_size 2 \
+    --expert_tensor_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/Qwen3-VL-235B-A22B-Instruct \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --max_length 2048 \
+    --packing true \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend flash

examples/models/qwen3_vl/mcore_full.sh

@@ -0,0 +1,48 @@
+# 8 * 80GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+OMP_NUM_THREADS=14 \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+megatron sft \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --moe_permute_fusion true \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 8 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-VL-30B-A3B-Instruct \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --packing true \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --moe_expert_capacity_factor 2 \
+    --optimizer_cpu_offload true \
+    --use_precision_aware_optimizer true \
+    --optimizer_offload_fraction 0.2 \
+    --attention_backend flash

examples/models/qwen3_vl/mixed.sh

@@ -0,0 +1,41 @@
+# 2 * 21GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift sft \
+    --model Qwen/Qwen3-VL-4B-Instruct \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --attn_impl flash_attn \
+    --padding_free true \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --packing true \
+    --gradient_checkpointing true \
+    --vit_gradient_checkpointing false \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/qwen3_vl/transformers.sh

@@ -0,0 +1,35 @@
+# use device_map
+# 8 * 70GiB
+IMAGE_MAX_TOKEN_NUM=1024 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+swift sft \
+    --model Qwen/Qwen3-VL-235B-A22B-Instruct \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 16 \
+    --attn_impl flash_attn \
+    --padding_free true \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --router_aux_loss_coef 1e-3 \
+    --experts_impl grouped_mm \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 4 \
+    --gradient_checkpointing true \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4

examples/models/qwen3_vl/zero3.sh

@@ -0,0 +1,37 @@
+# 2 * 42GiB
+IMAGE_MAX_TOKEN_NUM=1024 \
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift sft \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --dataset 'AI-ModelScope/LaTeX_OCR:human_handwrite#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 16 \
+    --attn_impl flash_attn \
+    --padding_free true \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --router_aux_loss_coef 1e-3 \
+    --experts_impl grouped_mm \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 1 \
+    --gradient_checkpointing true \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --deepspeed zero3 \
+    --use_liger_kernel true \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4