chore: import upstream snapshot with attribution

examples/train/agent/deepseek_r1.sh

@@ -0,0 +1,29 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \
+    --tuner_type full \
+    --dataset AI-ModelScope/function-calling-chatml \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --agent_template react_en \
+    --loss_scale react \
+    --response_prefix '' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 2 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 8 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --save_only_model true \
+    --packing true \
+    --use_liger_kernel true \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --attn_impl flash_attn \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 16

examples/train/agent/glm4.sh

@@ -0,0 +1,30 @@
+# 4 * 80GiB
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model ZhipuAI/GLM-4-9B-0414 \
+    --tuner_type full \
+    --dataset AI-ModelScope/function-calling-chatml \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --agent_template hermes \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 2 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --save_only_model true \
+    --packing true \
+    --deepspeed zero3 \
+    --use_liger_kernel true \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --attn_impl flash_attn \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 16

examples/train/agent/loss_scale/infer_lora.py

@@ -0,0 +1,89 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+# os.environ['SWIFT_DEBUG'] = '1'
+
+
+def infer(engine: 'InferEngine', infer_request: 'InferRequest'):
+    stop = [engine.template.agent_template.keyword.observation]  # compat react_en
+    request_config = RequestConfig(max_tokens=512, temperature=0, stop=stop)
+    resp_list = engine.infer([infer_request], request_config)
+    query = infer_request.messages[0]['content']
+    response = resp_list[0].choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    print(f'tool_calls: {resp_list[0].choices[0].message.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    infer_request.messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    resp_list = engine.infer([infer_request], request_config)
+    response2 = resp_list[0].choices[0].message.content
+    print(f'response2: {response2}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    stop = [engine.template.agent_template.keyword.observation]
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True, stop=stop)
+    gen_list = engine.infer([infer_request], request_config)
+    query = infer_request.messages[0]['content']
+    response = ''
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        delta = resp.choices[0].delta.content
+        response += delta
+        print(delta, end='', flush=True)
+    print()
+    print(f'tool_calls: {resp.choices[0].delta.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}\nresponse2: ', end='')
+    infer_request.messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    gen_list = engine.infer([infer_request], request_config)
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def get_infer_request():
+    return InferRequest(
+        messages=[{
+            'role': 'user',
+            'content': "How's the weather in Beijing today?"
+        }],
+        tools=[{
+            'name': 'get_current_weather',
+            'description': 'Get the current weather in a given location',
+            'parameters': {
+                'type': 'object',
+                'properties': {
+                    'location': {
+                        'type': 'string',
+                        'description': 'The city and state, e.g. San Francisco, CA'
+                    },
+                    'unit': {
+                        'type': 'string',
+                        'enum': ['celsius', 'fahrenheit']
+                    }
+                },
+                'required': ['location']
+            }
+        }])
+
+
+if __name__ == '__main__':
+    from swift.agent_template import agent_template_map
+    from swift.infer_engine import InferEngine, InferRequest, RequestConfig, TransformersEngine
+    model = 'Qwen/Qwen2.5-3B'
+    adapters = ['output/vx-xxx/checkpoint-xxx']
+    engine = TransformersEngine(model, adapters=adapters, max_batch_size=8)
+
+    # engine.template._agent_template = 'hermes'  # react_en/qwen_en/qwen_en_parallel
+
+    infer(engine, get_infer_request())
+    infer_stream(engine, get_infer_request())

examples/train/agent/loss_scale/train.sh

@@ -0,0 +1,30 @@
+# 20GB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-3B \
+    --tuner_type lora \
+    --dataset AI-ModelScope/function-calling-chatml#10000 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --loss_scale hermes \
+    --agent_template hermes \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 2 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --modules_to_save embed_tokens lm_head \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --use_liger_kernel true \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 16

examples/train/agent/qwen2_5.sh

@@ -0,0 +1,28 @@
+# 35GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-3B \
+    --tuner_type full \
+    --dataset AI-ModelScope/function-calling-chatml \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --agent_template hermes \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 2 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 8 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --save_only_model true \
+    --packing true \
+    --use_liger_kernel true \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --attn_impl flash_attn \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 16

examples/train/all_to_all/infer.sh

@@ -0,0 +1,9 @@
+# 53GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model BAAI/Emu3-Gen \
+    --infer_backend transformers \
+    --stream False \
+    --use_chat_template False \
+    --top_k 2048 \
+    --max_new_tokens 40960

examples/train/all_to_all/train.sh

@@ -0,0 +1,23 @@
+# 70 GiB * 2
+nproc_per_node=2
+NPROC_PER_NODE=$nproc_per_node \
+CUDA_VISIBLE_DEVICES=0,2 \
+max_position_embeddings=10240 \
+image_area=518400 \
+swift sft \
+    --model BAAI/Emu3-Gen \
+    --tuner_type lora \
+    --dataset 'swift/TextCaps#40' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --warmup_ratio 0.03 \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 1024 \
+    --weight_decay 0.1 \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}'

examples/train/base_to_chat/full.sh

@@ -0,0 +1,28 @@
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-1.5B \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero2

examples/train/base_to_chat/lora.sh

@@ -0,0 +1,34 @@
+# Use `--template default`
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+MASTER_PORT=29501 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-1.5B \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition' \
+    --torch_dtype bfloat16 \
+    --template default \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero2

examples/train/base_to_chat/lora2.sh

@@ -0,0 +1,33 @@
+# Use `--target_modules all-linear embed_tokens lm_head`
+# Please adjust the `lm_head` according to the model.
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-1.5B \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear embed_tokens lm_head \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero2

examples/train/cached_dataset/dpo.sh

@@ -0,0 +1,69 @@
+# ms-swift>=3.11
+OMP_NUM_THREADS=14 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+swift export \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --dataset swift/RLAIF-V-Dataset \
+    --split_dataset_ratio 0.01 \
+    --dataset_num_proc 8 \
+    --to_cached_dataset true \
+    --template_mode rlhf \
+    --output_dir ./qwen3_vl_cached_dataset
+
+
+# 16s/it; 8 * 65GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+megatron rlhf \
+    --rlhf_type dpo \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --save_safetensors true \
+    --cached_dataset qwen3_vl_cached_dataset/train \
+    --cached_val_dataset qwen3_vl_cached_dataset/val \
+    --load_from_cache_file true \
+    --tuner_type full \
+    --tensor_model_parallel_size 4 \
+    --expert_tensor_parallel_size 1 \
+    --pipeline_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --packing true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-VL-30B-A3B-Instruct \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 8192 \
+    --num_train_epochs 1 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --optimizer_cpu_offload true \
+    --use_precision_aware_optimizer true \
+    --optimizer_offload_fraction 0.4 \
+    --attention_backend flash \
+    --rpo_alpha 0.1 \
+    --beta 0.1 \
+    --loss_type sigmoid

examples/train/cached_dataset/mcore.sh

@@ -0,0 +1,60 @@
+# ms-swift>=3.11
+swift export \
+    --model Qwen/Qwen3-30B-A3B-Base \
+    --dataset 'swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT' \
+    --split_dataset_ratio 0.01 \
+    --dataset_num_proc 64 \
+    --to_cached_dataset true \
+    --output_dir ./qwen3_cached_dataset
+
+
+# 4 * 48GiB; 17s/it
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+megatron sft \
+    --model Qwen/Qwen3-30B-A3B-Base \
+    --save_safetensors true \
+    --merge_lora false \
+    --cached_dataset './qwen3_cached_dataset/train' \
+    --cached_val_dataset './qwen3_cached_dataset/val' \
+    --tuner_type lora \
+    --lora_rank 32 \
+    --lora_alpha 64 \
+    --target_modules all-linear \
+    --moe_permute_fusion true \
+    --expert_model_parallel_size 4 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 3 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/Qwen3-30B-A3B-Base \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --packing true \
+    --max_length 8192 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend flash
+
+
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters megatron_output/Qwen3-30B-A3B-Base/vx-xxx/checkpoint-xxx \
+    --load_data_args true \
+    --attn_impl flash_attn \
+    --stream true \
+    --max_new_tokens 512

examples/train/cached_dataset/pretrained.sh

@@ -0,0 +1,42 @@
+# ms-swift>=3.11
+swift export \
+    --model Qwen/Qwen2.5-7B \
+    --dataset 'AI-ModelScope/ruozhiba:all' \
+    --dataset_num_proc 64 \
+    --to_cached_dataset true \
+    --truncation_strategy split \
+    --max_length 8192 \
+    --split_dataset_ratio 0.01 \
+    --use_chat_template false \
+    --loss_scale all \
+    --output_dir ./pretrain_cached_dataset
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift pt \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --cached_dataset './pretrain_cached_dataset/train' \
+    --cached_val_dataset './pretrain_cached_dataset/val' \
+    --truncation_strategy split \
+    --num_train_epochs 3 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --packing true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-7B \
+    --deepspeed zero3 \
+    --use_liger_kernel true \
+    --attn_impl flash_attn

examples/train/cached_dataset/reranker.sh

@@ -0,0 +1,41 @@
+# ms-swift>=3.12
+swift export \
+    --model Qwen/Qwen3-Reranker-4B \
+    --task_type generative_reranker \
+    --dataset MTEB/scidocs-reranking \
+    --dataset_num_proc 64 \
+    --split_dataset_ratio 0.01 \
+    --to_cached_dataset true \
+    --output_dir ./qwen3_reranker_cached_dataset
+
+# 4 * 24GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen3-Reranker-4B \
+    --task_type generative_reranker \
+    --loss_type pointwise_reranker \
+    --tuner_type full \
+    --cached_dataset './qwen3_reranker_cached_dataset/train' \
+    --cached_val_dataset './qwen3_reranker_cached_dataset/val' \
+    --num_train_epochs 1 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 6e-6 \
+    --gradient_accumulation_steps 8 \
+    --packing true \
+    --eval_steps 50 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen3-Reranker-4B \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --dataloader_drop_last true

examples/train/cached_dataset/seq_cls.sh

@@ -0,0 +1,60 @@
+# ms-swift>=3.12
+OMP_NUM_THREADS=14 \
+MAX_PIXELS=1003520 \
+swift export \
+    --model Qwen/Qwen2.5-Omni-3B \
+    --dataset 'tany0699/garbage265#20000' \
+    --task_type seq_cls \
+    --num_labels 265 \
+    --problem_type single_label_classification \
+    --split_dataset_ratio 0.01 \
+    --dataset_num_proc 16 \
+    --to_cached_dataset true \
+    --output_dir ./seq_cls_cached_dataset
+
+
+# 18GiB
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+swift sft \
+    --model Qwen/Qwen2.5-Omni-3B \
+    --tuner_type lora \
+    --cached_dataset 'seq_cls_cached_dataset/train' \
+    --cached_val_dataset 'seq_cls_cached_dataset/val' \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 2 \
+    --packing true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --task_type seq_cls \
+    --num_labels 265 \
+    --problem_type single_label_classification \
+    --use_chat_template true \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-Omni-3B \
+    --attn_impl flash_attn
+
+# Use the validation set
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+swift infer \
+    --adapters output/Qwen2.5-Omni-3B/vx-xxx/checkpoint-xxx \
+    --load_data_args true \
+    --attn_impl flash_attn

examples/train/cached_dataset/sft.sh

@@ -0,0 +1,38 @@
+# ms-swift>=3.11
+swift export \
+    --model Qwen/Qwen2.5-7B \
+    --dataset 'swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT' \
+    --dataset_num_proc 64 \
+    --split_dataset_ratio 0.01 \
+    --to_cached_dataset true \
+    --output_dir ./qwen2_5_cached_dataset
+
+# 4 * 44GiB; 15.5s/it
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --cached_dataset './qwen2_5_cached_dataset/train' \
+    --cached_val_dataset './qwen2_5_cached_dataset/val' \
+    --num_train_epochs 3 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --packing true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-7B \
+    --deepspeed zero3 \
+    --use_liger_kernel true \
+    --attn_impl flash_attn

examples/train/cached_dataset/vlm.sh

@@ -0,0 +1,66 @@
+# ms-swift>=3.11
+OMP_NUM_THREADS=14 \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+swift export \
+    --model Qwen/Qwen2.5-Omni-7B \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'speech_asr/speech_asr_aishell1_trainsets:validation#5000' \
+    --split_dataset_ratio 0.01 \
+    --dataset_num_proc 16 \
+    --to_cached_dataset true \
+    --output_dir ./qwen2_5_omni_cached_dataset
+
+# 4 * 70GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+NPROC_PER_NODE=4 \
+ENABLE_AUDIO_OUTPUT=0 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen2.5-Omni-7B \
+    --tuner_type full \
+    --cached_dataset './qwen2_5_omni_cached_dataset/train' \
+    --cached_val_dataset './qwen2_5_omni_cached_dataset/val' \
+    --num_train_epochs 1 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --packing true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-Omni-7B \
+    --deepspeed zero2 \
+    --use_liger_kernel true \
+    --attn_impl flash_attn
+
+# Use the validation set
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+VIDEO_MAX_PIXELS=50176 \
+FPS_MAX_FRAMES=12 \
+ENABLE_AUDIO_OUTPUT=0 \
+swift infer \
+    --model output/Qwen2.5-Omni-7B/vx-xxx/checkpoint-xxx \
+    --load_data_args true \
+    --max_length 4096 \
+    --attn_impl flash_attn \
+    --stream true \
+    --temperature 0 \
+    --max_new_tokens 512

examples/train/early_stop/lora_sft.sh

@@ -0,0 +1,40 @@
+# 22GB
+# Change: https://github.com/modelscope/ms-swift/blob/main/swift/callbacks/early_stop.py
+# If you have custom implementations
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --split_dataset_ratio 0.1 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --early_stop_interval 3 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --metric_for_best_model loss \
+
+# a sample result
+# Train:  83%|██████████████████████████████████████████████████████████████████████████████████████████▊                  | 10/12 [00:42<00:06,  3.14s/it]
+#{'eval_loss': 4.26491737, 'eval_token_acc': 0.57142857, 'eval_runtime': 20.3945, 'eval_samples_per_second': 0.049, 'eval_steps_per_second': 0.049, 'epoch': 2.5, 'global_step/max_steps': '10/12', 'percentage': '83.33%', 'elapsed_time': '1m 2s', 'remaining_time': '12s'}
+#Val: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 28.85it/s]
+#[INFO:swift] Saving model checkpoint to output/xxx/checkpoint-10
+#[INFO:swift] Training stop because of eval metric is stable at step 10

examples/train/embedding/qwen3/infer.py

@@ -0,0 +1,45 @@
+# For full-parameter training, please refer to:
+# https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_embedding.py
+
+import torch
+
+from swift.infer_engine import InferRequest, TransformersEngine
+
+
+def run_qwen3_emb():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-Embedding-4B',
+        task_type='embedding',
+        attn_impl='flash_attention_2',
+        adapters=['output/vx-xxx/checkpoint-xxx'])
+
+    infer_requests = [
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'A dog sleeping under a table.'
+            },
+        ]),
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'a dog napping under a small table.'
+            },
+        ]),
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'a cat napping under a small tree.'
+            },
+        ])
+    ]
+    resp_list = engine.infer(infer_requests)
+    embedding0 = torch.tensor(resp_list[0].data[0].embedding)
+    embedding1 = torch.tensor(resp_list[1].data[0].embedding)
+    embedding2 = torch.tensor(resp_list[2].data[0].embedding)
+    embedding = torch.stack([embedding0, embedding1, embedding2])
+    print(f'scores: {embedding @ embedding.T}')
+
+
+if __name__ == '__main__':
+    run_qwen3_emb()

examples/train/embedding/qwen3/qwen3_emb.sh

@@ -0,0 +1,39 @@
+# 2*10GiB
+# losses: swift/loss
+# data format: docs/source_en/BestPractices/Embedding.md
+# --dataloader_drop_last must be true or eval gather will throw error
+# --model iic/gte-modernbert-base iic/gte_Qwen2-7B-instruct also supported
+CUDA_VISIBLE_DEVICES=0,1 \
+INFONCE_TEMPERATURE=0.1 \
+NPROC_PER_NODE=2 \
+swift sft \
+    --model Qwen/Qwen3-Embedding-4B \
+    --task_type embedding \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 5e-5 \
+    --target_modules all-linear \
+    --dataset sentence-transformers/stsb:positive \
+    --attn_impl flash_attn \
+    --padding_free true \
+    --torch_dtype bfloat16 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.02 \
+    --eval_strategy steps \
+    --output_dir output \
+    --save_steps 50 \
+    --eval_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --num_train_epochs 5 \
+    --max_length 8192 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 8 \
+    --gradient_accumulation_steps 1 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --warmup_ratio 0.05 \
+    --loss_type infonce \
+    --dataloader_drop_last true \
+    --deepspeed zero2

examples/train/embedding/qwen3/qwen3_vl_emb.sh

@@ -0,0 +1,35 @@
+# 2*30GiB
+CUDA_VISIBLE_DEVICES=0,1 \
+INFONCE_TEMPERATURE=0.1 \
+NPROC_PER_NODE=2 \
+swift sft \
+    --model Qwen/Qwen3-VL-Embedding-8B \
+    --task_type embedding \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 5e-5 \
+    --target_modules all-linear \
+    --dataset swift/TextCaps:emb \
+    --attn_impl flash_attn \
+    --padding_free true \
+    --torch_dtype bfloat16 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.02 \
+    --eval_strategy steps \
+    --output_dir output \
+    --save_steps 50 \
+    --eval_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --num_train_epochs 1 \
+    --max_length 8192 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 8 \
+    --gradient_accumulation_steps 1 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --warmup_ratio 0.05 \
+    --loss_type infonce \
+    --dataloader_drop_last true \
+    --deepspeed zero2

examples/train/embedding/train_gme.sh

@@ -0,0 +1,31 @@
+nproc_per_node=8
+
+# losses: swift/loss
+# 8*40G
+MAX_PIXELS=1003520 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model iic/gme-Qwen2-VL-2B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/TextCaps:emb' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 2 \
+    --gradient_accumulation_steps $(expr 64 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --eval_strategy steps \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --lazy_tokenize true \
+    --warmup_ratio 0.05 \
+    --learning_rate 5e-5 \
+    --deepspeed zero3 \
+    --dataloader_num_workers 4 \
+    --task_type embedding \
+    --loss_type infonce \
+    --dataloader_drop_last true

examples/train/flash_attention_3/mcore.sh

@@ -0,0 +1,47 @@
+# test_env: 4 * H20
+# fa2: 4 * 49GiB; 17.2s/it
+# fa3: 4 * 49GiB; 15.2s/it
+# https://github.com/Dao-AILab/flash-attention/tree/main#flashattention-3-beta-release
+# pip install "transformers==4.53.*"
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+megatron sft \
+    --model Qwen/Qwen3-30B-A3B-Base \
+    --save_safetensors true \
+    --merge_lora false \
+    --dataset 'swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT' \
+    --load_from_cache_file true \
+    --tuner_type lora \
+    --lora_rank 32 \
+    --lora_alpha 64 \
+    --target_modules all-linear \
+    --split_dataset_ratio 0.01 \
+    --moe_permute_fusion true \
+    --expert_model_parallel_size 4 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-3 \
+    --micro_batch_size 1 \
+    --global_batch_size 16 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 3 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --output_dir megatron_output/Qwen3-30B-A3B-Base \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --packing true \
+    --max_length 8192 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --attention_backend flash

examples/train/flash_attention_3/transformers.sh

@@ -0,0 +1,35 @@
+# test_env: 4 * H20
+# fa2: 4 * 43GiB; 35.5s/it
+# fa3: 4 * 43GiB; 32.4s/it
+# https://github.com/Dao-AILab/flash-attention/tree/main#flashattention-3-beta-release
+# pip install "transformers==4.53.*"
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/LongAlpaca-12k' \
+    --load_from_cache_file true \
+    --attn_impl flash_attention_3 \
+    --num_train_epochs 3 \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --packing true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 16384 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-7B \
+    --deepspeed zero3 \
+    --use_liger_kernel true

examples/train/full/dft.sh

@@ -0,0 +1,23 @@
+# 4*80G
+# exp: https://github.com/modelscope/ms-swift/pull/5355
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift sft \
+    --model Qwen/Qwen2.5-Math-1.5B \
+    --tuner_type full \
+    --dataset AI-MO/NuminaMath-CoT#100000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --enable_dft_loss true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --learning_rate 5e-5 \
+    --gradient_accumulation_steps 32 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.1 \
+    --deepspeed zero2 \
+    --dataloader_num_workers 4

examples/train/full/infer.sh

@@ -0,0 +1,7 @@
+# If you are using the validation set for inference, add the parameter `--load_data_args true`.
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model output/vx-xxx/checkpoint-xxx \
+    --stream true \
+    --temperature 0 \
+    --max_new_tokens 2048

examples/train/full/qwen2_5_32b.sh

@@ -0,0 +1,29 @@
+# 8 * 80GiB
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+swift sft \
+    --model Qwen/Qwen2.5-32B \
+    --tuner_type full \
+    --dataset 'liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT' \
+    --torch_dtype bfloat16 \
+    --max_steps 2000 \
+    --streaming true \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 2 \
+    --packing true \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-32B \
+    --deepspeed zero3 \
+    --use_liger_kernel true \
+    --attn_impl flash_attn

examples/train/full/train.sh

@@ -0,0 +1,25 @@
+# 76GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/grpo/external/README.md

@@ -0,0 +1,72 @@
+# README: GRPO External(Async) Mode Execution Scripts
+
+---
+
+> **Note**: External mode requires
+
+1. vLLM version 0.8.3 or higher.
+2. trl version 0.17.0 or higher
+
+For LoRA Training, set following parameters to speed up weight update
+```bash
+  --vllm_enable_lora true
+  --vllm_max_lora_rank xxx # same as lora_rank in training script
+```
+
+## **Introduction**
+
+The GRPO (Group Relative Policy Optimization) training framework supports high-performance inference engines like vLLM to accelerate the sampling process. The **External Mode** allows you to connect to an external vLLM inference server, separating the inference service from the training process. This mode is ideal for scenarios where you want to offload inference to dedicated hardware or servers, improving resource utilization and scalability.
+
+This folder contains scripts and instructions for running GRPO in **External Mode**, enabling integration with an external vLLM server.
+
+Before running the scripts, ensure the following:
+
+1. **vLLM Server Deployment**:
+   - An external vLLM server must be deployed and accessible.
+   - Use the `swift rollout` command to deploy the vLLM server.
+
+2. **Network Connectivity**:
+   - Ensure the training nodes can communicate with the vLLM server over the network.
+
+## **Deploying the vLLM Server**
+
+To deploy an external vLLM server, use the following command:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 \
+swift rollout \
+  --model Qwen/Qwen3-8B
+
+# tp
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rollout \
+  --model Qwen/Qwen3-8B \
+  --vllm_tensor_parallel_size 2
+
+# dp
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rollout \
+  --model Qwen/Qwen3-8B \
+  --vllm_data_parallel_size 2
+
+# tp + dp
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rollout \
+  --model Qwen/Qwen3-8B \
+  --vllm_tensor_parallel_size 2 \
+  --vllm_data_parallel_size 2
+```
+
+## Training with External vLLM Server
+Configuration Parameters
+
+```bash
+--use_vllm true \
+--vllm_mode server \
+--vllm_server_host <server ip> \
+--vllm_server_port <server port> \
+--vllm_server_timeout <Timeout duration> \
+```
+
+## Multi-Node Training
+On each node, execute the original single-node training script, using the environment variables `NNODES` and `NODE_RANK`, and ensure consistent use of configuration parameters across all nodes.

examples/train/grpo/external/agent.sh

@@ -0,0 +1,43 @@
+# wandb result link: https://wandb.ai/tastelikefeet/tastelikefeet?nw=nwuseryuzezyz
+# model link: https://www.modelscope.cn/models/swift/Qwen2-7B-Agent-GRPO
+# WANDB_API_KEY=xxx \
+
+# CUDA_VISIBLE_DEVICES=7 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-7B
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6 \
+NPROC_PER_NODE=7 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --dataset LLM-Research/xlam-function-calling-60k:grpo \
+    --load_from_cache_file true \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --per_device_train_batch_size 7 \
+    --per_device_eval_batch_size 7 \
+    --eval_steps 2000 \
+    --save_steps 2000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 1024 \
+    --reward_funcs toolbench react_format \
+    --num_generations 49 \
+    --deepspeed zero3 \
+    --temperature 1.0 \
+    --stop_words Observation: \
+    --agent_template react_grpo \
+    --top_p 0.85 \
+    --top_k 50 \
+    --log_completions true \
+    --report_to wandb

examples/train/grpo/external/grpo_32b_full.sh

@@ -0,0 +1,40 @@
+# run in another node
+# CUDA_VISIBLE_DEVICES=0,1 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-32B-Instruct \
+#     --vllm_tensor_parallel_size 2
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-32B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host xxx \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0 \
+    --max_completion_length 2048 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --top_p 0.9 \
+    --top_k 50 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --num_iterations 1 \
+    --report_to tensorboard wandb \
+    --beta 0.0

examples/train/grpo/external/grpo_7b.sh

@@ -0,0 +1,40 @@
+# 8 * 80 G (6 for training, 2 for rollout)
+# CUDA_VISIBLE_DEVICES=6,7 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-7B-Instruct \
+#     --data_parallel_size 2
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \
+NPROC_PER_NODE=6 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0 \
+    --max_completion_length 2048 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --top_p 0.9 \
+    --top_k 50 \
+    --deepspeed zero2 \
+    --log_completions true \
+    --num_iterations 1 \
+    --report_to tensorboard wandb \
+    --beta 0.04

examples/train/grpo/external/moe_full.sh

@@ -0,0 +1,44 @@
+# 8*80G
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift rollout \
+#     --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+#     --vllm_max_model_len 16384 \
+#     --vllm_enable_prefix_caching true
+
+CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7 \
+NPROC_PER_NODE=7 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --max_length 12000 \
+    --max_completion_length 8192 \
+    --overlong_filter true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_strategy 'steps' \
+    --eval_strategy 'steps' \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 4 \
+    --num_generations 14 \
+    --temperature 1.0 \
+    --deepspeed zero3_offload \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --move_model_batches 5

examples/train/grpo/external/moe_lora.sh

@@ -0,0 +1,44 @@
+# 8*80G
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift rollout \
+#     --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+#     --vllm_max_model_len 16384 \
+#     --vllm_enable_prefix_caching true
+
+CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7 \
+NPROC_PER_NODE=7 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --max_length 12000 \
+    --max_completion_length 8192 \
+    --overlong_filter true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_strategy 'steps' \
+    --eval_strategy 'steps' \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 4 \
+    --num_generations 14 \
+    --temperature 1.0 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --move_model_batches 5

examples/train/grpo/external/vllm_gym.sh

@@ -0,0 +1,45 @@
+# exp: https://github.com/modelscope/ms-swift/pull/4890
+
+# CUDA_VISIBLE_DEVICES=7 \
+# swift rollout \
+#   --model Qwen/Qwen2.5-3B-Instruct \
+#   --max_turns 3\
+#   --multi_turn_scheduler gym_scheduler \
+#   --use_gym_env true \
+#   --gym_env math_env
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \
+NPROC_PER_NODE=6 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --tuner_type full \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --vllm_server_pass_dataset true \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0 \
+    --max_completion_length 2048 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --steps_per_generation 2 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 6 \
+    --temperature 1.0 \
+    --top_p 0.9 \
+    --top_k 50 \
+    --log_completions true \
+    --num_iterations 1 \
+    --report_to tensorboard \
+    --beta 0 \
+    --loss_scale default

examples/train/grpo/external/vllm_multi_turn.sh

@@ -0,0 +1,57 @@
+# Exp: https://github.com/modelscope/ms-swift/pull/5307#issuecomment-3219803922
+# Before running this script, please run the following `swift rollout` script first
+# This script is a example for multi-turn training with dynamic num of rollout outputs
+# which means a trajectory of multi turn rollout is split into multiple data
+#       see details in thinking_tips_scheduler
+# NOTE: for same trajectory, the reward is supported to be the same,
+#       here we use the last turn data of each trajectory to compute accuracy reward
+#       see details in thinking_tips reward function
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift rollout \
+#     --model Qwen/Qwen3-1.7B \
+#     --vllm_use_async_engine true \
+#     --multi_turn_scheduler thinking_tips_scheduler \
+#     --vllm_max_model_len 32768 \
+#     --vllm_gpu_memory_utilization 0.8 \
+#     --max_turns 3
+
+CUDA_VISIBLE_DEVICES=1,2 \
+NPROC_PER_NODE=2 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-1.7B \
+    --tuner_type full \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs thinking_tips \
+    --loss_scale last_round \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --vllm_server_pass_dataset true \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0 \
+    --max_completion_length 8192 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --steps_per_generation 8 \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}' \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --deepspeed zero2 \
+    --log_completions true \
+    --log_entropy true \
+    --importance_sampling_level sequence \
+    --top_entropy_quantile 0.2 \
+    --num_iterations 1 \
+    --report_to tensorboard swanlab

examples/train/grpo/internal/README.md

@@ -0,0 +1,20 @@
+# README: GRPO Internal(Colocate) Mode Execution Scripts
+
+---
+**NOTE**
+
+## **Introduction**
+
+The GRPO (Group Relative Policy Optimization) training framework supports high-performance inference engines like vLLM to accelerate the sampling process. The **Internal Mode** allows you to deploy vLLM and perform training using the same GPU resources.
+
+This folder contains scripts and instructions for running GRPO in **Internal Mode**
+
+## Training with Internal mode
+```bash
+--use_vllm true \
+--vllm_mode colocate \
+--vllm_gpu_memory_utilization [ut_ratio] \
+```
+
+## Multi-Node Training
+On each node, execute the original single-node training script, using the environment variables `NNODES` and `NODE_RANK`, and ensure consistent use of configuration parameters across all nodes.

examples/train/grpo/internal/chord.sh

@@ -0,0 +1,53 @@
+# 8*80G GPU
+# CHORD https://arxiv.org/abs/2508.11408
+# GRPO total batch = 32(prompts)*8(num_generations) = 256 =  8(gpus) * 4(per_device_train_batch_size) * 8(gradient_accumulation_steps)
+# SFT total batch = 64 = 8(gpus) * 1(chord_sft_per_device_train_batch_size) * 8(gradient_accumulation_steps)
+
+# NOTE: We use the same dataset for GRPO and SFT, which may cause overlap (i.e., the same examples to be selected).
+# You can pre-download the dataset and manually split it to avoid this.
+
+export CHORD_SYSTEM_PROMPT="You are a helpful assistant that solves MATH problems.
+You should first think about the reasoning process in mind and then provide the user with the answer.
+You should present your reasoning process using the format: <think>\n...your reasoning process here... </think>\n"
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --dataset AI-MO/NuminaMath-TIR \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --beta 0.0 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --chord_sft_per_device_train_batch_size 1 \
+    --chord_sft_dataset AI-MO/NuminaMath-TIR \
+    --chord_enable_phi_function false \
+    --chord_mu_warmup_steps 0 \
+    --chord_mu_decay_steps 200 \
+    --chord_mu_peak 0.9 \
+    --chord_mu_valley 0.05 \
+    --num_generations 8 \
+    --tuner_type full \
+    --reward_funcs accuracy \
+    --system "$CHORD_SYSTEM_PROMPT" \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_max_model_len 8192 \
+    --max_completion_length 4096 \
+    --overlong_filter true \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --report_to tensorboard swanlab

examples/train/grpo/internal/fipo.sh

@@ -0,0 +1,46 @@
+CUDA_VISIBLE_DEVICES=2 \
+swift rollout \
+    --model Qwen/Qwen2.5-1.5B-Instruct
+
+# 2 GPUS for sequence parallel
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --dataset 'AI-MO/NuminaMath-TIR'  \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --load_from_cache_file true \
+    --max_completion_length 4096 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --save_total_limit 3 \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --system """You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}.""" \
+    --log_completions true \
+    --num_iterations 3 \
+    --padding_free true \
+    --sequence_parallel_size 2 \
+    --attn_impl flash_attn \
+    --beta 0 \
+    --dynamic_sample true \
+    --loss_type fipo \
+    --delta 10.0 \
+    --epsilon_high 0.28 \
+    --fipo_decay_rate 32 \
+    --fipo_clip_range 0.2 \
+    --fipo_clip_high_only false \
+    --fipo_safety_threshold 3.0

examples/train/grpo/internal/full_lmdeploy.sh

@@ -0,0 +1,3 @@
+# The LMDeploy backend in GRPO has been deprecated in Swift 3.5.
+# You can install Swift 3.4 to continue using it with the following script:
+# https://github.com/modelscope/ms-swift/blob/v3.4.1/examples/train/grpo/internal/full_lmdeploy.sh

examples/train/grpo/internal/gspo.sh

@@ -0,0 +1,43 @@
+# 8*80G GPU
+# GSPO https://arxiv.org/pdf/2507.18071
+# hyperparameter
+# - epsilon = 3e-4 from paper serction 5.1
+# - epsilon_high = 4e-4 from paper serction 5.1
+# - steps_per_generation = 32 (= 4 * gradient_accumulation_steps): paper section 5.1 partitions each batch of rollout data into four minibatches for gradient updates; in swift HF GRPO, steps_per_generation counts micro-batches, so it must be multiplied by gradient_accumulation_steps
+# - beta = 0: zero kl regularization https://github.com/volcengine/verl/pull/2775#issuecomment-3131807306
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --beta 0.0 \
+    --epsilon 3e-4 \
+    --epsilon_high 4e-4 \
+    --steps_per_generation 32 \
+    --importance_sampling_level sequence \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --gradient_accumulation_steps 8 \
+    --num_generations 16 \
+    --tuner_type full \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.6 \
+    --vllm_max_model_len 16384 \
+    --max_completion_length 8192 \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero3 \
+    --log_completions true

examples/train/grpo/internal/moe_full.sh

@@ -0,0 +1,40 @@
+# 8*80G
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_max_model_len 16384 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --max_length 12000 \
+    --max_completion_length 8192 \
+    --overlong_filter true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_strategy 'steps' \
+    --eval_strategy 'steps' \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --deepspeed zero3_offload \
+    --log_completions true \
+    --sleep_level 1 \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --move_model_batches 10

examples/train/grpo/internal/moe_lora.sh

@@ -0,0 +1,42 @@
+# 8*80G
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_max_model_len 16384 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --max_length 12000 \
+    --max_completion_length 8192 \
+    --overlong_filter true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_strategy 'steps' \
+    --eval_strategy 'steps' \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --move_model_batches 10

examples/train/grpo/internal/qlora.sh

@@ -0,0 +1,49 @@
+SYSTEM_PROMPT="You are a helpful math assistant. Solve the problem step by step. Show your reasoning in <think> </think> tags, then give the final numerical answer after ####.
+For example:
+<think> ... reasoning ... </think>
+#### 42"
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/gsm8k/gsm8k_plugin.py \
+    --reward_funcs gsm8k_accuracy gsm8k_format \
+    --columns '{"answer": "solution"}' \
+    --enable_thinking false \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 10240 \
+    --vllm_enable_lora true \
+    --sleep_level 1 \
+    --tuner_type lora \
+    --quant_method bnb \
+    --quant_bits 4 \
+    --bnb_4bit_quant_type nf4 \
+    --torch_dtype bfloat16 \
+    --dataset 'modelscope/gsm8k' \
+    --load_from_cache_file true \
+    --max_length 2048 \
+    --max_completion_length 8192 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --learning_rate 1e-5 \
+    --lr_scheduler_type cosine \
+    --save_steps 10 \
+    --save_total_limit 100 \
+    --logging_steps 1 \
+    --warmup_ratio 0.0 \
+    --dataloader_num_workers 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --system "$SYSTEM_PROMPT" \
+    --deepspeed zero2 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --max_grad_norm 1.0 \
+    --epsilon 0.2 \
+    --epsilon_high 0.28 \
+    --scale_rewards none

examples/train/grpo/internal/real.sh

@@ -0,0 +1,38 @@
+CUDA_VISIBLE_DEVICES=2 \
+swift rollout \
+    --model Qwen/Qwen3-1.7B
+
+
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-1.7B \
+    --dataset 'AI-MO/NuminaMath-TIR#5000'  \
+    --enable_thinking false \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --load_from_cache_file true \
+    --max_completion_length 4096 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --learning_rate 2e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_total_limit 2 \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --num_generations 8 \
+    --temperature 0.6 \
+    --system """You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}.""" \
+    --log_completions true \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --loss_type real \
+    --deepspeed zero2

examples/train/grpo/internal/reinforce_plus_plus.sh

@@ -0,0 +1,49 @@
+# Reinforce++-Baseline in https://arxiv.org/abs/2501.03262
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+MASTER_PORT=29900 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --advantage_estimator reinforce_plus_plus \
+    --scale_rewards batch \
+    --kl_in_reward true \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --dataset 'AI-ModelScope/clevr_cogen_a_train' \
+    --reward_funcs external_r1v_acc format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.8 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 16384 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --overlong_filter false \
+    --importance_sampling_level sequence \
+    --epsilon 3e-4 \
+    --epsilon_high 4e-4 \
+    --max_completion_length 1024 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero1 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --async_generate false \
+    --beta 0.001 \
+    --attn_impl flash_attention_2

examples/train/grpo/internal/rloo.sh

@@ -0,0 +1,45 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --advantage_estimator rloo \
+    --kl_in_reward true \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs external_r1v_acc format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 16384 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-ModelScope/clevr_cogen_a_train' \
+    --overlong_filter false \
+    --epsilon 3e-4 \
+    --epsilon_high 4e-4 \
+    --max_completion_length 1024 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --async_generate false \
+    --beta 0.001 \
+    --attn_impl flash_attention_2 \
+    --padding_free true \
+    --loss_type grpo

examples/train/grpo/internal/sapo.sh

@@ -0,0 +1,43 @@
+# SAPO https://arxiv.org/abs/2511.20347
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+MAX_PIXELS=602112 \
+swift rlhf \
+    --rlhf_type grpo \
+    --loss_type sapo \
+    --tau_pos 1 \
+    --tau_neg 1.05 \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs external_r1v_acc format \
+    --learning_rate 1e-6 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.6 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 8192 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-ModelScope/clevr_cogen_a_train \
+    --overlong_filter false \
+    --importance_sampling_level token \
+    --max_length 4096 \
+    --max_completion_length 4096 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --num_generations 8 \
+    --steps_per_generation 32 \
+    --save_steps 1000 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero1 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --beta 0.001 \
+    --attn_impl flash_attention_2

examples/train/grpo/internal/transformers.sh

@@ -0,0 +1,37 @@
+# pip install math_verify # reward function
+# pip install -U trl
+# GPU memory: 80GiB
+# You can set `--reward_model` to use a reward model to provide rewards.
+# TransformersEngine to rollout
+
+CUDA_VISIBLE_DEVICES=0 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B \
+    --reward_funcs accuracy format \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-MO/NuminaMath-TIR#1000' \
+    --load_from_cache_file true \
+    --max_completion_length 1024 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --temperature 0.9 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --log_completions true

examples/train/grpo/internal/vllm_72b_4gpu.sh

@@ -0,0 +1,42 @@
+# 4*80G GPU
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-72B-Instruct \
+    --tuner_type lora \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.55 \
+    --vllm_max_model_len 2048 \
+    --vllm_tensor_parallel_size 4 \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 1024 \
+    --reward_funcs accuracy format \
+    --num_generations 4 \
+    --system examples/train/grpo/prompt.txt \
+    --deepspeed zero3_offload \
+    --temperature 1.0 \
+    --top_p 1.0 \
+    --top_k 80 \
+    --log_completions true \
+    --move_model_batches 16 \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1

examples/train/grpo/internal/vllm_lora_qwenvl72b.sh

@@ -0,0 +1,48 @@
+# pip install math_verify # reward function
+# GPU memory: 8 * 80GiB
+
+MAX_PIXELS=602112 \
+WANDB_API_KEY=xxx \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+  --rlhf_type grpo \
+  --model Qwen/Qwen2.5-VL-72B-Instruct \
+  --tuner_type lora \
+  --use_vllm true \
+  --vllm_mode colocate \
+  --vllm_gpu_memory_utilization 0.5 \
+  --vllm_max_model_len 8192 \
+  --vllm_tensor_parallel_size 4 \
+  --dataset lmms-lab/multimodal-open-r1-8k-verified#1000 \
+  --load_from_cache_file true \
+  --external_plugins examples/train/grpo/plugin/plugin.py \
+  --reward_funcs external_r1v_acc format \
+  --reward_weights 1 0.1 \
+  --torch_dtype bfloat16 \
+  --attn_impl flash_attn \
+  --num_train_epochs 1 \
+  --max_length 8192 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 1 \
+  --gradient_accumulation_steps 1 \
+  --eval_steps 500 \
+  --save_steps 500 \
+  --learning_rate 1e-6 \
+  --save_total_limit 2 \
+  --logging_steps 1 \
+  --warmup_ratio 0.05 \
+  --dataloader_num_workers 4 \
+  --max_completion_length 2048 \
+  --num_generations 8 \
+  --deepspeed zero3 \
+  --temperature 1.1 \
+  --top_p 1.0 \
+  --top_k 80 \
+  --log_completions true \
+  --offload_optimizer true \
+  --offload_model true \
+  --move_model_batches 40 \
+  --sleep_level 1 \
+  --report_to wandb \
+  --system examples/train/grpo/prompt.txt

examples/train/grpo/internal/vllm_multi_turn.sh

@@ -0,0 +1,43 @@
+# It's recommended to use server mode for multi-turn training
+# Colocate multi-turn does not support rollouts with dynamic rollout outputs
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --tuner_type full \
+    --reward_funcs accuracy \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_max_model_len 2048 \
+    --vllm_tensor_parallel_size 4 \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 2048 \
+    --num_generations 32 \
+    --deepspeed zero3 \
+    --temperature 1.0 \
+    --top_p 1.0 \
+    --top_k 80 \
+    --log_completions true \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --multi_turn_scheduler math_tip_trick \
+    --max_turns 3

examples/train/grpo/internal/vllm_vl7b.sh

@@ -0,0 +1,30 @@
+MAX_PIXELS=1003520 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --tuner_type lora \
+    --dataset AI-ModelScope/chartqa_digit_r1v_format \
+    --load_from_cache_file true \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_tensor_parallel_size 4 \
+    --torch_dtype bfloat16 \
+    --system examples/train/grpo/prompt.txt \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --gradient_accumulation_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 1024 \
+    --reward_funcs accuracy format \
+    --num_generations 8 \
+    --sleep_level 1 \
+    --temperature 1.0 \
+    --top_p 0.85

examples/train/grpo/multi_node/Qwen2_5_32B_full.sh

@@ -0,0 +1,49 @@
+# External vLLM
+
+# Assume we have two nodes, one with 8 GPUs of 80GB each (880G) and another with 2 GPUs of 80GB each (2 80G).
+#   NODE1. The node with 2*80G will be used to deploy the vLLM server.
+#   NODE2. The node with 8*80G will be used for full-parameter fine-tuning of the 32B model.
+
+# Note : Use beta=0 to disable the reference model; otherwise, it may lead to Out-of-Memory (OOM) errors.
+
+# NODE1 for vLLM Server
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rollout \
+    --model Qwen/Qwen2.5-32B-Instruct \
+    --vllm_tensor_parallel_size 2
+
+# NODE2 for Training
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-32B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host xxx \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --top_p 0.9 \
+    --top_k 50 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --num_iterations 1 \
+    --report_to tensorboard wandb \
+    --beta 0.0

examples/train/grpo/multi_node/colocate_multi_node1.sh

@@ -0,0 +1,44 @@
+# Internal vLLM
+
+# pip install math_verify # reward function
+# pip install -U trl
+# note: Note: The parameters of each node need to be consistent.
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export NNODES=2
+export NODE_RANK=0
+export MASTER_ADDR=127.0.0.1
+export MASTER_PORT=29500
+export NPROC_PER_NODE=4
+
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-Math-7B \
+    --reward_funcs accuracy format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_max_model_len 4096 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-MO/NuminaMath-TIR#5000' \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 0.9 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true

examples/train/grpo/multi_node/colocate_multi_node2.sh

@@ -0,0 +1,39 @@
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export NNODES=2
+export NODE_RANK=1
+export MASTER_ADDR=xxx.xxx.xxx.xxx
+export MASTER_PORT=29500
+export NPROC_PER_NODE=4
+
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-Math-7B \
+    --reward_funcs accuracy format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_max_model_len 4096 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-MO/NuminaMath-TIR#5000' \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 0.9 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true

examples/train/grpo/multi_node/server_multi_node.sh

@@ -0,0 +1,83 @@
+# NOTE: Requires NCCL connectivity between the training master node and rollout nodes
+# This script demonstrates multi-node rollout and multi-node training with swift.
+# node1 and node2: multi-node rollout servers
+# node3 and node4: distributed training nodes
+
+# --- Rollout Section ---
+# For rollout, you can launch any number of servers on different nodes
+# Start rollout server on node1:
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rollout \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_data_parallel_size 2 \
+    --port <node1_port>
+
+# Start rollout server on node2:
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rollout \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_data_parallel_size 2 \
+    --port <node2_port>
+
+# --- Training Section ---
+# node3: Master training node (rank 0)
+NNODES=2 \
+NODE_RANK=0 \
+MASTER_ADDR=127.0.0.1 \
+MASTER_PORT=29500 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host <node1_ip> <node2_ip> \
+    --vllm_server_port <node1_port> <node2_port> \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --deepspeed zero2 \
+    --log_completions true \
+
+# node4: Secondary training node (rank 1)
+NNODES=2 \
+NODE_RANK=1 \
+MASTER_ADDR=<node3_ip> \
+MASTER_PORT=29500 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host <node1_ip> <node2_ip> \
+    --vllm_server_port <node1_port> <node2_port> \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --deepspeed zero2 \
+    --log_completions true \

examples/train/grpo/multi_node/train_dlc.sh

@@ -0,0 +1,40 @@
+# This script is used in DLC (Deep Learning Containers)
+# For more information, visit: https://www.aliyun.com/activity/bigdata/pai-dlc
+# https://help.aliyun.com/zh/pai/user-guide/general-environment-variables
+NNODES=$WORLD_SIZE \
+NODE_RANK=$RANK \
+torchrun \
+    --nproc_per_node=8 \
+    --nnodes=${WORLD_SIZE} \
+    --node_rank=${RANK} \
+    swift/cli/rlhf.py \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --system examples/train/grpo/prompt.txt \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_max_model_len 2048 \
+    --vllm_gpu_memory_utilization 0.3 \
+    --vllm_tensor_parallel_size 4 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --gradient_accumulation_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 2048 \
+    --reward_funcs accuracy format \
+    --num_generations 48 \
+    --sleep_level 1 \
+    --deepspeed zero3_offload \
+    --temperature 1.0 \
+    --top_p 0.85

examples/train/grpo/opd_rl.sh

@@ -0,0 +1,36 @@
+SYSTEM_PROMPT="""You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}."""
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3.5-2B \
+    --teacher_model Qwen/Qwen3.5-9B \
+    --enable_thinking false \
+    --tuner_type lora \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 4096 \
+    --sleep_level 1 \
+    --dataset 'modelscope/gsm8k' \
+    --torch_dtype bfloat16 \
+    --num_generations 1 \
+    --per_device_train_batch_size 2 \
+    --gradient_accumulation_steps 2 \
+    --learning_rate 5e-5 \
+    --logging_steps 1 \
+    --save_steps 100 \
+    --save_total_limit 10 \
+    --max_length 2048 \
+    --max_completion_length 2048 \
+    --deepspeed zero2 \
+    --warmup_ratio 0.1 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --attn_impl flash_attn \
+    --log_completions true \
+    --log_rollout_offpolicy_metrics true \
+    --report_to tensorboard swanlab

examples/train/grpo/plugin/deepeyes/deepeyes.sh

@@ -0,0 +1,58 @@
+# 8 * 80G
+# docs: https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/deepeyes.html
+
+# First: Deploy Qwen2.5-VL-72B-Instruct for verify
+# CUDA_VISIBLE_DEVICES=4,5,6,7 \
+# swift deploy \
+#     --model Qwen/Qwen2.5-VL-72B-Instruct \
+#     --vllm_tensor_parallel_size 4
+
+# Second: Run swift rollout to deploy rollout server
+# MAX_PIXELS=602112 \
+# CUDA_VISIBLE_DEVICES=3 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-VL-7B-Instruct \
+#     --vllm_use_async_engine true \
+#     --external_plugins examples/train/grpo/plugin/deepeyes/deepeyes_plugin.py  \
+#     --multi_turn_scheduler deepeyes_scheduler \
+#     --vllm_max_model_len 8192 \
+#     --vllm_gpu_memory_utilization 0.8 \
+#     --max_turns 5
+
+# Third: Run swift rlhf to train GRPO model
+MAX_PIXELS=602112 \
+CUDA_VISIBLE_DEVICES=0,1,2 \
+NPROC_PER_NODE=3 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --dataset "path/to/data_0.1.2_visual_toolbox_v2.parquet"\
+    "path/to/data_v0.8_visual_toolbox_v2.parquet"\
+    "path/to/data_thinklite_reasoning_acc.parquet" \
+    --load_from_cache_file true \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8001 \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --external_plugins examples/train/grpo/plugin/deepeyes/deepeyes_plugin.py \
+    --reward_funcs deepeyes_reward \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --deepspeed zero3 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --logging_steps 5 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 12 \
+    --beta 0 \
+    --temperature 0.9 \
+    --report_to tensorboard swanlab \
+    --log_completions true

examples/train/grpo/plugin/deepeyes/deepeyes_plugin.py

@@ -0,0 +1,443 @@
+# some code borrowed from https://github.com/Visual-Agent/DeepEyes/blob/main
+
+import base64
+import io
+import json
+import os
+import random
+import re
+from math import ceil, floor
+from openai import OpenAI
+from PIL import Image
+from typing import Any, Dict, List
+
+from swift.rewards.orm import ORM, orms
+from swift.rollout.multi_turn import MultiTurnScheduler, multi_turns
+
+try:
+    from math_verify import parse, verify
+except ImportError as e:
+    raise ImportError('please install math_verify by `pip install math_verify`') from e
+"""
+3 dataset file
+    1. data_v0.8_visual_toolbox_v2.parquet:  data_source == 'chart' (vl_agent.compute_score)
+    2. data_0.1.2_visual_toolbox_v2.parquet : data_source == 'vstar' (vl_agent.compute_score)
+    3. data_thinklite_reasoning_acc.parquet: data_source == 'thinklite_eureka' (vl_agent.compute_score_math)
+
+tool:
+    image_zoom_in_tool: zoom in the image, return a cropped image
+"""
+
+MATH_VERIFY_PROMPT = """# CONTEXT #
+I am a teacher, and I have some high-level math problems. I am tasked with evaluating the correctness of a student's answer.
+Below, I am provided with a problem and a reference answer. Additionally, a student's answer is provided. My job is to assess whether the student's answer captures the same meaning as the reference answer, even when expressed with different wording or format.
+
+# OBJECTIVE #
+I need you to judge whether the student's answer is correct given the ground truth answer.
+
+Your tasks include:
+1. Identify Mathematical or Notational Equivalence: Pay special attention to any LaTeX expressions in both answers. Confirm that the mathematical relationships, variables, and operations conveyed are equivalent.
+
+# TONE #
+Professional, scientific.
+
+# RESPONSE: MARKDOWN REPORT #
+## Equivalence Judgement
+[Whether the student's answer share the same meaning with the reference answer. (TRUE or FALSE)]
+
+# ATTENTION #
+ - The reference answer is ALWAYS correct. You should carefully judge whether the student gives the same answer as reference answer.
+ - The Equivalence Judgement is only TRUE or FALSE. The answer is FALSE even if the student's final answer almost correct with a minor mistakes.
+ - Don't give extra explanation.
+
+**Question**:
+{query}
+
+**Reference Answer**
+{gold_ans}
+
+## Student Final Answer
+{pred_ans}"""# noqa
+
+
+def extract_answer(action_string: str) -> Dict[str, any]:
+    answer = re.findall(r'<answer>(.*?)</answer>', action_string, re.DOTALL)
+    return answer[-1] if answer else None
+
+
+def extract_action(action_string: str) -> Dict[str, Any]:
+    tool_call_match = re.findall(r'<tool_call>(.*?)</tool_call>', action_string, re.DOTALL)
+    return tool_call_match[-1] if tool_call_match else None
+
+
+def get_chat_template():
+    chat_template = """
+Below are two answers to a question. Question is [Question], [Standard Answer] is the standard answer to the question, and [Model_answer] is the answer extracted from a model's output to this question.  Determine whether these two answers are consistent.
+Note that [Model Answer] is consistent with [Standard Answer] whenever they are essentially the same. If the meaning is expressed in the same way, it is considered consistent, for example, 'pink' and 'it is pink'.
+If they are consistent, Judement is 1; if they are different, Judement is 0. Just output Judement and don't output anything else.\n\n
+"""# noqa
+    return chat_template
+
+
+def get_gpt4_score_ICE():
+    example_1 = """
+[Question]: Is the countertop tan or blue?
+[Standard Answer]: The countertop is tan.
+[Model_answer] : tan
+Judgement: 1
+""" # noqa
+
+    example_2 = """
+[Question]: On which side of the picture is the barrier?
+[Standard Answer]: The barrier is on the left side of the picture.
+[Model_answer] : left
+Judgement: 1
+""" # noqa
+
+    example_3 = """
+[Question]: Is the kite brown and large?
+[Standard Answer]: Yes, the kite is brown and large.
+[Model_answer] : Yes
+Judgement: 1
+""" # noqa
+
+    example_4 = """
+[Question]: Are the spots on a giraffe?
+[Standard Answer]: No, the spots are on a banana.
+[Model_answer] : no
+Judgement: 1
+""" # noqa
+
+    example_5 = """
+[Question]: Who is wearing pants?
+[Standard Answer]: The boy is wearing pants.
+[Model_answer] : The person in the picture is wearing pants.
+Judgement: 1
+""" # noqa
+
+    example_6 = """
+[Question]: Is the man phone both blue and closed?
+[Standard Answer]: Yes, the man phone is both blue and closed.
+[Model_answer] : No.
+Judgement: 0
+""" # noqa
+
+    example_7 = """
+[Question]: What color is the towel in the center of the picture?
+[Standard Answer]: The towel in the center of the picture is blue.
+[Model_answer] : The towel in the center of the picture is pink.
+Judgement: 0
+""" # noqa
+
+    return [example_1, example_2, example_3, example_4, example_5, example_6, example_7]
+
+
+def get_prompt(predict_str, ground_truth, question):
+    examples = get_gpt4_score_ICE()
+    chat_template = get_chat_template()
+    demo_prompt = chat_template
+    for example in examples:
+        demo_prompt += example + '\n\n'
+    test_prompt = f"""
+[Question]: {question}
+[Standard Answer]: {ground_truth}
+[Model_answer] : {predict_str}
+Judgement:"""
+    full_prompt = f'{demo_prompt}{test_prompt}'
+
+    return full_prompt
+
+
+def load_pil_image(img):
+    try:
+        if isinstance(img, Image.Image):
+            return img
+
+        elif isinstance(img, Dict):
+            return Image.open(io.BytesIO(img['bytes']))
+
+        elif isinstance(img, str):
+            if os.path.exists(img):
+                return Image.open(img)
+
+            if ',' in img:
+                img_data = img.split(',')[1]
+            else:
+                img_data = img
+            img_bytes = base64.b64decode(img_data)
+            return Image.open(io.BytesIO(img_bytes))
+
+        elif isinstance(img, bytes):
+            return Image.open(io.BytesIO(img))
+
+        elif hasattr(img, 'read'):
+            return Image.open(img)
+        else:
+            return img
+
+    except Exception:
+        return img
+
+
+def rule_math_verify(ground_truth, model_answer):
+    gold = parse(ground_truth)
+    answer = parse(model_answer)
+    return verify(gold, answer)
+
+
+class DeepEyesReward(ORM):
+
+    def __init__(self, args, **kwargs):
+        super().__init__(args)
+        try:
+            self.client = OpenAI(
+                api_key='EMPTY',
+                base_url='http://127.0.0.1:8000/v1',
+            )
+            self.verify_model_name = self.client.models.list().data[0].id
+        except Exception as e:
+            raise RuntimeError('Failed to connect to the model service. Please deploy the model '
+                               "using 'swift deploy' or 'vllm serve'.") from e
+
+    def __call__(self, completions, reward_model, extra_info, data_source, **kwargs) -> List[float]:
+        # reference: https://github.com/Visual-Agent/DeepEyes/blob/main/verl/utils/reward_score/vl_agent.py
+        # NOTE: reward_model is a column name from the dataset, which contains the ground truth answer
+        rewards = []
+        messages = kwargs.get('messages')
+        for completion, solution, info, source, message in zip(completions, reward_model, extra_info, data_source,
+                                                               messages):
+            sol = solution['ground_truth']
+            info['messages'] = message
+            if source in ['vstar', 'chart']:
+                rewards.append(self.compute_score(completion, sol, info))
+            elif source in ['thinklite_eureka']:
+                rewards.append(self.compute_score_math(completion, sol, info))
+            else:
+                raise NotImplementedError
+
+        return rewards
+
+    def compute_score(self, predict_str: str, ground_truth: str, extra_info) -> float:
+        is_format_error = False
+        # predict_str = "<think>" + predict_str
+        count_think_1 = predict_str.count('<think>')
+        count_think_2 = predict_str.count('</think>')
+        if count_think_1 != count_think_2:
+            is_format_error = True
+        count_tool_1 = predict_str.count('<tool_call>')
+        count_tool_2 = predict_str.count('</tool_call>')
+        if count_tool_1 != count_tool_2:
+            is_format_error = True
+
+        predict_no_think = predict_str.split('</think>')[-1].strip()
+        count_answer_1 = predict_no_think.count('<answer>')
+        count_answer_2 = predict_no_think.count('</answer>')
+        if count_answer_1 != count_answer_2:
+            is_format_error = True
+
+        answer_text = predict_str.split('<answer>')[-1].split('</answer>')[0].strip()
+
+        question_text = extra_info['question']
+        full_prompt = get_prompt(answer_text, ground_truth, question_text)
+
+        chat_response = self.client.chat.completions.create(
+            model=self.verify_model_name,
+            messages=[
+                {
+                    'role': 'system',
+                    'content': 'You are a helpful assistant.'
+                },
+                {
+                    'role': 'user',
+                    'content': full_prompt
+                },
+            ],
+            seed=random.randint(0, 1000000),
+            temperature=0.3,
+        )
+        response = chat_response.choices[0].message.content.strip()
+        if 'Judgement:' in response:
+            response = response.split('Judgement:')[-1].strip()
+            if '1' in response:
+                acc_reward = 1.0
+            elif '0' in response:
+                acc_reward = 0.0
+            else:
+                acc_reward = 0.0
+        else:
+            if response == '1':
+                acc_reward = 1.0
+            elif response == '0':
+                acc_reward = 0.0
+            else:
+                acc_reward = 0.0
+
+        # Penalize for model trying to predict longer answer to hack llm-as-judge
+        if len(answer_text) >= 1000:
+            acc_reward = 0.0
+            is_format_error = True
+
+        num_image = 0
+        for message in extra_info['messages']:
+            if message['role'] == 'user' and '<image>' in message['content']:
+                num_image += 1
+        # More than one image indicates a successful tool call.
+        tool_reward = 1.0 if num_image > 1 and acc_reward > 0.5 else 0.0
+        format_reward = -1.0 if is_format_error else 0.0
+
+        return 0.8 * acc_reward + 0.2 * format_reward + 1.2 * tool_reward
+
+    def compute_score_math(self, predict_str: str, ground_truth: str, extra_info=None) -> float:
+        is_format_error = False
+        # predict_str = "<think>" + predict_str
+        count_think_1 = predict_str.count('<think>')
+        count_think_2 = predict_str.count('</think>')
+        if count_think_1 != count_think_2:
+            is_format_error = True
+
+        model_answer = ''
+        predict_no_think = predict_str.split('</think>')[-1].strip()
+        answer_pattern = r'\\boxed{([^}]+)}'
+        answer_list = re.findall(answer_pattern, predict_no_think, flags=re.DOTALL)
+        if len(answer_list) == 0:
+            acc_reward = 0.0
+            is_format_error = True
+        else:
+            if len(answer_list) > 1:
+                is_format_error = True
+
+            model_answer = answer_list[-1]
+            if rule_math_verify(ground_truth, model_answer):
+                acc_reward = 1.0
+            else:
+                acc_reward = 0
+                full_prompt = MATH_VERIFY_PROMPT.format(
+                    query=extra_info['question'],
+                    gold_ans=ground_truth,
+                    pred_ans=model_answer,
+                )
+                response = ''
+                for _ in range(8):
+                    try:
+                        chat_response = self.client.chat.completions.create(
+                            model=self.verify_model_name,
+                            messages=[
+                                {
+                                    'role': 'user',
+                                    'content': full_prompt
+                                },
+                            ],
+                            seed=random.randint(0, 1000000),
+                            temperature=0.0,
+                        )
+                        response = chat_response.choices[0].message.content.strip()
+                        break
+                    except Exception:
+                        continue
+                judgement = response.split('## Equivalence Judgement')[-1].lower()
+                if 'true' in judgement and 'false' not in judgement:
+                    acc_reward = 1.0
+
+        format_reward = -1.0 if is_format_error else 0.0
+        return 1.2 * acc_reward + 0.4 * format_reward
+
+
+orms['deepeyes_reward'] = DeepEyesReward
+
+
+class VisualToolBoxScheduler(MultiTurnScheduler):
+    user_prompt = ('\nThink first, call **image_zoom_in_tool** if needed, then answer. '
+                   'Format strictly as:  <think>...</think>  <tool_call>...</tool_call> (if tools needed)'
+                   '  <answer>...</answer> ')
+
+    def __init__(self, infer_engine=None, max_turns=None, *args, **kwargs):
+        super().__init__(infer_engine, max_turns, *args, **kwargs)
+
+    def check_finished(self, infer_request, response_choice, current_turn):
+        should_stop = super().check_finished(infer_request, response_choice, current_turn)
+        if should_stop:
+            return True
+
+        last_completion = infer_request.messages[-1]['content']
+
+        action = extract_action(last_completion)
+        # if the last completion is a tool call, do not finished yet
+        if action:
+            return False
+
+        return True
+
+    def step(self, infer_request, response_choice, current_turn):
+        from qwen_vl_utils import fetch_image
+        completion = response_choice.message.content
+        action = extract_action(completion)
+        cropped_img = None
+        extra_info = {}
+        try:
+            tool_call = json.loads(action.strip())
+            tool_name = tool_call['name']
+            if tool_name != 'image_zoom_in_tool':
+                raise ValueError(f'Unknown tool name: {tool_name}')
+            args = tool_call['arguments']
+            bbox = args['bbox_2d']
+            # NOTE: this function is only compatible with the QwenVL series models
+            # If you use another MLLM, please adjust the fetch_image function accordingly
+            # ensure the returned img is of type PIL.Image.Image and
+            # has been processed to a maximum size of max_pixels
+            img = fetch_image({'image': load_pil_image(infer_request.images[0])})
+
+            origin_height = img.height
+            origin_width = img.width
+            bbox = self.maybe_resize_bbox(bbox=bbox, origin_width=origin_width, origin_height=origin_height)
+            # for invalid bbox, the exception will be catched in except block
+            cropped_img = img.crop(bbox)
+            query = '<tool_response>' + '<image>' + self.user_prompt + '</tool_response>'
+        except Exception as e:
+            error_msg = f'Invalid tool call format: {action.strip()}. Error: {e}'
+            query = f'Error: {str(error_msg)}'
+
+        infer_request.messages.append({'role': 'user', 'content': query})
+        if cropped_img:
+            infer_request.images.append(cropped_img)
+        # override the images
+        extra_info['images'] = infer_request.images
+
+        # Return dictionary format according to new MultiTurnScheduler interface
+        return {'infer_request': infer_request, 'rollout_infos': extra_info}
+
+    def validate_bbox(self, left, top, right, bottom):
+        assert left < right and bottom > top, f'invalid shape for {left=}, {top=}, {right=}, {bottom=}'
+        height = bottom - top
+        width = right - left
+        assert max(height, width) / min(height,
+                                        width) <= 100, f'aspect ratio error: {left=}, {top=}, {right=}, {bottom=}'
+        assert min(height, width) > 30, f'{height=}, {width=} is too small'
+        return True
+
+    def maybe_resize_bbox(self, bbox, origin_width, origin_height):
+        left, top, right, bottom = bbox
+
+        left = max(0, left)
+        top = max(0, top)
+        right = min(origin_width, right)
+        bottom = min(origin_height, bottom)
+        self.validate_bbox(left, top, right, bottom)
+
+        height = bottom - top
+        width = right - left
+        if height < 28 or width < 28:
+            center_x = (left + right) / 2.0
+            center_y = (top + bottom) / 2.0
+            ratio = 28 / min(height, width)
+            new_half_height = ceil(height * ratio * 0.5)
+            new_half_width = ceil(width * ratio * 0.5)
+            new_left = floor(center_x - new_half_width)
+            new_right = ceil(center_x + new_half_width)
+            new_top = floor(center_y - new_half_height)
+            new_bottom = ceil(center_y + new_half_height)
+            self.validate_bbox(new_left, new_top, new_right, new_bottom)
+            return [new_left, new_top, new_right, new_bottom]
+        return [left, top, right, bottom]
+
+
+multi_turns['deepeyes_scheduler'] = VisualToolBoxScheduler

examples/train/grpo/plugin/gsm8k/gsm8k.sh

@@ -0,0 +1,43 @@
+SYSTEM_PROMPT="""You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}."""
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3.5-2B \
+    --external_plugins examples/train/grpo/plugin/gsm8k/gsm8k_plugin.py \
+    --reward_funcs gsm8k_accuracy gsm8k_format \
+    --columns '{"answer": "solution"}' \
+    --enable_thinking false \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 10240 \
+    --sleep_level 1 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'modelscope/gsm8k' \
+    --load_from_cache_file true \
+    --max_length 2048 \
+    --max_completion_length 8192 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --learning_rate 1e-6 \
+    --lr_scheduler_type cosine \
+    --save_steps 10 \
+    --save_total_limit 100 \
+    --logging_steps 1 \
+    --warmup_ratio 0.0 \
+    --dataloader_num_workers 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --system "$SYSTEM_PROMPT" \
+    --deepspeed zero2 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --max_grad_norm 1.0 \
+    --epsilon 0.2 \
+    --epsilon_high 0.28 \
+    --scale_rewards none

examples/train/grpo/plugin/gsm8k/gsm8k_plugin.py

@@ -0,0 +1,49 @@
+import re
+from typing import List
+
+from swift.rewards import ORM, orms
+
+
+class GSM8KAccuracy(ORM):
+
+    @staticmethod
+    def extract_answer(text: str) -> str:
+        """Extract the last #### number from text."""
+        text = text[-500:] if len(text) > 500 else text
+        # Prefer \boxed{} format
+        boxed = re.findall(r'\\boxed\{([^}]+)\}', text)
+        if boxed:
+            return boxed[-1].replace(',', '').replace(' ', '').strip()
+        # Fallback to #### format
+        matches = re.findall(r'####\s*([\-\d,\.\s]+)', text)
+        if matches:
+            return matches[-1].replace(',', '').replace(' ', '').strip()
+        return ''
+
+    def __call__(self, completions, solution, **kwargs) -> List[float]:
+        rewards = []
+        for completion, gt_answer in zip(completions, solution):
+            gt_num = self.extract_answer(gt_answer)
+            pred_num = self.extract_answer(completion)
+            correct = False
+            if pred_num and gt_num:
+                try:
+                    correct = abs(float(pred_num) - float(gt_num)) < 1e-5
+                except (ValueError, OverflowError):
+                    correct = pred_num == gt_num
+            rewards.append(1.0 if correct else 0.0)
+        return rewards
+
+
+class GSM8KFormat(ORM):
+
+    def __call__(self, completions, **kwargs) -> List[float]:
+        rewards = []
+        for completion in completions:
+            has_answer = bool(re.search(r'\\boxed\{[^}]+\}', completion) or re.search(r'####\s*[\-\d,\.]+', completion))
+            rewards.append(1.0 if has_answer else 0.0)
+        return rewards
+
+
+orms['gsm8k_accuracy'] = GSM8KAccuracy
+orms['gsm8k_format'] = GSM8KFormat

examples/train/grpo/plugin/openenv/run_grpo_sudoku.sh

@@ -0,0 +1,52 @@
+# ============================================================
+#  Swift GRPO training with OpenEnv TextArena Sudoku
+#
+#  Prerequisites:
+#    1. Start Sudoku server (separate terminal):
+#       TEXTARENA_ENV_ID=Sudoku-v0 MAX_CONCURRENT_ENVS=8 \
+#         python examples/train/grpo/plugin/openenv/start_sudoku_server.py
+#
+#    2. This script uses colocate mode:
+#       - vLLM and training share the same GPUs
+#       - No separate rollout server needed
+#
+#  Environment:  TextArena Sudoku (local server, port 8000)
+#  Model:        Qwen3.5-4B (enable_thinking=false)
+#  Scheduler:    SudokuScheduler (multi-turn, content diff tracking)
+#  Multi-turn:   max_turns=20 (20 moves per game)
+#  Rewards:      5-component (empty_cell/valid_move/repetition/progress/correct)
+#  Hints:        Board parsing + guaranteed moves + candidates
+#
+# ============================================================
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3.5-4B \
+    --dataset examples/train/grpo/plugin/openenv/sudoku.jsonl#1000 \
+    --external_plugins examples/train/grpo/plugin/openenv/sudoku_scheduler.py \
+    --enable_thinking false \
+    --torch_dtype bfloat16 \
+    --max_completion_length 256 \
+    --max_length 8192 \
+    --learning_rate 5e-6 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --num_generations 4 \
+    --generation_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --temperature 1 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_max_model_len 12288 \
+    --vllm_gpu_memory_utilization 0.35 \
+    --gradient_checkpointing true \
+    --use_gym_env true \
+    --multi_turn_scheduler sudoku_scheduler \
+    --max_turns 20 \
+    --save_strategy steps \
+    --save_steps 50 \
+    --logging_steps 1 \
+    --log_completions true \
+    --report_to tensorboard swanlab

examples/train/grpo/plugin/openenv/run_grpo_sudoku_server.sh

@@ -0,0 +1,65 @@
+# ============================================================
+#  Swift GRPO training with OpenEnv TextArena Sudoku (Server Mode)
+#
+#  Prerequisites:
+#    1. Start Sudoku server (separate terminal):
+#       TEXTARENA_ENV_ID=Sudoku-v0 MAX_CONCURRENT_ENVS=8 \
+#         python examples/train/grpo/plugin/openenv/start_sudoku_server.py
+#
+#    2. Start vLLM rollout server (separate terminal):
+#       CUDA_VISIBLE_DEVICES=0 \
+#       swift rollout \
+#           --model Qwen/Qwen3.5-4B \
+#           --external_plugins examples/train/grpo/plugin/openenv/sudoku_scheduler.py \
+#           --enable_thinking false \
+#           --max_length 8192 \
+#           --vllm_max_model_len 12288 \
+#           --vllm_gpu_memory_utilization 0.9 \
+#           --use_gym_env true \
+#           --multi_turn_scheduler sudoku_scheduler \
+#           --max_turns 20
+#
+#    3. This script starts training in server mode:
+#       - vLLM rollout server handles multi-turn + env interaction
+#       - Training process sends generation requests to rollout server
+#       - --multi_turn_scheduler / --max_turns go to BOTH rollout and rlhf
+#
+#  Environment:  TextArena Sudoku (local server, port 8000)
+#  Model:        Qwen3.5-4B (enable_thinking=false)
+#  Scheduler:    SudokuScheduler (multi-turn, content diff tracking)
+#  Multi-turn:   max_turns=20 (20 moves per game)
+#  Rewards:      5-component (empty_cell/valid_move/repetition/progress/correct)
+#  Hints:        Board parsing + guaranteed moves + candidates
+#
+# ============================================================
+
+CUDA_VISIBLE_DEVICES=1,2,3 \
+NPROC_PER_NODE=3 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3.5-4B \
+    --dataset examples/train/grpo/plugin/openenv/sudoku.jsonl \
+    --external_plugins examples/train/grpo/plugin/openenv/sudoku_scheduler.py \
+    --enable_thinking false \
+    --torch_dtype bfloat16 \
+    --max_completion_length 256 \
+    --max_length 8192 \
+    --learning_rate 5e-6 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --num_generations 6 \
+    --gradient_accumulation_steps 4 \
+    --temperature 1 \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8001 \
+    --gradient_checkpointing true \
+    --use_gym_env true \
+    --multi_turn_scheduler sudoku_scheduler \
+    --max_turns 20 \
+    --save_strategy steps \
+    --save_steps 50 \
+    --logging_steps 1 \
+    --log_completions true \
+    --report_to tensorboard swanlab

examples/train/grpo/plugin/openenv/start_sudoku_server.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Start the TextArena Sudoku server with configurable concurrent sessions.
+
+The default OpenEnv server only allows 1 concurrent session because
+TextArenaEnvironment is not marked as SUPPORTS_CONCURRENT_SESSIONS.
+Since each WebSocket session creates an independent game instance,
+it is safe to enable concurrent sessions.
+
+Usage:
+    TEXTARENA_ENV_ID=Sudoku-v0 python start_sudoku_server.py
+    TEXTARENA_ENV_ID=Sudoku-v0 MAX_CONCURRENT_ENVS=8 python start_sudoku_server.py
+"""
+import os
+import uvicorn
+from openenv.core.env_server.http_server import create_app
+from textarena_env.server.app import (TextArenaAction, TextArenaObservation, build_textarena_gradio_app,
+                                      create_textarena_environment)
+from textarena_env.server.environment import TextArenaEnvironment
+
+# Read config from environment
+# Note: TEXTARENA_ENV_ID is read by create_textarena_environment factory,
+# not by this script directly.
+max_concurrent_envs = int(os.getenv('MAX_CONCURRENT_ENVS', '8'))
+host = os.getenv('HOST', '0.0.0.0')
+port = int(os.getenv('PORT', '8000'))
+
+# Mark TextArenaEnvironment as supporting concurrent sessions.
+# Each WebSocket session creates an independent game instance via the factory,
+# so concurrent sessions are safe.
+
+TextArenaEnvironment.SUPPORTS_CONCURRENT_SESSIONS = True
+
+# Build the app with custom max_concurrent_envs
+
+app = create_app(
+    create_textarena_environment,
+    TextArenaAction,
+    TextArenaObservation,
+    env_name='textarena_env',
+    max_concurrent_envs=max_concurrent_envs,
+    gradio_builder=build_textarena_gradio_app,
+)
+
+if __name__ == '__main__':
+    env_id = os.getenv('TEXTARENA_ENV_ID', 'Sudoku-v0')
+    print(f'Starting server: env={env_id}, max_concurrent_envs={max_concurrent_envs}')
+    uvicorn.run(app, host=host, port=port)

examples/train/grpo/plugin/openenv/sudoku.jsonl

@@ -0,0 +1,10 @@
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}
+{"messages": [{"role": "user", "content": "Play"}], "env_config": {"name": "openenv", "base_url": "http://127.0.0.1:8000", "reset_kwargs": {}}}

examples/train/grpo/plugin/openenv/sudoku_scheduler.py

@@ -0,0 +1,416 @@
+"""Sudoku scheduler for OpenEnv TextArena Sudoku environment.
+
+Reference: TRL openenv_sudoku_grpo.ipynb
+Key features:
+1. Multiple reward functions (empty_cell, valid_move, repetition, progress, correct)
+2. Hints system: parse board, provide guaranteed moves and candidates
+3. Board state tracking with content diff for bounded context
+"""
+import asyncio
+import re
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Union
+
+from swift.rollout.multi_turn import OpenEnvScheduler, multi_turns
+
+SUDOKU_SYSTEM_PROMPT = """You are an expert Sudoku player with deep knowledge of logical deduction strategies.
+
+## GAME RULES
+1. The puzzle is a 9x9 grid divided into nine 3x3 subgrids (boxes)
+2. Some cells are pre-filled with numbers 1-9
+3. Fill empty cells ('.') with numbers 1-9
+4. Each row, column, and 3x3 box must contain 1-9 without repetition
+5. Cannot overwrite pre-filled cells
+6. Invalid moves result in penalties
+
+## HOW TO PLAY
+Output your move in this format: [row col number]
+Example: [3 5 7] means place 7 at row 3, column 5.
+You may reason before your move, but always end with [row col number].
+
+## STRATEGIC APPROACH
+- Naked Singles: If a cell has only one possible candidate, fill it immediately.
+- Hidden Singles: If a number can only go in one cell within a row/column/box, place it there.
+- Scanning: Look at each row, column, and box to find where numbers can go.
+
+## COMMON PITFALLS
+- Don't guess randomly - Sudoku is pure logic
+- Don't overwrite pre-filled cells
+- Don't repeat a move that was already made
+- Coordinates are 1-indexed (1-9)
+
+## BOARD READING
+- Rows labeled R1-R9 (top to bottom)
+- Columns labeled C1-C9 (left to right)
+- Empty cells shown as '.'"""
+
+
+def _is_valid_board_state(board_str: str) -> bool:
+    return 'R1' in board_str and 'R9' in board_str and '|' in board_str
+
+
+def _parse_board(board_str: str) -> list:
+    grid = [[0] * 9 for _ in range(9)]
+    if not _is_valid_board_state(board_str):
+        return grid
+    for line in board_str.split('\n'):
+        line_stripped = line.strip()
+        if line_stripped and line_stripped[0] == 'R' and len(line_stripped) > 1 and line_stripped[1].isdigit():
+            row = int(line_stripped[1]) - 1
+            col = 0
+            for char in line_stripped[2:]:
+                if col >= 9:
+                    break
+                if char == '.':
+                    grid[row][col] = 0
+                    col += 1
+                elif char.isdigit():
+                    grid[row][col] = int(char)
+                    col += 1
+    return grid
+
+
+def _count_filled_cells(board_str: str) -> int:
+    grid = _parse_board(board_str)
+    return sum(1 for row in grid for cell in row if cell != 0)
+
+
+def _get_valid_numbers(grid: list, row: int, col: int) -> set:
+    if grid[row][col] != 0:
+        return set()
+    used = set()
+    for c in range(9):
+        if grid[row][c] != 0:
+            used.add(grid[row][c])
+    for r in range(9):
+        if grid[r][col] != 0:
+            used.add(grid[r][col])
+    box_row, box_col = 3 * (row // 3), 3 * (col // 3)
+    for r in range(box_row, box_row + 3):
+        for c in range(box_col, box_col + 3):
+            if grid[r][c] != 0:
+                used.add(grid[r][c])
+    return set(range(1, 10)) - used
+
+
+def _extract_empty_cells_with_candidates(board_str: str, sort_by_difficulty: bool = True):
+    grid = _parse_board(board_str)
+    cells = []
+    for row in range(9):
+        for col in range(9):
+            if grid[row][col] == 0:
+                candidates = _get_valid_numbers(grid, row, col)
+                cells.append((row + 1, col + 1, candidates))
+    if sort_by_difficulty:
+        cells.sort(key=lambda x: len(x[2]))
+    return cells
+
+
+def _extract_empty_cells(board_str: str) -> list:
+    """Return list of (row, col) tuples for empty cells, 0-indexed."""
+    grid = _parse_board(board_str)
+    return [(r, c) for r in range(9) for c in range(9) if grid[r][c] == 0]
+
+
+def _extract_board_only(text: str) -> str:
+    if not text:
+        return ''
+    lines = text.split('\n')
+    board_lines = []
+    in_board = False
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith('C1') or (stripped and stripped[0] == 'R' and len(stripped) > 1
+                                         and stripped[1].isdigit()):
+            in_board = True
+        if in_board and (stripped.startswith('-') or stripped.startswith('R') or stripped.startswith('C1')):
+            board_lines.append(line)
+        elif (in_board and stripped and not stripped.startswith('-')
+              and not (stripped[0] == 'R' and len(stripped) > 1 and stripped[1].isdigit())):
+            break
+    return '\n'.join(board_lines) if board_lines else ''
+
+
+def _make_hints(board_str: str, successful_moves: list, failed_moves: list, difficulty: str = 'easy') -> str:
+    parts = []
+    all_tried = successful_moves + failed_moves
+    if all_tried:
+        parts.append(f"\nMOVES ALREADY TRIED (do not repeat): {', '.join(all_tried[:10])}")
+    if not board_str or not _is_valid_board_state(board_str):
+        return '\n'.join(parts)
+
+    cells = _extract_empty_cells_with_candidates(board_str, sort_by_difficulty=True)
+    if cells:
+        guaranteed = []
+        other = []
+        for r, c, candidates in cells[:10]:
+            if len(candidates) == 1:
+                guaranteed.append(f'[{r} {c} {list(candidates)[0]}]')
+            elif len(candidates) <= 3:
+                nums = ','.join(str(n) for n in sorted(candidates))
+                other.append(f'({r},{c})->{nums}')
+        if guaranteed:
+            parts.append(f"\nGUARANTEED MOVES (only one option): {', '.join(guaranteed[:5])}")
+        if other:
+            parts.append(f"Other options: {' | '.join(other[:5])}")
+
+    return '\n'.join(parts)
+
+
+class SudokuScheduler(OpenEnvScheduler):
+    """Sudoku scheduler with multi-reward and hints system.
+
+    Tracks 5 reward components per trajectory:
+    - empty_cell_reward: Did the model target empty cells? (+1/-1)
+    - valid_move_reward: Were moves accepted by env? (1.0/-0.5/0.0)
+    - repetition_reward: Penalty for repeating moves (exponential)
+    - progress_reward: How much of the puzzle was filled (0-1)
+    - correct_reward: Environment's reward (0 or 1)
+
+    Combined reward = sum of all components.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._last_content_len: Dict[str, int] = {}
+        # Per-uuid state tracking
+        self._board_states: Dict[str, str] = {}
+        self._move_counts: Dict[str, defaultdict] = {}
+        self._successful_moves: Dict[str, list] = {}
+        self._failed_moves: Dict[str, list] = {}
+        self._valid_move_scores: Dict[str, list] = {}
+        self._empty_cell_scores: Dict[str, list] = {}
+        self._correct_scores: Dict[str, list] = {}
+        self._repetition_scores: Dict[str, list] = {}
+        self._initial_filled: Dict[str, int] = {}
+        self._max_filled: Dict[str, int] = {}
+
+    async def on_trajectory_start(self, requests):
+        """Initialize env, parse board, compute hints."""
+        semaphore = asyncio.Semaphore(getattr(self, 'max_concurrent_envs', 4))
+
+        async def _init_single(req):
+            async with semaphore:
+                uuid = req.uuid
+                if uuid in self._envs:
+                    await self._close_and_remove(uuid)
+
+                row_env_config = (req.data_dict or {}).get('env_config', {}) if hasattr(req, 'data_dict') else {}
+                env_config = {**getattr(self, 'env_config_defaults', {}), **row_env_config}
+                wrapper = self._create_env(env_config)
+
+                obs, metadata = await asyncio.to_thread(wrapper.reset)
+                system_message = env_config.get('system_message', SUDOKU_SYSTEM_PROMPT)
+
+                content = self._extract_content(obs)
+                self._last_content_len[uuid] = len(content)
+
+                # Parse initial board state
+                board = _extract_board_only(content) if _is_valid_board_state(content) else content
+                self._board_states[uuid] = content if _is_valid_board_state(content) else ''
+                initial_filled = _count_filled_cells(self._board_states[uuid]) if self._board_states[uuid] else 0
+
+                # Initialize tracking state
+                self._move_counts[uuid] = defaultdict(int)
+                self._successful_moves[uuid] = []
+                self._failed_moves[uuid] = []
+                self._valid_move_scores[uuid] = []
+                self._empty_cell_scores[uuid] = []
+                self._correct_scores[uuid] = []
+                self._repetition_scores[uuid] = []
+                self._initial_filled[uuid] = initial_filled
+                self._max_filled[uuid] = initial_filled
+
+                # Build initial message with board + hints
+                hints = _make_hints(self._board_states[uuid], [], [])
+                user_content = f'{board}{hints}' if board else content
+
+                from swift.rollout.multi_turn import Messages
+                messages = []
+                if system_message:
+                    messages.append({'role': 'system', 'content': system_message})
+                messages.append({'role': 'user', 'content': user_content})
+                req.messages = messages
+
+                self._envs[uuid] = wrapper
+                self._total_rewards[uuid] = 0.0
+                self._step_rewards[uuid] = []
+                self._pending_obs[uuid] = None
+
+        await asyncio.gather(*[_init_single(req) for req in requests])
+
+    async def _close_and_remove(self, uuid):
+        """Override to clean up all tracking state."""
+        await super()._close_and_remove(uuid)
+        self._last_content_len.pop(uuid, None)
+        self._board_states.pop(uuid, None)
+        self._move_counts.pop(uuid, None)
+        self._successful_moves.pop(uuid, None)
+        self._failed_moves.pop(uuid, None)
+        self._valid_move_scores.pop(uuid, None)
+        self._empty_cell_scores.pop(uuid, None)
+        self._correct_scores.pop(uuid, None)
+        self._repetition_scores.pop(uuid, None)
+        self._initial_filled.pop(uuid, None)
+        self._max_filled.pop(uuid, None)
+
+    def _extract_content(self, observation: Any) -> str:
+        if isinstance(observation, dict):
+            messages = observation.get('messages', [])
+            if messages:
+                return messages[0].get('content', '')
+            prompt = observation.get('prompt', '')
+            if prompt:
+                return prompt
+        return str(observation)
+
+    async def on_turn_end(self, infer_request, response_choice, current_turn):
+        """Parse move, step env, compute multi-reward, generate hints."""
+        uuid = infer_request.uuid
+        wrapper = self._envs.get(uuid)
+        if wrapper is None:
+            return {'done': True, 'rollout_infos': {}}
+
+        action_text = response_choice.message.content
+        action_dict = self.parse_action(action_text)
+        if action_dict is None:
+            # Parse failed: end trajectory with penalty instead of polluting env
+            self._total_rewards[uuid] = self._total_rewards.get(uuid, 0.0) - 1.0
+            self._step_rewards.setdefault(uuid, []).append(-1.0)
+            await self._close_and_remove(uuid)
+            return {
+                'done': True,
+                'rollout_infos': {
+                    'total_reward': self._total_rewards[uuid],
+                    'step_rewards': list(self._step_rewards.get(uuid, [])),
+                    'gym_done': True,
+                }
+            }
+        move = action_dict.get('message', '')
+
+        # Step environment
+        obs, env_reward, done, metadata = await asyncio.to_thread(wrapper.step, action_dict)
+        correct_score = float(env_reward or 0.0)
+
+        # Extract new content (diff from last seen)
+        full_content = self._extract_content(obs)
+        last_len = self._last_content_len.get(uuid, 0)
+        new_content = full_content[last_len:] if len(full_content) > last_len else full_content
+        self._last_content_len[uuid] = len(full_content)
+
+        # Check if env says invalid
+        new_content_lower = new_content.lower()
+        env_says_invalid = any(kw in new_content_lower
+                               for kw in ['invalid', 'error', 'cannot', 'already', 'violation', 'lost'])
+
+        # Check if move targets an empty cell
+        if self._board_states.get(uuid):
+            empty_cells = _extract_empty_cells(self._board_states[uuid])
+            # Convert move coords (1-indexed from model) to 0-indexed for comparison
+            move_nums = re.findall(r'\d+', move)
+            targets_empty = tuple(int(x) - 1 for x in move_nums[:2]) in empty_cells if len(move_nums) >= 3 else True
+        else:
+            targets_empty = True
+
+        # Empty cell reward: +1 if targeted empty, -1 if tried to overwrite
+        empty_cell_score = 1.0 if targets_empty else -1.0
+
+        # Repetition tracking
+        is_new_move = self._move_counts[uuid][move] == 0
+        repetition_count = self._move_counts[uuid][move]
+        self._move_counts[uuid][move] += 1
+        repetition_score = -min(2**repetition_count, 10.0) if repetition_count > 0 else 0.0
+
+        # Valid move score
+        is_valid = not env_says_invalid and targets_empty
+        if is_valid and is_new_move:
+            valid_move_score = 1.0
+            self._successful_moves[uuid].append(move)
+        elif 'please resubmit' in new_content_lower or 'avoid penalties' in new_content_lower:
+            valid_move_score = -0.5
+            self._failed_moves[uuid].append(move)
+        else:
+            valid_move_score = 0.0
+            if not is_valid:
+                self._failed_moves[uuid].append(move)
+
+        # Update board state if valid and new content has board
+        if is_valid and _is_valid_board_state(new_content):
+            self._board_states[uuid] = new_content
+            current_filled = _count_filled_cells(new_content)
+            if current_filled > self._max_filled[uuid]:
+                self._max_filled[uuid] = current_filled
+
+        # Progress reward
+        remaining = 81 - self._initial_filled[uuid]
+        if remaining > 0:
+            progress_score = (self._max_filled[uuid] - self._initial_filled[uuid]) / remaining
+        else:
+            progress_score = 1.0
+
+        # Track all scores
+        self._valid_move_scores[uuid].append(valid_move_score)
+        self._empty_cell_scores[uuid].append(empty_cell_score)
+        self._correct_scores[uuid].append(correct_score)
+        self._repetition_scores[uuid].append(repetition_score)
+
+        combined_reward = (
+            sum(self._empty_cell_scores[uuid]) / max(len(self._empty_cell_scores[uuid]), 1)
+            + sum(self._valid_move_scores[uuid]) / max(len(self._valid_move_scores[uuid]), 1)
+            + sum(self._repetition_scores[uuid]) / max(len(self._repetition_scores[uuid]), 1) + progress_score
+            + correct_score)
+
+        self._total_rewards[uuid] = combined_reward
+        self._step_rewards.setdefault(uuid, []).append(combined_reward)
+
+        # Build next observation with board + hints
+        if not done:
+            board_str = self._board_states.get(uuid, '')
+            board = _extract_board_only(board_str) if board_str else ''
+            hints = _make_hints(
+                board_str,
+                self._successful_moves[uuid],
+                self._failed_moves[uuid],
+            )
+            step_num = len(self._successful_moves[uuid])
+            next_obs = f'Step {step_num}. Progress: {step_num} cells filled.\n\nBoard:\n{board}{hints}'
+        else:
+            next_obs = None
+
+        self._pending_obs[uuid] = next_obs
+
+        rollout_infos = {
+            'total_reward': self._total_rewards[uuid],
+            'step_rewards': list(self._step_rewards.get(uuid, [])),
+            'gym_done': done,
+            'empty_cell_reward': sum(self._empty_cell_scores[uuid]) / max(len(self._empty_cell_scores[uuid]), 1),
+            'valid_move_reward': sum(self._valid_move_scores[uuid]) / max(len(self._valid_move_scores[uuid]), 1),
+            'repetition_reward': sum(self._repetition_scores[uuid]) / max(len(self._repetition_scores[uuid]), 1),
+            'progress_reward': progress_score,
+            'correct_reward': correct_score,
+        }
+        if done:
+            await self._close_and_remove(uuid)
+
+        return {'done': done, 'rollout_infos': rollout_infos}
+
+    def parse_action(self, text: str) -> Optional[Dict[str, Any]]:
+        """Extract [row col number] from model output. Returns None if parse fails."""
+        match = re.search(r'\[\s*(\d+)\s+(\d+)\s+(\d+)\s*\]', text)
+        if match:
+            row, col, num = match.groups()
+            return {'message': f'[{row} {col} {num}]'}
+
+        numbers = re.findall(r'\d+', text)
+        if len(numbers) >= 3:
+            return {'message': f'[{numbers[0]} {numbers[1]} {numbers[2]}]'}
+
+        return None
+
+    def format_observation(self, observation: Any) -> Union[str, List[Dict]]:
+        return self._extract_content(observation)
+
+
+# Register scheduler so --external_plugins can load it
+
+multi_turns['sudoku_scheduler'] = SudokuScheduler

examples/train/grpo/plugin/run_external_reward_func.sh

@@ -0,0 +1,49 @@
+# register customized plugins in plugin.py file
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+MAX_PIXELS=602112 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs external_r1v_acc format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.6 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 16384 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-ModelScope/clevr_cogen_a_train' \
+    --overlong_filter false \
+    --importance_sampling_level token \
+    --epsilon 0.2 \
+    --epsilon_high 0.28 \
+    --max_completion_length 8192 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --steps_per_generation 4 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero1 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --async_generate false \
+    --beta 0.001 \
+    --loss_type grpo \
+    --vllm_enable_lora false \
+    --advantage_estimator grpo

examples/train/grpo/plugin/run_external_reward_model.sh

@@ -0,0 +1,23 @@
+# see rm_plugin example in swift/rewards/rm_plugin.py
+# register customized plugin in external_plugins file
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B \
+    --dataset AI-MO/NuminaMath-TIR#5000 \
+    --load_from_cache_file true \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs format \
+    --reward_model Qwen/Qwen2.5-3B-Instruct Shanghai_AI_Laboratory/internlm2-7b-reward \
+    --reward_model_plugin genrm my_rmplugin \
+    --reward_weights 0.1 1 1 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --log_completions true \
+    --deepspeed zero2

examples/train/grpo/plugin/run_external_scheduler.sh

@@ -0,0 +1,71 @@
+# This script require main branch ms-swift
+# This script is intended solely as a Tool Calling training example
+# The calculator tool implemented here can perform only basic arithmetic operations and may not be able to solve all math problems in the dataset.
+# Before running this script, please run the following `swift rollout` script first
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-7B-Instruct \
+#     --vllm_use_async_engine true \
+#     --external_plugins examples/train/grpo/plugin/plugin.py \
+#     --multi_turn_scheduler tool_call_scheduler \
+#     --vllm_max_model_len 8192 \
+#     --vllm_gpu_memory_utilization 0.8 \
+#     --max_turns 5
+
+SYSTEM_PROMPT='
+Answer the following questions as best you can. You have access to the following tools:
+
+calculator
+Purpose: Perform basic arithmetic (+, -, *, /, parentheses) and return the result as text.
+Input (single string): the math expression to evaluate, e.g. "2*(3+4)".
+Only digits, spaces, and the characters +-*/(). are allowed.
+
+Use the following format:
+
+Question: the input question you must answer
+Thought: you should always think about what to do
+Action: the action to take, should be one of [calculator]
+Action Input: the input to the action
+Observation: the result of the action
+... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
+Thought: I now know the final answer
+Final Answer: the final answer to the original input question, the answer should be written as \(\boxed{<answer>}\), e.g. \(\boxed{10}\)
+
+Begin!
+'
+
+CUDA_VISIBLE_DEVICES=1,2,3 \
+NPROC_PER_NODE=3 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --reward_funcs accuracy \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --dataset 'AI-MO/NuminaMath-TIR#1000' \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --temperature 0.9 \
+    --system "$SYSTEM_PROMPT" \
+    --log_completions true \
+    --deepspeed zero3 \
+    --stop_words "Observation:" \
+    --report_to swanlab tensorboard

examples/train/grpo/plugin/treepo/tree_rollout.py

@@ -0,0 +1,214 @@
+import re
+import torch
+from copy import deepcopy
+from dataclasses import dataclass, field
+from enum import Enum
+from modelscope.preprocessors.templates.utils import Messages
+from typing import List
+
+from swift.infer_engine.protocol import ChatCompletionResponseChoice
+
+
+class SampleStatus(Enum):
+    INITIAL = 'initial'
+    TO_INFER = 'to_infer'
+    FINISH_NEXT_INFER = 'finish_next_infer'
+    FINISHED = 'finished'
+    ROLLBACK = 'rollback'
+
+
+class FinishedReason(Enum):
+    ANSWER = 'finished_with_answer'
+    MAX_INFER_STEP = 'finished_with_max_infer_steps'
+    UNFINISHED = 'unfinished'
+
+
+@dataclass
+class DataSampleTree:
+    """
+    Attributes:
+        tree_idx (str):
+            for example 0/1-2/2-3/4-0, root_node = 0, next node = 1-2 infer batch 1 and index 2 sample
+
+        last_response (ChatCompletionResponseChoice):
+            vllm previous round output
+    """
+    tree_idx: str
+    request_id: str
+
+    messages: Messages
+    logprobs: List[List[float]] = field(default_factory=list)
+
+    all_response_ids: List[List[int]] = field(default_factory=list)
+    last_response: ChatCompletionResponseChoice = None
+
+    token_count_per_step: List[int] = field(default_factory=list)
+
+    status: SampleStatus = SampleStatus.INITIAL
+    finished_reason: FinishedReason = FinishedReason.UNFINISHED
+
+    @property
+    def root_node(self):
+        return int(self.tree_idx.split('/')[0])
+
+    @property
+    def depth(self):
+        return len(self.tree_idx.split('/')) - 1
+
+    @property
+    def response_num(self):
+        return len(self.all_response_ids)
+
+    def response_truncate(self, truncate_len: int):
+        """
+        Before rollback, truncate the response.
+        """
+
+        if truncate_len < 1:
+            return
+
+        self.logprobs = self.logprobs[:-truncate_len]
+        self.all_response_ids = self.all_response_ids[:-truncate_len]
+        self.messages = self.messages[:-(truncate_len * 2 - 1)]
+        self.last_response = None
+
+    def extend_response(self, choice: ChatCompletionResponseChoice):
+        self.extend_response_text(choice.message.content)
+        self.extend_logprobs([item['logprob'] for item in choice.logprobs['content']])
+
+        self.all_response_ids.append(choice.token_ids)
+        self.token_count_per_step.append(len(choice.token_ids))
+
+        choice.logprobs = None
+        self.last_response = deepcopy(choice)
+
+    def extend_response_text(self, response_text: str):
+        self.messages.append({'role': 'assistant', 'content': response_text})
+
+    def extend_logprobs(self, logprobs: List[float]):
+        self.logprobs.append(logprobs)
+
+
+def _repeat_list_interleave(any_list, repeat_times):
+    # return [item for sublist in [[item] * repeat_times for item in any_list] for item in sublist]
+    return [deepcopy(item) for sublist in [[item] * repeat_times for item in any_list] for item in sublist]
+
+
+def _increment_tree_idx_depth(
+    samples: list[DataSampleTree],
+    next_infer_step: int,
+) -> list[DataSampleTree]:
+    for infer_batch_idx, sample in enumerate(samples):
+        sample.tree_idx = sample.tree_idx + '/' + f'{next_infer_step}-{infer_batch_idx}'
+    return samples
+
+
+def extract_last_boxed(text):
+    pattern = r'\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}'
+
+    matches = list(re.finditer(pattern, text))
+    if matches:
+        return matches[-1].group(0)
+    return None
+
+
+class AbstractDivergence:
+
+    @classmethod
+    def calc_weights(cls, root_idx, samples_to_go_deeper, **kwargs) -> List[float]:
+        pass
+
+    @classmethod
+    def allocate_with_weights(cls, weights, budget, max_divergence) -> List[int]:
+        n = len(weights)
+        alloc = [0] * n
+
+        w = [float(wi) if wi is not None and wi > 0 else 0.0 for wi in weights]
+        total_w = sum(w)
+        if total_w == 0:
+            return alloc
+
+        # first round of allocation by weight ratio
+        ideals = [(w[i] / total_w) * budget if w[i] > 0 else 0.0 for i in range(n)]
+        for i in range(n):
+            if w[i] <= 0:
+                continue
+            f = int(ideals[i])
+            alloc[i] = min(f, max_divergence)
+
+        # second round of allocation by greedy allocation
+        remain = budget - sum(alloc)
+        if remain <= 0:
+            return alloc
+
+        # weights desc, index asc
+        remainders = [(ideals[i] - int(ideals[i]), i) for i in range(n) if w[i] > 0 and alloc[i] < max_divergence]
+        remainders.sort(key=lambda x: (-x[0], x[1]))
+
+        idx = 0
+        while remain > 0 and remainders:
+            frac, i = remainders[idx % len(remainders)]
+            if alloc[i] < max_divergence:
+                alloc[i] += 1
+                remain -= 1
+
+                if alloc[i] >= max_divergence:
+                    remainders = [r for r in remainders if r[1] != i]
+                    idx = 0
+                    continue
+            idx += 1
+
+        return alloc
+
+    @classmethod
+    def apply(cls, root_idx, samples_to_go_deeper, divergence_budget, max_divergence, **kwargs) -> List[DataSampleTree]:
+        """
+        Args:
+            root_idx: current root node idx
+            samples_to_go_deeper: go deeper samples which root_node = root_idx
+            divergence_budget: total divergence
+            max_divergence: each sample max divergence
+        """
+        weights = cls.calc_weights(root_idx, samples_to_go_deeper, **kwargs)
+        allocate_divergence = cls.allocate_with_weights(weights, divergence_budget, max_divergence)
+
+        divergence_samples = []
+        for sample, divergence in zip(samples_to_go_deeper, allocate_divergence):
+            for _ in range(divergence):
+                divergence_samples.append(deepcopy(sample))
+
+        return divergence_samples
+
+
+class LogProbDivergence(AbstractDivergence):
+
+    @classmethod
+    def calc_weights(cls, root_idx, samples_to_go_deeper, **kwargs) -> List[float]:
+        """
+        In this strategy, weight is proportional to entropy
+        """
+        entropies = []
+        for sample in samples_to_go_deeper:
+            log_probs = torch.tensor(sample.logprobs[-1])
+
+            probs = torch.exp(log_probs)
+            entropy = -torch.sum(probs * log_probs)
+            entropies.append(entropy)
+
+        entropies_tensor = torch.stack(entropies)
+        weights = torch.softmax(entropies_tensor, dim=0)
+
+        return weights.tolist()
+
+
+class AvgDivergence(AbstractDivergence):
+
+    @classmethod
+    def calc_weights(cls, root_idx, samples_to_go_deeper, **kwargs) -> List[float]:
+        avg = torch.ones(len(samples_to_go_deeper))
+        weights = torch.softmax(avg, dim=0)
+
+        return weights.tolist()
+
+
+DivergenceStrategyMapping = {'logprobs': LogProbDivergence, 'average': AvgDivergence}

examples/train/grpo/plugin/treepo/tree_rollout.sh

@@ -0,0 +1,50 @@
+# This script is a example for multi-turn training with tree-rollout.
+# Regarding parameter configuration, currently tree_rollout, acting as the inference side, cannot receive relevant training parameters. Please note the following:
+#   1.Ensure that max_tree_width in tree_rollout is equal to num_generations.
+#   2.If DP (Data Parallelism) is enabled during the rollout stage, ensures that data within the same group is allocated to the same inference device.
+#     For example: If generation_batch_size(per_device_batch_size * gradient_accumulation_steps * num_processes) = 32 and num_generations = 8,
+#     then the rollout DP num should equal 4/2/1.
+# For more details on tool invocation, dialogue termination criteria, and other logic, please refer to the TreeRolloutScheduler implementation.
+
+# First: Run swift rollout to deploy rollout server
+CUDA_VISIBLE_DEVICES=0 \
+swift rollout \
+    --model Qwen/Qwen2.5-0.5B \
+    --vllm_use_async_engine true \
+    --external_plugins examples/train/grpo/plugin/treepo/tree_rollout_plugin.py \
+    --multi_turn_scheduler tree_rollout_scheduler \
+    --max_turns 6
+
+
+# Second: Run swift rlhf to train GRPO model
+CUDA_VISIBLE_DEVICES=1 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-0.5B \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --external_plugins examples/train/grpo/plugin/treepo/tree_rollout_plugin.py \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --split_dataset_ratio 0 \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --top_p 0.9 \
+    --top_k 50 \
+    --log_completions true \
+    --num_iterations 1 \
+    --beta 0.04

examples/train/grpo/plugin/treepo/tree_rollout_plugin.py

@@ -0,0 +1,254 @@
+import asyncio
+import json
+import random
+from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait
+from copy import deepcopy
+from tree_rollout import (DataSampleTree, DivergenceStrategyMapping, FinishedReason, SampleStatus,
+                          _increment_tree_idx_depth, _repeat_list_interleave, extract_last_boxed)
+from typing import Any, Dict, List, Optional, Union
+
+from swift.infer_engine import RequestConfig
+from swift.infer_engine.protocol import ChatCompletionResponse, RolloutInferRequest, RolloutOutput
+from swift.rewards import MultiTurnScheduler, multi_turns
+
+
+class TreeRolloutScheduler(MultiTurnScheduler):
+    """
+    Base class for multi-turn tree-rollout scheduling.
+
+    Provides default implementation for multi-turn conversation management.
+
+    CUSTOMIZATION:
+        Implement the required `step()` method and optionally override `check_finished()`
+        - Uses TreeRolloutScheduler's run() method infrastructure
+        - Only need to implement turn transition logic in step()
+        - Optionally customize termination conditions
+
+    Attributes:
+        max_tree_width (int):
+            For GRPO, it must be equal to num_generations.
+        max_tree_depth (int):
+            Controls the maximum number of reasoning turns for a single prompt.
+        root_divergence (int):
+            Number of branches generated in the first-round inference at the root node.
+        max_divergence (int):
+            Maximum number of branches allowed for each node.
+        divergence_strategy (str):
+            Strategy for selecting branch nodes; defaults to logprobs.
+    """
+
+    def __init__(self, infer_engine=None, max_turns=None, *args, **kwargs):
+        super().__init__(infer_engine, max_turns, *args, **kwargs)
+        self.max_tree_width = 8
+        self.max_tree_depth = max_turns | 6
+        self.max_divergence = 2
+        self.divergence_strategy = 'logprobs'
+        self.root_divergence = 1
+
+        self.executor = ThreadPoolExecutor(max_workers=self.max_tree_width)
+
+    async def async_infer(self,
+                          infer_requests: List[Union['RolloutInferRequest', Dict[str, Any]]],
+                          request_config: 'RequestConfig',
+                          *,
+                          use_tqdm: Optional[bool] = None,
+                          **kwargs) -> List['RolloutOutput']:
+        # dedup_requests_by_messages
+        processed_request = []
+        seen = set()
+        uuids = []
+
+        for item in infer_requests:
+            if isinstance(item, dict):
+                req = RolloutInferRequest(**item)
+            else:
+                req = item
+
+            msg_key = json.dumps(req.messages, sort_keys=True)
+            uuids.append(req.uuid)
+
+            if msg_key not in seen:
+                seen.add(msg_key)
+                processed_request.append(req)
+
+        request_config.logprobs = True
+
+        outputs = await super().async_infer(processed_request, request_config, use_tqdm=use_tqdm, **kwargs)
+
+        assert len(outputs) == len(uuids), '[Tree Rollout] Please check the max_tree_width is equal to num_generations.'
+
+        for idx, output in enumerate(outputs):
+            output.response.id = uuids[idx]
+
+        return outputs
+
+    async def run(self, infer_request: Union[List[RolloutInferRequest], RolloutInferRequest],
+                  request_config: 'RequestConfig', **kwargs) -> List['RolloutOutput']:
+        if isinstance(infer_request, RolloutInferRequest):
+            infer_request = [infer_request]
+        else:
+            infer_request = list(infer_request)
+
+        request_config.logprobs = True
+
+        finished_rollout_by_root: Dict[int, List[RolloutOutput]] = {i: [] for i in range(len(infer_request))}
+        finished_samples: Dict[int, List[DataSampleTree]] = {i: [] for i in range(len(infer_request))}
+
+        samples_to_infer = []
+
+        for root_idx in range(len(infer_request)):
+            samples_to_infer.append(
+                DataSampleTree(
+                    tree_idx=str(root_idx),
+                    request_id=infer_request[root_idx].uuid,
+                    messages=infer_request[root_idx].messages,
+                    status=SampleStatus.TO_INFER))
+
+        # first step
+        next_infer_step = 1
+        samples_to_infer = _repeat_list_interleave(samples_to_infer, self.root_divergence)
+        samples_to_infer = _increment_tree_idx_depth(samples_to_infer, next_infer_step)
+
+        while len(samples_to_infer) > 0:
+            # resolve the error: Request id xxx already running
+            vllm_inputs = [
+                RolloutInferRequest(messages=sample.messages, uuid=f'{sample.request_id}-{sample.tree_idx}')
+                for sample in samples_to_infer
+            ]
+
+            # Get model response
+            tasks = [self.infer_engine.infer_async(request, request_config, **kwargs) for request in vllm_inputs]
+            outputs: List[ChatCompletionResponse] = await asyncio.gather(*tasks)
+
+            assert len(vllm_inputs) == len(
+                outputs), f'outputs length {len(outputs)} != inputs length {len(vllm_inputs)}'
+
+            samples_last_step = deepcopy(samples_to_infer)
+            samples_to_infer = []
+
+            for idx, (sample, output) in enumerate(zip(samples_last_step, outputs)):
+                assert len(output.choices) == 1, 'vllm should only generate one output'
+                self.check_finished(sample, output)
+
+                # bind the output and request
+                output.id = sample.request_id
+                choice = output.choices[0]
+                child_sample = deepcopy(sample)
+                child_sample.extend_response(choice)
+
+                if child_sample.status == SampleStatus.FINISHED:
+                    finished_samples[child_sample.root_node].append(child_sample)
+                    finished_rollout_by_root[child_sample.root_node].append(
+                        RolloutOutput(
+                            response=output,
+                            messages=deepcopy(child_sample.messages),
+                            response_token_ids=deepcopy(child_sample.all_response_ids),
+                            # If we use intermediate reasoning results when computing the reward,
+                            # but loss_mask is not explicitly set,
+                            # only the loss of the final round of reasoning will be computed.
+                            response_loss_mask=[[1] * len(response_ids)
+                                                for response_ids in child_sample.all_response_ids],
+                            rollout_infos={'num_turns': next_infer_step},
+                        ))
+                else:
+                    samples_to_infer.append(child_sample)
+
+            # if we have budget, do divergence
+            if len(samples_to_infer) > 0 and self.max_divergence > 1:
+                for root_idx in finished_samples.keys():
+                    root_to_infer_samples = [sample for sample in samples_to_infer if sample.root_node == root_idx]
+                    root_finished_samples = finished_samples[root_idx]
+
+                    budget = self.max_tree_width - len(root_finished_samples) - len(root_to_infer_samples)
+
+                    if budget > 0 and len(root_to_infer_samples) > 0:
+                        divergence_executor = DivergenceStrategyMapping[self.divergence_strategy]
+                        if not divergence_executor:
+                            raise ValueError(
+                                f"[Tree Rollout] The divergence strategy: {self.divergence_strategy} doesn't exist.")
+
+                        divergence_samples = divergence_executor.apply(root_idx, root_to_infer_samples, budget,
+                                                                       self.max_divergence - 1)
+                        samples_to_infer.extend(divergence_samples)
+
+            # before end loop, if finished_count < max_tree_width, rollback
+            if len(samples_to_infer) == 0 and any(count < self.max_tree_width
+                                                  for count in [len(value) for value in finished_samples.values()]):
+                samples_to_infer = self.roll_back_to_divergence(finished_samples)
+
+            # tools call etc
+            futures = [self.executor.submit(self.step, sample) for sample in samples_to_infer]
+            wait(futures, return_when=ALL_COMPLETED)
+
+            next_infer_step += 1
+            samples_to_infer = _increment_tree_idx_depth(samples_to_infer, next_infer_step)
+
+        # flatten finished outputs
+        return [traj for lst in finished_rollout_by_root.values() for traj in lst]
+
+    def step(self, sample: DataSampleTree, **kwargs):
+        """
+        You need to rewrite or modify this method to customize the next round of prompts, such as tools call.
+        """
+
+        # Special handling has already been done in the rollback.
+        if sample.status == SampleStatus.ROLLBACK:
+            sample.status = SampleStatus.TO_INFER
+            return
+        elif sample.status == SampleStatus.FINISH_NEXT_INFER:
+            prompt = 'In this round of responses, you must generate an answer.'
+        else:
+            prompt = 'The answer is not correct, It seems You made a mistake, you need to recheck very carefully.'
+
+        sample.messages.append({'role': 'user', 'content': prompt})
+
+    def check_finished(self, sample: DataSampleTree, output: ChatCompletionResponse, **kwargs) -> bool:
+        """
+        Rewrite this method to add custom check logic
+        """
+
+        boxed_answer = extract_last_boxed(output.choices[0].message.content)
+
+        if boxed_answer is not None:
+            sample.status = SampleStatus.FINISHED
+            sample.finished_reason = FinishedReason.ANSWER
+
+        elif sample.status == SampleStatus.FINISH_NEXT_INFER:
+            sample.status = SampleStatus.FINISHED
+            sample.finished_reason = FinishedReason.MAX_INFER_STEP
+
+        elif sample.depth >= self.max_tree_depth - 1:
+            sample.status = SampleStatus.FINISH_NEXT_INFER
+
+        return sample.status == SampleStatus.FINISHED
+
+    def roll_back_to_divergence(
+        self,
+        finished_samples: Dict[int, List[DataSampleTree]],
+    ) -> List[DataSampleTree]:
+        """
+        All nodes have completed inference, but there is still budget available, rollback.
+        """
+
+        sample_to_infer = []
+        for root_idx, sample_list in finished_samples.items():
+            if len(sample_list) >= self.max_tree_width:
+                continue
+
+            diff_count = self.max_tree_width - len(sample_list)
+            result = random.sample(sample_list, min(diff_count, len(sample_list)))
+
+            result_copy = deepcopy(result)
+
+            # Randomly rollback several inference iterations; The rollback strategy can be optimized subsequently.
+            for sample in result_copy:
+                sample.status = SampleStatus.ROLLBACK
+                truncate_len = sample.response_num
+                sample.response_truncate(random.randint(1, truncate_len))
+
+            sample_to_infer.extend(result_copy)
+
+        return sample_to_infer
+
+
+multi_turns['tree_rollout_scheduler'] = TreeRolloutScheduler

examples/train/grpo/prompt.txt

@@ -0,0 +1 @@
+A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within <think> </think> and <answer> </answer> tags, respectively, i.e., <think> reasoning process here </think><answer> answer here </answer>

examples/train/grpo/qwen2_5_omni/grpo.sh

@@ -0,0 +1,42 @@
+# 4 * 50GiB
+pip install transformers math_verify trl -U
+
+MAX_PIXELS=1003520 \
+NPROC_PER_NODE=4 \
+ENABLE_AUDIO_OUTPUT=1 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-Omni-7B \
+    --reward_funcs external_r1v_acc format \
+    --reward_weights 1 0.5 \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --torch_dtype bfloat16 \
+    --dataset lmms-lab/multimodal-open-r1-8k-verified#1000 \
+    --load_from_cache_file true \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 2 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1. \
+    --top_p 0.99 \
+    --top_k 50 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true

examples/train/grpo/qwen2_5_omni/infer.sh

@@ -0,0 +1,7 @@
+MAX_PIXELS=1003520 \
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters vx-xxx/checkpoint-xxx \
+    --load_data_args true \
+    --stream true \
+    --max_new_tokens 2048

examples/train/infer.sh

@@ -0,0 +1,8 @@
+# If it's full parameter training, use `--model xxx` instead of `--adapters xxx`.
+# If you are using the validation set for inference, add the parameter `--load_data_args true`.
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --stream true \
+    --temperature 0 \
+    --max_new_tokens 2048

examples/train/liger/sft.sh

@@ -0,0 +1,33 @@
+# test env: 4 * A100
+# Using use_liger_kernel and packing: 4 * 42GB, 1 hour 35 minutes
+# Not using use_liger_kernel: 4 * 54GB, 1 hour 40 minutes
+# Not using use_liger_kernel and packing: 4 * 52GB, 3 hours 30 minutes
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --dataset 'liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT#10000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --num_train_epochs 5 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output/Qwen2.5-7B \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --packing true \
+    --use_liger_kernel true

examples/train/lora_sft.sh

@@ -0,0 +1,30 @@
+# 22GB
+# qwen3: https://github.com/modelscope/ms-swift/blob/main/examples/train/think_model/qwen3_demo1.sh
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataset_num_proc 4 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/moe/llama4.sh

@@ -0,0 +1,32 @@
+# If you don't want to train the router, set:
+# `--target_regex '^(language_model).*\.(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)$'`
+NPROC_PER_NODE=4 \
+USE_HF=1 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift sft \
+    --model meta-llama/Llama-4-Scout-17B-16E-Instruct \
+    --dataset 'linxy/LaTeX_OCR:full#5000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --router_aux_loss_coef 1e-3 \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 4 \
+    --gradient_checkpointing true \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --deepspeed zero3 \
+    --dataloader_num_workers 4

examples/train/moe/qwen3_moe.sh

@@ -0,0 +1,34 @@
+# If you don't want to train the router, set:
+# `--target_modules q_proj k_proj v_proj o_proj gate_proj up_proj down_proj`
+
+# Note: If you need to use DeepSpeed ZeRO-2/ZeRO-3 but encounter hangs
+# try using transformers==4.51.3
+
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --tuner_type lora \
+    --dataset 'swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT#2000' \
+              'swift/self-cognition#1000' \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --router_aux_loss_coef 1e-3 \
+    --experts_impl grouped_mm \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-gpu/ddp/train.sh

@@ -0,0 +1,30 @@
+# 27.5GiB * 2
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}'

examples/train/multi-gpu/ddp_device_map/train.sh

@@ -0,0 +1,30 @@
+# 14GiB * 4
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}'

examples/train/multi-gpu/deepspeed/train_zero2.sh

@@ -0,0 +1,30 @@
+# 18GiB * 2
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero2

examples/train/multi-gpu/deepspeed/train_zero3.sh

@@ -0,0 +1,30 @@
+# 16GiB * 2
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero3

examples/train/multi-gpu/device_map/train.sh

@@ -0,0 +1,28 @@
+# 2 * 76GiB
+CUDA_VISIBLE_DEVICES=0,1 \
+MAX_PIXELS=1003520 \
+swift sft \
+    --model Qwen/Qwen2.5-VL-72B-Instruct \
+    --dataset 'modelscope/coco_2014_caption:validation#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4

examples/train/multi-gpu/fsdp2_lora/fsdp2.json

@@ -0,0 +1,25 @@
+{
+  "compute_environment": "LOCAL_MACHINE",
+  "debug": false,
+  "distributed_type": "FSDP",
+  "downcast_bf16": "no",
+  "fsdp_config": {
+    "fsdp_auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+    "fsdp_cpu_ram_efficient_loading": true,
+    "fsdp_reshard_after_forward": true,
+    "fsdp_state_dict_type": "FULL_STATE_DICT",
+    "fsdp_activation_checkpointing": true,
+    "fsdp_version": 2
+  },
+  "machine_rank": 0,
+  "main_training_function": "main",
+  "mixed_precision": "bf16",
+  "num_machines": 1,
+  "num_processes": 2,
+  "rdzv_backend": "static",
+  "same_network": true,
+  "tpu_env": [],
+  "tpu_use_cluster": false,
+  "tpu_use_sudo": false,
+  "use_cpu": false
+}

examples/train/multi-gpu/fsdp2_lora/train.sh

@@ -0,0 +1,32 @@
+# 14.7GiB * 2
+# NOTE: for swift>=3.12, you can use --fsdp fsdp2 instead of accelerate launch
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+accelerate launch --config_file "./examples/train/multi-gpu/fsdp2_lora/fsdp2.json" \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_checkpointing false \
+    --weight_decay 0.1 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-gpu/fsdp_qlora/fsdp_offload.json

@@ -0,0 +1,28 @@
+{
+  "compute_environment": "LOCAL_MACHINE",
+  "debug": false,
+  "distributed_type": "FSDP",
+  "downcast_bf16": "no",
+  "fsdp_config": {
+    "fsdp_auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+    "fsdp_backward_prefetch": "BACKWARD_PRE",
+    "fsdp_cpu_ram_efficient_loading": true,
+    "fsdp_forward_prefetch": false,
+    "fsdp_offload_params": true,
+    "fsdp_sharding_strategy": "FULL_SHARD",
+    "fsdp_state_dict_type": "FULL_STATE_DICT",
+    "fsdp_sync_module_states": true,
+    "fsdp_use_orig_params": false
+  },
+  "machine_rank": 0,
+  "main_training_function": "main",
+  "mixed_precision": "no",
+  "num_machines": 1,
+  "num_processes": 2,
+  "rdzv_backend": "static",
+  "same_network": true,
+  "tpu_env": [],
+  "tpu_use_cluster": false,
+  "tpu_use_sudo": false,
+  "use_cpu": false
+}

examples/train/multi-gpu/fsdp_qlora/train.sh

@@ -0,0 +1,36 @@
+# 80GiB * 2
+# NOTE: for swift>=3.12, you can use --fsdp fsdp2 instead of accelerate launch
+
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+accelerate launch --config_file "./examples/train/multi-gpu/fsdp_qlora/fsdp_offload.json" \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-72B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --quant_bits 4 \
+    --bnb_4bit_compute_dtype bfloat16 \
+    --bnb_4bit_quant_storage bfloat16 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_checkpointing true \
+    --weight_decay 0.1 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/accelerate/multi_node.yaml

@@ -0,0 +1,17 @@
+compute_environment: LOCAL_MACHINE
+deepspeed_config:
+    deepspeed_multinode_launcher: standard
+    gradient_accumulation_steps: 16
+    offload_optimizer_device: none
+    offload_param_device: none
+    zero3_init_flag: false
+    zero_stage: 3
+distributed_type: DEEPSPEED
+main_process_ip: 'xxx.xxx.xxx.xxx'
+main_process_port: 29500
+main_training_function: main
+mixed_precision: bf16
+num_machines: 2
+num_processes: 8  # world size
+rdzv_backend: static
+use_cpu: false

examples/train/multi-node/accelerate/train_node1.sh

@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+accelerate launch --config_file ./examples/train/multi-node/accelerate/multi_node.yaml --machine_rank 0 \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/accelerate/train_node2.sh

@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+accelerate launch --config_file ./examples/train/multi-node/accelerate/multi_node.yaml --machine_rank 1 \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/deepspeed/README.md

@@ -0,0 +1,42 @@
+# How to run
+
+## 1. Install pdsh in your nodes
+
+```shell
+# https://code.google.com/archive/p/pdsh/downloads
+# For example, download to /root:
+cd /root
+wget https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pdsh/pdsh-2.29.tar.bz2
+tar -xvf pdsh-2.29.tar.bz2
+cd pdsh-2.29
+./configure --prefix=/root/pdsh-2.29 --with-ssh --without-rsh --with-exec --with-timeout=60 --with-nodeupdown --with-rcmd-rank-list=ssh
+make
+make install
+```
+
+In case of the privilege is correct:
+```shell
+chown root:root /root/pdsh-2.29
+```
+
+## Configure the ssh
+
+vim your ~/.ssh/config and input:
+```text
+Host worker-0
+    HostName your-worker-0-ip-here
+    User root
+Host worker-1
+    HostName your-worker-1-ip-here
+    User root
+```
+Say you have two nodes, when doing this, make sure your other nodes can be logined with `ssh root@worker-x` without password(with ssh-key).
+
+## Clone swift repo and run
+
+```shell
+git clone https://github.com/modelscope/ms-swift.git
+cd ms-swift
+# If your node number is different, edit examples/train/multi-node/deepspeed/host.txt
+sh examples/train/multi-node/deepspeed/train.sh
+```

examples/train/multi-node/deepspeed/host.txt

@@ -0,0 +1,2 @@
+worker-0 slots=2
+worker-1 slots=2

examples/train/multi-node/deepspeed/train.sh

@@ -0,0 +1,20 @@
+# If your need only a part of the GPUs in every node, try:
+# --include="worker-0:0,1@worker-1:2,3"
+deepspeed --hostfile=./examples/train/multi-node/deepspeed/host.txt \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --load_from_cache_file true \
+    --num_train_epochs 1 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/dlc/train.sh

@@ -0,0 +1,27 @@
+# For more information, visit: https://www.aliyun.com/activity/bigdata/pai-dlc
+# https://help.aliyun.com/zh/pai/user-guide/general-environment-variables
+NNODES=$WORLD_SIZE \
+NODE_RANK=$RANK \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2

examples/train/multi-node/ray/sft.sh

@@ -0,0 +1 @@
+swift sft examples/train/multi-node/ray/sft.yaml