chore: import upstream snapshot with attribution

examples/train/grpo/internal/README.md

@@ -0,0 +1,20 @@
+# README: GRPO Internal(Colocate) Mode Execution Scripts
+
+---
+**NOTE**
+
+## **Introduction**
+
+The GRPO (Group Relative Policy Optimization) training framework supports high-performance inference engines like vLLM to accelerate the sampling process. The **Internal Mode** allows you to deploy vLLM and perform training using the same GPU resources.
+
+This folder contains scripts and instructions for running GRPO in **Internal Mode**
+
+## Training with Internal mode
+```bash
+--use_vllm true \
+--vllm_mode colocate \
+--vllm_gpu_memory_utilization [ut_ratio] \
+```
+
+## Multi-Node Training
+On each node, execute the original single-node training script, using the environment variables `NNODES` and `NODE_RANK`, and ensure consistent use of configuration parameters across all nodes.

examples/train/grpo/internal/chord.sh

@@ -0,0 +1,53 @@
+# 8*80G GPU
+# CHORD https://arxiv.org/abs/2508.11408
+# GRPO total batch = 32(prompts)*8(num_generations) = 256 =  8(gpus) * 4(per_device_train_batch_size) * 8(gradient_accumulation_steps)
+# SFT total batch = 64 = 8(gpus) * 1(chord_sft_per_device_train_batch_size) * 8(gradient_accumulation_steps)
+
+# NOTE: We use the same dataset for GRPO and SFT, which may cause overlap (i.e., the same examples to be selected).
+# You can pre-download the dataset and manually split it to avoid this.
+
+export CHORD_SYSTEM_PROMPT="You are a helpful assistant that solves MATH problems.
+You should first think about the reasoning process in mind and then provide the user with the answer.
+You should present your reasoning process using the format: <think>\n...your reasoning process here... </think>\n"
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --dataset AI-MO/NuminaMath-TIR \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --beta 0.0 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --chord_sft_per_device_train_batch_size 1 \
+    --chord_sft_dataset AI-MO/NuminaMath-TIR \
+    --chord_enable_phi_function false \
+    --chord_mu_warmup_steps 0 \
+    --chord_mu_decay_steps 200 \
+    --chord_mu_peak 0.9 \
+    --chord_mu_valley 0.05 \
+    --num_generations 8 \
+    --tuner_type full \
+    --reward_funcs accuracy \
+    --system "$CHORD_SYSTEM_PROMPT" \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_max_model_len 8192 \
+    --max_completion_length 4096 \
+    --overlong_filter true \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --report_to tensorboard swanlab

examples/train/grpo/internal/fipo.sh

@@ -0,0 +1,46 @@
+CUDA_VISIBLE_DEVICES=2 \
+swift rollout \
+    --model Qwen/Qwen2.5-1.5B-Instruct
+
+# 2 GPUS for sequence parallel
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --dataset 'AI-MO/NuminaMath-TIR'  \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --load_from_cache_file true \
+    --max_completion_length 4096 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --save_total_limit 3 \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 8 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --system """You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}.""" \
+    --log_completions true \
+    --num_iterations 3 \
+    --padding_free true \
+    --sequence_parallel_size 2 \
+    --attn_impl flash_attn \
+    --beta 0 \
+    --dynamic_sample true \
+    --loss_type fipo \
+    --delta 10.0 \
+    --epsilon_high 0.28 \
+    --fipo_decay_rate 32 \
+    --fipo_clip_range 0.2 \
+    --fipo_clip_high_only false \
+    --fipo_safety_threshold 3.0

examples/train/grpo/internal/full_lmdeploy.sh

@@ -0,0 +1,3 @@
+# The LMDeploy backend in GRPO has been deprecated in Swift 3.5.
+# You can install Swift 3.4 to continue using it with the following script:
+# https://github.com/modelscope/ms-swift/blob/v3.4.1/examples/train/grpo/internal/full_lmdeploy.sh

examples/train/grpo/internal/gspo.sh

@@ -0,0 +1,43 @@
+# 8*80G GPU
+# GSPO https://arxiv.org/pdf/2507.18071
+# hyperparameter
+# - epsilon = 3e-4 from paper serction 5.1
+# - epsilon_high = 4e-4 from paper serction 5.1
+# - steps_per_generation = 32 (= 4 * gradient_accumulation_steps): paper section 5.1 partitions each batch of rollout data into four minibatches for gradient updates; in swift HF GRPO, steps_per_generation counts micro-batches, so it must be multiplied by gradient_accumulation_steps
+# - beta = 0: zero kl regularization https://github.com/volcengine/verl/pull/2775#issuecomment-3131807306
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --beta 0.0 \
+    --epsilon 3e-4 \
+    --epsilon_high 4e-4 \
+    --steps_per_generation 32 \
+    --importance_sampling_level sequence \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --gradient_accumulation_steps 8 \
+    --num_generations 16 \
+    --tuner_type full \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.6 \
+    --vllm_max_model_len 16384 \
+    --max_completion_length 8192 \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero3 \
+    --log_completions true

examples/train/grpo/internal/moe_full.sh

@@ -0,0 +1,40 @@
+# 8*80G
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_max_model_len 16384 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --max_length 12000 \
+    --max_completion_length 8192 \
+    --overlong_filter true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_strategy 'steps' \
+    --eval_strategy 'steps' \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --deepspeed zero3_offload \
+    --log_completions true \
+    --sleep_level 1 \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --move_model_batches 10

examples/train/grpo/internal/moe_lora.sh

@@ -0,0 +1,42 @@
+# 8*80G
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-30B-A3B-Instruct-2507 \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_max_model_len 16384 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --max_length 12000 \
+    --max_completion_length 8192 \
+    --overlong_filter true \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --save_strategy 'steps' \
+    --eval_strategy 'steps' \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --move_model_batches 10

examples/train/grpo/internal/qlora.sh

@@ -0,0 +1,49 @@
+SYSTEM_PROMPT="You are a helpful math assistant. Solve the problem step by step. Show your reasoning in <think> </think> tags, then give the final numerical answer after ####.
+For example:
+<think> ... reasoning ... </think>
+#### 42"
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/gsm8k/gsm8k_plugin.py \
+    --reward_funcs gsm8k_accuracy gsm8k_format \
+    --columns '{"answer": "solution"}' \
+    --enable_thinking false \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 10240 \
+    --vllm_enable_lora true \
+    --sleep_level 1 \
+    --tuner_type lora \
+    --quant_method bnb \
+    --quant_bits 4 \
+    --bnb_4bit_quant_type nf4 \
+    --torch_dtype bfloat16 \
+    --dataset 'modelscope/gsm8k' \
+    --load_from_cache_file true \
+    --max_length 2048 \
+    --max_completion_length 8192 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --learning_rate 1e-5 \
+    --lr_scheduler_type cosine \
+    --save_steps 10 \
+    --save_total_limit 100 \
+    --logging_steps 1 \
+    --warmup_ratio 0.0 \
+    --dataloader_num_workers 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --system "$SYSTEM_PROMPT" \
+    --deepspeed zero2 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --max_grad_norm 1.0 \
+    --epsilon 0.2 \
+    --epsilon_high 0.28 \
+    --scale_rewards none

examples/train/grpo/internal/real.sh

@@ -0,0 +1,38 @@
+CUDA_VISIBLE_DEVICES=2 \
+swift rollout \
+    --model Qwen/Qwen3-1.7B
+
+
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen3-1.7B \
+    --dataset 'AI-MO/NuminaMath-TIR#5000'  \
+    --enable_thinking false \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --load_from_cache_file true \
+    --max_completion_length 4096 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --learning_rate 2e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_total_limit 2 \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --num_generations 8 \
+    --temperature 0.6 \
+    --system """You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}.""" \
+    --log_completions true \
+    --num_iterations 1 \
+    --beta 0.001 \
+    --loss_type real \
+    --deepspeed zero2

examples/train/grpo/internal/reinforce_plus_plus.sh

@@ -0,0 +1,49 @@
+# Reinforce++-Baseline in https://arxiv.org/abs/2501.03262
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+MASTER_PORT=29900 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --advantage_estimator reinforce_plus_plus \
+    --scale_rewards batch \
+    --kl_in_reward true \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --dataset 'AI-ModelScope/clevr_cogen_a_train' \
+    --reward_funcs external_r1v_acc format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.8 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 16384 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --overlong_filter false \
+    --importance_sampling_level sequence \
+    --epsilon 3e-4 \
+    --epsilon_high 4e-4 \
+    --max_completion_length 1024 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero1 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --async_generate false \
+    --beta 0.001 \
+    --attn_impl flash_attention_2

examples/train/grpo/internal/rloo.sh

@@ -0,0 +1,45 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --advantage_estimator rloo \
+    --kl_in_reward true \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs external_r1v_acc format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.4 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 16384 \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-ModelScope/clevr_cogen_a_train' \
+    --overlong_filter false \
+    --epsilon 3e-4 \
+    --epsilon_high 4e-4 \
+    --max_completion_length 1024 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --save_total_limit 10 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --num_generations 16 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --num_iterations 1 \
+    --async_generate false \
+    --beta 0.001 \
+    --attn_impl flash_attention_2 \
+    --padding_free true \
+    --loss_type grpo

examples/train/grpo/internal/sapo.sh

@@ -0,0 +1,43 @@
+# SAPO https://arxiv.org/abs/2511.20347
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+MAX_PIXELS=602112 \
+swift rlhf \
+    --rlhf_type grpo \
+    --loss_type sapo \
+    --tau_pos 1 \
+    --tau_neg 1.05 \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --external_plugins examples/train/grpo/plugin/plugin.py \
+    --reward_funcs external_r1v_acc format \
+    --learning_rate 1e-6 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.6 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len 8192 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-ModelScope/clevr_cogen_a_train \
+    --overlong_filter false \
+    --importance_sampling_level token \
+    --max_length 4096 \
+    --max_completion_length 4096 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --num_generations 8 \
+    --steps_per_generation 32 \
+    --save_steps 1000 \
+    --sleep_level 1 \
+    --offload_model true \
+    --offload_optimizer true \
+    --logging_steps 1 \
+    --dataloader_num_workers 4 \
+    --temperature 1.0 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero1 \
+    --log_completions true \
+    --report_to tensorboard swanlab \
+    --beta 0.001 \
+    --attn_impl flash_attention_2

examples/train/grpo/internal/transformers.sh

@@ -0,0 +1,37 @@
+# pip install math_verify # reward function
+# pip install -U trl
+# GPU memory: 80GiB
+# You can set `--reward_model` to use a reward model to provide rewards.
+# TransformersEngine to rollout
+
+CUDA_VISIBLE_DEVICES=0 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B \
+    --reward_funcs accuracy format \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-MO/NuminaMath-TIR#1000' \
+    --load_from_cache_file true \
+    --max_completion_length 1024 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --temperature 0.9 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --log_completions true

examples/train/grpo/internal/vllm_72b_4gpu.sh

@@ -0,0 +1,42 @@
+# 4*80G GPU
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-72B-Instruct \
+    --tuner_type lora \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.55 \
+    --vllm_max_model_len 2048 \
+    --vllm_tensor_parallel_size 4 \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 1024 \
+    --reward_funcs accuracy format \
+    --num_generations 4 \
+    --system examples/train/grpo/prompt.txt \
+    --deepspeed zero3_offload \
+    --temperature 1.0 \
+    --top_p 1.0 \
+    --top_k 80 \
+    --log_completions true \
+    --move_model_batches 16 \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1

examples/train/grpo/internal/vllm_lora_qwenvl72b.sh

@@ -0,0 +1,48 @@
+# pip install math_verify # reward function
+# GPU memory: 8 * 80GiB
+
+MAX_PIXELS=602112 \
+WANDB_API_KEY=xxx \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+  --rlhf_type grpo \
+  --model Qwen/Qwen2.5-VL-72B-Instruct \
+  --tuner_type lora \
+  --use_vllm true \
+  --vllm_mode colocate \
+  --vllm_gpu_memory_utilization 0.5 \
+  --vllm_max_model_len 8192 \
+  --vllm_tensor_parallel_size 4 \
+  --dataset lmms-lab/multimodal-open-r1-8k-verified#1000 \
+  --load_from_cache_file true \
+  --external_plugins examples/train/grpo/plugin/plugin.py \
+  --reward_funcs external_r1v_acc format \
+  --reward_weights 1 0.1 \
+  --torch_dtype bfloat16 \
+  --attn_impl flash_attn \
+  --num_train_epochs 1 \
+  --max_length 8192 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 1 \
+  --gradient_accumulation_steps 1 \
+  --eval_steps 500 \
+  --save_steps 500 \
+  --learning_rate 1e-6 \
+  --save_total_limit 2 \
+  --logging_steps 1 \
+  --warmup_ratio 0.05 \
+  --dataloader_num_workers 4 \
+  --max_completion_length 2048 \
+  --num_generations 8 \
+  --deepspeed zero3 \
+  --temperature 1.1 \
+  --top_p 1.0 \
+  --top_k 80 \
+  --log_completions true \
+  --offload_optimizer true \
+  --offload_model true \
+  --move_model_batches 40 \
+  --sleep_level 1 \
+  --report_to wandb \
+  --system examples/train/grpo/prompt.txt

examples/train/grpo/internal/vllm_multi_turn.sh

@@ -0,0 +1,43 @@
+# It's recommended to use server mode for multi-turn training
+# Colocate multi-turn does not support rollouts with dynamic rollout outputs
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --tuner_type full \
+    --reward_funcs accuracy \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_max_model_len 2048 \
+    --vllm_tensor_parallel_size 4 \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 2048 \
+    --num_generations 32 \
+    --deepspeed zero3 \
+    --temperature 1.0 \
+    --top_p 1.0 \
+    --top_k 80 \
+    --log_completions true \
+    --offload_optimizer true \
+    --offload_model true \
+    --sleep_level 1 \
+    --multi_turn_scheduler math_tip_trick \
+    --max_turns 3

examples/train/grpo/internal/vllm_vl7b.sh

@@ -0,0 +1,30 @@
+MAX_PIXELS=1003520 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --tuner_type lora \
+    --dataset AI-ModelScope/chartqa_digit_r1v_format \
+    --load_from_cache_file true \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_tensor_parallel_size 4 \
+    --torch_dtype bfloat16 \
+    --system examples/train/grpo/prompt.txt \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --gradient_accumulation_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 1024 \
+    --reward_funcs accuracy format \
+    --num_generations 8 \
+    --sleep_level 1 \
+    --temperature 1.0 \
+    --top_p 0.85