chore: import upstream snapshot with attribution

examples/train/rlhf/README.md

@@ -0,0 +1,3 @@
+# TIPS
+
+Multi-modal models' RLHF are also supported! Check the multimodal folder for details.

examples/train/rlhf/cpo.sh

@@ -0,0 +1,30 @@
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type cpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4

examples/train/rlhf/dpo/full.sh

@@ -0,0 +1,31 @@
+# with padding_free: 4 * 47GiB, 1.90s/it
+# without padding_free: 4 * 57GiB 3.32s/it
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type dpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --rpo_alpha 0.1 \
+    --padding_free true

examples/train/rlhf/dpo/lora.sh

@@ -0,0 +1,30 @@
+# 24GiB
+# It is recommended to use padding_free. For more details, please refer to:
+# https://github.com/modelscope/ms-swift/blob/main/examples/train/padding_free/dpo.sh
+CUDA_VISIBLE_DEVICES=0 \
+swift rlhf \
+    --rlhf_type dpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --rpo_alpha 0.1 \
+    --dataset_num_proc 4

examples/train/rlhf/gkd/fast.sh

@@ -0,0 +1,48 @@
+
+export teacher_model='Qwen/Qwen2.5-14B-Instruct'
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model $teacher_model \
+    --infer_backend vllm \
+    --val_dataset 'AI-ModelScope/alpaca-gpt4-data-en#5000' 'AI-ModelScope/alpaca-gpt4-data-zh#5000' \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --write_batch_size 10000 \
+    --result_path new_dataset.jsonl
+
+
+# 4 * 56GiB, 1.55s/it
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model $teacher_model \
+    --tuner_type full \
+    --dataset 'new_dataset.jsonl' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --padding_free true \
+    --attn_impl flash_attn \
+    --lmbda 0

examples/train/rlhf/gkd/full.sh

@@ -0,0 +1,30 @@
+# 4 * 66GiB, 46s/it
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model Qwen/Qwen2.5-14B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-en#2000' 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --max_completion_length 512 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn

examples/train/rlhf/gkd/multi_turn.sh

@@ -0,0 +1,41 @@
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3.5-2B \
+    --teacher_model Qwen/Qwen3.5-9B \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/NuminaMath-TIR#5000' \
+    --lmbda 1 \
+    --beta 0.5 \
+    --temperature 1.0 \
+    --torch_dtype bfloat16 \
+    --max_steps 200 \
+    --per_device_train_batch_size 4 \
+    --learning_rate 5e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_steps 200 \
+    --save_total_limit 1 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --max_completion_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --teacher_deepspeed zero3_offload \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.3 \
+    --vllm_max_model_len 8192 \
+    --sleep_level 1 \
+    --enable_thinking false \
+    --loss_scale last_round \
+    --multi_turn_scheduler math_tip_trick \
+    --max_turns 2 \
+    --truncation_strategy delete \
+    --log_completions true

examples/train/rlhf/gkd/teacher_server.sh

@@ -0,0 +1,51 @@
+top_k=64
+max_prompt_length=2048
+max_completion_length=2048
+max_total_length=$((max_prompt_length + max_completion_length))
+
+export IMAGE_MAX_TOKEN_NUM=1024
+
+# Teacher server must be running first:
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift deploy \
+#     --model Qwen/Qwen3.5-4B \
+#     --infer_backend vllm \
+#     --port 8000 \
+#     --max_logprobs $top_k \
+#     --max_length $max_total_length \
+#     --vllm_max_model_len $max_total_length
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3.5-4B \
+    --teacher_model_server http://localhost:8000 \
+    --gkd_logits_topk $top_k \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len $max_total_length \
+    --sleep_level 0 \
+    --dataset 'modelscope/gsm8k' \
+    --lmbda 1 \
+    --beta 0.5 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 2 \
+    --gradient_accumulation_steps 4 \
+    --learning_rate 5e-5 \
+    --logging_steps 1 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --max_length $max_prompt_length \
+    --max_completion_length $max_completion_length \
+    --warmup_ratio 0.1 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --temperature 1.0 \
+    --attn_impl flash_attn \
+    --report_to tensorboard swanlab

examples/train/rlhf/gkd/think_model.sh

@@ -0,0 +1,50 @@
+
+export teacher_model='Qwen/Qwen3-8B'
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model $teacher_model \
+    --infer_backend vllm \
+    --val_dataset 'AI-ModelScope/alpaca-gpt4-data-en#5000' 'AI-ModelScope/alpaca-gpt4-data-zh#5000' \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --write_batch_size 10000 \
+    --result_path new_dataset.jsonl
+
+
+# 4 * 67GiB, 2.50s/it
+# You need to additionally add sft_loss, because tokens like '<think>' have not been trained.
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3-8B-Base \
+    --teacher_model $teacher_model \
+    --tuner_type full \
+    --dataset 'new_dataset.jsonl' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --learning_rate 1e-5 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --packing true \
+    --attn_impl flash_attn \
+    --sft_alpha 0.1 \
+    --lmbda 0

examples/train/rlhf/gkd/vllm_colocate.sh

@@ -0,0 +1,36 @@
+# 4 * 73GiB, 11s/it
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model Qwen/Qwen2.5-14B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-en#2000' 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \
+    --split_dataset_ratio 0.01 \
+    --lmbda 0.5 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --max_completion_length 512 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --teacher_deepspeed zero3_offload \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.3 \
+    --sleep_level 1

examples/train/rlhf/gkd/vllm_server.sh

@@ -0,0 +1,42 @@
+# CUDA_VISIBLE_DEVICES=5 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-7B \
+#     --vllm_max_model_len 2560
+
+# 4 * 54GiB
+# 5s/it (with vLLM)
+# 14s/it (without vLLM)
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model Qwen/Qwen2.5-14B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-en#2000' 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \
+    --split_dataset_ratio 0.01 \
+    --lmbda 0.5 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --max_completion_length 512 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000

examples/train/rlhf/kto.sh

@@ -0,0 +1,29 @@
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type kto \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/ultrafeedback-binarized-preferences-cleaned-kto#10000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4

examples/train/rlhf/mpo.sh

@@ -0,0 +1,36 @@
+# https://arxiv.org/abs/2411.10442
+# modified from swift/examples/train/multimodal/rlhf/dpo/full.sh
+nproc_per_node=4
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=$nproc_per_node \
+MAX_PIXELS=1003520 \
+swift rlhf \
+    --rlhf_type dpo \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --dataset 'swift/RLAIF-V-Dataset#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --loss_type sigmoid bco_pair sft \
+    --loss_weights 0.8 0.2 1.0 \
+    --rpo_alpha 0.0 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --deepspeed zero3 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --save_only_model true

examples/train/rlhf/opsd/opsd.sh

@@ -0,0 +1,67 @@
+# OPSD Training Script
+# Paper: https://arxiv.org/abs/2601.18734
+#
+# ## Configuration
+# - **Teacher**: Base model (disable_adapter)
+# - **Student**: LoRA-adapted model
+# - **Dataset**: open-r1/OpenThoughts-114k-math
+# - **Model**: Qwen3-4B
+#
+# ## Hyperparameters (follow paper)
+# ```
+# lr=2e-5, lora_r=64, lora_alpha=128, temp=1.2, beta=0.5, lambda=1
+# max_completion_length=2048, effective_batch=32 (1×8×4)
+# ```
+#
+# ## AIME2025 Results (OVERALL)
+# | Checkpoint | Accuracy | Improvement |
+# |------------|----------|-------------|
+# | Base       | 0.1667   | -           |
+# | 100 steps  | 0.2667   | +60%        |
+#
+# ## Evaluation
+# ```bash
+# swift eval --model Qwen/Qwen3-4B \
+#     --adapters output/Qwen3-4B/xxx/checkpoint-xxx \
+#     --eval_dataset aime25 --eval_backend Native --infer_backend vllm \
+#     --vllm_max_lora_rank 64 \
+#     --eval_generation_config '{"max_tokens":8192,"temperature":0.0,"do_sample":false}'
+# ```
+
+
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3-4B \
+    --teacher_model Qwen/Qwen3-4B \
+    --tuner_type lora \
+    --lora_rank 64 \
+    --lora_alpha 128 \
+    --target_modules all-linear \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.7 \
+    --vllm_max_model_len 10240 \
+    --sleep_level 1 \
+    --external_plugins examples/train/rlhf/opsd/opsd_plugin.py \
+    --dataset 'open-r1/OpenThoughts-114k-math' \
+    --lmbda 1.0 \
+    --beta 0.5 \
+    --temperature 1.2 \
+    --sft_alpha 0 \
+    --torch_dtype bfloat16 \
+    --max_steps 1000 \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --learning_rate 2e-5 \
+    --save_steps 100 \
+    --save_total_limit 10 \
+    --logging_steps 1 \
+    --max_length 8192 \
+    --max_completion_length 2048 \
+    --save_only_model true \
+    --gradient_checkpointing true \
+    --deepspeed zero0 \
+    --attn_impl flash_attn \
+    --report_to tensorboard swanlab

examples/train/rlhf/opsd/opsd_plugin.py

@@ -0,0 +1,62 @@
+"""OPSD dataset plugin for open-r1/OpenThoughts-114k-math.
+
+Prepares the dataset for On-Policy Self-Distillation:
+- Student sees only the problem.
+- Teacher sees the problem + reference solution (privileged info via teacher_prompt).
+- Only verified-correct examples are used.
+
+Usage:
+    # GKD path (teacher KL as a direct loss):
+    swift rlhf --rlhf_type gkd --external_plugins opsd_plugin.py ...
+    # OPD-RL path (teacher KL as a per-token RL advantage):
+    swift rlhf --rlhf_type grpo --teacher_model <same-as-model> --external_plugins opsd_plugin.py ...
+"""
+from typing import Any, Dict, List, Optional
+
+from swift.dataset import DatasetMeta, RowPreprocessor, register_dataset
+
+SYSTEM_PROMPT = 'Please reason step by step, and put your final answer within \\boxed{}.'
+
+TRANSITION_PROMPT = ('After understanding the reference solution and the rationale behind each step, '
+                     'now articulate your own step-by-step reasoning that derives the final answer.')
+
+
+class OpenThoughtsOPSDPreprocessor(RowPreprocessor):
+    """Preprocessor that builds teacher_prompt from the reference solution.
+
+    Both student and teacher share the same system prompt for format guidance.
+    The teacher's user message additionally includes the reference solution as privileged info.
+    """
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        if not row.get('correct', True):
+            return None
+
+        problem = row.get('problem', '')
+        solution = row.get('solution', '')
+
+        teacher_prompt = (f'{problem}\n\n'
+                          f'Here is a reference solution to this problem:\n{solution}\n\n'
+                          f'{TRANSITION_PROMPT}')
+
+        messages: List[Dict[str, str]] = [
+            {
+                'role': 'system',
+                'content': SYSTEM_PROMPT
+            },
+            {
+                'role': 'user',
+                'content': problem
+            },
+        ]
+
+        return {'messages': messages, 'teacher_prompt': teacher_prompt}
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='open-r1/OpenThoughts-114k-math',
+        hf_dataset_id='open-r1/OpenThoughts-114k-math',
+        preprocess_func=OpenThoughtsOPSDPreprocessor(),
+        tags=['math', 'opsd'],
+    ))

examples/train/rlhf/orpo.sh

@@ -0,0 +1,30 @@
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type orpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4

examples/train/rlhf/ppo/full.sh

@@ -0,0 +1,37 @@
+# 8 * 65 GiB
+# Currently, it only supports the case where the model and reward_model use the same template/tokenizer.
+# Currently, multimodal model PPO is not supported.
+
+# pip install "deepspeed==0.14.*"
+nproc_per_node=8
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type ppo \
+    --model LLM-Research/Meta-Llama-3.1-8B-Instruct \
+    --reward_model 'AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2' \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' 'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero3 \
+    --response_length 512 \
+    --temperature 0.7 \
+    --dataset_num_proc 4 \
+    --save_only_model true

examples/train/rlhf/ppo/lora.sh

@@ -0,0 +1,40 @@
+# 4 * 50GiB
+# Currently, it only supports the case where the model and reward_model use the same template/tokenizer.
+# Currently, multimodal model PPO is not supported.
+
+# pip install "deepspeed==0.14.*"
+nproc_per_node=4
+
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type ppo \
+    --model LLM-Research/Meta-Llama-3.1-8B-Instruct \
+    --reward_model 'AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2' \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' 'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --response_length 512 \
+    --temperature 0.7 \
+    --dataset_num_proc 4 \
+    --save_only_model true

examples/train/rlhf/rm/infer.py

@@ -0,0 +1,74 @@
+from swift.infer_engine import InferRequest, TransformersEngine
+
+query = '甲、乙、丙三个人在一起做作业，有一道数学题比较难，当他们三个人都把自己的解法说出来以后，' \
+        '甲说："我做错了。"乙说："甲做对了。"丙说："我做错了。"' \
+        '在一旁的丁看到他们的答案并听了她们的意见后说："你们三个人中有一个人做对了，有一个人说对了。"' \
+        '请问，他们三人中到底谁做对了？'
+
+infer_requests = [
+    InferRequest(messages=[
+        {
+            'role': 'user',
+            'content': query
+        },
+        {
+            'role':
+            'assistant',
+            'content':
+            '天哪，这道题真的很有趣！ 🤔 让我们来分析一下他们的对话：'
+            ' 1. 甲说："我做错了。"如果甲真的做错了，那么乙说"甲做对了"就是错误的。'
+            '但是，如果甲做对了，那么乙说"甲做对了"就是正确的。'
+            ' 2. 乙说："甲做对了。"如果乙真的认为甲做对了，那么甲说"我做错了"就是错误的。'
+            ' 3. 丙说："我做错了。"这个信息暂时不太重要，但我们可以注意到丙没有说其他人做对或做错。'
+            ' 现在，丁出现了，说："你们三个人中有一个人做对了，有一个人说对了。"'
+            ' 如果甲做对了，那么乙说对了，丙做错了。但是，这样的话，丁的说法就不对了，因为有两个说对了。'
+            ' 如果甲做错了，那么乙说错了，丙做对了。这时候，丁的说法才正确，因为有一个人做对了（丙），有一个人说对了（乙）。'
+            ' 所以，答案是：丙做对了！ 👏'
+            ' PS：这道题真的很有趣，需要我们细心分析他们的对话！ 😄'
+        },
+    ]),
+    InferRequest(messages=[
+        {
+            'role': 'user',
+            'content': query
+        },
+        {
+            'role':
+            'assistant',
+            'content':
+            "What a delightful puzzle! Let's dive into the magical world of logic and figure out "
+            "who's correct and who's not. So, we have three friends, 甲 (Ka), 乙 (Otsu), and 丙 (Chi), "
+            'working on a tricky '
+            "math problem together. They each share their solutions, and then they start discussing who's correct. "
+            "Here's what they say: * 甲 (Ka) says, \"I did it wrong.\" * 乙 (Otsu) says, \"Ka did it right.\" "
+            "* 丙 (Chi) says, \"I did it wrong.\" Now, 丁 (Tei), who's been observing from the side, chimes in and says, "
+            "\"Among the three of you, one person got it right, and one person said it correctly.\" "
+            "Let's break it down step by step: "
+            '1. If 甲 (Ka) did it wrong, then 乙 (Otsu) must be telling the truth, because 乙 says 甲 did it right. '
+            '2. But wait, if 乙 is telling the truth, that means 甲 actually did it right! '
+            "This contradicts 甲's statement, "
+            'so 甲 must be lying. '
+            '3. Now, 丙 (Chi) says they did it wrong. If 丙 did it wrong, then 丙 is telling the truth. '
+            'But if 丙 did it right, then 丙 would be lying. '
+            '4. Since 丁 says one person got it right and one person said it correctly, '
+            'it means 丙 must be the one who did '
+            "it right. If 丙 did it right, then 丙's statement \"I did it wrong\" is actually a lie. "
+            '5. So, we have 甲 lying about doing it wrong, 乙 telling the truth about 甲, '
+            'and 丙 lying about doing it wrong. '
+            'The magical conclusion is that 丙 (Chi) is the one who actually got the math problem correct!'
+        },
+    ]),
+]
+
+engine = TransformersEngine(
+    'Qwen/Qwen2.5-7B-Instruct',
+    task_type='seq_cls',
+    num_labels=1,
+    problem_type='regression',
+    adapters=['output/vx-xxx/checkpoint-xxx'])
+
+responses = engine.infer(infer_requests)
+rewards = [response.choices[0].message.content for response in responses]
+# rewards: [23.875, -13.5625]
+# Correctly demonstrates Chinese preference.
+print(f'rewards: {rewards}')

examples/train/rlhf/rm/train.sh

@@ -0,0 +1,30 @@
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type rm \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4

examples/train/rlhf/simpo.sh

@@ -0,0 +1,28 @@
+# 2*50GB
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type simpo \
+    --model Qwen/Qwen2.5-3B-Instruct \
+    --tuner_type full \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --dataset_num_proc 4