chore: import upstream snapshot with attribution

examples/train/rlhf/gkd/fast.sh

@@ -0,0 +1,48 @@
+
+export teacher_model='Qwen/Qwen2.5-14B-Instruct'
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model $teacher_model \
+    --infer_backend vllm \
+    --val_dataset 'AI-ModelScope/alpaca-gpt4-data-en#5000' 'AI-ModelScope/alpaca-gpt4-data-zh#5000' \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --write_batch_size 10000 \
+    --result_path new_dataset.jsonl
+
+
+# 4 * 56GiB, 1.55s/it
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model $teacher_model \
+    --tuner_type full \
+    --dataset 'new_dataset.jsonl' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --padding_free true \
+    --attn_impl flash_attn \
+    --lmbda 0

examples/train/rlhf/gkd/full.sh

@@ -0,0 +1,30 @@
+# 4 * 66GiB, 46s/it
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model Qwen/Qwen2.5-14B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-en#2000' 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --max_completion_length 512 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn

examples/train/rlhf/gkd/multi_turn.sh

@@ -0,0 +1,41 @@
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3.5-2B \
+    --teacher_model Qwen/Qwen3.5-9B \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/NuminaMath-TIR#5000' \
+    --lmbda 1 \
+    --beta 0.5 \
+    --temperature 1.0 \
+    --torch_dtype bfloat16 \
+    --max_steps 200 \
+    --per_device_train_batch_size 4 \
+    --learning_rate 5e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_steps 200 \
+    --save_total_limit 1 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --max_completion_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --teacher_deepspeed zero3_offload \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.3 \
+    --vllm_max_model_len 8192 \
+    --sleep_level 1 \
+    --enable_thinking false \
+    --loss_scale last_round \
+    --multi_turn_scheduler math_tip_trick \
+    --max_turns 2 \
+    --truncation_strategy delete \
+    --log_completions true

examples/train/rlhf/gkd/teacher_server.sh

@@ -0,0 +1,51 @@
+top_k=64
+max_prompt_length=2048
+max_completion_length=2048
+max_total_length=$((max_prompt_length + max_completion_length))
+
+export IMAGE_MAX_TOKEN_NUM=1024
+
+# Teacher server must be running first:
+
+# CUDA_VISIBLE_DEVICES=0 \
+# swift deploy \
+#     --model Qwen/Qwen3.5-4B \
+#     --infer_backend vllm \
+#     --port 8000 \
+#     --max_logprobs $top_k \
+#     --max_length $max_total_length \
+#     --vllm_max_model_len $max_total_length
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3.5-4B \
+    --teacher_model_server http://localhost:8000 \
+    --gkd_logits_topk $top_k \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_tensor_parallel_size 1 \
+    --vllm_max_model_len $max_total_length \
+    --sleep_level 0 \
+    --dataset 'modelscope/gsm8k' \
+    --lmbda 1 \
+    --beta 0.5 \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 2 \
+    --gradient_accumulation_steps 4 \
+    --learning_rate 5e-5 \
+    --logging_steps 1 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --max_length $max_prompt_length \
+    --max_completion_length $max_completion_length \
+    --warmup_ratio 0.1 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --temperature 1.0 \
+    --attn_impl flash_attn \
+    --report_to tensorboard swanlab

examples/train/rlhf/gkd/think_model.sh

@@ -0,0 +1,50 @@
+
+export teacher_model='Qwen/Qwen3-8B'
+
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model $teacher_model \
+    --infer_backend vllm \
+    --val_dataset 'AI-ModelScope/alpaca-gpt4-data-en#5000' 'AI-ModelScope/alpaca-gpt4-data-zh#5000' \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --write_batch_size 10000 \
+    --result_path new_dataset.jsonl
+
+
+# 4 * 67GiB, 2.50s/it
+# You need to additionally add sft_loss, because tokens like '<think>' have not been trained.
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen3-8B-Base \
+    --teacher_model $teacher_model \
+    --tuner_type full \
+    --dataset 'new_dataset.jsonl' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --learning_rate 1e-5 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --packing true \
+    --attn_impl flash_attn \
+    --sft_alpha 0.1 \
+    --lmbda 0

examples/train/rlhf/gkd/vllm_colocate.sh

@@ -0,0 +1,36 @@
+# 4 * 73GiB, 11s/it
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model Qwen/Qwen2.5-14B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-en#2000' 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \
+    --split_dataset_ratio 0.01 \
+    --lmbda 0.5 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --max_completion_length 512 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --teacher_deepspeed zero3_offload \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.3 \
+    --sleep_level 1

examples/train/rlhf/gkd/vllm_server.sh

@@ -0,0 +1,42 @@
+# CUDA_VISIBLE_DEVICES=5 \
+# swift rollout \
+#     --model Qwen/Qwen2.5-7B \
+#     --vllm_max_model_len 2560
+
+# 4 * 54GiB
+# 5s/it (with vLLM)
+# 14s/it (without vLLM)
+NPROC_PER_NODE=4 \
+PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rlhf \
+    --rlhf_type gkd \
+    --model Qwen/Qwen2.5-7B \
+    --teacher_model Qwen/Qwen2.5-14B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-en#2000' 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \
+    --split_dataset_ratio 0.01 \
+    --lmbda 0.5 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 1 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --max_completion_length 512 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --save_only_model true \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero3 \
+    --attn_impl flash_attn \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host 127.0.0.1 \
+    --vllm_server_port 8000