chore: import upstream snapshot with attribution

2026-07-13 13:34:58 +08:00
commit a203934033
1368 changed files with 175001 additions and 0 deletions
@@ -0,0 +1,49 @@
+# External vLLM
+
+# Assume we have two nodes, one with 8 GPUs of 80GB each (880G) and another with 2 GPUs of 80GB each (2 80G).
+#   NODE1. The node with 2*80G will be used to deploy the vLLM server.
+#   NODE2. The node with 8*80G will be used for full-parameter fine-tuning of the 32B model.
+
+# Note : Use beta=0 to disable the reference model; otherwise, it may lead to Out-of-Memory (OOM) errors.
+
+# NODE1 for vLLM Server
+CUDA_VISIBLE_DEVICES=0,1 \
+swift rollout \
+    --model Qwen/Qwen2.5-32B-Instruct \
+    --vllm_tensor_parallel_size 2
+
+# NODE2 for Training
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=8 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-32B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host xxx \
+    --vllm_server_port 8000 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 1 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 1.0 \
+    --top_p 0.9 \
+    --top_k 50 \
+    --deepspeed zero3 \
+    --log_completions true \
+    --num_iterations 1 \
+    --report_to tensorboard wandb \
+    --beta 0.0
@@ -0,0 +1,44 @@
+# Internal vLLM
+
+# pip install math_verify # reward function
+# pip install -U trl
+# note: Note: The parameters of each node need to be consistent.
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export NNODES=2
+export NODE_RANK=0
+export MASTER_ADDR=127.0.0.1
+export MASTER_PORT=29500
+export NPROC_PER_NODE=4
+
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-Math-7B \
+    --reward_funcs accuracy format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_max_model_len 4096 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-MO/NuminaMath-TIR#5000' \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 0.9 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true
@@ -0,0 +1,39 @@
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export NNODES=2
+export NODE_RANK=1
+export MASTER_ADDR=xxx.xxx.xxx.xxx
+export MASTER_PORT=29500
+export NPROC_PER_NODE=4
+
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-Math-7B \
+    --reward_funcs accuracy format \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_gpu_memory_utilization 0.5 \
+    --vllm_max_model_len 4096 \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'AI-MO/NuminaMath-TIR#5000' \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-6 \
+    --gradient_accumulation_steps 2 \
+    --eval_steps 200 \
+    --save_steps 200 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 8 \
+    --temperature 0.9 \
+    --system 'examples/train/grpo/prompt.txt' \
+    --deepspeed zero2 \
+    --log_completions true
@@ -0,0 +1,83 @@
+# NOTE: Requires NCCL connectivity between the training master node and rollout nodes
+# This script demonstrates multi-node rollout and multi-node training with swift.
+# node1 and node2: multi-node rollout servers
+# node3 and node4: distributed training nodes
+
+# --- Rollout Section ---
+# For rollout, you can launch any number of servers on different nodes
+# Start rollout server on node1:
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rollout \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_data_parallel_size 2 \
+    --port <node1_port>
+
+# Start rollout server on node2:
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift rollout \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_data_parallel_size 2 \
+    --port <node2_port>
+
+# --- Training Section ---
+# node3: Master training node (rank 0)
+NNODES=2 \
+NODE_RANK=0 \
+MASTER_ADDR=127.0.0.1 \
+MASTER_PORT=29500 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host <node1_ip> <node2_ip> \
+    --vllm_server_port <node1_port> <node2_port> \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --deepspeed zero2 \
+    --log_completions true \
+
+# node4: Secondary training node (rank 1)
+NNODES=2 \
+NODE_RANK=1 \
+MASTER_ADDR=<node3_ip> \
+MASTER_PORT=29500 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=4 \
+swift rlhf \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --reward_funcs accuracy \
+    --use_vllm true \
+    --vllm_mode server \
+    --vllm_server_host <node1_ip> <node2_ip> \
+    --vllm_server_port <node1_port> <node2_port> \
+    --dataset AI-MO/NuminaMath-TIR#1000 \
+    --load_from_cache_file true \
+    --max_completion_length 2048 \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --num_generations 4 \
+    --deepspeed zero2 \
+    --log_completions true \
@@ -0,0 +1,40 @@
+# This script is used in DLC (Deep Learning Containers)
+# For more information, visit: https://www.aliyun.com/activity/bigdata/pai-dlc
+# https://help.aliyun.com/zh/pai/user-guide/general-environment-variables
+NNODES=$WORLD_SIZE \
+NODE_RANK=$RANK \
+torchrun \
+    --nproc_per_node=8 \
+    --nnodes=${WORLD_SIZE} \
+    --node_rank=${RANK} \
+    swift/cli/rlhf.py \
+    --rlhf_type grpo \
+    --model Qwen/Qwen2.5-7B \
+    --tuner_type full \
+    --dataset AI-MO/NuminaMath-TIR#10000 \
+    --load_from_cache_file true \
+    --torch_dtype bfloat16 \
+    --system examples/train/grpo/prompt.txt \
+    --num_train_epochs 1 \
+    --max_length 2048 \
+    --use_vllm true \
+    --vllm_mode colocate \
+    --vllm_max_model_len 2048 \
+    --vllm_gpu_memory_utilization 0.3 \
+    --vllm_tensor_parallel_size 4 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --learning_rate 1e-6 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --output_dir output \
+    --gradient_accumulation_steps 1 \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --max_completion_length 2048 \
+    --reward_funcs accuracy format \
+    --num_generations 48 \
+    --sleep_level 1 \
+    --deepspeed zero3_offload \
+    --temperature 1.0 \
+    --top_p 0.85