This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,64 @@
|
||||
# DP size = world_size // (context_parallel_size * tensor_model_parallel_size * pipeline_model_parallel_size)
|
||||
# = 8 // (1 * 1 * 1) = 8
|
||||
|
||||
# NOTE: global_batch_size and micro_batch_size are completion-level
|
||||
# global_batch_size = micro_batch_size * DP size * gradient_accumulation_steps (128)
|
||||
# generation_batch_size = global_batch_size * steps_per_generation (128 * 4 = 512)
|
||||
# num_of_prompt_to_rollout = generation_batch_size / num_generations (512 / 8 = 64)
|
||||
# num_of_prompt_to_train = generation_batch_size / num_generations (128 / 8 = 16)
|
||||
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
|
||||
NPROC_PER_NODE=8 \
|
||||
MAX_PIXELS=602112 \
|
||||
MASTER_PORT=29600 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--save_safetensors true \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 1 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--dataset AI-ModelScope/clevr_cogen_a_train#10000 \
|
||||
--num_train_epochs 1 \
|
||||
--global_batch_size 128 \
|
||||
--micro_batch_size 4 \
|
||||
--steps_per_generation 4 \
|
||||
--num_generations 8 \
|
||||
--external_plugins examples/train/grpo/plugin/plugin.py \
|
||||
--reward_funcs external_r1v_acc format \
|
||||
--use_vllm true \
|
||||
--vllm_mode colocate \
|
||||
--vllm_gpu_memory_utilization 0.7 \
|
||||
--vllm_max_model_len 10240 \
|
||||
--max_length 8192 \
|
||||
--max_completion_length 2048 \
|
||||
--tuner_type full \
|
||||
--lr 1e-6 \
|
||||
--bf16 true \
|
||||
--beta 0.001 \
|
||||
--importance_sampling_level token \
|
||||
--epsilon 0.2 \
|
||||
--epsilon_high 0.2 \
|
||||
--dynamic_sample false \
|
||||
--overlong_filter true \
|
||||
--loss_type grpo \
|
||||
--sleep_level 2 \
|
||||
--offload_model true \
|
||||
--offload_bridge false \
|
||||
--offload_optimizer true \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--system examples/train/grpo/prompt.txt \
|
||||
--padding_free true \
|
||||
--log_completions true \
|
||||
--report_to wandb \
|
||||
--train_iters 100 \
|
||||
--eval_steps 1000 \
|
||||
--save_steps 1000
|
||||
@@ -0,0 +1,71 @@
|
||||
# MAX_PIXELS=602112 \
|
||||
# CUDA_VISIBLE_DEVICES=6,7 \
|
||||
# swift rollout \
|
||||
# --model Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
# --vllm_data_parallel_size 2 \
|
||||
# --vllm_max_model_len 10240
|
||||
|
||||
# DP size = world_size // (context_parallel_size * tensor_model_parallel_size * pipeline_model_parallel_size)
|
||||
# = 6 // (1 * 1 * 1) = 6
|
||||
|
||||
# NOTE: global_batch_size and micro_batch_size are completion-level
|
||||
# global_batch_size = micro_batch_size * DP size * gradient_accumulation_steps (96)
|
||||
# generation_batch_size = global_batch_size * steps_per_generation (96 * 4 = 384)
|
||||
# num_of_prompt_to_rollout = generation_batch_size / num_generations (384 / 8 = 48)
|
||||
# num_of_prompt_to_train = generation_batch_size / num_generations (96 / 8 = 12)
|
||||
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \
|
||||
NPROC_PER_NODE=6 \
|
||||
MAX_PIXELS=602112 \
|
||||
MASTER_PORT=29600 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--save_safetensors true \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 1 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--dataset AI-ModelScope/clevr_cogen_a_train#10000 \
|
||||
--num_train_epochs 1 \
|
||||
--global_batch_size 96 \
|
||||
--micro_batch_size 4 \
|
||||
--steps_per_generation 4 \
|
||||
--num_generations 8 \
|
||||
--external_plugins examples/train/grpo/plugin/plugin.py \
|
||||
--reward_funcs external_r1v_acc format \
|
||||
--use_vllm true \
|
||||
--vllm_mode server \
|
||||
--vllm_server_host 127.0.0.1 \
|
||||
--vllm_server_port 8000 \
|
||||
--max_length 8192 \
|
||||
--max_completion_length 2048 \
|
||||
--tuner_type full \
|
||||
--lr 1e-6 \
|
||||
--bf16 true \
|
||||
--beta 0.001 \
|
||||
--importance_sampling_level token \
|
||||
--epsilon 0.2 \
|
||||
--epsilon_high 0.2 \
|
||||
--dynamic_sample false \
|
||||
--overlong_filter true \
|
||||
--loss_type grpo \
|
||||
--sleep_level 2 \
|
||||
--offload_model true \
|
||||
--offload_bridge false \
|
||||
--offload_optimizer true \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--system examples/train/grpo/prompt.txt \
|
||||
--padding_free true \
|
||||
--log_completions true \
|
||||
--report_to wandb \
|
||||
--train_iters 100 \
|
||||
--eval_steps 1000 \
|
||||
--save_steps 1000
|
||||
@@ -0,0 +1,54 @@
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
|
||||
NPROC_PER_NODE=8 \
|
||||
PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen3-30B-A3B-Instruct-2507 \
|
||||
--save_safetensors true \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 4 \
|
||||
--expert_model_parallel_size 4 \
|
||||
--pipeline_model_parallel_size 2 \
|
||||
--dataset open-r1/DAPO-Math-17k-Processed \
|
||||
--num_train_epochs 1 \
|
||||
--global_batch_size 8 \
|
||||
--micro_batch_size 1 \
|
||||
--steps_per_generation 1 \
|
||||
--num_generations 8 \
|
||||
--reward_funcs accuracy format \
|
||||
--use_vllm true \
|
||||
--vllm_mode colocate \
|
||||
--vllm_gpu_memory_utilization 0.4 \
|
||||
--vllm_tensor_parallel_size 8 \
|
||||
--vllm_max_model_len 16384 \
|
||||
--max_length 8192 \
|
||||
--max_completion_length 8192 \
|
||||
--tuner_type full \
|
||||
--lr 1e-6 \
|
||||
--bf16 true \
|
||||
--beta 0.00 \
|
||||
--importance_sampling_level sequence \
|
||||
--epsilon 3e-4 \
|
||||
--epsilon_high 4e-4 \
|
||||
--dynamic_sample false \
|
||||
--overlong_filter true \
|
||||
--loss_type grpo \
|
||||
--sleep_level 2 \
|
||||
--offload_model true \
|
||||
--offload_bridge false \
|
||||
--offload_optimizer true \
|
||||
--optimizer_cpu_offload true \
|
||||
--use_precision_aware_optimizer \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--padding_free true \
|
||||
--sequence_parallel true \
|
||||
--log_completions true \
|
||||
--report_to wandb
|
||||
@@ -0,0 +1,53 @@
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
|
||||
NPROC_PER_NODE=8 \
|
||||
PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen3-30B-A3B-Instruct-2507 \
|
||||
--save_safetensors true \
|
||||
--merge_lora false \
|
||||
--context_parallel_size 2 \
|
||||
--tensor_model_parallel_size 2 \
|
||||
--expert_model_parallel_size 4 \
|
||||
--pipeline_model_parallel_size 2 \
|
||||
--dataset open-r1/DAPO-Math-17k-Processed \
|
||||
--num_train_epochs 1 \
|
||||
--global_batch_size 64 \
|
||||
--micro_batch_size 2 \
|
||||
--steps_per_generation 2 \
|
||||
--num_generations 8 \
|
||||
--reward_funcs accuracy format \
|
||||
--use_vllm true \
|
||||
--vllm_mode colocate \
|
||||
--vllm_gpu_memory_utilization 0.3 \
|
||||
--vllm_tensor_parallel_size 4 \
|
||||
--vllm_max_model_len 16384 \
|
||||
--max_length 8192 \
|
||||
--max_completion_length 8192 \
|
||||
--tuner_type lora \
|
||||
--lr 5e-5 \
|
||||
--bf16 true \
|
||||
--beta 0.00 \
|
||||
--importance_sampling_level sequence \
|
||||
--epsilon 3e-4 \
|
||||
--epsilon_high 4e-4 \
|
||||
--dynamic_sample false \
|
||||
--overlong_filter true \
|
||||
--loss_type grpo \
|
||||
--sleep_level 2 \
|
||||
--offload_model true \
|
||||
--offload_bridge false \
|
||||
--offload_optimizer true \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--padding_free true \
|
||||
--sequence_parallel true \
|
||||
--log_completions true \
|
||||
--report_to wandb
|
||||
@@ -0,0 +1,128 @@
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":0}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":1}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":2}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":3}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":4}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":5}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":6}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":7}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":8}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":9}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":10}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":11}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":12}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":13}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":14}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":15}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":16}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":17}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":18}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":19}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":20}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":21}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":22}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":23}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":24}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":25}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":26}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":27}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":28}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":29}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":30}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":31}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":32}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":33}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":34}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":35}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":36}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":37}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":38}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":39}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":40}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":41}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":42}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":43}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":44}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":45}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":46}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":47}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":48}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":49}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":50}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":51}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":52}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":53}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":54}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":55}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":56}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":57}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":58}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":59}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":60}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":61}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":62}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":63}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":64}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":65}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":66}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":67}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":68}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":69}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":70}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":71}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":72}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":73}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":74}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":75}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":76}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":77}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":78}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":79}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":80}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":81}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":82}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":83}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":84}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":85}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":86}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":87}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":88}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":89}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":90}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":91}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":92}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":93}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":94}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":95}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":96}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":97}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":98}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":99}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":100}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":101}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":102}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":103}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":104}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":105}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":106}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":107}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":108}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":109}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":110}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":111}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":112}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":113}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":114}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":115}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":116}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":117}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":118}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":119}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":120}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":121}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":122}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":123}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":124}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":125}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":126}}
|
||||
{"messages":[{"role":"user","content":"<placeholder>"}],"env_config":{"seed":127}}
|
||||
@@ -0,0 +1,66 @@
|
||||
# Multi-turn GRPO with the FrozenLake env.
|
||||
# Env lives in frozen_lake_plugin.py (loaded via --external_plugins);
|
||||
# with --use_gym_env true, the env's total_reward is consumed directly — no reward_funcs needed.
|
||||
# To prevent excessively long generations, max_completion_length is capped at 512 (per turn);
|
||||
# since prompts are short, max_length (first 9 turns + prompt) is capped at 6120.
|
||||
# vllm_max_model_len = max_length + last-turn length = 6632
|
||||
# reward improves from 0.2 → 0.6 within 120 steps: https://github.com/modelscope/ms-swift/pull/9405
|
||||
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
|
||||
NPROC_PER_NODE=8 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen3.5-2B \
|
||||
--enable_thinking false \
|
||||
--save_safetensors true \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 1 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--dataset 'examples/megatron/grpo/multi_turn/frozen_lake.jsonl#1024' \
|
||||
--load_from_cache_file false \
|
||||
--train_iters 120 \
|
||||
--global_batch_size 64 \
|
||||
--micro_batch_size 1 \
|
||||
--steps_per_generation 4 \
|
||||
--num_generations 8 \
|
||||
--external_plugins examples/megatron/grpo/multi_turn/frozen_lake_plugin.py \
|
||||
--use_vllm true \
|
||||
--vllm_mode colocate \
|
||||
--vllm_gpu_memory_utilization 0.5 \
|
||||
--vllm_max_model_len 6632 \
|
||||
--max_length 6120 \
|
||||
--max_completion_length 512 \
|
||||
--multi_turn_scheduler gym_scheduler \
|
||||
--gym_env frozen_lake \
|
||||
--use_gym_env true \
|
||||
--max_turns 10 \
|
||||
--tuner_type lora \
|
||||
--lr 5e-5 \
|
||||
--bf16 true \
|
||||
--beta 0.001 \
|
||||
--importance_sampling_level token \
|
||||
--epsilon 0.2 \
|
||||
--epsilon_high 0.2 \
|
||||
--dynamic_sample false \
|
||||
--overlong_filter true \
|
||||
--loss_type grpo \
|
||||
--sleep_level 2 \
|
||||
--offload_model true \
|
||||
--offload_bridge false \
|
||||
--offload_optimizer true \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 4 \
|
||||
--dataset_num_proc 4 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--top_p 1.0 \
|
||||
--top_k 80 \
|
||||
--padding_free true \
|
||||
--log_completions true \
|
||||
--report_to tensorboard swanlab \
|
||||
--eval_steps 1000 \
|
||||
--save_steps 1000
|
||||
@@ -0,0 +1,194 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
"""Text-based FrozenLake env for multi-turn GRPO training.
|
||||
|
||||
Each turn the LLM sees an ASCII grid (S/G/H/F/P) and must reply with a
|
||||
single move inside ``<action>...</action>``. Reaching G yields reward 1.0;
|
||||
stepping into H ends the episode with 0. The outer step budget comes from
|
||||
swift's ``--max_turns`` flag. Per-row ``env_config.seed`` controls map
|
||||
generation so all ``num_generations`` rollouts of a row share the same map.
|
||||
|
||||
Register via ``--external_plugins`` and select with ``--gym_env frozen_lake``.
|
||||
"""
|
||||
# code borrowed from ROLL/roll/pipeline/agentic/env/frozen_lake
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from swift.infer_engine.protocol import RolloutInferRequest
|
||||
from swift.rollout.gym_env import Env, envs
|
||||
from swift.template import Messages
|
||||
from swift.utils import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def _is_valid(board: List[List[str]], size: int) -> bool:
|
||||
"""BFS from S; return True iff G is reachable through non-hole cells."""
|
||||
start = None
|
||||
for r in range(size):
|
||||
for c in range(size):
|
||||
if board[r][c] == 'S':
|
||||
start = (r, c)
|
||||
break
|
||||
if start is not None:
|
||||
break
|
||||
if start is None:
|
||||
return False
|
||||
|
||||
frontier = [start]
|
||||
discovered = set()
|
||||
while frontier:
|
||||
r, c = frontier.pop()
|
||||
if (r, c) in discovered:
|
||||
continue
|
||||
discovered.add((r, c))
|
||||
for dr, dc in ((1, 0), (0, 1), (-1, 0), (0, -1)):
|
||||
nr, nc = r + dr, c + dc
|
||||
if 0 <= nr < size and 0 <= nc < size:
|
||||
cell = board[nr][nc]
|
||||
if cell == 'G':
|
||||
return True
|
||||
if cell != 'H':
|
||||
frontier.append((nr, nc))
|
||||
return False
|
||||
|
||||
|
||||
def generate_random_map(size: int = 4, p: float = 0.8, seed: Optional[int] = None) -> List[str]:
|
||||
"""Generate a random solvable FrozenLake map.
|
||||
|
||||
Args:
|
||||
size: side length of the square grid.
|
||||
p: probability of a non-hole (F) cell — 1 - p of any cell being H.
|
||||
seed: RNG seed; same seed always yields the same map.
|
||||
|
||||
Returns:
|
||||
List of `size` strings, each of length `size`, using chars S/G/F/H.
|
||||
Both S and G positions are randomised (unlike gym's ``FrozenLake-v1``
|
||||
which pins S to top-left and G to bottom-right); BFS-validated.
|
||||
"""
|
||||
rng = random.Random(seed)
|
||||
while True:
|
||||
board = [['F' if rng.random() < p else 'H' for _ in range(size)] for _ in range(size)]
|
||||
start_r, start_c = rng.randrange(size), rng.randrange(size)
|
||||
goal_r, goal_c = rng.randrange(size), rng.randrange(size)
|
||||
if (start_r, start_c) == (goal_r, goal_c):
|
||||
continue
|
||||
board[start_r][start_c] = 'S'
|
||||
board[goal_r][goal_c] = 'G'
|
||||
if _is_valid(board, size):
|
||||
return [''.join(row) for row in board]
|
||||
|
||||
|
||||
# (row_delta, col_delta) for each canonical action token.
|
||||
ACTIONS: Dict[str, Tuple[int, int]] = {
|
||||
'up': (-1, 0),
|
||||
'down': (1, 0),
|
||||
'left': (0, -1),
|
||||
'right': (0, 1),
|
||||
}
|
||||
|
||||
SYSTEM_PROMPT = ('You are playing FrozenLake. You see a grid where:\n'
|
||||
' P = your current position, S = start, G = goal, H = hole, F = safe ice.\n'
|
||||
'Move one cell per turn. Reach G to win (+1 reward). Stepping into H ends '
|
||||
'the episode with 0 reward. Moves that would go off the grid leave you in '
|
||||
'place.\n\n'
|
||||
'On every turn, output your move inside <action>...</action>. '
|
||||
'The action must be exactly one of: up, down, left, right.\n\n'
|
||||
'Example: <action>down</action>')
|
||||
|
||||
_ACTION_TAG_RE = re.compile(r'<action>\s*(up|down|left|right)\s*</action>', re.IGNORECASE)
|
||||
_BARE_ACTION_RE = re.compile(r'\b(up|down|left|right)\b', re.IGNORECASE)
|
||||
|
||||
|
||||
def _render(grid: List[str], row: int, col: int) -> str:
|
||||
"""Render the grid with the player position marked as 'P'."""
|
||||
rendered = []
|
||||
for r, line in enumerate(grid):
|
||||
if r == row:
|
||||
chars = list(line)
|
||||
chars[col] = 'P' if chars[col] != 'G' else '*' # '*' = player on goal
|
||||
rendered.append(' '.join(chars))
|
||||
else:
|
||||
rendered.append(' '.join(line))
|
||||
return '\n'.join(rendered)
|
||||
|
||||
|
||||
def _parse_action(completion: str) -> Optional[str]:
|
||||
"""Extract the action from the assistant message. Returns None if missing."""
|
||||
m = _ACTION_TAG_RE.search(completion)
|
||||
if m:
|
||||
return m.group(1).lower()
|
||||
matches = _BARE_ACTION_RE.findall(completion)
|
||||
if matches:
|
||||
return matches[-1].lower()
|
||||
return None
|
||||
|
||||
|
||||
class FrozenLakeEnv(Env):
|
||||
|
||||
def __init__(self, env_config: Dict[str, Any]):
|
||||
super().__init__(env_config)
|
||||
self.size: int = int(env_config.get('size', 4))
|
||||
self.p: float = float(env_config.get('p', 0.8))
|
||||
seed = env_config.get('seed')
|
||||
self.seed: Optional[int] = int(seed) if seed is not None else None
|
||||
self.grid: List[str] = []
|
||||
self.row: int = 0
|
||||
self.col: int = 0
|
||||
self.steps: int = 0
|
||||
|
||||
async def reset(self, config: RolloutInferRequest) -> Tuple[str, Dict[str, Any], str]:
|
||||
self.grid = generate_random_map(size=self.size, p=self.p, seed=self.seed)
|
||||
for r, line in enumerate(self.grid):
|
||||
if 'S' in line:
|
||||
self.row, self.col = r, line.index('S')
|
||||
break
|
||||
self.steps = 0
|
||||
|
||||
observation = (f'FrozenLake {self.size}x{self.size}:\n'
|
||||
f'{_render(self.grid, self.row, self.col)}\n\n'
|
||||
f'You are at row {self.row}, col {self.col}. Output your first move.')
|
||||
info = {'seed': self.seed, 'size': self.size}
|
||||
return observation, info, SYSTEM_PROMPT
|
||||
|
||||
async def step(self, action: Messages) -> Tuple[str, float, bool, Dict[str, Any]]:
|
||||
completion = action[-1].get('content', '') if action else ''
|
||||
move = _parse_action(completion)
|
||||
self.steps += 1
|
||||
|
||||
info: Dict[str, Any] = {'seed': self.seed, 'step': self.steps, 'parsed_action': move}
|
||||
|
||||
if move is None:
|
||||
obs = (f'Invalid response: could not find a move. Reply with '
|
||||
f'<action>up|down|left|right</action>.\n\n'
|
||||
f'{_render(self.grid, self.row, self.col)}')
|
||||
return obs, 0.0, False, {**info, 'status': 'invalid_action'}
|
||||
|
||||
dr, dc = ACTIONS[move]
|
||||
new_row = max(0, min(len(self.grid) - 1, self.row + dr))
|
||||
new_col = max(0, min(len(self.grid[0]) - 1, self.col + dc))
|
||||
cell = self.grid[new_row][new_col]
|
||||
self.row, self.col = new_row, new_col
|
||||
|
||||
if cell == 'G':
|
||||
obs = (f'You moved {move} and reached the goal!\n'
|
||||
f'{_render(self.grid, self.row, self.col)}')
|
||||
return obs, 1.0, True, {**info, 'status': 'goal'}
|
||||
if cell == 'H':
|
||||
obs = (f'You moved {move} and fell into a hole. Episode over.\n'
|
||||
f'{_render(self.grid, self.row, self.col)}')
|
||||
return obs, 0.0, True, {**info, 'status': 'hole'}
|
||||
|
||||
obs = (f'You moved {move}. Now at row {self.row}, col {self.col} (step {self.steps}).\n'
|
||||
f'{_render(self.grid, self.row, self.col)}\n'
|
||||
f'Output your next move.')
|
||||
return obs, 0.0, False, {**info, 'status': 'ok'}
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
|
||||
envs['frozen_lake'] = FrozenLakeEnv
|
||||
@@ -0,0 +1,71 @@
|
||||
# server mode version of frozen_lake.sh
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3 \
|
||||
swift rollout \
|
||||
--model Qwen/Qwen3.5-2B \
|
||||
--enable_thinking false \
|
||||
--vllm_data_parallel_size 4 \
|
||||
--multi_turn_scheduler gym_scheduler \
|
||||
--gym_env frozen_lake \
|
||||
--use_gym_env true \
|
||||
--max_turns 10 \
|
||||
--max_length 6120 \
|
||||
--vllm_max_model_len 6632 \
|
||||
--external_plugins examples/megatron/grpo/multi_turn/frozen_lake_plugin.py > frozen_lake_rollout.log 2>&1 &
|
||||
|
||||
# Wait for rollout server to be ready
|
||||
echo "Waiting for rollout server to start..."
|
||||
until curl -s http://127.0.0.1:8000/health/ > /dev/null 2>&1; do
|
||||
sleep 10
|
||||
done
|
||||
echo "Rollout server is ready!"
|
||||
|
||||
CUDA_VISIBLE_DEVICES=4,5,6,7 \
|
||||
NPROC_PER_NODE=4 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen3.5-2B \
|
||||
--save_safetensors true \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 1 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--train_iters 120 \
|
||||
--dataset 'examples/megatron/grpo/multi_turn/frozen_lake.jsonl#1024' \
|
||||
--load_from_cache_file false \
|
||||
--global_batch_size 32 \
|
||||
--micro_batch_size 1 \
|
||||
--steps_per_generation 4 \
|
||||
--num_generations 8 \
|
||||
--external_plugins examples/megatron/grpo/multi_turn/frozen_lake_plugin.py \
|
||||
--use_vllm true \
|
||||
--vllm_mode server \
|
||||
--vllm_server_host 127.0.0.1 \
|
||||
--vllm_server_port 8000 \
|
||||
--max_length 6120 \
|
||||
--max_completion_length 512 \
|
||||
--enable_thinking false \
|
||||
--tuner_type lora \
|
||||
--lr 5e-5 \
|
||||
--bf16 true \
|
||||
--beta 0.001 \
|
||||
--importance_sampling_level token \
|
||||
--epsilon 0.2 \
|
||||
--epsilon_high 0.2 \
|
||||
--dynamic_sample false \
|
||||
--overlong_filter true \
|
||||
--loss_type grpo \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 4 \
|
||||
--dataset_num_proc 4 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--top_p 1.0 \
|
||||
--top_k 80 \
|
||||
--padding_free true \
|
||||
--log_completions true \
|
||||
--report_to tensorboard swanlab \
|
||||
--eval_steps 1000 \
|
||||
--save_steps 1000 \
|
||||
@@ -0,0 +1,61 @@
|
||||
SYSTEM_PROMPT="""You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}."""
|
||||
|
||||
CUDA_VISIBLE_DEVICES=0,1 \
|
||||
NPROC_PER_NODE=2 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen3.5-2B \
|
||||
--teacher_model Qwen/Qwen3.5-9B \
|
||||
--enable_thinking false \
|
||||
--save_safetensors true \
|
||||
--expert_model_parallel_size 1 \
|
||||
--tensor_model_parallel_size 2 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--sequence_parallel true \
|
||||
--system "$SYSTEM_PROMPT" \
|
||||
--dataset modelscope/gsm8k \
|
||||
--num_train_epochs 1 \
|
||||
--global_batch_size 16 \
|
||||
--micro_batch_size 2 \
|
||||
--steps_per_generation 2 \
|
||||
--num_generations 1 \
|
||||
--use_vllm true \
|
||||
--vllm_mode colocate \
|
||||
--vllm_gpu_memory_utilization 0.6 \
|
||||
--vllm_tensor_parallel_size 1 \
|
||||
--vllm_max_model_len 12288 \
|
||||
--sleep_level 1 \
|
||||
--freeze_vit false \
|
||||
--max_length 4096 \
|
||||
--max_completion_length 2048 \
|
||||
--tuner_type lora \
|
||||
--lora_rank 8 \
|
||||
--lora_alpha 32 \
|
||||
--target_modules all-linear \
|
||||
--lr 3e-5 \
|
||||
--bf16 true \
|
||||
--beta 0 \
|
||||
--epsilon 0.2 \
|
||||
--epsilon_high 0.2 \
|
||||
--loss_type grpo \
|
||||
--offload_model true \
|
||||
--offload_bridge false \
|
||||
--offload_optimizer true \
|
||||
--log_rollout_offpolicy_metrics true \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1 \
|
||||
--padding_free false \
|
||||
--log_completions true \
|
||||
--report_to swanlab \
|
||||
--train_iters 1000 \
|
||||
--eval_steps 1000 \
|
||||
--save_steps 100 \
|
||||
--top_k 0 \
|
||||
--top_p 1
|
||||
@@ -0,0 +1,48 @@
|
||||
# REAL, https://arxiv.org/abs/2602.05630
|
||||
|
||||
CUDA_VISIBLE_DEVICES=2 \
|
||||
swift rollout \
|
||||
--model Qwen/Qwen2.5-0.5B-Instruct
|
||||
|
||||
CUDA_VISIBLE_DEVICES=0,1 \
|
||||
NPROC_PER_NODE=2 \
|
||||
MASTER_PORT=29600 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--model Qwen/Qwen2.5-0.5B-Instruct \
|
||||
--save_safetensors true \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 1 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--dataset 'AI-MO/NuminaMath-TIR#4000' \
|
||||
--num_train_epochs 1 \
|
||||
--micro_batch_size 8 \
|
||||
--global_batch_size 128 \
|
||||
--num_generations 8 \
|
||||
--reward_funcs accuracy \
|
||||
--use_vllm true \
|
||||
--vllm_mode server \
|
||||
--vllm_server_host 127.0.0.1 \
|
||||
--vllm_server_port 8000 \
|
||||
--max_completion_length 2048 \
|
||||
--tuner_type full \
|
||||
--gradient_accumulation_fusion false \
|
||||
--lr 2e-6 \
|
||||
--bf16 true \
|
||||
--beta 0.001 \
|
||||
--dynamic_sample false \
|
||||
--loss_type real \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--padding_free true \
|
||||
--attention_backend flash \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--temperature 0.6 \
|
||||
--system """You are a helpful math assistant. Solve the problem step by step and put your final answer within \\boxed{}.""" \
|
||||
--log_completions true \
|
||||
--eval_steps 100 \
|
||||
--save_steps 100
|
||||
@@ -0,0 +1,54 @@
|
||||
# SAPO https://arxiv.org/abs/2511.20347
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
|
||||
NPROC_PER_NODE=8 \
|
||||
MAX_PIXELS=602112 \
|
||||
megatron rlhf \
|
||||
--rlhf_type grpo \
|
||||
--loss_type sapo \
|
||||
--tau_pos 1 \
|
||||
--tau_neg 1.05 \
|
||||
--model Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--context_parallel_size 1 \
|
||||
--tensor_model_parallel_size 1 \
|
||||
--pipeline_model_parallel_size 1 \
|
||||
--dataset AI-ModelScope/clevr_cogen_a_train \
|
||||
--save_safetensors true \
|
||||
--external_plugins examples/train/grpo/plugin/plugin.py \
|
||||
--reward_funcs external_r1v_acc format \
|
||||
--dynamic_sample false \
|
||||
--steps_per_generation 4 \
|
||||
--micro_batch_size 2 \
|
||||
--global_batch_size 128 \
|
||||
--num_generations 8 \
|
||||
--use_vllm true \
|
||||
--vllm_mode colocate \
|
||||
--vllm_gpu_memory_utilization 0.7 \
|
||||
--vllm_max_model_len 8192 \
|
||||
--max_length 4096 \
|
||||
--max_completion_length 4096 \
|
||||
--tuner_type full \
|
||||
--bf16 true \
|
||||
--importance_sampling_level token \
|
||||
--epsilon 0.2 \
|
||||
--epsilon_high 0.2 \
|
||||
--overlong_filter true \
|
||||
--num_train_epochs 1 \
|
||||
--eval_steps 1000 \
|
||||
--save_steps 1000 \
|
||||
--sleep_level 2 \
|
||||
--offload_model true \
|
||||
--offload_optimizer true \
|
||||
--logging_steps 1 \
|
||||
--recompute_granularity selective \
|
||||
--finetune \
|
||||
--lr 1e-6 \
|
||||
--dataloader_num_workers 8 \
|
||||
--dataset_num_proc 8 \
|
||||
--no_save_optim \
|
||||
--no_save_rng \
|
||||
--attention_backend flash \
|
||||
--temperature 1.0 \
|
||||
--system examples/train/grpo/prompt.txt \
|
||||
--beta 0.001 \
|
||||
--padding_free true \
|
||||
--report_to wandb
|
||||
Reference in New Issue
Block a user