chore: import upstream snapshot with attribution

examples/ray/grpo/multi_turn/frozen_lake_colocate.sh

@@ -0,0 +1 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 megatron rlhf --use_ray --config frozen_lake_colocate.yaml

examples/ray/grpo/multi_turn/frozen_lake_colocate.yaml

@@ -0,0 +1,92 @@
+# Ray + Megatron GRPO on FrozenLake (multi-turn, colocate mode).
+# Parameters aligned with examples/megatron/grpo/multi_turn/frozen_lake.sh
+rlhf_type: grpo
+
+model: Qwen/Qwen3.5-2B
+dataset: 'examples/megatron/grpo/multi_turn/frozen_lake.jsonl#1024'
+external_plugins: examples/megatron/grpo/multi_turn/frozen_lake_plugin.py
+load_from_cache_file: false
+split_dataset_ratio: 0
+dataset_num_proc: 4
+dataloader_num_workers: 4
+
+# Multi-turn frozen_lake env
+multi_turn_scheduler: gym_scheduler
+gym_env: frozen_lake
+use_gym_env: true
+max_turns: 10
+
+# Batch config
+micro_batch_size: 1
+global_batch_size: 64
+num_generations: 8
+steps_per_generation: 4
+train_iters: 120
+logging_steps: 1
+seed: 42
+
+# Length params
+max_length: 6120
+max_completion_length: 512
+vllm_max_model_len: 6632
+
+enable_thinking: false
+
+# Training config
+padding_free: true
+cross_entropy_loss_fusion: true
+gradient_accumulation_fusion: false
+attention_backend: flash
+recompute_granularity: selective
+
+# Optimizer
+lr: 5e-5
+bf16: true
+
+# Generation config
+temperature: 1.0
+top_p: 1.0
+top_k: 80
+
+# GRPO params
+beta: 0.001
+epsilon: 0.2
+epsilon_high: 0.2
+loss_type: grpo
+advantage_estimator: grpo
+importance_sampling_level: token
+dynamic_sample: false
+overlong_filter: true
+log_completions: true
+log_rollout_offpolicy_metrics: true
+
+# vLLM + colocate
+use_vllm: true
+colocate_groups: [[train, rollout]]
+offload_model: true
+offload_optimizer: true
+sleep_level: 1
+
+# Checkpointing
+eval_steps: 1000
+save_steps: 1000
+no_save_optim: true
+no_save_rng: true
+
+# Reporting
+report_to: swanlab
+
+train:
+  gpus: 8
+  tuner_type: lora
+  lora_rank: 8
+  lora_alpha: 32
+  tensor_model_parallel_size: 1
+  pipeline_model_parallel_size: 1
+  context_parallel_size: 1
+  output_dir: megatron_output/ray_grpo_frozen_lake_colocate
+
+rollout:
+  gpus: 8
+  vllm_tensor_parallel_size: 1
+  vllm_gpu_memory_utilization: 0.5

examples/ray/grpo/opd_rl_colocate.yaml

@@ -0,0 +1,53 @@
+rlhf_type: grpo
+
+model: Qwen/Qwen3.5-2B
+teacher_model: Qwen/Qwen3.5-9B
+
+dataset: modelscope/gsm8k
+dataset_num_proc: 4
+split_dataset_ratio: 0
+
+micro_batch_size: 2
+global_batch_size: 16
+num_generations: 1
+steps_per_generation: 4
+num_train_epochs: 1
+logging_steps: 1
+seed: 42
+max_length: 2048
+max_completion_length: 2048
+padding_free: false
+cross_entropy_loss_fusion: true
+gradient_accumulation_fusion: false
+lr: 3e-5
+lr_warmup_fraction: 0.0
+attention_backend: flash
+temperature: 1.0
+beta: 0
+teacher_kl_coef: 1.0
+
+use_vllm: true
+
+colocate_groups: [[train, rollout]]
+offload_model: true
+offload_optimizer: true
+offload_teacher_model: true
+sleep_level: 1
+
+save_steps: 100
+no_save_optim: true
+no_save_rng: true
+
+train:
+  gpus: 4
+  tuner_type: lora
+  lora_rank: 8
+  lora_alpha: 32
+  tensor_model_parallel_size: 1
+  output_dir: megatron_output/ray_opd_rl_colocate
+
+rollout:
+  gpus: 4
+  vllm_tensor_parallel_size: 1
+  vllm_gpu_memory_utilization: 0.4
+  vllm_max_model_len: 4096

examples/ray/grpo/ray_grpo_colocate.yaml

@@ -0,0 +1,56 @@
+rlhf_type: grpo
+
+model: Qwen/Qwen2.5-VL-3B-Instruct
+dataset: AI-ModelScope/clevr_cogen_a_train#2000
+external_plugins: examples/train/grpo/plugin/plugin.py
+reward_funcs: [external_r1v_acc, format]
+overlong_filter: false
+dataset_num_proc: 4
+split_dataset_ratio: 0
+
+micro_batch_size: 2
+global_batch_size: 16
+num_generations: 8
+steps_per_generation: 4
+num_train_epochs: 1
+logging_steps: 1
+seed: 42
+max_length: 4096
+max_completion_length: 4096
+padding_free: false
+cross_entropy_loss_fusion: true
+gradient_accumulation_fusion: false
+lr: 5e-5
+lr_warmup_fraction: 0.0
+min_lr: 5e-5
+attention_backend: flash
+temperature: 1.0
+beta: 0
+epsilon: 0.2
+epsilon_high: 0.28
+loss_type: grpo
+advantage_estimator: grpo
+importance_sampling_level: token
+log_rollout_offpolicy_metrics: true
+system: examples/train/grpo/prompt.txt
+
+use_vllm: true
+
+colocate_groups: [[train, rollout]]
+offload_model: true
+offload_optimizer: true
+sleep_level: 1
+
+train:
+  gpus: 4
+  tuner_type: lora
+  lora_rank: 8
+  lora_alpha: 32
+  tensor_model_parallel_size: 1
+  output_dir: megatron_output/ray_grpo_colocate
+
+rollout:
+  gpus: 4
+  vllm_tensor_parallel_size: 1
+  vllm_gpu_memory_utilization: 0.4
+  vllm_max_model_len: 8192

examples/ray/grpo/ray_grpo_separate.yaml

@@ -0,0 +1,53 @@
+rlhf_type: grpo
+
+model: Qwen/Qwen2.5-VL-3B-Instruct
+dataset: AI-ModelScope/clevr_cogen_a_train#2000
+external_plugins: examples/train/grpo/plugin/plugin.py
+reward_funcs: [external_r1v_acc, format]
+overlong_filter: false
+dataset_num_proc: 4
+split_dataset_ratio: 0
+load_from_cache_file: false
+
+micro_batch_size: 2
+global_batch_size: 16
+num_generations: 8
+steps_per_generation: 4
+num_train_epochs: 1
+train_iters: 100
+logging_steps: 1
+seed: 42
+max_length: 4096
+max_completion_length: 4096
+padding_free: false
+cross_entropy_loss_fusion: true
+gradient_accumulation_fusion: false
+lr: 5e-5
+lr_warmup_fraction: 0.0
+min_lr: 5e-5
+attention_backend: flash
+temperature: 1.0
+beta: 0
+epsilon: 0.2
+epsilon_high: 0.28
+loss_type: grpo
+advantage_estimator: grpo
+importance_sampling_level: token
+log_rollout_offpolicy_metrics: true
+system: examples/train/grpo/prompt.txt
+
+use_vllm: true
+
+train:
+  gpus: 4
+  tuner_type: lora
+  lora_rank: 8
+  lora_alpha: 32
+  tensor_model_parallel_size: 1
+  output_dir: megatron_output/ray_grpo_separate
+
+rollout:
+  gpus: 4
+  vllm_tensor_parallel_size: 1
+  vllm_gpu_memory_utilization: 0.8
+  vllm_max_model_len: 8192

examples/ray/grpo/run_colocate.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Ray-based Megatron GRPO — colocate mode
+export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+megatron rlhf --use_ray true --config "$SCRIPT_DIR/ray_grpo_colocate.yaml"

examples/ray/grpo/run_opd.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Ray Megatron OPD-RL (On-Policy Distillation as RL) — colocate mode + colocated teacher.
+export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+megatron rlhf --use_ray true --config "$SCRIPT_DIR/opd_rl_colocate.yaml"

examples/ray/grpo/run_separate.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Ray-based Megatron GRPO — separate mode
+export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+megatron rlhf --use_ray true --config "$SCRIPT_DIR/ray_grpo_separate.yaml"