chore: import upstream snapshot with attribution

rdagent/components/coder/rl/__init__.py

@@ -0,0 +1 @@
+from rdagent.components.coder.rl.costeer import RLCoSTEER

rdagent/components/coder/rl/costeer.py

@@ -0,0 +1,140 @@
+"""RL CoSTEER - Code generation component for RL post-training"""
+
+from typing import Generator
+
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.core.evolving_agent import EvolvingStrategy, EvoStep
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+
+class RLCoderCoSTEERSettings(CoSTEERSettings):
+    """RL Coder settings."""
+
+    pass
+
+
+class RLEvolvingStrategy(EvolvingStrategy):
+    """RL code generation strategy using LLM."""
+
+    def __init__(self, scen: Scenario, settings: CoSTEERSettings):
+        self.scen = scen
+        self.settings = settings
+
+    def evolve_iter(
+        self,
+        *,
+        evo: EvolvingItem,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        evolving_trace: list[EvoStep] = [],
+        **kwargs,
+    ) -> Generator[EvolvingItem, EvolvingItem, None]:
+        """Generate code for all tasks using LLM."""
+        for index, target_task in enumerate(evo.sub_tasks):
+            code = self._generate_code(target_task, evolving_trace)
+            if evo.sub_workspace_list[index] is None:
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code)
+
+        evo = yield evo
+        return
+
+    def _generate_code(self, task: Task, evolving_trace: list[EvoStep] = []) -> dict[str, str]:
+        """Generate RL training code using LLM."""
+        from rdagent.app.rl.conf import RL_RD_SETTING
+
+        # 获取上轮反馈
+        feedback = None
+        if evolving_trace:
+            last_step = evolving_trace[-1]
+            if hasattr(last_step, "feedback") and last_step.feedback:
+                feedback = str(last_step.feedback)
+
+        # 构造 prompt
+        system_prompt = T(".prompts:rl_coder.system").r()
+        user_prompt = T(".prompts:rl_coder.user").r(
+            task_description=task.description if hasattr(task, "description") else str(task),
+            base_model=RL_RD_SETTING.base_model or "",
+            benchmark=RL_RD_SETTING.benchmark or "",
+            hypothesis=str(task.name) if hasattr(task, "name") else "Train RL model",
+            feedback=feedback,
+        )
+
+        # 调用 LLM
+        session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
+        code = session.build_chat_completion(
+            user_prompt=user_prompt,
+            json_mode=False,
+            code_block_language="python",
+        )
+        logger.info(f"LLM generated code:\n{code[:200]}...")
+        return {"main.py": code}
+
+    def _mock_code(self) -> dict[str, str]:
+        """Fallback mock code."""
+        return {"main.py": """import gymnasium as gym
+from stable_baselines3 import PPO
+
+env = gym.make("CartPole-v1")
+model = PPO("MlpPolicy", env, verbose=1)
+model.learn(total_timesteps=1000)
+model.save("ppo_cartpole")
+print("Training completed!")
+"""}
+
+
+class RLCoderEvaluator:
+    """RL code evaluator (mock implementation)."""
+
+    def __init__(self, scen: Scenario) -> None:
+        self.scen = scen
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace | None,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+    ) -> CoSTEERSingleFeedback:
+        """Evaluate RL code. Currently returns mock success."""
+        # TODO: 实现真正的评估逻辑
+        return CoSTEERSingleFeedback(
+            execution="Mock: executed successfully",
+            return_checking=None,
+            code="Mock: code looks good",
+            final_decision=True,
+        )
+
+
+class RLCoSTEER(CoSTEER):
+    """RL CoSTEER - orchestrates code generation and evaluation."""
+
+    def __init__(self, scen: Scenario, *args, **kwargs) -> None:
+        settings = RLCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator([RLCoderEvaluator(scen=scen)], scen=scen)
+        es = RLEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            scen=scen,
+            max_loop=1,
+            stop_eval_chain_on_fail=False,
+            with_knowledge=False,
+            knowledge_self_gen=False,
+            **kwargs,
+        )

rdagent/components/coder/rl/prompts.yaml

@@ -0,0 +1,94 @@
+rl_coder:
+  system: |-
+    你是 RL post-training 专家，负责生成训练代码。
+
+    ## 运行环境
+    代码会被部署到 `$WORKSPACE/code/main.py` 并在该目录下执行。
+    以下环境变量已由框架设置，代码中用 `os.environ["..."]` 读取：
+    - `MODEL_PATH`: 基础模型绝对路径（只读）
+    - `DATA_PATH`: 训练数据目录绝对路径（只读）
+    - `OUTPUT_DIR`: 模型输出目录绝对路径（`$WORKSPACE/output/`）
+    - `GRADING_SERVER_URL`: 评测服务地址（训练完后系统自动提交，代码不需要调用）
+
+    ## 框架: trl (版本 0.27+)
+
+    ## 可用算法
+    - **GRPO**: 推荐，只需 reward function，不需要预构建偏好对
+    - **DPO**: 需要 (prompt, chosen, rejected) 偏好对
+
+    ## API 要点
+
+    ### GRPOTrainer
+    ```python
+    from trl import GRPOConfig, GRPOTrainer
+
+    trainer = GRPOTrainer(
+        model=MODEL_PATH,               # 模型路径
+        reward_funcs=reward_fn,          # reward 函数
+        args=GRPOConfig(
+            output_dir=OUTPUT_DIR,       # 输出目录
+            ...
+        ),
+        train_dataset=dataset,           # 必须有 "prompt" 列
+        processing_class=tokenizer,
+    )
+    ```
+
+    ### reward function 签名（重要！）
+    ```python
+    def reward_fn(completions, answer, **kwargs):
+        # completions: list[str] - 模型生成的回复
+        # answer: list[str] - 数据集中的 answer 列（自动传入）
+        # kwargs: 数据集其他列（如 question）
+        return [float(...) for ...]  # 返回 reward 列表
+    ```
+
+    ### GRPOConfig 关键参数
+    - `num_generations`: 每个 prompt 采样次数，必须 >= 2
+    - `max_completion_length`: 生成最大长度
+    - `per_device_train_batch_size`: 批次大小
+
+    ## 输出要求
+    - 生成完整的 `main.py`，可直接运行
+    - 路径全部通过 `os.environ` 获取，**不要硬编码路径**
+    - 数据从 `$DATA_PATH` 下的 jsonl 文件加载
+    - 模型保存到 `$OUTPUT_DIR`（可用子目录如 `$OUTPUT_DIR/v1`）
+
+    ## 评测机制
+    训练完成后，系统自动将 `$OUTPUT_DIR` 下最新的模型提交到 Grading Server。
+    - 有模型 → 自动评测，返回 score
+    - 为空 → 跳过评测
+    代码只需负责训练和保存模型，**不需要**自行调用评测 API。
+
+    ## 代码模板
+    ```python
+    import os
+    MODEL_PATH = os.environ["MODEL_PATH"]
+    DATA_PATH = os.environ["DATA_PATH"]
+    OUTPUT_DIR = os.environ["OUTPUT_DIR"]
+    # ... 训练逻辑 ...
+    trainer.save_model(OUTPUT_DIR)
+    ```
+
+  user: |-
+    ## 任务
+    {{ task_description }}
+
+    ## 基础模型
+    - 名称: {{ base_model }}
+    - 路径: 通过 $MODEL_PATH 环境变量获取
+
+    ## 训练数据
+    - 数据集: {{ benchmark }}
+    - 路径: 通过 $DATA_PATH 环境变量获取
+
+    ## 假设
+    {{ hypothesis }}
+
+    {% if feedback %}
+    ## 上轮反馈
+    {{ feedback }}
+    {% endif %}
+
+    请根据数据格式和假设，生成完整的训练代码（main.py）。
+    注意：路径全部通过 os.environ 获取，不要硬编码。