chore: import upstream snapshot with attribution

rdagent/components/coder/CoSTEER/__init__.py

@@ -0,0 +1,178 @@
+from copy import deepcopy
+from datetime import datetime
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERRAGStrategyV1,
+    CoSTEERRAGStrategyV2,
+)
+from rdagent.core.developer import Developer
+from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvaluator, RAGEvoAgent
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import Experiment
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.backend.base import RD_Agent_TIMER_wrapper
+
+
+class CoSTEER(Developer[Experiment]):
+    def __init__(
+        self,
+        settings: CoSTEERSettings,
+        eva: RAGEvaluator,
+        es: EvolvingStrategy,
+        *args,
+        evolving_version: int = 2,
+        with_knowledge: bool = True,
+        knowledge_self_gen: bool = True,
+        max_loop: int | None = None,
+        stop_eval_chain_on_fail: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.settings = settings
+
+        self.max_loop = settings.max_loop if max_loop is None else max_loop
+        self.knowledge_base_path = (
+            Path(settings.knowledge_base_path) if settings.knowledge_base_path is not None else None
+        )
+        self.new_knowledge_base_path = (
+            Path(settings.new_knowledge_base_path) if settings.new_knowledge_base_path is not None else None
+        )
+
+        self.with_knowledge = with_knowledge
+        self.knowledge_self_gen = knowledge_self_gen
+        self.evolving_strategy = es
+        self.evaluator = eva
+        self.evolving_version = evolving_version
+        self.stop_eval_chain_on_fail = stop_eval_chain_on_fail
+
+        # init rag method
+        self.rag = (
+            CoSTEERRAGStrategyV2(
+                settings=settings,
+                former_knowledge_base_path=self.knowledge_base_path,
+                dump_knowledge_base_path=self.new_knowledge_base_path,
+                evolving_version=self.evolving_version,
+            )
+            if self.evolving_version == 2
+            else CoSTEERRAGStrategyV1(
+                settings=settings,
+                former_knowledge_base_path=self.knowledge_base_path,
+                dump_knowledge_base_path=self.new_knowledge_base_path,
+                evolving_version=self.evolving_version,
+            )
+        )
+
+    def get_develop_max_seconds(self) -> int | None:
+        """
+        Get the maximum seconds for the develop task.
+        Sub classes might override this method to provide a different value.
+        """
+        return None
+
+    def _get_last_fb(self) -> CoSTEERMultiFeedback:
+        fb = self.evolve_agent.evolving_trace[-1].feedback
+        assert fb is not None, "feedback is None"
+        assert isinstance(fb, CoSTEERMultiFeedback), "feedback must be of type CoSTEERMultiFeedback"
+        return fb
+
+    def should_use_new_evo(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback) -> bool:
+        """
+        Compare new feedback with the fallback feedback.
+
+        Returns:
+            bool: True if the new feedback better and False if the new feedback is worse or invalid.
+        """
+        if new_fb is not None and new_fb.is_acceptable():
+            return True
+        return False
+
+    def develop(self, exp: Experiment) -> Experiment:
+
+        # init intermediate items
+        max_seconds = self.get_develop_max_seconds()
+        evo_exp = EvolvingItem.from_experiment(exp)
+
+        self.evolve_agent = RAGEvoAgent[EvolvingItem](
+            max_loop=self.max_loop,
+            evolving_strategy=self.evolving_strategy,
+            rag=self.rag,
+            with_knowledge=self.with_knowledge,
+            knowledge_self_gen=self.knowledge_self_gen,
+            enable_filelock=self.settings.enable_filelock,
+            filelock_path=self.settings.filelock_path,
+            stop_eval_chain_on_fail=self.stop_eval_chain_on_fail,
+        )
+
+        # Evolving the solution
+        start_datetime = datetime.now()
+        fallback_evo_exp = None
+        fallback_evo_fb = None
+        reached_max_seconds = False
+
+        evo_fb = None
+        for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator):
+            assert isinstance(evo_exp, Experiment)  # multiple inheritance
+            evo_fb = self._get_last_fb()
+            update_fallback = self.should_use_new_evo(
+                base_fb=fallback_evo_fb,
+                new_fb=evo_fb,
+            )
+            if update_fallback:
+                fallback_evo_exp = deepcopy(evo_exp)
+                fallback_evo_fb = deepcopy(evo_fb)
+                fallback_evo_exp.create_ws_ckp()  # NOTE: creating checkpoints for saving files in the workspace to prevent inplace mutation.
+
+            logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
+            for sw in evo_exp.sub_workspace_list:
+                logger.info(f"evolving workspace: {sw}")
+            if max_seconds is not None and (datetime.now() - start_datetime).total_seconds() > max_seconds:
+                logger.info(f"Reached max time limit {max_seconds} seconds, stop evolving")
+                reached_max_seconds = True
+                break
+            if RD_Agent_TIMER_wrapper.timer.started and RD_Agent_TIMER_wrapper.timer.is_timeout():
+                logger.info("Global timer is timeout, stop evolving")
+                break
+
+        try:
+            # Fallback is required because we might not choose the last acceptable evo to submit.
+            if fallback_evo_exp is not None:
+                logger.info("Fallback to the fallback solution.")
+                evo_exp = fallback_evo_exp
+                evo_exp.recover_ws_ckp()
+                evo_fb = fallback_evo_fb
+            assert evo_fb is not None  # multistep_evolve should run at least once
+            evo_exp = self._exp_postprocess_by_feedback(evo_exp, evo_fb)
+        except CoderError as e:
+            e.caused_by_timeout = reached_max_seconds
+            raise e
+
+        exp.sub_workspace_list = evo_exp.sub_workspace_list
+        exp.experiment_workspace = evo_exp.experiment_workspace
+        return exp
+
+    def _exp_postprocess_by_feedback(self, evo: Experiment, feedback: CoSTEERMultiFeedback) -> Experiment:
+        """
+        Responsibility:
+        - Raise Error if it failed to handle the develop task
+        -
+        """
+        assert isinstance(evo, Experiment)
+        assert isinstance(feedback, CoSTEERMultiFeedback)
+        assert len(evo.sub_workspace_list) == len(feedback)
+
+        # FIXME: when whould the feedback be None?
+        failed_feedbacks = [
+            f"- feedback{index + 1:02d}:\n  - execution: {f.execution}\n  - return_checking: {f.return_checking}\n  - code: {f.code}"
+            for index, f in enumerate(feedback)
+            if f is not None and not f.is_acceptable()
+        ]
+
+        if len(failed_feedbacks) == len(feedback):
+            feedback_summary = "\n".join(failed_feedbacks)
+            raise CoderError(f"All tasks are failed:\n{feedback_summary}")
+
+        return evo

rdagent/components/coder/CoSTEER/config.py

@@ -0,0 +1,42 @@
+from typing import Union
+
+from rdagent.core.conf import ExtendedBaseSettings
+
+
+class CoSTEERSettings(ExtendedBaseSettings):
+    """CoSTEER settings, this setting is supposed not to be used directly!!!"""
+
+    class Config:
+        env_prefix = "CoSTEER_"
+
+    coder_use_cache: bool = False
+    """Indicates whether to use cache for the coder"""
+
+    max_loop: int = 10
+    """Maximum number of task implementation loops"""
+
+    fail_task_trial_limit: int = 20
+
+    v1_query_former_trace_limit: int = 3
+    v1_query_similar_success_limit: int = 3
+
+    v2_query_component_limit: int = 1
+    v2_query_error_limit: int = 1
+    v2_query_former_trace_limit: int = 3
+    v2_add_fail_attempt_to_latest_successful_execution: bool = False
+    v2_error_summary: bool = False
+    v2_knowledge_sampler: float = 1.0
+
+    knowledge_base_path: Union[str, None] = None
+    """Path to the knowledge base"""
+
+    new_knowledge_base_path: Union[str, None] = None
+    """Path to the new knowledge base"""
+
+    enable_filelock: bool = False
+    filelock_path: Union[str, None] = None
+
+    max_seconds_multiplier: int = 10**6
+
+
+CoSTEER_SETTINGS = CoSTEERSettings()

rdagent/components/coder/CoSTEER/evaluators.py

@@ -0,0 +1,333 @@
+import json
+from abc import abstractmethod
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Dict, Generator, List
+
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evaluation import Evaluator, Feedback
+from rdagent.core.evolving_agent import RAGEvaluator
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import Task, Workspace
+from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.log import rdagent_logger as logger
+
+if TYPE_CHECKING:
+    from rdagent.core.scenario import Scenario
+
+# TODO:
+# 1. It seems logically sound, but we currently lack a scenario to apply it.
+# 2. If it proves to be useful, relocate it to a more general location.
+#
+# class FBWorkspaceExeFeedback(Feedback):
+#     """
+#     It pairs with FBWorkspace in the abstract level.
+#     """
+#     # ws: FBWorkspace   # potential
+#     stdout: str
+
+
+@dataclass
+class CoSTEERSingleFeedback(Feedback):
+    # TODO: (xiao)
+    # it should be more general class for FBWorkspaceExeFeedback
+    # A better name of it may be NormalFeedback
+    # TODO: It should be a general feeddback for CoSTEERR
+    """
+    The feedback for the data loader evaluation.
+    It is design align the phases of the implemented code
+    - Execution -> Return Value -> Code -> Final Decision
+    """
+    execution: str  # Summarized execution feedback
+    # execution_feedback
+    return_checking: str | None  # including every check in the testing (constraints about the generated value)
+    # value_feedback, shape_feedback, value_generated_flag
+    code: str
+    final_decision: bool | None = None
+    raw_execution: str = ""  # Full raw stdout for UI display
+    source_feedback: Dict[str, bool] = field(
+        default_factory=dict
+    )  # Record the source of the feedback since it might be merged from multiple feedbacks, stores the mapping from source tag to its final_decision, this dict also includes the feedback source of itself
+
+    @staticmethod
+    def val_and_update_init_dict(data: dict) -> dict:
+        # TODO: (bowen) use a more general method to validate and update the data dictionary before init, like pydantic
+        """
+        Validates and converts the 'final_decision' field in the given data dictionary.
+
+        Args:
+            data (dict): The data dictionary containing the 'final_decision' field.
+
+        Returns:
+            dict: The updated data dictionary with 'final_decision' as a boolean.
+
+        Raises:
+            ValueError: If 'final_decision' is not present or not a boolean.
+        """
+        if "final_decision" not in data:
+            raise ValueError("'final_decision' is required")
+
+        if isinstance(data["final_decision"], str):
+            if data["final_decision"] == "false" or data["final_decision"] == "False":
+                data["final_decision"] = False
+            elif data["final_decision"] == "true" or data["final_decision"] == "True":
+                data["final_decision"] = True
+
+        if not isinstance(data["final_decision"], bool):
+            raise ValueError(f"'final_decision' must be a boolean, not {type(data['final_decision'])}")
+
+        for attr in "execution", "return_checking", "code":
+            if data.get(attr) is not None and not isinstance(data[attr], str):
+                data[attr] = json.dumps(data[attr], indent=2, ensure_ascii=False)
+        return data
+
+    @classmethod
+    def merge(cls, feedback_li: list["CoSTEERSingleFeedback"]) -> "CoSTEERSingleFeedback":
+        # NOTE:
+        # Here we don't know the detailed design of each feedback, we just know they are CoSTEERSingleFeedback
+        # So we merge them only based on CoSTEERSingleFeedback's attributes
+        # **So some information may be lost when we have different types of feedbacks**
+        # If you have more sophisticated sub class of CoSTEERSingleFeedback, you should override this method
+        # to avoid the loss of information.
+
+        fb = deepcopy(feedback_li[0])
+
+        # for all the evaluators, aggregate the final_decision from `task_id`
+        fb.final_decision = all(fb.final_decision for fb in feedback_li)
+        for attr in "execution", "return_checking", "code":
+            setattr(
+                fb,
+                attr,
+                "\n\n".join([getattr(_fb, attr) for _fb in feedback_li if getattr(_fb, attr) is not None]),
+            )
+        fb.source_feedback = {}
+        for _fb in feedback_li:
+            for tag, decision in _fb.source_feedback.items():
+                fb.source_feedback[tag] = decision
+        return fb
+
+    def __str__(self) -> str:
+        return f"""------------------Execution------------------
+{self.execution}
+------------------Return Checking------------------
+{self.return_checking if self.return_checking is not None else 'No return checking'}
+------------------Code------------------
+{self.code}
+------------------Final Decision------------------
+This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
+"""
+
+    def __bool__(self):
+        return self.final_decision
+
+
+class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback):
+    """This class is a base class for all code generator feedback to single implementation"""
+
+    def __init__(
+        self,
+        execution_feedback: str = None,
+        shape_feedback: str = None,
+        code_feedback: str = None,
+        value_feedback: str = None,
+        final_decision: bool = None,
+        final_feedback: str = None,
+        value_generated_flag: bool = None,
+        final_decision_based_on_gt: bool = None,
+        source_feedback: dict = None,
+    ) -> None:
+        self.execution_feedback = execution_feedback
+        self.code_feedback = code_feedback
+        self.value_feedback = value_feedback
+        self.final_decision = final_decision
+        self.final_feedback = final_feedback
+        self.value_generated_flag = value_generated_flag
+        self.final_decision_based_on_gt = final_decision_based_on_gt
+        self.source_feedback = source_feedback if source_feedback is not None else {}
+
+        # TODO:
+        # Not general enough. So we should not put them in the general costeer feedback
+        # Instead, we should create subclass for it.
+        self.shape_feedback = shape_feedback  # Not general enough. So
+
+    @property
+    def execution(self):
+        return self.execution_feedback
+
+    @execution.setter
+    def execution(self, value):
+        self.execution_feedback = value
+
+    @property
+    def return_checking(self):
+        if self.value_generated_flag:
+            return f"value feedback: {self.value_feedback}\n\nshape feedback: {self.shape_feedback}"
+        return None
+
+    @return_checking.setter
+    def return_checking(self, value):
+        # Since return_checking is derived from value_feedback and shape_feedback,
+        # we don't need to do anything here
+        self.value_feedback = value
+        self.shape_feedback = value
+
+    @property
+    def code(self):
+        return self.code_feedback
+
+    @code.setter
+    def code(self, value):
+        self.code_feedback = value
+
+    def __str__(self) -> str:
+        return f"""------------------Execution Feedback------------------
+{self.execution_feedback if self.execution_feedback is not None else 'No execution feedback'}
+------------------Shape Feedback------------------
+{self.shape_feedback if self.shape_feedback is not None else 'No shape feedback'}
+------------------Code Feedback------------------
+{self.code_feedback if self.code_feedback is not None else 'No code feedback'}
+------------------Value Feedback------------------
+{self.value_feedback if self.value_feedback is not None else 'No value feedback'}
+------------------Final Feedback------------------
+{self.final_feedback if self.final_feedback is not None else 'No final feedback'}
+------------------Final Decision------------------
+This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
+"""
+
+
+class CoSTEERMultiFeedback(Feedback):
+    """Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
+
+    def __init__(self, feedback_list: List[CoSTEERSingleFeedback]) -> None:
+        self.feedback_list = feedback_list
+
+    def __getitem__(self, index: int) -> CoSTEERSingleFeedback:
+        return self.feedback_list[index]
+
+    def __len__(self) -> int:
+        return len(self.feedback_list)
+
+    def append(self, feedback: CoSTEERSingleFeedback) -> None:
+        self.feedback_list.append(feedback)
+
+    def __iter__(self):
+        return iter(self.feedback_list)
+
+    def is_acceptable(self) -> bool:
+        return all(feedback.is_acceptable() for feedback in self.feedback_list)
+
+    def finished(self) -> bool:
+        """
+        In some implementations, tasks may fail multiple times, leading agents to skip the implementation.
+        This results in None feedback. However, we want to accept the correct parts and ignore None feedback.
+        """
+        return all(feedback.final_decision for feedback in self.feedback_list if feedback is not None)
+
+    def __bool__(self) -> bool:
+        return all(feedback.final_decision for feedback in self.feedback_list)
+
+
+class CoSTEEREvaluator(Evaluator):
+    def __init__(
+        self,
+        scen: "Scenario",
+    ) -> None:
+        self.scen = scen
+
+    # TODO:
+    # I think we should have unified interface for all evaluates, for examples.
+    # So we should adjust the interface of other factors
+    # Based on the implementation, I think a better name is some name like task-implement evaluator
+    @abstractmethod
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        **kwargs,
+    ) -> CoSTEERSingleFeedback:
+        raise NotImplementedError("Please implement the `evaluator` method")
+
+
+class CoSTEERMultiEvaluator(RAGEvaluator):
+    """This is for evaluation of experiment. Due to we have multiple tasks, so we will return a list of evaluation feebacks"""
+
+    def __init__(self, single_evaluator: CoSTEEREvaluator | list[CoSTEEREvaluator], scen: "Scenario") -> None:
+        super().__init__()
+        self.scen = scen
+        self.single_evaluator = single_evaluator
+
+    def evaluate_iter(
+        self,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> Generator[CoSTEERMultiFeedback, EvolvingItem | None, CoSTEERMultiFeedback]:
+        evo = yield CoSTEERMultiFeedback(
+            []
+        )  # it will receive the evo first, so the first yield is for get the sent evo instead of generate useful feedback
+
+        eval_l = self.single_evaluator if isinstance(self.single_evaluator, list) else [self.single_evaluator]
+
+        # 1) Evaluate each sub_task
+        task_li_feedback_li = []
+        # task_li_feedback_li: List[List[CoSTEERSingleFeedback]]
+        # Example:
+        # If there are 2 evaluators and 3 sub_tasks in evo, and each evaluator's evaluate returns a list of 3 CoSTEERSingleFeedbacks,
+        # Then task_li_feedback_li will be:
+        # [
+        #   [feedback_1_1, feedback_1_2, feedback_1_3],  # results from the 1st evaluator for all sub_tasks
+        #   [feedback_2_1, feedback_2_2, feedback_2_3],  # results from the 2nd evaluator for all sub_tasks
+        # ]
+        # Where feedback_i_j is the feedback from the i-th evaluator for the j-th sub_task.
+        for ev in eval_l:
+            multi_implementation_feedback = multiprocessing_wrapper(
+                [
+                    (
+                        ev.evaluate,
+                        (
+                            evo.sub_tasks[index],
+                            evo.sub_workspace_list[index],
+                            evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
+                            queried_knowledge,
+                        ),
+                    )
+                    for index in range(len(evo.sub_tasks))
+                ],
+                n=RD_AGENT_SETTINGS.multi_proc_n,
+            )
+            # None received, we skip the rest and return the overall feedback directly
+            evo_next_iter = yield CoSTEERMultiFeedback(multi_implementation_feedback)
+            task_li_feedback_li.append(multi_implementation_feedback)
+            if evo_next_iter is None:
+                break
+            evo = evo_next_iter
+
+        # 2) merge the feedbacks along the sub_tasks to aggregate the multiple evaluation feedbacks
+        merged_task_feedback = []
+        # task_li_feedback_li[0] is a list of feedbacks of different tasks for the 1st evaluator
+        for task_id, fb in enumerate(task_li_feedback_li[0]):
+            fb = fb.merge([fb_li[task_id] for fb_li in task_li_feedback_li])
+            merged_task_feedback.append(fb)
+        # merged_task_feedback: List[CoSTEERSingleFeedback]
+        # Example:
+        # [
+        #   CoSTEERSingleFeedback(final_decision=True, execution="...", return_checking="...", code="..."),
+        #   CoSTEERSingleFeedback(final_decision=False, execution="...", return_checking="...", code="..."),
+        #   ...
+        # ]
+        # Each element corresponds to the merged feedback for one sub-task across all evaluators.
+        # merged_task_feedback[i] is the merged feedback for the i-th sub_task
+
+        final_decision = [
+            None if single_feedback is None else single_feedback.final_decision
+            for single_feedback in merged_task_feedback
+        ]
+        logger.info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
+
+        # TODO: this is to be compatible with factor_implementation;
+        for index in range(len(evo.sub_tasks)):
+            if final_decision[index]:
+                evo.sub_tasks[index].factor_implementation = True
+
+        return CoSTEERMultiFeedback(merged_task_feedback)

rdagent/components/coder/CoSTEER/evolvable_subjects.py

@@ -0,0 +1,32 @@
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+
+
+class EvolvingItem(Experiment, EvolvableSubjects):
+    """
+    Intermediate item of factor implementation.
+    """
+
+    def __init__(
+        self,
+        sub_tasks: list[Task],
+        sub_gt_implementations: list[FBWorkspace] = None,
+    ):
+        Experiment.__init__(self, sub_tasks=sub_tasks)
+        if sub_gt_implementations is not None and len(
+            sub_gt_implementations,
+        ) != len(self.sub_tasks):
+            self.sub_gt_implementations = None
+            logger.warning(
+                "The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
+            )
+        else:
+            self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: Experiment) -> "EvolvingItem":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/CoSTEER/evolving_strategy.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Callable, Generator
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiFeedback,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep, QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import multiprocessing_wrapper
+
+
+class MultiProcessEvolvingStrategy(EvolvingStrategy):
+    KEY_CHANGE_SUMMARY = "__change_summary__"  # Optional key for the summary of the change of evolving subjects
+
+    def __init__(self, scen: Scenario, settings: CoSTEERSettings, improve_mode: bool = False):
+        super().__init__(scen)
+        self.settings = settings
+        self.improve_mode = improve_mode  # improve mode means we only implement the task which has failed before. The main diff is the first loop will not implement all tasks.
+
+    def implement_one_task(
+        self,
+        target_task: Task,
+        queried_knowledge: QueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:  # FIXME: fix interface of previous implement
+        """
+        This method will input the task & current workspace,
+        and output the modification to applied to the workspace.
+        (i.e. replace the content <filename> with <content>)
+
+        Parameters
+        ----------
+        target_task : Task
+
+        queried_knowledge : QueriedKnowledge | None
+
+        workspace : FBWorkspace | None
+
+        prev_task_feedback : CoSTEERSingleFeedback | None
+            task feedback for previous evolving step
+            None indicate it is the first loop.
+
+        Return
+        ------
+        The new files {<filename>: <content>} to update the workspace.
+        - Special Keys: self.KEY_CHANGE_SUMMARY;
+        """
+        raise NotImplementedError
+
+    def implement_func_list(self) -> list[Callable]:
+        """
+        One evolve solution will be divided into multiple implement functions.
+        The functions will be called sequentially.
+
+        `implement_one_task` is the default implementation.  Please refer to its signature for more details.
+        """
+        return [self.implement_one_task]
+
+    @abstractmethod
+    def assign_code_list_to_evo(self, code_list: list[dict], evo: EvolvingItem) -> None:
+        """
+        Assign the code list to the evolving item.
+
+        Due to the implement_one_task take `workspace` as input and output the `modification`.
+        We should apply implementation to evo
+
+        Assumptions:
+        - The modidication on evo should happen in-place!!
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        raise NotImplementedError
+
+    def assign_code_list_to_evo(self, code_list: list[dict | None], evo) -> None:
+        """Assign code modifications to evolving item.
+
+        For runner, coder already generated full training config, so typically no modifications.
+        But this method is required by the abstract base class.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+
+            # If there are any modifications (usually empty for runner)
+            if code_list[index]:
+                # Handle change summary if present
+                if self.KEY_CHANGE_SUMMARY in code_list[index]:
+                    evo.sub_workspace_list[index].change_summary = code_list[index].pop(self.KEY_CHANGE_SUMMARY)
+                # Inject any modified files
+                evo.sub_workspace_list[index].inject_files(**code_list[index])
+
+        return evo
+
+    def evolve_iter(
+        self,
+        *,
+        evo: EvolvingItem,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        evolving_trace: list[EvoStep] = [],
+        **kwargs,
+    ) -> Generator[EvolvingItem, EvolvingItem, None]:
+        if queried_knowledge is None:
+            raise ValueError(
+                "MultiProcessEvolvingStrategy requires queried_knowledge for efficient implementation. Please set with_knowledge=True in CoSTEER constructor."
+            )
+        code_list = [None for _ in range(len(evo.sub_tasks))]
+
+        last_feedback = None
+        if len(evolving_trace) > 0:
+            last_feedback = evolving_trace[-1].feedback
+            assert isinstance(last_feedback, CoSTEERMultiFeedback)
+
+        # 1.找出需要evolve的task
+        to_be_finished_task_index: list[int] = []
+        for index, target_task in enumerate(evo.sub_tasks):
+            target_task_desc = target_task.get_task_information()
+            if target_task_desc in queried_knowledge.success_task_to_knowledge_dict:
+                # NOTE: very weird logic:
+                # it depends on the knowledge to set the already finished task
+                code_list[index] = queried_knowledge.success_task_to_knowledge_dict[
+                    target_task_desc
+                ].implementation.file_dict
+            else:
+                # Schedule the task only if:
+                # - it is not marked failed
+                # - and (in improve mode) we actually have prior failure feedback to act on
+                skip_for_improve_mode = self.improve_mode and (
+                    last_feedback is None
+                    or (isinstance(last_feedback, CoSTEERMultiFeedback) and last_feedback[index] is None)
+                )
+                if target_task_desc not in queried_knowledge.failed_task_info_set and not skip_for_improve_mode:
+                    to_be_finished_task_index.append(index)
+                if skip_for_improve_mode:
+                    code_list[index] = (
+                        {}
+                    )  # empty implementation for skipped task, but assign_code_list_to_evo will still assign it
+
+        for implement_func in self.implement_func_list():
+            result = multiprocessing_wrapper(
+                [
+                    (
+                        implement_func,
+                        (
+                            evo.sub_tasks[target_index],
+                            queried_knowledge,
+                            evo.experiment_workspace,
+                            None if last_feedback is None else last_feedback[target_index],
+                        ),
+                    )
+                    for target_index in to_be_finished_task_index
+                ],
+                n=RD_AGENT_SETTINGS.multi_proc_n,
+            )
+            for index, target_index in enumerate(to_be_finished_task_index):
+                code_list[target_index] = result[index]
+
+            self.assign_code_list_to_evo(code_list, evo)
+            yield evo

rdagent/components/coder/CoSTEER/prompts.yaml

@@ -0,0 +1,10 @@
+
+analyze_component_prompt_v1_system: |-
+  User is getting a new task that might consist of the components below (given in component_index: component_description):
+  {{all_component_content}}
+
+  You should find out what components does the new task have, and put their indices in a list.
+  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+  {
+      "component_no_list": the list containing indices of components.
+  }

rdagent/components/coder/CoSTEER/task.py

@@ -0,0 +1,9 @@
+from rdagent.core.experiment import Task
+
+
+class CoSTEERTask(Task):
+    def __init__(self, base_code: str = None, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # TODO: we may upgrade the base_code into a workspace-like thing to know previous.
+        # NOTE: (xiao) think we don't need the base_code anymore. The information should be retrieved from the workspace.
+        self.base_code = base_code

rdagent/components/coder/data_science/conf.py

@@ -0,0 +1,87 @@
+from typing import Literal
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.utils.env import (
+    CondaConf,
+    DockerEnv,
+    DSDockerConf,
+    Env,
+    LocalEnv,
+    MLEBDockerConf,
+    MLECondaConf,
+)
+
+
+class DSCoderCoSTEERSettings(CoSTEERSettings):
+    """Data Science CoSTEER settings"""
+
+    class Config:
+        env_prefix = "DS_Coder_CoSTEER_"
+
+    max_seconds_multiplier: int = 4
+    env_type: str = "docker"
+    # TODO: extract a function for env and conf.
+    extra_evaluator: list[str] = []
+    """Extra evaluators to use"""
+
+    extra_eval: list[str] = []
+    """
+    Extra evaluators
+
+    The evaluator follows the following assumptions:
+    - It runs after previous evaluator (So the running results are already there)
+
+    It is not a complete feature due to it is only implemented in DS Pipeline & Coder.
+
+    TODO: The complete version should be implemented in the CoSTEERSettings.
+    """
+
+
+def get_ds_env(
+    conf_type: Literal["kaggle", "mlebench"] = "kaggle",
+    extra_volumes: dict = {},
+    running_timeout_period: int | None = DS_RD_SETTING.debug_timeout,
+    enable_cache: bool | None = None,
+) -> Env:
+    """
+    Retrieve the appropriate environment configuration based on the env_type setting.
+
+    Returns:
+        Env: An instance of the environment configured either as DockerEnv or LocalEnv.
+
+    Raises:
+        ValueError: If the env_type is not recognized.
+    """
+    conf = DSCoderCoSTEERSettings()
+    assert conf_type in ["kaggle", "mlebench"], f"Unknown conf_type: {conf_type}"
+
+    if conf.env_type == "docker":
+        env_conf = DSDockerConf() if conf_type == "kaggle" else MLEBDockerConf()
+        env = DockerEnv(conf=env_conf)
+    elif conf.env_type == "conda":
+        env = LocalEnv(
+            conf=(
+                CondaConf(conda_env_name=conf_type) if conf_type == "kaggle" else MLECondaConf(conda_env_name=conf_type)
+            )
+        )
+    else:
+        raise ValueError(f"Unknown env type: {conf.env_type}")
+    env.conf.extra_volumes = extra_volumes.copy()
+    env.conf.running_timeout_period = running_timeout_period
+    if enable_cache is not None:
+        env.conf.enable_cache = enable_cache
+    env.prepare()
+    return env
+
+
+def get_clear_ws_cmd(stage: Literal["before_training", "before_inference"] = "before_training") -> str:
+    """
+    Clean the files in workspace to a specific stage
+    """
+    assert stage in ["before_training", "before_inference"], f"Unknown stage: {stage}"
+    if DS_RD_SETTING.enable_model_dump and stage == "before_training":
+        cmd = "rm -r submission.csv scores.csv models trace.log"
+    else:
+        cmd = "rm submission.csv scores.csv trace.log"
+    return cmd

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -0,0 +1,164 @@
+"""
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.ensemble.eval import EnsembleCoSTEEREvaluator
+from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: EnsembleTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # Get task information for knowledge querying
+        ensemble_information_str = target_task.get_task_information()
+
+        # Query knowledge
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[ensemble_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[ensemble_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("ensemble.py") != workspace.file_dict.get("ensemble.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # Generate code with knowledge integration
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        system_prompt = T(".prompts:ensemble_coder.system").r(
+            task_desc=ensemble_information_str,
+            competition_info=competition_info,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=(
+                queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None
+            ),
+            all_code=workspace.all_codes,
+            out_spec=PythonAgentOut.get_spec(),
+        )
+
+        if DS_RD_SETTING.spec_enabled:
+            code_spec = workspace.file_dict["spec/ensemble.md"]
+        else:
+            test_code = (
+                Environment(undefined=StrictUndefined)
+                .from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text())
+                .render(
+                    model_names=[
+                        fn[:-3] for fn in workspace.file_dict.keys() if fn.startswith("model_") and "test" not in fn
+                    ],
+                    metric_name=self.scen.metric_name,
+                )
+            )
+            code_spec = T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.Ensemble").r(), test_code=test_code
+            )
+        user_prompt = T(".prompts:ensemble_coder.user").r(
+            code_spec=code_spec,
+            latest_code=workspace.file_dict.get("ensemble.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            ensemble_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if ensemble_code != workspace.file_dict.get("ensemble.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new ensemble code.")
+
+        return {
+            "ensemble.py": ensemble_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class EnsembleCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(EnsembleCoSTEEREvaluator(scen=scen), scen=scen)
+        es = EnsembleMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/ensemble/conf.py

@@ -0,0 +1,2 @@
+# Configuration file for ensemble component
+# Currently empty as no specific configuration is needed

rdagent/components/coder/data_science/ensemble/eval.py

@@ -0,0 +1,100 @@
+import json
+import re
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+EnsembleEvalFeedback = CoSTEERSingleFeedback
+
+
+class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> EnsembleEvalFeedback:
+
+        target_task_information = target_task.get_task_information()
+        metric_name = self.scen.metric_name
+
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return EnsembleEvalFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        fname = "test/ensemble_test.txt"
+        test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()
+        test_code = (
+            Environment(undefined=StrictUndefined)
+            .from_string(test_code)
+            .render(
+                model_names=[
+                    fn[:-3] for fn in implementation.file_dict.keys() if fn.startswith("model_") and "test" not in fn
+                ],
+                metric_name=metric_name,
+            )
+        )
+
+        implementation.inject_files(**{fname: test_code})
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.exit_code
+
+        stdout += f"\nNOTE: the above scripts run with return code {ret_code}"
+
+        if "main.py" in implementation.file_dict and ret_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        system_prompt = T(".prompts:ensemble_eval.system").r(
+            task_desc=target_task_information,
+            test_code=test_code,
+            metric_name=metric_name,
+            code=implementation.file_dict["ensemble.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:ensemble_eval.user").r(
+            stdout=stdout,
+            workflow_stdout=workflow_stdout,
+        )
+        efb = build_cls_from_json_with_retry(
+            EnsembleEvalFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=EnsembleEvalFeedback.val_and_update_init_dict,
+        )
+        efb.final_decision = efb.final_decision and ret_code == 0
+        return efb

rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt

@@ -0,0 +1,137 @@
+"""
+Tests for `ensemble_workflow` in ensemble.py
+
+A qualified ensemble_workflow implementation should:
+- Return predictions
+- Have correct shapes for inputs and outputs
+- Use validation data appropriately
+- Generate a scores.csv file
+"""
+
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from sklearn.model_selection import train_test_split
+import torch
+import tensorflow as tf
+from load_data import load_data
+from feature import feat_eng
+from ensemble import ensemble_workflow
+
+def print_preds_info(model_name, data_type, preds):
+    if preds is None:
+        print(f"Model {model_name} {data_type} predictions: None")
+    else:
+        print(f"Model {model_name} {data_type} predictions shape: {preds.shape}")
+
+        print("Showing a preview of the predictions (first few entries only):")
+        if isinstance(preds, (pd.DataFrame, pd.Series)):
+            print(preds.head())
+        elif isinstance(preds, (np.ndarray, torch.Tensor, tf.Tensor)):
+            print(preds[:2])
+        elif isinstance(preds, list):
+            print(pd.DataFrame(preds[:5]))
+        else:
+            print(f"Unknown prediction type: {type(preds)}")
+
+def get_length(data):
+    return data.shape[0] if hasattr(data, 'shape') else len(data)
+
+X, y, test_X, test_ids = load_data()
+X, y, test_X = feat_eng(X, y, test_X)
+train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# Print the types of train_y and val_y
+print(f"train_y type: {type(train_y)}, val_y type: {type(val_y)}")
+
+test_preds_dict = {}
+val_preds_dict = {}
+{% for mn in model_names %}
+from {{mn}} import model_workflow as {{mn}}_workflow
+val_preds_dict["{{mn}}"], test_preds_dict["{{mn}}"], _ = {{mn}}_workflow(
+    X=train_X,
+    y=train_y,
+    val_X=val_X,
+    val_y=val_y,
+    test_X=test_X
+)
+
+print_preds_info("{{mn}}", "test", test_preds_dict["{{mn}}"])
+{% endfor %}
+
+for key in val_preds_dict.keys():
+    if val_preds_dict[key] is None: 
+        print(f"Model {key} validation predictions (val_preds_dict[key]) is None.")
+    elif isinstance(val_preds_dict[key], list):
+        print(f"Model {key} validation predictions (val_preds_dict[key]) (list type) length: {len(val_preds_dict[key])}")
+    else:
+        print(f"Model {key} validation predictions (val_preds_dict[key]) shape: {val_preds_dict[key].shape}")
+
+    if test_preds_dict[key] is None: 
+        print(f"Model {key} test predictions (test_preds_dict[key]) is None.")
+    elif isinstance(test_preds_dict[key], list):
+        print(f"Model {key} test predictions (test_preds_dict[key]) (list type) length: {len(test_preds_dict[key])}")
+    else:
+        print(f"Model {key} test predictions (test_preds_dict[key]) shape: {test_preds_dict[key].shape}")
+
+print(f"val_y.shape: {val_y.shape}" if not isinstance(val_y, list) else f"val_y(list)'s length: {len(val_y)}")
+
+import sys
+import reprlib
+def debug_info_print(func):
+    aRepr = reprlib.Repr()
+    aRepr.maxother=300
+    def wrapper(*args, **kwargs):
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == func.__code__:
+                print("\n" + "="*20 + "Running ensemble code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+
+
+# Run ensemble
+final_pred = debug_info_print(ensemble_workflow)(test_preds_dict, val_preds_dict, val_y)
+
+print_preds_info("ensemble", "test", final_pred)
+
+# Check type
+pred_type = type(next(iter(test_preds_dict.values())))
+assert isinstance(final_pred, pred_type), (
+    f"Type mismatch: 'final_pred' is of type {type(final_pred)}, but expected {pred_type} "
+)
+
+# Check shape
+if isinstance(final_pred, (list, np.ndarray, pd.DataFrame, torch.Tensor, tf.Tensor)):
+    assert get_length(final_pred) == get_length(test_X), (
+        f"Wrong output sample size: get_length(final_pred)={get_length(final_pred)} "
+        f"vs. get_length(test_X)={get_length(test_X)}"
+    )
+
+# check scores.csv
+assert Path("scores.csv").exists(), "scores.csv is not generated"
+score_df = pd.read_csv("scores.csv", index_col=0)
+model_set_in_scores = set(score_df.index)
+
+assert model_set_in_scores == set({{model_names}}).union({"ensemble"}), (
+    f"The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {{model_names}} + ['ensemble']\nscore_df is:\n{score_df}"
+)
+assert score_df.index.is_unique, "The scores dataframe has duplicate model names."
+assert score_df.columns.tolist() == ["{{metric_name}}"], f"The column names of the scores dataframe should be ['{{metric_name}}'], but is '{score_df.columns.tolist()}'"
+
+# Check for NaN values in score_df
+assert not score_df.isnull().values.any(), (
+    f"The scores dataframe contains NaN values at the following locations:\n{score_df[score_df.isnull().any(axis=1)]}"
+)
+
+
+print("Ensemble test end.")

rdagent/components/coder/data_science/ensemble/exp.py

@@ -0,0 +1,13 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.core.utils import cache_with_pickle
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class EnsembleTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/ensemble/prompts.yaml

@@ -0,0 +1,124 @@
+ensemble_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    Currently, you are working on model ensemble implementation. Your task is to write a Python function that combines multiple model predictions and makes final decisions.
+
+    Your specific task as follows:
+    {{ task_desc }}
+
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict["ensemble.py"] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict["ensemble.py"] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. The function's code is associated with several other functions including a data loader, feature engineering, and model training. all codes are as follows:
+    {{ all_code }}
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %}
+
+
+ensemble_eval:
+  system: |-
+    You are a data scientist responsible for evaluating ensemble implementation code generation.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## Ensemble Code
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The ensemble code is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+    You will analyze the execution results based on the test output provided.
+    
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The ensemble code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the ensemble test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+
+    The metric used for scoring the predictions:
+    **{{ metric_name }}**
+
+    ## Evaluation Criteria
+    - You will be given the standard output (`stdout`) from the ensemble test and, if applicable, the workflow test.
+    - Code should have no try-except blocks because they can hide errors.
+    - Check whether the code implement the scoring process using the given metric.
+    - The stdout includes the local variable values from the ensemble code execution. Check whether the validation score is calculated correctly.
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the ensemble executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Detail the checks performed on the ensemble results, including shape and value validation.",
+        "code": "Assess code quality, readability, and adherence to specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+  user: |-    
+    --------- Ensemble test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/ensemble/test.py

@@ -0,0 +1,58 @@
+"""
+Helper functions for testing the ensemble coder(CoSTEER-based) component.
+"""
+
+import sys
+from pathlib import Path
+
+from rdagent.components.coder.data_science.ensemble import EnsembleCoSTEER
+from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+# Add the competition folder to path
+COMPETITION_PATH = (
+    Path(__file__).parent.parent.parent.parent.parent
+    / "scenarios"
+    / "kaggle"
+    / "tpl_ex"
+    / "aerial-cactus-identification"
+)
+sys.path.append(str(COMPETITION_PATH))
+
+EnsembleExperiment = DSExperiment
+
+
+def load_ensemble_spec():
+    spec_path = COMPETITION_PATH / "spec" / "ensemble.md"
+    with open(spec_path, "r") as f:
+        return f.read()
+
+
+def develop_one_competition(competition: str):
+    # Initialize scenario and coder
+    scen = KaggleScen(competition=competition)
+    ensemble_coder = EnsembleCoSTEER(scen)
+    # Load ensemble specification
+    ensemble_spec = load_ensemble_spec()
+
+    # Create the ensemble task with actual data context and specification
+    task = EnsembleTask(
+        name="EnsembleTask",
+        description="""
+        Implement ensemble and decision making for model predictions.
+        """,
+    )
+
+    exp = EnsembleExperiment(pending_tasks_list=[task])
+
+    # Injecting the corresponding specification
+    exp.experiment_workspace.inject_files(**{"spec/ensemble.md": ensemble_spec})
+
+    # Develop the experiment
+    exp = ensemble_coder.develop(exp)
+    return exp
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")

rdagent/components/coder/data_science/feature/__init__.py

@@ -0,0 +1,140 @@
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.feature.eval import FeatureCoSTEEREvaluator
+from rdagent.components.coder.data_science.feature.exp import FeatureTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: FeatureTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # return a workspace with "load_data.py", "spec/load_data.md" inside
+        # assign the implemented code to the new workspace.
+        feature_information_str = target_task.get_task_information()
+
+        # 1. query
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[feature_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[feature_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("feature.py") != workspace.file_dict.get("feature.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 2. code
+        system_prompt = T(".prompts:feature_coder.system").r(
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
+            task_desc=feature_information_str,
+            data_loader_code=workspace.file_dict.get("load_data.py"),
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+        )
+        code_spec = (
+            workspace.file_dict["spec/feature.md"]
+            if DS_RD_SETTING.spec_enabled
+            else T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.FeatureEng").r(),
+                test_code=(DIRNAME / "eval_tests" / "feature_test.txt").read_text(),
+            )
+        )
+        user_prompt = T(".prompts:feature_coder.user").r(
+            code_spec=code_spec,
+            latest_code=workspace.file_dict.get("feature.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            feature_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if feature_code != workspace.file_dict.get("feature.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new feature code.")
+
+        return {
+            "feature.py": feature_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class FeatureCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            FeatureCoSTEEREvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = FeatureMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/feature/eval.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+FeatureEvalFeedback = CoSTEERSingleFeedback
+
+
+class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> FeatureEvalFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return FeatureEvalFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        # TODO: do we need to clean the generated temporary content?
+        fname = "test/feature_test.py"
+        test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text()
+        implementation.inject_files(**{fname: test_code})
+
+        result = implementation.run(env=env, entry=f"python {fname}")
+
+        if "main.py" in implementation.file_dict and result.exit_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        system_prompt = T(".prompts:feature_eval.system").r(
+            task_desc=target_task.get_task_information(),
+            test_code=test_code,
+            code=implementation.file_dict["feature.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:feature_eval.user").r(
+            stdout=result.stdout,
+            workflow_stdout=workflow_stdout,
+        )
+
+        fb = build_cls_from_json_with_retry(
+            FeatureEvalFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=FeatureEvalFeedback.val_and_update_init_dict,
+        )
+        fb.final_decision = fb.final_decision and result.exit_code == 0
+
+        return fb

rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt

@@ -0,0 +1,114 @@
+"""
+Tests for `feat_eng` in feature.py
+"""
+
+
+from copy import deepcopy
+import sys
+import numpy as np
+import pandas as pd
+from feature import feat_eng
+from load_data import load_data
+import reprlib
+aRepr = reprlib.Repr()
+aRepr.maxother=300
+
+X, y, X_test, test_ids = load_data()
+print("X:", aRepr.repr(X))
+print("y:", aRepr.repr(y))
+print("X_test:", aRepr.repr(X_test))
+print("test_ids", aRepr.repr(test_ids))
+
+print(f"X.shape: {X.shape}" if hasattr(X, 'shape') else f"X length: {len(X)}")
+print(f"y.shape: {y.shape}" if hasattr(y, 'shape') else f"y length: {len(y)}")
+print(f"X_test.shape: {X_test.shape}" if hasattr(X_test, 'shape') else f"X_test length: {len(X_test)}")
+print(f"test_ids length: {len(test_ids)}")
+
+X_loaded = deepcopy(X)
+y_loaded = deepcopy(y)
+X_test_loaded = deepcopy(X_test)
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+
+def debug_info_print(func):
+    def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == original_code:
+                print("\n" + "="*20 + "Running feat_eng code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+X, y, X_test = debug_info_print(feat_eng)(X, y, X_test)
+
+
+def get_length(data):
+    return data.shape[0] if hasattr(data, 'shape') else len(data)
+
+
+def get_width(data):
+    return 1 if isinstance(data, list) else data.shape[1:]
+
+
+def get_column_list(data):
+    return data.columns.tolist() if isinstance(data, pd.DataFrame) else None
+
+
+assert X is not None, "The feature engineering function returned None for X."
+assert y is not None, "The feature engineering function returned None for y."
+assert X_test is not None, "The feature engineering function returned None for X_test."
+
+assert get_length(X_test) == get_length(
+    test_ids
+), f"Mismatch in length of test images and test IDs: X_test ({get_length(X_test)}) and test_ids ({get_length(test_ids)})"
+assert get_length(X) == get_length(
+    y
+), f"Mismatch in length of training images and labels: X ({get_length(X)}) and y ({get_length(y)})"
+
+assert get_length(X) != 0, f"Training data is empty."
+assert get_length(y) != 0, f"Training labels are empty."
+assert get_length(X_test) != 0, f"Test data is empty."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+if isinstance(X, pd.DataFrame) and isinstance(X_test, pd.DataFrame):
+    assert get_column_list(X) == get_column_list(X_test), "Mismatch in column names of training and test data."
+
+if isinstance(X, pd.DataFrame):
+    def normalize_dtype(dtype):
+        return "numeric" if np.issubdtype(dtype, np.number) else str(dtype)
+
+    X_dtypes_unique_sorted = sorted(set(normalize_dtype(dt) for dt in X.dtypes.unique()))
+    X_loaded_dtypes_unique_sorted = sorted(set(normalize_dtype(dt) for dt in X_loaded.dtypes.unique()))
+
+    X_dtypes_unique_sorted_new = [
+        dt for dt in X_dtypes_unique_sorted if dt not in X_loaded_dtypes_unique_sorted and dt != "object"
+    ]
+    assert (
+        np.dtypes.ObjectDType in X_loaded_dtypes_unique_sorted or len(X_dtypes_unique_sorted_new) == 0
+    ), f"feature engineering has produced new data types which is not allowed, data loader data types are {X_loaded_dtypes_unique_sorted} and feature engineering data types are {X_dtypes_unique_sorted}"
+
+
+print(
+    "Feature Engineering test passed successfully. All checks including length, width, and data types have been validated."
+)

rdagent/components/coder/data_science/feature/exp.py

@@ -0,0 +1,13 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.core.utils import cache_with_pickle
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class FeatureTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/feature/prompts.yaml

@@ -0,0 +1,131 @@
+feature_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict["feature.py"] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict["feature.py"] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. If feature engineering is unnecessary or should be combined with model training, you may skip this step.
+    2. Be cautious of any column drop in the code. Dropping a column easily without any more attempts, it may not be a good practice.
+    3. The function input is the output of the following data loader:
+    ```python
+    {{ data_loader_code }}
+    ```
+    4. **Additional Guidance:**
+      - If a previous attempt exists, improve upon it without repeating mistakes.
+      - If errors indicate a missing file, find a way to download it or implement an alternative solution.
+      - You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    5. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache```
+    6. Coding tricks:
+      - If the input consists of a batch of file paths and you need to modify the file contents to complete your feature engineering task, you can accomplish your feature engineering task by modifying these files and creating new files in a subfolder within "{% include "scenarios.data_science.share:scen.cache_path" %}" (this path is persistent, otherwise you may lose your created file). Then the new file paths are returned.
+
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+  
+  user: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %}
+
+
+feature_eval:
+  system: |-
+    You are a data scientist responsible for evaluating feature engineering code generation.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## Feature Engineering Code
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The feature engineering code is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+    You will analyze the execution results based on the test output provided.
+
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The feature engineering code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the feature engineering test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the feature engineering test and, if applicable, the workflow test.
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the feature engineering executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Evaluate the correctness and integrity of processed data, checking for missing values, incorrect transformations, and data consistency.",
+        "code": "Assess code quality, readability, and adherence to specifications. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for optimization.",
+        "final_decision": <true/false>
+    }
+    ```
+  
+  user: |-
+    --------- Feature engineering test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/feature/test.py

@@ -0,0 +1,37 @@
+"""
+Helper functions for testing the feature coder(CoSTEER-based) component.
+- Does the developer loop work correctly
+
+It is NOT:
+- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop)
+"""
+
+from rdagent.components.coder.data_science.feature import FeatureCoSTEER
+from rdagent.components.coder.data_science.feature.exp import FeatureTask
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+def develop_one_competition(competition: str):  # -> experiment
+    scen = KaggleScen(competition=competition)
+    feature_coder = FeatureCoSTEER(scen)
+
+    with open("./rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/feature.md", "r") as file:
+        feat_spec = file.read()
+
+    # Create the experiment
+    ft = FeatureTask(name="FeatureTask", description=scen.get_competition_full_desc())
+    exp = DSExperiment(
+        sub_tasks=[ft],
+    )
+
+    with open("./rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/load_data.py", "r") as file:
+        load_data_code = file.read()
+    exp.experiment_workspace.inject_files(**{"load_data.py": load_data_code, "spec/feature.md": feat_spec})
+
+    # Develop the experiment
+    exp = feature_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")

rdagent/components/coder/data_science/model/__init__.py

@@ -0,0 +1,173 @@
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.model.eval import (
+    ModelGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.model.exp import ModelTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonBatchEditOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: ModelTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        model_information_str = target_task.get_task_information()
+
+        # 1. query
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[model_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[model_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get(f"{target_task.name}.py")
+                != workspace.file_dict.get(f"{target_task.name}.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 2. code
+        system_prompt = T(".prompts:model_coder.system").r(
+            task_desc=model_information_str,
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
+            data_loader_code=workspace.file_dict.get("load_data.py"),
+            feature_code=workspace.file_dict["feature.py"],
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonBatchEditOut.get_spec(),
+        )
+        # user_prompt = T(".prompts:model_coder.user").r(
+        #     model_spec=workspace.file_dict["spec/model.md"],
+        #     feature_code=workspace.file_dict["feature.py"],
+        #     latest_code=workspace.file_dict.get(f"{target_task.name}.py", None),
+        # )
+        # We want to use a simpler way to
+        code_spec = (
+            workspace.file_dict["spec/model.md"]
+            if DS_RD_SETTING.spec_enabled
+            else T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.Model").r(),
+                test_code=(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name),
+            )
+        )
+        user_prompt = T(".prompts:model_coder.user_general").r(
+            code_spec=code_spec,
+            latest_model_code=workspace.get_codes(
+                r"^model_(?!test)\w+\.py$"
+            ),  # TODO: If we have high failure rate here, we should clean this step with less information.
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            batch_edit = PythonBatchEditOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+
+            if not all(i.startswith("model_") for i in batch_edit.keys()):
+                user_prompt += "\nYou should only update model codes!"
+                continue
+
+            # 3. post process to align file name to the task name
+            # we assumpt batch_edit only contains one model file update.
+            batch_edit = {
+                (f"{target_task.name}.py" if value != "__DEL__" and key != f"{target_task.name}.py" else key): value
+                for key, value in batch_edit.items()
+            }
+
+            user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+            # TODO: besides same code problem, we should also consider other problems lead to retry.
+            if f"{target_task.name}.py" not in batch_edit:
+                continue
+
+            if batch_edit and max(len(i.encode("utf-8")) for i in batch_edit.keys()) > 255:
+                continue
+
+            if batch_edit[f"{target_task.name}.py"] != "__DEL__" and batch_edit[
+                f"{target_task.name}.py"
+            ] != workspace.file_dict.get(f"{target_task.name}.py"):
+                break
+
+            # If the task involves model removal, assume it can only process one model at a time.
+            if len(batch_edit) == 1 and batch_edit[f"{target_task.name}.py"] == "__DEL__":
+                break
+        else:
+            raise CoderError("Failed to generate a new model code.")
+
+        return batch_edit
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class ModelCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            ModelGeneralCaseSpecEvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        # eva = ModelGeneralCaseSpecEvaluator(scen=scen)
+        es = ModelMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/model/eval.py

@@ -0,0 +1,123 @@
+"""
+Beyond previous tests
+-
+"""
+
+import json
+import re
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+ModelSingleFeedback = CoSTEERSingleFeedback
+
+
+# Below are unit tests for testing the specification of the implemented model ------------------
+class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
+    """
+    Motivation case:
+    - Simplest case, we already split the data into train_data, valid_data, and test_data. We require the model to learn (optionally validate on valid data), and infer on test data.
+
+    Test workflow:
+    - Build train, valid, and test data to run it, and test the output (e.g., shape, etc.)
+    """
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> ModelSingleFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return ModelSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        if_model_removed = False
+
+        if f"{target_task.name}.py" in implementation.file_dict:
+            fname = "test/model_test.py"
+            test_code = (
+                (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
+            )  # only check the model changed this time
+            implementation.inject_files(**{fname: test_code})
+            result = implementation.run(env=env, entry=f"python {fname}")
+            stdout = result.stdout
+            ret_code = result.exit_code
+
+            if stdout is None:
+                raise CoderError(
+                    "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit."
+                )
+        else:
+            ret_code = 0
+            if_model_removed = True
+            stdout = f"Model {target_task.name} removal succeeded."
+
+        if "main.py" in implementation.file_dict and ret_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        if if_model_removed:
+            system_prompt = T(".prompts:model_eval_rm.system").r(
+                task_desc=target_task.get_task_information(),
+                workflow_stdout=workflow_stdout,
+                workflow_code=implementation.all_codes,
+            )
+            user_prompt = T(".prompts:model_eval_rm.user").r(
+                stdout=stdout,
+                workflow_stdout=workflow_stdout,
+            )
+        else:
+            system_prompt = T(".prompts:model_eval.system").r(
+                task_desc=target_task.get_task_information(),
+                test_code=test_code,
+                code=implementation.file_dict[f"{target_task.name}.py"],
+                workflow_stdout=workflow_stdout,
+                workflow_code=implementation.all_codes,
+            )
+            user_prompt = T(".prompts:model_eval.user").r(
+                stdout=stdout,
+                workflow_stdout=workflow_stdout,
+            )
+
+        fb = build_cls_from_json_with_retry(
+            ModelSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict,
+        )
+        fb.final_decision = fb.final_decision and ret_code == 0
+
+        return fb

rdagent/components/coder/data_science/model/eval_tests/model_test.txt

@@ -0,0 +1,105 @@
+"""
+Tests for `model_workflow` in model01.py
+"""
+import sys
+import time
+
+from feature import feat_eng
+from load_data import load_data
+from model01 import model_workflow
+from sklearn.model_selection import train_test_split
+
+
+def log_execution_results(start_time, val_pred, test_pred, hypers, execution_label):
+    """Log the results of a single model execution."""
+    feedback_str = f"{execution_label} end.\n"
+    feedback_str += f"Validation predictions shape: {val_pred.shape if val_pred is not None else 'None'}\n"
+    feedback_str += f"Test predictions shape: {test_pred.shape if test_pred is not None else 'None'}\n"
+    feedback_str += f"Hyperparameters: {hypers if hypers is not None else 'None'}\n"
+    feedback_str += f"Execution time: {time.time() - start_time:.2f} seconds.\n"
+    print(feedback_str)
+
+
+import reprlib
+aRepr = reprlib.Repr()
+aRepr.maxother=300
+
+# Load and preprocess data
+X, y, test_X, test_ids = load_data()
+X, y, test_X = feat_eng(X, y, test_X)
+
+print(f"X.shape: {X.shape}" if hasattr(X, 'shape') else f"X length: {len(X)}")
+print(f"y.shape: {y.shape}" if hasattr(y, 'shape') else f"y length: {len(y)}")
+print(f"test_X.shape: {test_X.shape}" if hasattr(test_X, 'shape') else f"test_X length: {len(test_X)}")
+print(f"test_ids length: {len(test_ids)}")
+
+train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.8, random_state=42)
+
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+print("train_X:", aRepr.repr(train_X))
+print("train_y:", aRepr.repr(train_y))
+print("val_X:", aRepr.repr(val_X))
+print("val_y:", aRepr.repr(val_y))
+
+print(f"train_X.shape: {train_X.shape}" if hasattr(train_X, 'shape') else f"train_X length: {len(train_X)}")
+print(f"train_y.shape: {train_y.shape}" if hasattr(train_y, 'shape') else f"train_y length: {len(train_y)}")
+print(f"val_X.shape: {val_X.shape}" if hasattr(val_X, 'shape') else f"val_X length: {len(val_X)}")
+print(f"val_y.shape: {val_y.shape}" if hasattr(val_y, 'shape') else f"val_y length: {len(val_y)}")
+
+
+
+def debug_info_print(func):
+    def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == original_code:
+                print("\n" + "="*20 + "Running model training code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+
+# First execution
+print("The first execution begins.\n")
+start_time = time.time()
+val_pred, test_pred, hypers = debug_info_print(model_workflow)(
+    X=train_X,
+    y=train_y,
+    val_X=val_X,
+    val_y=val_y,
+    test_X=None,
+)
+log_execution_results(start_time, val_pred, test_pred, hypers, "The first execution")
+
+# Second execution
+print("The second execution begins.\n")
+start_time = time.time()
+val_pred, test_pred, final_hypers = debug_info_print(model_workflow)(
+    X=train_X,
+    y=train_y,
+    val_X=None,
+    val_y=None,
+    test_X=test_X,
+    hyper_params=hypers,
+)
+log_execution_results(start_time, val_pred, test_pred, final_hypers, "The second execution")
+
+print("Model code test end.")

rdagent/components/coder/data_science/model/exp.py

@@ -0,0 +1,21 @@
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class ModelTask(CoSTEERTask):
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(name=name, description=description, *args, **kwargs)
+
+    def get_task_information(self):
+        task_desc = f"""name: {self.name}
+description: {self.description}
+"""
+        return task_desc

rdagent/components/coder/data_science/model/prompts.yaml

@@ -0,0 +1,186 @@
+model_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict[similar_successful_knowledge.target_task.name ~ '.py'] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict[former_failed_knowledge.target_task.name ~ '.py'] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. The function's input is from the output of a feature engineering function whose input is the output of a data loading function. The data loader function and feature engineering function code is as follows:
+    --------- Data Loader Code ---------
+    {{ data_loader_code }}
+    --------- Feature Engineering Code ---------
+    {{ feature_code }}
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. If the model can both be implemented by PyTorch and Tensorflow, please use pytorch for broader compatibility.
+    4. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache``
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    The file name should be the model name described in the model task in the format "{task_name}.py". You should always follow this name format.
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user_general: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    --------- Former model code ---------
+    {% if latest_model_code|length == 0 %}
+    So far the workspace is empty. No model code has been implemented yet.
+    {% else %}
+    {{ latest_model_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    {% endif %}
+
+model_eval:
+  system: |-
+    You are a data scientist responsible for evaluating model building code generation.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## Model Building Code
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The model building code is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+
+    ### Execution Phases
+    The model is tested in two phases:
+
+    1. Initial Training Phase:
+       - The model receives **train and valid inputs** with **empty hyperparameters**.
+       - The focus is on verifying whether the model successfully trains and produces **valid outputs and hyperparameter outputs**.
+
+    2. Retraining Phase:
+       - The model receives **train and test inputs** (without valid inputs).
+       - The hyperparameters generated from the first phase are passed back for **retraining**.
+
+
+    ### Key Requirements for Approval
+    A model can only be approved if it meets all of the following conditions:
+    1. Hyperparameter Handling
+      - If hyperparameters are returned, they must include an early stop round.
+      - The hyperparameters must be correctly utilized in the model for retraining.
+      - If the early stop round is provided, it must be used in the model implementation.
+    2. The model output shape must strictly match the specifications in `spec.md`.
+
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The model building code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the model building test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the model building test and, if applicable, the workflow test.
+    [Note] If no stdout for model buidling test is provided, the model failed due to a timeout or out-of-memory error. You should analyze potential optimizations.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the model building executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in spec.md. You also need to check whether the hyperparameters used for retraining are correctly returned during the test execution of the model.",
+        "code": "Assess code quality, readability, and adherence to specifications. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for optimization.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Model building test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}
+
+model_eval_rm:
+  system: |-
+    You are a data scientist responsible for evaluating model removal process.
+
+    ## Task Description
+    {{ task_desc }}
+
+    {% if workflow_stdout is not none %}
+    ## Whole Workflow Consideration
+    The model building code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the model removal test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the model removal test and, if applicable, the workflow test.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the model removal executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in spec.md.",
+        "code": "Assess code quality, readability, and adherence to specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Model removal test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/model/test.py

@@ -0,0 +1,67 @@
+"""
+Generate dataset to test the model workflow output
+"""
+
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
+from rdagent.components.coder.data_science.model import ModelCoSTEER
+from rdagent.components.coder.data_science.model.eval import (
+    ModelGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.model.exp import ModelTask
+from rdagent.core.experiment import FBWorkspace
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+# Take tasks, spec.md and feat as input, generate a feedback as output
+def develop_one_competition(competition: str):
+    scen = KaggleScen(competition=competition)
+    model_coder = ModelCoSTEER(scen)
+
+    # Create the task
+    mt = ModelTask(
+        name="ModelTask",
+        description="A CNN Model",
+        model_type="CNN",
+        architecture="\hat{y}_u = CNN(X_u)",
+        # variables="variables: {'\\hat{y}_u': 'The predicted output for node u', 'X_u': 'The input features for node u'}",
+        hyperparameters="...",
+        base_code="",
+    )
+
+    tpl_ex_path = Path(__file__).resolve() / Path("rdagent/scenarios/kaggle/tpl_ex").resolve() / competition
+    injected_file_names = ["spec/model.md", "load_data.py", "feature.py", "model01.py"]
+
+    modelexp = FBWorkspace()
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        modelexp.inject_files(**{file_name: file_path.read_text()})
+
+    mt.base_code += modelexp.file_dict["model01.py"]
+    exp = DSExperiment(
+        sub_tasks=[mt],
+    )
+
+    # Test the evaluator:
+    """eva = ModelGeneralCaseSpecEvaluator(scen=scen)
+    exp.feedback = eva.evaluate(target_task=mt, queried_knowledge=None, implementation=modelexp, gt_implementation=None)
+    print(exp.feedback)"""
+
+    # Test the evolving strategy:
+    """es = ModelMultiProcessEvolvingStrategy(scen=scen, settings=CoSTEER_SETTINGS)
+    new_code = es.implement_one_task(target_task=mt, queried_knowledge=None, workspace=modelexp)
+    print(new_code)"""
+
+    # Run the experiment
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        exp.experiment_workspace.inject_files(**{file_name: file_path.read_text()})
+
+    exp = model_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")
+    # dotenv run -- python rdagent/components/coder/data_science/model/test.py

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -0,0 +1,165 @@
+"""
+
+Loop should not large change exclude
+- Action Choice[current data loader & spec]
+- other should share
+    - Propose[choice] => Task[Choice] => CoSTEER =>
+        -
+
+Extra feature:
+- cache
+
+
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator
+from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: PipelineTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        data_folder_info = self.scen.processed_data_folder_description
+        pipeline_task_info = target_task.get_task_information()
+
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[pipeline_task_info] if queried_knowledge is not None else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        system_prompt = T(".prompts:pipeline_coder.system").r(
+            task_desc=pipeline_task_info,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+            runtime_environment=self.scen.get_runtime_environment(),
+            package_info=target_task.package_info,
+            enable_model_dump=DS_RD_SETTING.enable_model_dump,
+            enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM,
+            spec=T("scenarios.data_science.share:component_spec.Pipeline").r(
+                metric_name=self.scen.metric_name,
+                enable_notebook_conversion=DS_RD_SETTING.enable_notebook_conversion,
+            ),
+        )
+        user_prompt = T(".prompts:pipeline_coder.user").r(
+            competition_info=competition_info,
+            folder_spec=data_folder_info,
+            latest_code=workspace.file_dict.get("main.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            pipeline_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if pipeline_code != workspace.file_dict.get("main.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new pipeline code.")
+
+        return {
+            "main.py": pipeline_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class PipelineCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eval_l = [PipelineCoSTEEREvaluator(scen=scen)]
+        if DS_RD_SETTING.enable_model_dump:
+            eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample"))
+        for evaluator in settings.extra_evaluator:
+            eval_l.append(import_class(evaluator)(scen=scen))
+
+        for extra_eval in DSCoderCoSTEERSettings().extra_eval:
+            kls = import_class(extra_eval)
+            eval_l.append(kls(scen=scen))
+
+        eva = CoSTEERMultiEvaluator(
+            single_evaluator=eval_l, scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/pipeline/eval.py

@@ -0,0 +1,348 @@
+# tess successfully running.
+# (GPT) if it aligns with the spec & rationality of the spec.
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.agent.context7 import Agent as DocAgent
+from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.share.notebook import NotebookConverter
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.data_science.test_eval import get_test_eval
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+@dataclass
+class DSCoderFeedback(CoSTEERSingleFeedback):
+    """
+    Feedback for Data Science CoSTEER evaluation.
+    This feedback is used to evaluate the code and execution of the Data Science CoSTEER task.
+    """
+
+    requires_documentation_search: bool | None = None  # Keep None means the feature is disabled
+    error_message: str | None = None
+
+    @staticmethod
+    def val_and_update_init_dict(data: dict) -> dict:
+        # First call parent class validation method to handle base fields
+        data = CoSTEERSingleFeedback.val_and_update_init_dict(data)
+
+        # Validate new fields
+        if "requires_documentation_search" in data:
+            if isinstance(data["requires_documentation_search"], str):
+                if data["requires_documentation_search"] == "false" or data["requires_documentation_search"] == "False":
+                    data["requires_documentation_search"] = False
+                elif data["requires_documentation_search"] == "true" or data["requires_documentation_search"] == "True":
+                    data["requires_documentation_search"] = True
+                else:
+                    raise ValueError(
+                        f"'requires_documentation_search' string value must be 'true', 'True', 'false', or 'False', not '{data['requires_documentation_search']}'"
+                    )
+            elif data["requires_documentation_search"] is not None and not isinstance(
+                data["requires_documentation_search"], bool
+            ):
+                raise ValueError(
+                    f"'requires_documentation_search' must be a boolean, string, or None, not {type(data['requires_documentation_search'])}"
+                )
+
+        if "error_message" in data:
+            if data["error_message"] is not None and not isinstance(data["error_message"], str):
+                raise ValueError(f"'error_message' must be a string or None, not {type(data['error_message'])}")
+
+        return data
+
+    def __str__(self) -> str:
+        base_str = super().__str__()
+
+        if self.requires_documentation_search is not None:
+            base_str += f"-------------------Documentation Search Required------------------\n{self.requires_documentation_search}\n"
+
+        if self.error_message is not None:
+            # Check if error_message contains Context7 documentation results
+            if "### API Documentation Reference:" in self.error_message:
+                base_str += f"-------------------Error Analysis & Documentation Search Results ------------------\n{self.error_message}\n"
+            else:
+                base_str += f"-------------------Error Message------------------\n{self.error_message}\n"
+
+        return base_str
+
+    @classmethod
+    def merge(cls, feedback_li: list[CoSTEERSingleFeedback]) -> "DSCoderFeedback":
+        # Call parent class merge method to handle base fields
+        merged_fb = super().merge(feedback_li)
+
+        # Convert to DSCoderFeedback type if needed
+        if not isinstance(merged_fb, DSCoderFeedback):
+            merged_fb = DSCoderFeedback(
+                execution=merged_fb.execution,
+                return_checking=merged_fb.return_checking,
+                code=merged_fb.code,
+                final_decision=merged_fb.final_decision,
+            )
+
+        # Merge error_message fields
+        error_messages = [
+            fb.error_message for fb in feedback_li if isinstance(fb, DSCoderFeedback) and fb.error_message is not None
+        ]
+        if error_messages:
+            merged_fb.error_message = "\n\n".join(error_messages)
+
+        # Merge requires_documentation_search fields (True if any is True)
+        requires_search = [
+            fb.requires_documentation_search
+            for fb in feedback_li
+            if isinstance(fb, DSCoderFeedback) and fb.requires_documentation_search is not None
+        ]
+        if requires_search:
+            merged_fb.requires_documentation_search = any(requires_search)
+
+        return merged_fb
+
+
+PipelineSingleFeedback = DSCoderFeedback  # Only for compatible
+PipelineMultiFeedback = CoSTEERMultiFeedback
+
+
+class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
+        **kwargs,
+    ) -> PipelineSingleFeedback:
+
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return PipelineSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                error_message="This task has failed too many times, skip implementation.",
+                requires_documentation_search=None,
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        stdout = ""
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
+        if DS_RD_SETTING.sample_data_by_LLM:
+            # Because coder runs on full data, we need to run debug mode in advance to save time
+            result = implementation.run(
+                env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py --debug"
+            )
+        else:
+            result = implementation.run(
+                env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py"
+            )
+        result_stdout = result.stdout
+
+        nb_conversion_ret_code = 0
+        nb_conversion_check_text = ""
+        if DS_RD_SETTING.enable_notebook_conversion:
+            notebook_converter = NotebookConverter()
+            code = implementation.file_dict["main.py"]
+            error_msg = notebook_converter.validate_code_format(code)
+            if error_msg is not None:
+                nb_conversion_check_text = error_msg
+                nb_conversion_ret_code = 1
+            else:
+                notebook_converter.convert(
+                    task=target_task,
+                    code=code,
+                    stdout=result_stdout,
+                    outfile=implementation.workspace_path / "main.ipynb",
+                    use_debug_flag=DS_RD_SETTING.sample_data_by_LLM,
+                )
+
+        sample_submission_check = True
+        test_eval = get_test_eval()
+        if (sample_submission_file_name := test_eval.get_sample_submission_name(self.scen.competition)) is not None:
+            # check whether code ever opens the sample submission file
+            if (implementation.workspace_path / "trace.log").exists():
+                opened_trace_lines = [
+                    line
+                    for line in (implementation.workspace_path / "trace.log").read_text().splitlines()
+                    if "openat" in line and sample_submission_file_name in line
+                ]
+                if len(opened_trace_lines) > 0:
+                    stdout += f"Code opened the sample submission file '{sample_submission_file_name}' during execution.\n Reject the implementation!\n"
+                    sample_submission_check = False
+
+        result_stdout = remove_eda_part(result_stdout)
+        if result.exit_code != 0:
+            stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result_stdout.strip()}\n"
+        else:
+            stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result_stdout.strip()}\n"
+        if DS_RD_SETTING.sample_data_by_LLM:
+            debug_time, full_estimated_time = None, None
+            if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result_stdout, re.DOTALL):
+                debug_time = float(match.group(1))
+            if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result_stdout, re.DOTALL):
+                full_estimated_time = float(match.group(1))
+            if debug_time is not None and full_estimated_time is not None:
+                stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds. The estimated time is {full_estimated_time / env.conf.running_timeout_period * 100:.2f}% the debug time."
+            else:
+                stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n"
+
+        score_fp = implementation.workspace_path / "scores.csv"
+        score_ret_code = 0
+        score_check_text = ""
+        if not score_fp.exists():
+            score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
+            score_ret_code = 1
+        else:
+            try:
+                score_df = pd.read_csv(score_fp, index_col=0)
+                model_set_in_scores = set(score_df.index)
+
+                # Check model names (index)
+                if not score_df.index.is_unique:
+                    score_check_text += "\n[Error] The file 'scores.csv' contains duplicate model names."
+                    score_ret_code = 1
+                if "ensemble" not in model_set_in_scores:
+                    score_check_text += "\n[Error] The file 'scores.csv' doesn't contain the ensemble model."
+                    score_ret_code = 1
+                if score_ret_code != 0:
+                    score_check_text += f"The dataframe in file 'scores.csv' is:\n{score_df}"
+
+                # Check metric name (columns) - case insensitive
+                if [col.lower() for col in score_df.columns.tolist()] != [self.scen.metric_name.lower()]:
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
+                    score_ret_code = 1
+
+                # Check if scores contain NaN (values)
+                if score_df.isnull().values.any():
+                    nan_locations = score_df[score_df.isnull().any(axis=1)]
+                    score_check_text += f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+                    score_ret_code = 1
+
+            except Exception as e:
+                score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
+                score_ret_code = 1
+
+        test_eval = get_test_eval()
+        if DS_RD_SETTING.sample_data_by_LLM and test_eval.enabled(self.scen.competition):
+            submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation)
+            stdout += f"\n### Submission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
+        elif not test_eval.is_sub_enabled(self.scen.competition):
+            submission_ret_code = 0
+        else:
+            # Check submission file
+            base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
+            implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+            # stdout += "----Submission Check 1-----\n"
+            submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+            submission_check_out = submission_result.stdout
+            submission_ret_code = submission_result.exit_code
+            stdout += "\n" + submission_check_out
+
+        if not isinstance(implementation, FBWorkspace):
+            eda_output = None
+        else:
+            eda_output = implementation.file_dict.get("EDA.md", None)
+
+        # extract enable_mcp_documentation_search from data science configuration
+        enable_mcp_documentation_search = DS_RD_SETTING.enable_mcp_documentation_search
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[target_task.get_task_information()]
+            if queried_knowledge is not None
+            else []
+        )
+
+        system_prompt = T(".prompts:pipeline_eval.system").r(
+            is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
+            debug_mode=DS_RD_SETTING.sample_data_by_LLM,
+            enable_mcp_documentation_search=enable_mcp_documentation_search,
+            mle_check=DS_RD_SETTING.sample_data_by_LLM,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+        )
+        user_prompt = T(".prompts:pipeline_eval.user").r(
+            scenario=self.scen.get_scenario_all_desc(eda_output=eda_output),
+            task_desc=target_task.get_task_information(),
+            stdout=stdout.strip(),
+            spec=T("scenarios.data_science.share:component_spec.Pipeline").r(
+                metric_name=self.scen.metric_name,
+                enable_notebook_conversion=DS_RD_SETTING.enable_notebook_conversion,
+            ),
+            code=implementation.file_dict["main.py"],
+        )
+        wfb = build_cls_from_json_with_retry(
+            PipelineSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=PipelineSingleFeedback.val_and_update_init_dict,
+        )
+
+        # judge whether we should perform documentation search
+        do_documentation_search = enable_mcp_documentation_search and wfb.requires_documentation_search
+
+        if do_documentation_search:
+            # Use MCPAgent for clean, user-friendly interface
+            try:
+                # Create agent targeting Context7 service - model config comes from mcp_config.json
+                doc_agent = DocAgent()
+
+                # Synchronous query - perfect for evaluation context
+                if wfb.error_message:  # Type safety check
+                    context7_result = doc_agent.query(query=wfb.error_message)
+
+                    if context7_result:
+                        logger.info("Context7: Documentation search completed successfully")
+                        wfb.error_message += f"\n\n### API Documentation Reference:\nThe following API documentation was retrieved based on the error. This provides factual information about API changes or parameter specifications only:\n\n{context7_result}"
+                    else:
+                        logger.warning("Context7: Documentation search failed or no results found")
+                else:
+                    logger.warning("Context7: No error message to search for")
+
+            # TODO: confirm what exception will be raised when timeout
+            # except concurrent.futures.TimeoutError:
+            #     logger.error("Context7: Query timed out after 180 seconds")
+            except Exception as e:
+                error_msg = str(e) if str(e) else type(e).__name__
+                logger.error(f"Context7: Query failed - {error_msg}")
+
+        if score_ret_code != 0 and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + score_check_text
+        if submission_ret_code != 0 and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += "\nSubmission file check failed."
+        if sample_submission_check is False and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += (
+                "\nSample submission file check failed. Code should not open the sample submission file."
+            )
+        if nb_conversion_ret_code != 0 and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + nb_conversion_check_text
+        return wfb

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -0,0 +1,94 @@
+import hashlib
+from pathlib import Path
+import pandas as pd
+
+
+def calculate_md5(file_path):
+    with open(file_path, "rb") as f:
+        file_hash = hashlib.md5(f.read()).hexdigest()
+    return file_hash
+
+
+if Path("scores.csv").exists():
+    file_md5 = calculate_md5("scores.csv")
+else:
+    print("Warning: scores.csv does not exist. MD5 check will be skipped.")
+    file_md5 = None
+    
+"""
+find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
+./denoising-dirty-documents/sampleSubmission.csv
+./the-icml-2013-whale-challenge-right-whale-redux/sampleSubmission.csv
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv.zip
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv
+./random-acts-of-pizza/sampleSubmission.csv
+./text-normalization-challenge-english-language/en_sample_submission_2.csv.zip
+./text-normalization-challenge-english-language/en_sample_submission_2.csv
+./detecting-insults-in-social-commentary/sample_submission_null.csv
+"""
+
+# Find sample submission file dynamically
+input_dir = Path('{% include "scenarios.data_science.share:scen.input_path" %}')
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
+    input_dir.glob("*sampleSubmission*.csv")
+) + list(input_dir.glob("*randomPredictions*.tsv"))
+
+if not sample_submission_files:
+    print(f'Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}')
+    sample_submission_name = None
+    SAMPLE_SUBMISSION_PATH = None
+else:
+    sample_submission_name = sample_submission_files[0].name
+    SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+    print(f"Using sample submission file: {sample_submission_name}")
+
+if SAMPLE_SUBMISSION_PATH is not None and not Path(SAMPLE_SUBMISSION_PATH).exists():
+    print(f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}")
+
+if not Path("submission.csv").exists():
+    print("Error: submission.csv not found")
+
+if SAMPLE_SUBMISSION_PATH is not None and Path(SAMPLE_SUBMISSION_PATH).exists() and Path("submission.csv").exists():
+    sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+    our_submission = pd.read_csv("submission.csv")
+
+    success = True
+    print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+    print("Columns in our_submission.csv:", our_submission.columns)
+
+    for col in sample_submission.columns:
+        if col not in our_submission.columns:
+            success = False
+            print(f"Column {col} not found in submission.csv")
+
+    if success:
+        print(f"submission.csv's columns aligns with {sample_submission_name} .")
+    else:
+        print(f"submission.csv's columns does not align with {sample_submission_name} .")
+
+
+    def print_first_rows(file_path, file_name, num_rows=5):
+        print(f"\nFirst {num_rows} rows of {file_name}:")
+        try:
+            with open(file_path, "r") as file:
+                for i, line in enumerate(file):
+                    if i < num_rows:
+                        print(line.strip())
+                    else:
+                        break
+        except FileNotFoundError:
+            print(f"Error: {file_name} not found.")
+
+
+    print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+    print_first_rows("submission.csv", "submission.csv")
+
+    if file_md5 is not None:
+        if calculate_md5("scores.csv") != file_md5:
+            print("Warning: scores.csv has been rewritten in the test script!")
+else:
+    print("Skipping comparison and preview due to missing files.")
+
+print(
+    f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). "
+)

rdagent/components/coder/data_science/pipeline/exp.py

@@ -0,0 +1,8 @@
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class PipelineTask(CoSTEERTask):
+    def __init__(self, name: str = "Pipeline", package_info: str | None = None, *args, **kwargs) -> None:
+        super().__init__(name=name, *args, **kwargs)
+        self.package_info = package_info

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -0,0 +1,347 @@
+pipeline_coder:
+  system: |-
+    You are a grandmaster-level data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+    Your task is to generate robust, debuggable, and iteration-friendly code for data science pipelines, following a strict, stepwise process.
+
+    **Important Context**: You are working on sample datasets and your code will go through automated iterations. Design your code to be iteration-friendly with comprehensive print statements and clear debugging information to facilitate the automatic improvement process.
+
+    # Task Description
+    {{ task_desc }}
+    
+    ## The runtime environment your code will running on
+    {{ runtime_environment }}
+
+    {% if package_info is not none %}
+    To help you write the runnable code, the user has provided the package information which contains the package names and versions.
+    You should be careful about the package versions, as the code will be executed in the environment with the specified version and the api might be different from the latest version.
+    The user might provide the packages the environment doesn't have, you should avoid using any of them.
+    ## Package Information
+    {{ package_info }}
+    {% endif %}
+    
+    ## Hyperparameters Specification
+    Follow the hyperparameter choices if they are specified in the task description, unless they are unreasonable or incorrect.
+    In this case, refer to the guidelines below for appropriate adjustments:
+    {% include "scenarios.data_science.share:spec.hyperparameter" %}
+    
+    # Specification your code should follow
+    {{ spec }}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    ## Previous Failed Attempts
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    # Workflow Overview
+    You must complete the following stages in order. 
+
+    ## Data Loading
+    - Load the dataset strictly from `{% include "scenarios.data_science.share:scen.input_path" %}` as described in the **Data Folder Description**. DO NOT attempt to load data from the current directory (`./`).
+    - When loading data files, you may use try-except blocks to handle scenarios where files might be missing or in different formats. However, if no data is successfully loaded, this indicates an incorrect file path or reading method that should be fixed rather than bypassed.
+    - **Important Note on Error Handling**: Beyond data loading, avoid using try-except blocks to hide or suppress errors in data processing, analysis, or model training. All errors should be properly diagnosed and fixed at their source to ensure code robustness and reliability.
+
+    ## Exploratory Data Analysis (EDA) (Required)
+    Please follow this systematic methodology (in the required schema) for your analysis.
+    1. Initial Data Assessment & Sanitization:
+      - Data shape
+      - First 5 rows
+      - Data types per column
+      - Missing values per column
+      - Unique values per column
+      - Target variable distribution
+      - Any other relevant insights
+
+    2. Detailed Feature Analysis (A Non-Exhaustive Guide):
+    For Numerical & Categorical Features:
+      - Central Tendency & Dispersion
+      - Distribution Shape & Imbalance
+      - Outliers & Anomalies
+      - Cardinality & Granularity
+    For Text Features:
+      - Text Granularity & Scale
+      - Core Content & Topicality
+      - Linguistic Structure & Style
+      - Vocabulary Richness & Redundancy
+
+    3. The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      {EDA content}
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - An evaluation agent will help to check whether the EDA part is added correctly.
+    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    {% if enable_model_dump %}
+    ## Model Dumping
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+    {% endif %}
+
+    {% if enable_debug_mode %}
+    ## Debug Mode
+    Your code will be executed in a debug mode with following command: 
+    ```bash
+    python main.py --debug
+    ```
+    Please simulate the following code to check whether the code is running in debug mode:
+    ```python
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--debug', action='store_true', help='Run in debug mode')
+    args = parser.parse_args()
+    DEBUG = False
+    if args.debug:
+      DEBUG = True
+    ```
+    In debug mode, you should only sample ten percent of the training data and run the minimum epochs to quickly test the correctness of the code.
+    In debug mode, you should implement a timer to measure the time taken for your debug configuration and estimate the time required for the full run. Your timer should only measure the time taken for the training part, not the data loading or feature engineering part.
+    For example:
+    ```python
+    # Read data, feature engineering, etc.
+    start_time = time.time()
+    # Train your model
+    end_time = time.time()
+    debug_time = end_time - start_time
+    # post processing, saving model, etc.
+    ```
+    In debug mode, your code should run faster, so the environment will set a shorter time limit than the standard time limit for your code.
+    For example, you can sample ten percent of the training data and run for one epoch, then the full run with ten epochs will take one hundred times the time taken for the debug run. The scale is calculated by yourself depending on the data sampling and epoch number you choose. If your full run enables early stopping, the scale should be smaller considering the early stopping will stop the training earlier than the full epochs.
+    Be careful about the train-valid split strategy. Stratified related split is highly risk since the data has some categories with only one sample. If you use Stratified related split, you should consider using a try-except block to catch the error and use a different split strategy if the error occurs. Example code:
+    ```python
+    try:
+      fold_indices = StratifiedKFold(...).split(train_X, train_y) or StratifiedShuffleSplit or StratifiedSubsetSampler etc.
+    except Exception as e:
+        fold_indices = KFold(...).split(train_X, train_y) or other split strategy
+    ```
+    You should sample the data after train valid split. When you split the data after sampling, you might get a class with only one sample which might cause the split strategy to fail. 
+    Your debug code should run exactly the same as the full run, except for the data sampling and epoch number, to ensure the correctness of the code.
+    You should print total time and estimated time in standard output using print function in the following schema:
+    === Start of Debug Information ===
+    debug_time: time_taken_for_debug_run_in_seconds (e.g., 'debug_time: 10.0')
+    estimated_time: estimated_time_for_full_run_in_seconds (e.g., 'estimated_time: 100.0')
+    === End of Debug Information ===
+    User will use the following code to match: re.search(r"(.*?)=== Start of Debug Information ===(.*)=== End of Debug Information ===", stdout, re.DOTALL).groups()[1]
+    Notice, data sampling should only be applied in debug mode. Always use the full data in the full run!
+    Example code:
+    ```python
+    if args.debug:
+      sample_size = int(0.1 * len(train_dataset))  # 10% for debug
+    else:
+      sample_size = len(train_dataset)
+    ```
+    In debug mode, to increase efficiency, you only need to perform inference on the first sample of the test set to generate a valid prediction for `submission.csv`. For all other samples in the test set, you should use a placeholder value (e.g., 0 or a default value) to fill the prediction column. This ensures that the generated `submission.csv` has the same number of rows as the full run and passes the format check.
+    Example code:
+    ```python
+    all_preds = []
+    for i, batch in enumerate(test_loader):
+        # In debug mode, use placeholders for all batches after the first one to improve efficiency.
+        if args.debug and i > 0:
+            # The shape and data type of the placeholder must match the model's actual output.
+            # Here, we assume `predictions` is a NumPy array.
+            placeholder = np.zeros_like(predictions)
+            all_preds.append(placeholder)
+            continue
+
+        # In full mode, or for the first batch in debug mode, perform actual model inference.
+        predictions = model.predict(batch)
+        all_preds.append(predictions)
+
+    # final_predictions = np.concatenate(all_preds)
+    # ... then create and save submission.csv
+    ```
+    You should be very careful about the label classes number in the debug mode. The label classes should be the same as the full run even when you are in the debug mode. The label classes number is often used to build the model.
+    {% endif %}
+
+    ## General Guidelines
+    1. Code correctness is the top priority. Ensure your code is runnable and produces the expected output even if some task requirements are not fully met because the task itself might contain some errors like the wrong package name or wrong package function names.
+    2. Use the print() function for all output; do not use the logging module.
+    3. **Avoid all hard-coded values (e.g., fixed dataset sizes)**. Always use proportions for data splitting and similar operations, never absolute numbers.
+    4. Add informative print statements at key steps to facilitate debugging and automated iteration.
+    5. For model training, use reasonable epoch numbers. ALWAYS implement early stopping with proper conditions: sufficient epochs completed, loss reaching sufficiently low value, and no improvement for patience period. Save best model checkpoints based on validation performance.
+    6. Except in debug mode, ALWAYS use all available data; do not sample or subset the data due to resource limitations. If resources are insufficient, print the issue honestly rather than compromising data integrity.
+    7. Do not use tqdm or similar progress bar tools.
+    8. **Try-except blocks are ONLY allowed when reading files. If no files are successfully read, it indicates incorrect file paths or reading methods, not a try-except issue. Try-except is PROHIBITED elsewhere in the code. Assert statements are PROHIBITED throughout the entire code.**
+    9. ATTENTION: ALWAYS use the best saved model (not necessarily final epoch) for predictions. **NEVER create dummy/placeholder submissions (e.g., all 1s, random values)**. If training fails, report failure honestly rather than generating fake submission files.
+    10. You should ALWAYS generate the complete code rather than partial code.
+    11. If the task contains any user instructions, you must strictly follow them. User instructions have the highest priority and should be followed even if they conflict with other specifications or guidelines.
+    12. Strictly follow all specifications and general guidelines described above.
+
+    ### Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    # Competition Information
+    {{ competition_info }}
+
+    # Data Folder Description (All path are relative to the data folder, i.e. "{% include "scenarios.data_science.share:scen.input_path" %}")
+    {{ folder_spec }}
+    
+    {% if latest_code %}
+    # Former code
+    ```
+    {{ latest_code }}
+    ```
+    {% if latest_code_feedback is not none %}
+    ## Feedback to former code
+    {{ latest_code_feedback }}
+    
+    ## Improvement Planning
+    Before modifying the code, carefully analyze the feedback and identify no more than three key areas requiring changes. Plan your modifications strategically:
+    1. Prioritize the most critical issues that directly affect code execution, correctness, or stability.
+    2. Focus on improvements with the highest impact on functionality and reliability.
+    3. Preserve existing working components. Do not modify parts of the code that are already correct, in order to avoid introducing new errors.
+    
+    The previous version of the code contained errors. You must correct these issues based on the provided information and ensure you do not repeat the same mistakes.
+    
+    {% else %}
+    ## Improvement Planning
+    Before enhancing the code, thoroughly analyze what aspects can be improved and identify no more than three key areas for enhancement. Plan your improvements strategically:
+    1. Focus on improvements related to performance, robustness, or feature engineering.
+    2. Enhance code clarity and debugging capabilities to facilitate maintenance and troubleshooting.
+    3. Optimize model configuration or validation strategy to improve overall effectiveness.
+    
+    The previous version of the code is correct. You should improve the code based on the provided task while ensuring that unrelated parts remain unchanged.
+    {% endif %}
+    {% endif %}
+
+pipeline_eval:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %}
+    You will be provided with:
+    1. A detailed competition scenario description.
+    2. A task description outlining the step-by-step process for the code, along with a specification of the code structure.
+    3. A code implementation and its execution output.
+    Your task is to rigorously evaluate the code implementation against the provided scenario and task description, ensuring it meets all requirements, adheres to the specified structure, and executes successfully.
+
+    ## Evaluation Aspects
+    
+    ### Execution Success
+    - Goal: Ensure the code executes successfully without any errors.
+    - Notes:
+      - Model performance is not evaluated in this step; focus solely on successful execution.
+      - Warnings are acceptable if they do not interfere with successful code execution.
+    - If the code execute successfully:
+      - Proceed to Step 2.
+    - If the code does not execute successfully:
+      - Set the "final_decision" to false.
+      {% if enable_mcp_documentation_search %}
+      - Given that my package/environment is fixed and unchangeable, first you should go through the code and the execution output,if the problem could be solved by looking up the official documentation to confirm feature/API availability, compatible usage, or official alternatives in the fixed environment, set the "requires_documentation_search" to true.
+      {% endif %}
+      - Write complete analysis in the "execution" field.
+
+    ### Competition Alignment
+    - Goal: Confirm strict adherence to the competition's evaluation rules and experimental setup.
+    - Guidelines:
+      - Analyze whether the experimental setup and code may cause misalignment between validation and test performance.
+      - Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
+        - The metric implementation must exactly match scenario requirements (metric value itself is not the focus).
+        - Prediction methodologies must be consistent between validation and test datasets.
+        - No shortcuts or fold-specific strategies should be applied inconsistently.
+      - Check for corner-case consistency.
+      - Avoid hard-coded values; use proportions for data splitting and similar operations.
+    - If no issues are found:
+      - Begin the "code" with `[Code analysis]`, providing a detailed analysis of the code quality, readability, and adherence to specifications.
+    - If discrepancies or risks are found:
+      - Set the "final_decision" to false.
+      - Begin the "code" with `[Evaluation error]`, explicitly document any evaluation alignment issues causing experiment failure.
+
+    {% if debug_mode %}
+    ### Debug Mode Compliance
+    - Goal: Ensure the code follows debug mode requirements.
+    - Guidelines:
+      - Sufficient debugging information (print statements, clear error messages) should be included to facilitate automatic improvement processes.
+      - The code should be executed in debug mode with the command `python main.py --debug`.
+      - In debug mode, the code should sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code.
+      - Check whether the code follows these requirements. If not, emphasize it in your feedback and reject this implementation.
+      - Execution time and estimated time for the full run should be checked. Estimated time should not be too large to finish in the given time limit.
+      - Consider the early stopping mechanism in the code. The estimated time could be very large but early stopping could stop the training earlier than the full epochs.
+      - Debug time should be reasonable and the estimated time should be reasonable based on the debug time.
+      - Data sampling should only be applied in debug mode. Always use the full data in the full run.
+      - The label classes number should be the same as the full run even in debug mode.
+    - If the code passes this step: Proceed to Next Aspects.
+    - If the code does not pass this step: Clearly document the debug mode compliance issues and reject the implementation.{% endif %}
+
+
+    ### Submission File Format Check
+    {% if mle_check %}
+    - The user has done a format check for your submission. Since you didn't sample any test data, your debug mode output should be the same format as the full run.
+    - The user will put the check result in the "Submission check" section of the execution output.
+    - If the submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should give the conclusion that the code executed successfully. If no other code related issues are found, set the "final_decision" to true.
+    - If the submission check returns an error message, you should set the "final_decision" to false and clearly document the issues in the "return_checking" field.
+    {% elif is_sub_enabled %}
+    - Goal: Verify that the code correctly generates the final submission in the expected format and that the submission is authentic.
+    - Guidelines:
+      - The submission file must strictly match the required structure (correct columns, index format, data types). The index names and column names must be identical to the format specified in the Competition Information's '====== Submission Format ======' section.
+      - Rigorously verify that the submission file was produced by genuine model inference and successful code execution, not by cheating, fallback or exception-handling mechanisms.
+        - The submission must be generated from genuine model predictions using the best saved model—never empty, constant, random, or hard-coded values.
+        - Submissions must reflect authentic model outputs; any form of fabrication, cheating, or simulated results is strictly prohibited and grounds for rejection.
+        - Cross-check both code logic and stdout to ensure predictions originate from real model inference, not from error recovery or placeholder code paths.
+      - Only check the format of the submission since only part of the data is provided; the submission might have a different index than expected due to data sampling.
+      - Verify honest failure reporting if training issues occur.
+    - If the code passes this step, Finalize evaluation.
+    - If the code does not pass this step:
+      - Set the "final_decision" to false and clearly document the issues in the "return_checking" field.
+    {% else %}
+      Submission File Format Check is not conducted since no target submission format is provided. You should consider this submission file is valid.
+    {% endif %}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    ### Step 6: Similar Successful Implementations to help Code Improvement
+    The user has done several similar tasks and get some successful implementations. These code might not be implemented to the same task, but they are similar to your task and they might work well on your dataset.
+    Please refer to these successful implementation and provide your suggestions in your response on how to correct your current code based on these successful implementations.
+    ## Successful Implementations for Similar Tasks
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Similar Task {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.all_codes }}
+    {% endfor %} 
+    {% endif %}
+
+    ## Output Format
+    Please respond with your feedback in the following JSON format without anything else.
+    ```json
+    {
+        {% if enable_mcp_documentation_search %}
+        "requires_documentation_search": <true/false>,
+        {% endif %}"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. If errors occurred, analyze the root causes: (1) Are they fundamental algorithmic/approach issues, or (2) Implementation details that can be easily fixed, or (3) Environment/dependency problems?",
+        "return_checking": "Examine the generated files by cross-referencing the code logic and stdout output. Verify: (1) Format matches required submission format (index, column names, CSV content); (2) **File generation authenticity**: Is the file genuinely produced by successful model execution, or is it a result of exception handling/fallback mechanisms? Cite specific code sections and stdout evidence.",
+        "code": "Begin explicitly with [Code analysis] or [Evaluation error]. Provide structured analysis: (1) **Technical Appropriateness**: Does the chosen approach (algorithms, data processing, validation strategy) match this problem's data characteristics and competition requirements? (2) **Effective Components**: What specific parts work well and why are they effective for this problem type? (3) **Issues & Improvements**: Identify concrete problems and suggest actionable improvement directions (without providing actual code). (4) **Code Quality**: Assess readability, structure, and adherence to specifications.",
+        {% if enable_mcp_documentation_search %}
+        "error_message": "If the code execution has problems, extract the error information in the following format, otherwise set to empty string: ### TRACEBACK: <full relevant traceback extracted from execution output> ### SUPPLEMENTARY_INFO: <only if TRACEBACK is unclear - copy exact code fragments: import statements, variable=value assignments, function calls with parameters as they appear in code>",
+        {% endif %}"final_decision": <true/false>
+    }
+    ```
+
+
+  user: |-
+    # Competition Information
+    {{ scenario }}
+
+    # Task Description
+    {{ task_desc }}
+
+    ## Task Specification for Code Structure
+    {{ spec }}
+
+    # Code
+    ```
+    {{ code }}
+    ```
+
+    ## Execution Output
+    ```
+    {{ stdout }}
+    ```

rdagent/components/coder/data_science/raw_data_loader/README.md

@@ -0,0 +1,15 @@
+# CoSTEER
+
+- subworkspace使用主experiment_workspace `RD-Agent/rdagent/scenarios/data_science/experiment/experiment.py`
+
+## evolving_strategy ( implement_one_task() )
+
+1. xxxTask (in exp.py)
+    - spec
+    - description
+2. 
+
+## evaluator
+
+1. queried_knowledge部分 共用
+2. eval_test脚本

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -0,0 +1,242 @@
+"""
+
+Loop should not large change exclude
+- Action Choice[current data loader & spec]
+- other should share
+    - Propose[choice] => Task[Choice] => CoSTEER =>
+        -
+
+Extra feature:
+- cache
+
+
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+import re
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import (
+    DSCoderCoSTEERSettings,
+    get_ds_env,
+)
+from rdagent.components.coder.data_science.raw_data_loader.eval import (
+    DataLoaderCoSTEEREvaluator,
+)
+from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: DataLoaderTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # return a workspace with "load_data.py", "spec/load_data.md" inside
+        # assign the implemented code to the new workspace.
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        data_folder_info = self.scen.processed_data_folder_description
+        data_loader_task_info = target_task.get_task_information()
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[data_loader_task_info]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[data_loader_task_info]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("load_data.py") != workspace.file_dict.get("load_data.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 1. specifications
+        # TODO: We may move spec into a separated COSTEER task
+        if DS_RD_SETTING.spec_enabled:
+            if "spec/data_loader.md" not in workspace.file_dict:  # Only generate the spec once
+                system_prompt = T(".prompts:spec.system").r(
+                    runtime_environment=self.scen.get_runtime_environment(),
+                    task_desc=data_loader_task_info,
+                    competition_info=competition_info,
+                    folder_spec=data_folder_info,
+                )
+                data_loader_prompt = T(".prompts:spec.user.data_loader").r(
+                    latest_spec=workspace.file_dict.get("spec/data_loader.md")
+                )
+                feature_prompt = T(".prompts:spec.user.feature").r(
+                    latest_spec=workspace.file_dict.get("spec/feature.md")
+                )
+                model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md"))
+                ensemble_prompt = T(".prompts:spec.user.ensemble").r(
+                    latest_spec=workspace.file_dict.get("spec/ensemble.md")
+                )
+                workflow_prompt = T(".prompts:spec.user.workflow").r(
+                    latest_spec=workspace.file_dict.get("spec/workflow.md")
+                )
+
+                spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
+
+                data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
+                feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
+                model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
+                ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
+                workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
+            else:
+                data_loader_spec = workspace.file_dict["spec/data_loader.md"]
+                feature_spec = workspace.file_dict["spec/feature.md"]
+                model_spec = workspace.file_dict["spec/model.md"]
+                ensemble_spec = workspace.file_dict["spec/ensemble.md"]
+                workflow_spec = workspace.file_dict["spec/workflow.md"]
+
+        # 2. code
+        system_prompt = T(".prompts:data_loader_coder.system").r(
+            task_desc=data_loader_task_info,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+        )
+        code_spec = (
+            data_loader_spec
+            if DS_RD_SETTING.spec_enabled
+            else T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.DataLoadSpec").r(),
+                test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(),
+            )
+        )
+        user_prompt = T(".prompts:data_loader_coder.user").r(
+            competition_info=competition_info,
+            code_spec=code_spec,
+            folder_spec=data_folder_info,
+            latest_code=workspace.file_dict.get("load_data.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            data_loader_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if data_loader_code != workspace.file_dict.get("load_data.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new data loader code.")
+
+        return (
+            {
+                "spec/data_loader.md": data_loader_spec,
+                "spec/feature.md": feature_spec,
+                "spec/model.md": model_spec,
+                "spec/ensemble.md": ensemble_spec,
+                "spec/workflow.md": workflow_spec,
+                "load_data.py": data_loader_code,
+            }
+            if DS_RD_SETTING.spec_enabled
+            else {
+                "load_data.py": data_loader_code,
+            }
+        )
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class DataLoaderCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            DataLoaderCoSTEEREvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = DataLoaderMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )
+
+    def develop(self, exp):
+        new_exp = super().develop(exp)
+
+        env = get_ds_env(
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            },
+            running_timeout_period=self.scen.real_full_timeout(),
+        )
+
+        stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py")
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
+        eda_output = match.groups()[1] if match else None
+        if eda_output is not None:
+            new_exp.experiment_workspace.inject_files(**{"EDA.md": eda_output})
+        else:
+            eda_output = "No EDA output."
+            new_exp.experiment_workspace.inject_files(**{"EDA.md": eda_output})
+        return new_exp

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -0,0 +1,94 @@
+# tess successfully running.
+# (GPT) if it aligns with the spec & rationality of the spec.
+import json
+import re
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+DataLoaderEvalFeedback = CoSTEERSingleFeedback
+
+
+class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
+        **kwargs,
+    ) -> DataLoaderEvalFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return DataLoaderEvalFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        # TODO: do we need to clean the generated temporary content?
+        fname = "test/data_loader_test.py"
+        test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text()
+        implementation.inject_files(**{fname: test_code})
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.exit_code
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL)
+        stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "")
+        stdout = stdout_part_1 + stdout_part_2
+        if eda_output is not None and len(eda_output.split(" ")) > 10000:
+            eda_output += "Length of EDA output is too long, truncated. Please reject this implementation and motivate it to reduce the length of EDA output."
+
+        if "main.py" in implementation.file_dict and ret_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        system_prompt = T(".prompts:data_loader_eval.system").r(
+            task_desc=target_task.get_task_information(),
+            test_code=test_code,
+            code=implementation.file_dict["load_data.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:data_loader_eval.user").r(
+            stdout=stdout,
+            eda_output=eda_output,
+            workflow_stdout=workflow_stdout,
+        )
+
+        fb = build_cls_from_json_with_retry(
+            DataLoaderEvalFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=DataLoaderEvalFeedback.val_and_update_init_dict,
+        )
+        fb.final_decision = fb.final_decision and ret_code == 0
+
+        return fb

rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt

@@ -0,0 +1,83 @@
+"""
+Tests for `load_data` in load_data.py
+"""
+
+import pickle
+
+import pandas as pd
+from load_data import load_data
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+
+def debug_info_print(func):
+    aRepr = reprlib.Repr()
+    aRepr.maxother=300
+    def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == original_code:
+                print("\n" + "="*20 + "Running data_load code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+
+X, y, X_test, test_ids = debug_info_print(load_data)()
+
+
+def get_length(data):
+    return data.shape[0] if hasattr(data, 'shape') else len(data)
+
+
+def get_width(data):
+    return data.shape[1:] if hasattr(data, 'shape') else 1
+
+
+def get_column_list(data):
+    return data.columns.tolist() if isinstance(data, pd.DataFrame) else None
+
+assert X is not None, "Training data (X) is None."
+assert y is not None, "Training labels (y) are None."
+assert X_test is not None, "Test data (X_test) is None."
+assert test_ids is not None, "Test IDs (test_ids) are None."
+
+assert get_length(X_test) == get_length(
+    test_ids
+), f"Mismatch in length of test images and test IDs: X_test ({get_length(X_test)}) and test_ids ({get_length(test_ids)})"
+assert get_length(X) == get_length(
+    y
+), f"Mismatch in length of training images and labels: X ({get_length(X)}) and y ({get_length(y)})"
+
+assert get_length(X) != 0, f"Training data is empty."
+assert get_length(y) != 0, f"Training labels are empty."
+assert get_length(X_test) != 0, f"Test data is empty."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+if isinstance(X, pd.DataFrame) and isinstance(X_test, pd.DataFrame):
+    assert get_column_list(X) == get_column_list(X_test), "Mismatch in column names of training and test data."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+print("Data loader test passed successfully. Length of test images matches length of test IDs.")

rdagent/components/coder/data_science/raw_data_loader/exp.py

@@ -0,0 +1,6 @@
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class DataLoaderTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/raw_data_loader/prompts.yaml

@@ -0,0 +1,402 @@
+
+spec:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    Currently, you are working on a Kaggle competition project. 
+    This project involves analyzing data and building models to beat other competitors, with the code being generated by large language models.
+
+    The runtime environment you are working in includes the following libraries and their respective versions:
+    {{ runtime_environment }}
+
+    Your overall task is provided below:
+    {{ task_desc }}
+    
+    Your task is to write five specification texts (in markdown format) for the following tasks, based on the competition information provided
+    - Data loading (and preprocessing)
+    - Feature Engineering
+    - Model Building
+    - Ensemble
+    - The overall workflow
+
+    The specifications for each step should be tailored to the competition information provided. 
+    
+    Your specification should consists two parts:
+    1. The function definition in code format, including type annotations and a clear, complete docstring that describes the function's purpose, input parameters, return value, and any relevant exceptions.
+    2. Additional information or notes that the coder should consider while implementing the function.
+    
+    Your specifications should include only the function definition and docstring, without any code implementation or inline comments.
+
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    ----------- Folder Description (All path are relative to the data folder) ---------
+    - Ensure that all columns in sample_submission can be generated.
+    {{ folder_spec }}
+
+  user:
+    data_loader: |-
+      Data loader specification text should follow these detailed requirements:
+      1. Function Interface:
+        - Function Name: `load_data`
+        - Input: No input arguments.
+        - Output:
+          - `X` (DT, define based on competition information): Feature matrix for training data.
+          - `y` (DT): Target vector for training data.
+          - `X_test` (DT): Feature matrix for test data.
+          - `test_ids` (DT): Identifiers for the test data.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Specify the data source location (`{% include "scenarios.data_science.share:scen.input_path" %}`).
+          - Clearly define the structure and type of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+      2. Notes:
+        - Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+
+      Responsibilities and notes of an implemented data loader that aligns with the generated specification.
+      {% include "scenarios.data_science.share:component_spec.DataLoadSpec" %}
+
+      {% if latest_spec %}
+      6. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    feature: |-
+      Feature engineering specification text should adhere to the following requirements:
+      1. Function Interface:
+        - Function Name: `feat_eng`
+        - Parameters:
+          - `X` (DT): Train data to be transformed.
+          - `y` (DT): Train label data.
+          - `X_test` (DT): Test data.
+        - Output:
+          - `X_transformed` (DT): Transformed train data.
+          - `y_transformed` (DT): Transformed train label data.
+          - `X_test_transformed` (DT): Transformed test data.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Clarify the input parameters and their data types.
+          - Define the structure and format of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+
+      2. Precautions for Feature Engineering:
+        - Well handle the shape of the data:
+          - The sample size of the train data and the test data should be the same in all scenarios.
+          - To some tabular or time-series data, you may add or remove some columns so your inferred column number may be unsure.
+          - For scenarios where each dimension does not have a special meaning (like image, audio, and so on), the input shape and the output shape should be exactly the same in most cases unless there is a compelling reason to change them.
+        - Integration with the Model Pipeline:
+          - If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage.
+            - Model-related operations should not be implemented in this step. (e.g., it uses tools combined with models like torch.Dataset with rich data transformation/augmentation)
+          - Otherwise, ensure this function applies all required transformations while avoiding data leakage.
+        - General Considerations:
+          - Ensure scalability for large datasets.
+          - Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
+          - Ensure consistency between feature data types and transformations.
+          - Prevent data leakage: Do not use information derived from the test set when transforming training data.
+        - Domain-Specific Features:
+          - Apply logic for competition-specific features (e.g., text vectorization, image augmentations, categorical encoding).
+
+      3. Code Standards:
+        - Avoid using progress bars (e.g., `tqdm`) in the implementation.          
+
+      4. Notes:
+        - Align `DT` (data type) definitions with those in the Data Loader specification.
+        - GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+      
+      {% if latest_spec %}
+      5. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    model: |-
+      Model building specification text should adhere to the following requirements:
+
+      1. Function Interface:
+        - Function Name: `model_workflow`
+        - Parameters:
+          - `X` (DT): Training feature data.
+          - `y` (DT): Training label data.
+          - `val_X` (Optional[DT]): Validation feature data.
+          - `val_y` (Optional[DT]): Validation label data.
+          - `test_X` (Optional[DT]): Test feature data.
+          - `hyper_params` (dict): Dictionary of hyperparameters for model configuration.
+        - Output:
+          - `pred_val` (Optional[DT]): Predictions on validation data.
+          - `pred_test` (Optional[DT]): Predictions on test data.
+          - `hyper_params` (dict): Updated dictionary of hyperparameters after training.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Clarify the input parameters and their data types.
+          - Define the structure and format of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+
+      2. Code Standards:
+        - Do not use progress bars (e.g., `tqdm`) in the implementation.
+
+      3. Precautions:
+        - Ensure input arrays (`X`, `y`, `val_X`, `val_y`, `test_X`) have consistent dimensions and shapes.
+        - Use default values for hyperparameters if `hyper_params` is not provided.
+        - Train the model on `X` and `y`.
+        - Evaluate the model using `val_X` and `val_y` if validation data is available.
+        - If `test_X` is provided, generate predictions for it.
+
+      4. Notes:
+        - Align `DT` (data type) with the definitions used in Feature Engineering specifications.
+        - The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process.
+        - Some data transformations/augmentations can be included in this step (e.g., data tools provided by TensorFlow and Torch)
+
+      {% if latest_spec %}
+      5. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    ensemble: |-
+      Ensemble specification text adhere to the following requirements:
+      1. Function Interface:
+        - Function Name: `ensemble_workflow`
+        - Parameters:
+          - `test_preds_dict` (Dict[str, DT]): A dictionary of test predictions from different models. The key is the model file name.
+          - `val_preds_dict` (Dict[str, DT]): A dictionary of validation predictions from different models. The key is the model file name.
+          - `val_label` (DT): Validation label.
+        - Output:
+          - `final_pred` (DT): Ensemble prediction for the test data.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Clarify the input parameters and their data types.
+          - Define the structure and format of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+
+      2. Precautions:
+        - Input Validation:
+          - Ensure all predictions in `test_preds_dict` and `val_preds_dict` have consistent shapes and dimensions.
+          - Verify that `val_label` is provided and matches the length of `val_preds_dict` predictions.
+          - Handle empty or invalid inputs gracefully with appropriate error messages.
+        - Metric Calculation and Storage:
+          - Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`, e.g.:
+            ```python
+            scores = {}
+            for model_name, val_pred in val_preds_dict.items():
+                scores[model_name] = calculate_metric(val_label, val_pred)
+            
+            ...
+            some code about ensemble strategy
+            ...
+            ensemble_val_pred = ...
+
+            ensemble_score = calculate_metric(val_label, ensemble_val_pred)
+            scores["ensemble"] = ensemble_score  # Ensure "ensemble" is explicitly stored
+            
+            scores_df = pd.DataFrame(scores.items(), columns=["Model", <metric_name>])
+            scores_df.to_csv("scores.csv", index=False)
+            ```
+          - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
+        
+      3. Code Standards:
+        - Do not use progress bars (e.g., tqdm) in the code.
+
+      4. Notes:
+        - Align `DT` (data type) definitions with those used in model specifications.
+        - Ensure flexibility to handle multiple ensemble strategies based on competition requirements.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+
+      {% if latest_spec %}
+      5. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    workflow: |-
+      {% include "scenarios.data_science.share:component_spec.Workflow" %}
+
+      {% if latest_spec %}
+      7. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly.
+      You should create the rules based on the competition information instead of copying the requirements.
+
+data_loader_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementation Examples for Similar Task ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Example {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.all_codes }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache```
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+    
+    ## Exploratory Data Analysis (EDA) part(Required):
+    - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - An evaluation agent will help to check whether the EDA part is added correctly.
+    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    --------- Competition Information ---------
+    {{ competition_info }}
+
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    --------- Data Folder Description (All path are relative to the data folder, i.e. "{% include "scenarios.data_science.share:scen.input_path" %}") ---------
+    {{ folder_spec }}
+    
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %} 
+
+    You should strictly follow the code specifications provided by the specification to implement the function.
+
+
+data_loader_eval:
+  system: |-
+    You are a data scientist responsible for evaluating data loader code for a Kaggle-style machine learning competition project.
+    
+    ## Task Description
+    {{ task_desc }}
+
+    ## Data Loader Code
+    The data loader code is located in `load_data.py`:
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The data loader is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The data loader is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    {{ workflow_code }}
+
+    You should evaluate both the data loader test results and the overall workflow execution. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the data loader test and, if applicable, the workflow test.
+
+    ## Exploratory Data Analysis (EDA) Part evaluation
+    - The code has also generated some EDA output to help understand the data better. 
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+    You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
+    
+    Your response must follow this structured JSON format:
+    ```json
+    {
+        "execution": "Describe how well the data loader executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Evaluate the correctness and integrity of the loaded data. Check for issues like missing values, incorrect data types, outliers, or formatting inconsistencies.",
+        "code": "Assess code quality, readability, and adherence to best practices. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for faster data loading.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Data loader test stdout ---------
+    {{ stdout }}   
+    --------- Data loader EDA stdout ---------
+    {% if eda_output is not none %}
+    {{ eda_output }}
+    {% else %}
+    No EDA output is provided.
+    {% endif %}
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/raw_data_loader/test.py

@@ -0,0 +1,30 @@
+"""
+Helper functions for testing the raw_data_loader coder(CoSTEER-based) component.
+- Does the developer loop work correctly
+
+It is NOT:
+- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop)
+"""
+
+from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER
+from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+def develop_one_competition(competition: str):  # -> experiment
+    scen = KaggleScen(competition=competition)
+    data_loader_coder = DataLoaderCoSTEER(scen)
+
+    # Create the experiment
+    dlt = DataLoaderTask(name="DataLoaderTask", description="")
+    exp = DSExperiment(
+        sub_tasks=[dlt],
+    )
+
+    # Develop the experiment
+    exp = data_loader_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")

rdagent/components/coder/data_science/share/doc.py

@@ -0,0 +1,37 @@
+"""
+Developers concentrating on writing documents for a workspace
+"""
+
+from rdagent.core.developer import Developer
+from rdagent.core.experiment import Experiment, FBWorkspace
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import MarkdownAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class DocDev(Developer[Experiment]):
+    """
+    The developer is responsible for writing documents for a workspace.
+    """
+
+    def develop(self, exp: Experiment) -> None:
+        """
+        Write documents for the workspace.
+        """
+        ws: FBWorkspace = exp.experiment_workspace
+
+        file_li = [str(file.relative_to(ws.workspace_path)) for file in ws.workspace_path.rglob("*") if file.is_file()]
+
+        key_file_list = ["main.py", "scores.csv"]
+
+        system_prompt = T(".prompts:docdev.system").r()
+        user_prompt = T(".prompts:docdev.user").r(
+            file_li=file_li,
+            key_files={f: (ws.workspace_path / f).read_text() for f in key_file_list},
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt, system_prompt=system_prompt
+        )
+        markdown = MarkdownAgentOut.extract_output(resp)
+        ws.inject_files(**{"README.md": markdown})

rdagent/components/coder/data_science/share/ds_costeer.py

@@ -0,0 +1,9 @@
+from rdagent.components.coder.CoSTEER import CoSTEER
+
+
+class DSCoSTEER(CoSTEER):
+    def get_develop_max_seconds(self) -> int | None:
+        """
+        The coder uses the scenario's real debug timeout as the maximum seconds for development.
+        """
+        return int(self.scen.real_debug_timeout() * self.settings.max_seconds_multiplier)

rdagent/components/coder/data_science/share/eval.py

@@ -0,0 +1,176 @@
+import re
+from pathlib import Path
+from typing import Literal
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+PipelineSingleFeedback = CoSTEERSingleFeedback
+PipelineMultiFeedback = CoSTEERMultiFeedback
+
+NO_SUB = "<No submission.csv file found.>"
+NO_SCORE = "<No scores.csv file found.>"
+
+
+class ModelDumpEvaluator(CoSTEEREvaluator):
+    """This evaluator assumes that it runs after the model"""
+
+    def __init__(self, scen: Scenario, data_type: Literal["sample", "full"]):
+        super().__init__(scen)
+        self.data_type = data_type
+
+    def evaluate(
+        self, target_task: Task, implementation: FBWorkspace, gt_implementation: FBWorkspace, *kargs, **kwargs
+    ) -> CoSTEERSingleFeedback:
+
+        model_folder = implementation.workspace_path / "models"
+        # 1) Check if the model_folder is not empty
+        if not model_folder.exists() or not any(model_folder.iterdir()):
+            err_msg = "Model folder (`models` sub folder) is empty or does not exist. The model is not dumped."
+            return CoSTEERSingleFeedback(
+                execution=err_msg,
+                return_checking=err_msg,
+                code=err_msg,
+                final_decision=False,
+            )
+
+        data_source_path = (
+            f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}"
+            if self.data_type == "full"
+            else self.scen.debug_path
+        )
+        env = get_ds_env(
+            extra_volumes={data_source_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=(
+                self.scen.real_full_timeout() if self.data_type == "full" else self.scen.real_debug_timeout()
+            ),
+        )
+
+        # 2) check the result and stdout after reruning the model.
+
+        # Read the content of files submission.csv and scores.csv before execution
+        submission_content_before = (
+            (implementation.workspace_path / "submission.csv").read_text()
+            if (implementation.workspace_path / "submission.csv").exists()
+            else NO_SUB
+        )
+        scores_content_before = (
+            (implementation.workspace_path / "scores.csv").read_text()
+            if (implementation.workspace_path / "scores.csv").exists()
+            else NO_SCORE
+        )
+
+        # Remove the files submission.csv and scores.csv
+        implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference"))
+
+        # Execute the main script
+        stdout = remove_eda_part(
+            implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference")
+        )
+
+        # walk model_folder and list the files
+        model_folder_files = [
+            str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
+        ]
+
+        opened_trace_lines = None
+        if (implementation.workspace_path / "trace.log").exists():
+            input_path = T("scenarios.data_science.share:scen.input_path").r()
+            abs_input_path = str(Path(input_path).resolve())
+            # matching path in string like `openat(AT_FDCWD, "/home/user/project/main.py", O_RDONLY) = 5`
+            path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"')
+            log_content = (implementation.workspace_path / "trace.log").read_text()
+
+            opened_files = set()
+            for line in log_content.splitlines():
+                if "openat" not in line or (abs_input_path not in line and input_path not in line):
+                    continue
+
+                match = path_regex.search(line)
+                if match:
+                    full_path = Path(match.group(1)).resolve()
+                    if str(full_path).startswith(abs_input_path):
+                        opened_files.add(Path(data_source_path).resolve() / full_path.relative_to(abs_input_path))
+
+            from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator
+
+            tree_gen = FileTreeGenerator(allowed_paths=opened_files)  # pass opened files filter
+            opened_trace_lines = tree_gen.generate_tree(Path(data_source_path).resolve())
+            # Limitation: training and test are expected to be different files.
+
+        # this will assert the generation of necessary files
+        for f in ["submission.csv", "scores.csv"]:
+            if not (implementation.workspace_path / f).exists():
+                err_msg = f"{f} does not exist. The model is not dumped. Make sure that the required files, like submission.csv and scores.csv, are created even if you bypass the model training step by loading the saved model file directly."
+                return CoSTEERSingleFeedback(
+                    execution=err_msg,
+                    return_checking=err_msg,
+                    code=err_msg,
+                    final_decision=False,
+                )
+
+        # Check if scores contain NaN (values)
+        score_df = pd.read_csv((implementation.workspace_path / "scores.csv"), index_col=0)
+        if score_df.isnull().values.any():
+            nan_locations = score_df[score_df.isnull().any(axis=1)]
+            err_msg = f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+            return CoSTEERSingleFeedback(
+                execution=err_msg,
+                return_checking=err_msg,
+                code=err_msg,
+                final_decision=False,
+            )
+
+        submission_content_after = (
+            (implementation.workspace_path / "submission.csv").read_text()
+            if (implementation.workspace_path / "submission.csv").exists()
+            else NO_SUB
+        )
+        scores_content_after = (
+            (implementation.workspace_path / "scores.csv").read_text()
+            if (implementation.workspace_path / "scores.csv").exists()
+            else NO_SCORE
+        )
+
+        system_prompt = T(".prompts:dump_model_eval.system").r()
+        user_prompt = T(".prompts:dump_model_eval.user").r(
+            stdout=stdout.strip(),
+            code=implementation.all_codes,
+            model_folder_files=model_folder_files,
+            scores_content_before=scores_content_before,
+            scores_content_after=scores_content_after,
+            opened_trace_lines=opened_trace_lines,
+        )
+
+        csfb = build_cls_from_json_with_retry(
+            CoSTEERSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+        )
+
+        if DS_RD_SETTING.model_dump_check_level == "high":
+            # Read the content of files submission.csv and scores.csv after execution
+            # Check if the content has changed
+            # excactly same checking. But it will take more user's time
+            if scores_content_before != scores_content_after:
+                return_msg = "\n[Error] The content of scores.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it."
+                return_msg += f"\nBefore:\n{scores_content_before}\nAfter:\n{scores_content_after}"
+                if submission_content_before != submission_content_after:
+                    # If the scores file changes, display the two contents and append it into the return_checking
+                    return_msg = "[Error] The content of submission.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it."
+                csfb.return_checking = (csfb.return_checking or "") + return_msg
+        return csfb

rdagent/components/coder/data_science/share/notebook.py

@@ -0,0 +1,135 @@
+"""
+Handles conversion from a Python file to a Jupyter notebook.
+"""
+
+import argparse
+from typing import Optional
+
+import nbformat
+
+from rdagent.components.coder.data_science.share.util import (
+    extract_first_section_name_from_code,
+    extract_function_body,
+    split_code_and_output_into_sections,
+)
+from rdagent.core.experiment import Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import MarkdownAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class NotebookConverter:
+    """
+    Builder responsible for writing a Jupyter notebook for a workspace.
+    """
+
+    def validate_code_format(self, code: str) -> str | None:
+        """
+        Returns None if the code format is valid, otherwise returns an error message.
+        """
+        main_function_body = extract_function_body(code, "main")
+        if not main_function_body:
+            return "[Error] No main function found in the code. Please ensure that the main function is defined and contains the necessary print statements to divide sections."
+
+        found_section_name = extract_first_section_name_from_code(main_function_body)
+        if not found_section_name:
+            return "[Error] No sections found in the code. Expected to see 'print(\"Section: <section name>\")' as section dividers. Also make sure that they are actually run and not just comments."
+
+        return None
+
+    def convert(
+        self,
+        task: Optional[Task],
+        code: str,
+        stdout: str,
+        outfile: Optional[str] = None,
+        use_debug_flag: bool = False,
+    ) -> str:
+        """
+        Build a notebook based on the current progression.
+        """
+        # Handle argparse in the code to ensure it works in a notebook environment
+        should_handle_argparse = "argparse" in code
+        sections = split_code_and_output_into_sections(code=code, stdout=stdout)
+        notebook = nbformat.v4.new_notebook()
+
+        # Use LLM to generate an intro cell for the notebook
+        if task:
+            system_prompt = T(".prompts:notebookconverter.system").r()
+            user_prompt = T(".prompts:notebookconverter.user").r(
+                plan=task.get_task_information(),
+                code=code,
+            )
+            resp = APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt, system_prompt=system_prompt
+            )
+            intro_content = MarkdownAgentOut.extract_output(resp)
+            notebook.cells.append(nbformat.v4.new_markdown_cell(intro_content))
+
+        if should_handle_argparse:
+            # Remove extra `import sys` since it will be added for argparse handling
+            if "import sys\n" in sections[0]["code"]:
+                sections[0]["code"] = sections[0]["code"].replace("import sys\n", "")
+
+            # Add sys.argv modification for argparse handling
+            sections[0]["code"] = (
+                "\n".join(
+                    [
+                        "import sys",
+                        "# hack to allow argparse to work in notebook",
+                        ('sys.argv = ["main.py", "--debug"]' if use_debug_flag else 'sys.argv = ["main.py"]'),
+                    ]
+                )
+                + "\n\n"
+                + sections[0]["code"].lstrip()
+            )
+
+        for section in sections:
+            # Create a markdown cell for the section name and comments
+            markdown_content = ""
+            if section["name"]:
+                markdown_content += f"## {section['name']}\n"
+            if section["comments"]:
+                markdown_content += f"{section['comments']}\n"
+            if markdown_content:
+                notebook.cells.append(nbformat.v4.new_markdown_cell(markdown_content))
+
+            # Create a code cell for the section code and output
+            if section["code"]:
+                cell = nbformat.v4.new_code_cell(section["code"])
+                if section["output"]:
+                    # For simplicity, treat all output as coming from stdout
+                    # TODO: support Jupyter kernel execution and handle outputs appropriately here
+                    cell.outputs = [nbformat.v4.new_output("stream", name="stdout", text=section["output"])]
+                notebook.cells.append(cell)
+
+        # Save the notebook or return it as a string
+        if outfile:
+            with open((outfile), "w", encoding="utf-8") as f:
+                nbformat.write(notebook, f)
+                logger.info(f"Notebook written to {outfile}")
+
+        return nbformat.writes(notebook)
+
+
+if __name__ == "__main__":
+    converter = NotebookConverter()
+    parser = argparse.ArgumentParser(description="Convert Python code to Jupyter notebook.")
+    parser.add_argument("inputfile", type=str, help="Path to the input Python file.")
+    parser.add_argument("outfile", type=str, help="Path to the output Notebook file.")
+    parser.add_argument(
+        "--stdout",
+        type=str,
+        default="",
+        help="Standard output from the code execution.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Use debug flag to modify sys.argv.")
+    args = parser.parse_args()
+    converter.convert(
+        task=None,
+        code=open(args.inputfile, "r").read(),
+        stdout=args.stdout,
+        outfile=args.outfile,
+        use_debug_flag=False,
+    )

rdagent/components/coder/data_science/share/prompts.yaml

@@ -0,0 +1,123 @@
+dump_model_coder:
+  guideline: |-
+    Your code will be executed in a inference mode with following command: 
+    ```bash
+    python main.py --inference
+    ```
+    Please dump the model in a "models/" subfolder in the first running, and the script rerun performs inference without needing to retrain the model when running the code again.
+    In inference Mode, the script MUST NOT load any training data. 
+    If there are parameters generated from the training data that might be needed for inference on test data, please save them in the "models/" subfolder as well.
+    If no test set is provided, reserve a portion of the data as your test set and save the generated test files in the models/ subfolder for use in submission and inference.
+    Make sure that the required files, like submission.csv and scores.csv, are created without model training step through loading the saved model and test data file directly.
+    
+
+dump_model_eval:
+  system: |-
+    You are a data scientist tasked with evaluating code generation. You've developed a Kaggle competition code that can produce a submission file.
+    The code should follow the guideline below:
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+
+    You will receive the following information:
+    - The implemented code
+    - The stdout from running the code
+    - The file list in "models/" subfolder
+    - The scores.csv file generated during both training and inference (if it exists)
+
+    Focus on these aspects:
+    - Check if the code saves the model in the "models/" subfolder.
+    - Check if the code saves the test data in the "models/" subfolder when there is no test data specified.
+    - Ensure that when the code is rerun in inference mode, it skips the training process and loads the model from the "models/" subfolder for direct inference.
+      - Verify that there is no training activity in the output.
+      - Verify that the script does not load the original training data.
+    - Ensure that even if you skip the model training by loading saved models, the files like scores.csv and submission.csv are still correctly created.
+    - The model's performance should remain consistent and not vary unreasonably between training and inference.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. Carefully check the stdout to ensure that when the code is rerun, it skips the training process and loads the model from the 'models/' subfolder for direct inference. Append the information that makes you think that the model is still being retrained when rerunning the code."
+        "return_checking": "Verify the generated files include necessary files. Make sure scores.csv file does not change unreasonably between training and inference",
+        "code": "The code has explicity dump the model into 'models/' subfolder; When the modes files are already in 'models/' subfolder, the code will explicity skip the training process.",
+        "final_decision": <true or false in boolean type; only return true when ensuring that the code saves the model in a 'models/' subfolder, and the script rerun performs inference without needing to retrain the model.>
+    }
+    ```
+
+  user: |-
+    ------------ The implemented code ------------ 
+    {{code}}
+
+    ------------ The stdout from running the code ------------ 
+    {{stdout}}
+
+    ------------ File opened by the code ------------
+    {{opened_trace_lines}}
+
+    ------------ The file list in "models/" subfolder ------------
+    {% for f in model_folder_files %}
+    - {{ f }}
+    {% endfor %}
+
+    ------------ The scores.csv file generated ------------
+    # Training:
+    {{scores_content_before}}
+
+    # Inference:
+    {{scores_content_after}}
+
+
+docdev:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %}  Your task is to create documentation for a data science solution.
+
+    You will be given:
+    - a list of files in the folder.
+    - content from some important files.
+
+    Please explain the trained models in the "models/" folder. The training and inference processes are detailed in the `main.py` file. The models' evaluation results are in `scores.csv`. Please respond with a markdown file that includes the following information:
+    - Explain the purpose of each model. If some models are part of a group (like those from cross-validation), describe them together.
+    - Provide key details for each model group:
+      - Important training parameters
+      - Model details
+      - Performance of each model
+
+    Be brief. Mention the file path when you introduce files.
+    Don't introduce anything other than models.
+
+    {% include "utils.agent.tpl:MarkdownOut" %}
+
+  user: |-
+    --------------- The file list in the workspace ---------------
+    {% for f in file_li %}
+    - {{ f }}
+    {% endfor %}
+
+    --------------- File content of each file ---------------
+    {% for fname, content in key_files.items() %}
+    File Path: {{fname}}
+    ```
+    {{content}}
+    ```
+    {% endfor %}
+
+notebookconverter:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %} Your task is to provide a summary for a data science solution.
+
+    You will be given:
+    - The original implementation plan for the script.
+    - A Python script that contains code and output.
+
+    Your task is to generate markdown content that includes a title and a short paragraph summarizing the technique in model training, the type of model produced and any other noteworthy details in the solution.
+
+    The return content should be like the format below(Please note that "````" is used to avoid confliction of "```" in markdown file)
+    ````markdown
+    # <The title of the notebook>
+    <the content of markdown file>
+    ````
+
+  user: |-
+    --------------- The implementation plan ---------------
+    {{plan}}
+
+    --------------- The Python script content ---------------
+    {{code}}

rdagent/components/coder/data_science/share/util.py

@@ -0,0 +1,365 @@
+import ast
+import io
+import re
+import tokenize
+from itertools import zip_longest
+from typing import List, Optional, Set, Tuple, TypedDict
+
+
+class CodeSection(TypedDict):
+    """
+    Represents a section of the original Python source code, to be converted to a notebook cell.
+    """
+
+    name: Optional[str]
+    code: Optional[str]
+    comments: Optional[str]
+    output: Optional[str]
+
+
+def extract_function_body(source_code: str, function_name: str) -> Optional[str]:
+    """
+    Extracts the body of a function from the source code.
+    Returns None if the function is not found.
+
+    Assumption: The function is multiline and defined at the top level.
+    """
+    tree = ast.parse(source_code)
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef) and node.name == function_name:
+            lines = source_code.splitlines()
+            start = node.body[0].lineno
+            end = node.body[-1].end_lineno
+            body_lines = lines[start - 1 : end]
+            indent_level = len(body_lines[0]) - len(body_lines[0].lstrip())
+            return "\n".join(line[indent_level:] for line in body_lines)
+    return None
+
+
+def split_sections(
+    text: str, section_header_regex: str, known_sections: Optional[list[str]] = None
+) -> tuple[Optional[str], list[str], list[str]]:
+    """
+    Split text into sections based on the section headers.
+    """
+    sections = []
+    section_names = []
+    current_section = []
+    next_section_name_index = 0
+    for line in text.splitlines():
+        match = re.match(section_header_regex, line)
+        extracted_section_name = match.group(1).strip() if match else None
+        if extracted_section_name and (
+            not known_sections
+            or (
+                next_section_name_index < len(known_sections)
+                and extracted_section_name == known_sections[next_section_name_index]
+            )
+        ):
+            if current_section:
+                sections.append("\n".join(current_section))
+                current_section = []
+            current_section.append(line)
+            section_names.append(extracted_section_name)
+            next_section_name_index += 1
+        else:
+            current_section.append(line)
+    if current_section:
+        sections.append("\n".join(current_section))
+
+    # If the first section does not match the header regex, treat it as a header section.
+    header_section = None
+    if sections and not re.search(section_header_regex, sections[0]):
+        header_section = sections[0]
+        sections = sections[1:]
+
+    return header_section, sections, section_names
+
+
+def split_code_sections(source_code: str) -> tuple[Optional[str], list[str]]:
+    """
+    Split code into sections based on the section headers.
+    """
+    return split_sections(source_code, r'^print\(["\']Section: (.+)["\']\)')
+
+
+def split_output_sections(stdout: str, known_sections: list[str]) -> tuple[Optional[str], list[str]]:
+    """
+    Split output into sections based on the section headers.
+    """
+    header_section, sections, _ = split_sections(stdout, r"^Section: (.+)", known_sections=known_sections)
+    return header_section, sections
+
+
+def extract_comment_under_first_print(source_code) -> tuple[Optional[str], str]:
+    """
+    Extract comments from the source code after the first print statement.
+    """
+    lines = source_code.splitlines()
+    lines_to_remove = set()
+    all_comments = []
+
+    parsed = ast.parse(source_code)
+    # Find the first print statement only
+    first_print_lineno = None
+    for node in ast.walk(parsed):
+        if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
+            if getattr(node.value.func, "id", None) == "print":
+                first_print_lineno = node.lineno
+                break
+
+    if first_print_lineno is None:
+        # No print statement found, return empty comments and original code
+        return None, source_code
+
+    for i in range(first_print_lineno, len(lines)):
+        stripped = lines[i].strip()
+        if stripped.startswith("#"):
+            comment_text = stripped.lstrip("# ").strip()
+            all_comments.append(comment_text)
+            lines_to_remove.add(i)
+        elif stripped == "":
+            continue
+        elif i > first_print_lineno:
+            break  # stop after hitting actual code line
+
+    cleaned_lines = [line for idx, line in enumerate(lines) if idx not in lines_to_remove]
+    cleaned_code = "\n".join(cleaned_lines)
+    comments_str = "\n".join(all_comments) if all_comments else None
+
+    return comments_str, cleaned_code
+
+
+def extract_first_section_name_from_code(source_code):
+    """
+    Extract the first section name from the source code.
+    """
+    parsed = ast.parse(source_code)
+    for node in ast.walk(parsed):
+        if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
+            call = node.value
+            if getattr(call.func, "id", None) == "print" and call.args:
+                arg0 = call.args[0]
+                if isinstance(arg0, ast.Constant) and isinstance(arg0.value, str):
+                    # Match "Section: ..." pattern
+                    m = re.match(r"Section:\s*(.+)", arg0.value)
+                    if m:
+                        return m.group(1).strip()
+    return None
+
+
+def extract_first_section_name_from_output(stdout: str) -> Optional[str]:
+    """
+    Extract the first section name from the output string.
+    """
+    match = re.search(r"Section:\s*(.+)", stdout)
+    if match:
+        return match.group(1).strip()
+    return None
+
+
+def is_function_called(source_code: str, func_name: str) -> bool:
+    """
+    Returns True if the function named `func_name` is called in `source_code`.
+    """
+    tree = ast.parse(source_code)
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Call):
+            # For simple function calls like func()
+            if isinstance(node.func, ast.Name) and node.func.id == func_name:
+                return True
+
+            # For calls like module.func()
+            elif isinstance(node.func, ast.Attribute) and node.func.attr == func_name:
+                return True
+    return False
+
+
+def remove_function(source_code: str, function_name: str) -> str:
+    """
+    Remove a function definition from the source code.
+    """
+    tree = ast.parse(source_code)
+    lines = source_code.splitlines()
+
+    for node in tree.body:
+        if isinstance(node, ast.FunctionDef) and node.name == function_name:
+            start_lineno = node.lineno - 1
+            end_lineno = node.end_lineno
+            return "\n".join(lines[:start_lineno] + lines[end_lineno:])
+
+    return source_code
+
+
+def remove_main_block(source_code: str) -> str:
+    """
+    Remove the if __name__ == "__main__": block from the source code.
+    """
+    tree = ast.parse(source_code)
+    lines = source_code.splitlines()
+
+    # Find the main block and note its line numbers
+    for node in tree.body:
+        if isinstance(node, ast.If):
+            test = node.test
+            if (
+                isinstance(test, ast.Compare)
+                and isinstance(test.left, ast.Name)
+                and test.left.id == "__name__"
+                and len(test.ops) == 1
+                and isinstance(test.ops[0], ast.Eq)
+                and len(test.comparators) == 1
+                and isinstance(test.comparators[0], ast.Constant)
+                and test.comparators[0].value == "__main__"
+            ):
+
+                # Remove lines corresponding to this block
+                start_lineno = node.lineno - 1
+                end_lineno = node.end_lineno
+                return "\n".join(lines[:start_lineno] + lines[end_lineno:])
+
+    return source_code
+
+
+def extract_top_level_functions_with_decorators_and_comments(
+    code: str,
+) -> List[Tuple[str, str]]:
+    """
+    Returns list of (function_name, source_segment) for top-level functions (excluding "main"),
+    including decorators and contiguous preceding comments.
+    """
+    # Parse AST to get function nodes
+    tree = ast.parse(code)
+    lines = code.splitlines(keepends=True)
+
+    # Precompute which line numbers have comment tokens
+    comment_lines: Set[int] = set()
+    lines = code.splitlines(keepends=True)  # preserve exact line content for prefix checks
+
+    tokgen = tokenize.generate_tokens(io.StringIO(code).readline)  # yields (type, string, start, end, line)
+    for tok_type, _, (srow, scol), _, _ in tokgen:
+        if tok_type == tokenize.COMMENT:
+            # everything before the comment on that line must be whitespace
+            prefix = lines[srow - 1][:scol]
+            if prefix.strip() == "":
+                comment_lines.add(srow)
+
+    functions = []
+
+    for node in tree.body:  # only top-level
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        if node.name == "main":
+            continue
+
+        # Determine the starting line: earliest decorator if present, else the def/async line
+        if node.decorator_list:
+            start_lineno = min(d.lineno for d in node.decorator_list)
+        else:
+            start_lineno = node.lineno
+
+        # Extend upward to include contiguous comment lines (no intervening non-blank/non-comment)
+        span_start = start_lineno
+        curr = span_start - 1  # check line above; lines are 1-based
+        while curr > 0:
+            line_text = lines[curr - 1]
+            if curr in comment_lines:
+                span_start = curr
+                curr -= 1
+                continue
+            if line_text.strip() == "":
+                # blank line: include it and keep scanning upward
+                span_start = curr
+                curr -= 1
+                continue
+            break  # encountered code or something else; stop
+
+        # Determine end line of the function definition including its body
+        # Prefer end_lineno if available (Python 3.8+)
+        if hasattr(node, "end_lineno") and node.end_lineno is not None:
+            span_end = node.end_lineno
+        else:
+            # Fallback: get last lineno from the deepest child in body
+            def _max_lineno(n):
+                max_ln = getattr(n, "lineno", 0)
+                for child in ast.iter_child_nodes(n):
+                    ln = _max_lineno(child)
+                    if ln > max_ln:
+                        max_ln = ln
+                return max_ln
+
+            span_end = _max_lineno(node)
+
+        # Slice the original source lines
+        segment = "".join(lines[span_start - 1 : span_end])
+        functions.append((node.name, segment))
+
+    return functions
+
+
+def split_code_and_output_into_sections(code: str, stdout: str) -> list[CodeSection]:
+    """
+    Converts a Python script and its output into a list of CodeSections.
+    Pre-condition: The code in the main() function contains print statements that indicate section names, e.g., `print("Section: <section name>")`.
+    """
+    # This will hold all top-level code and by default all function definitions.
+    # Functions will later be moved to more relevant sections if needed.
+    # The first step is to remove both the if __name__ == "__main__": block and the main function
+    top_level_code = remove_main_block(remove_function(code, "main"))
+
+    main_function_body = extract_function_body(code, "main")
+    functions = extract_top_level_functions_with_decorators_and_comments(top_level_code)
+
+    # Split the main function body into sections based on print("Section: <section name>") code
+    main_fn_top_level_section, main_fn_sections, known_section_names = (
+        split_code_sections(main_function_body) if main_function_body else (None, [], [])
+    )
+
+    # Split the output into sections based on "Section: " headers
+    output_top_level_section, output_sections = split_output_sections(stdout, known_section_names)
+
+    # Merge code and outputs into code sections
+    result_sections: list[CodeSection] = []
+    for output_section, code_section in zip_longest(output_sections, main_fn_sections):
+        name = None
+        if code_section is not None:
+            # If code section is available, extract the section name from it
+            name = extract_first_section_name_from_code(code_section)
+        elif output_section:
+            # If only output section is available, extract the section name from it
+            name = extract_first_section_name_from_output(output_section)
+        comments, cleaned_code = (
+            extract_comment_under_first_print(code_section) if code_section is not None else (None, None)
+        )
+        # Strip whitespaces for the cell
+        if cleaned_code is not None:
+            cleaned_code = cleaned_code.strip()
+        result_sections.append(CodeSection(name=name, code=cleaned_code, comments=comments, output=output_section))
+
+    # Small optimization: move function definitions to the sections where they are first called
+    # TODO: this doesn't handle nested function references, e.g., fn A calls fn B which calls fn C
+    # currently will not move C to the section where A is called
+    for name, segment in functions:
+        for section in result_sections:
+            if section["code"] and is_function_called(section["code"], name):
+                section["code"] = segment.strip() + "\n\n" + section["code"].lstrip()
+                top_level_code = top_level_code.replace(segment, "")
+                break
+
+    # Inject the top-level code at the beginning of the sections
+    top_level_code = (
+        top_level_code.rstrip() + "\n\n" + main_fn_top_level_section.lstrip()
+        if main_fn_top_level_section
+        else top_level_code
+    )
+    result_sections.insert(
+        0,
+        CodeSection(
+            name=None,
+            code=top_level_code,
+            comments=None,
+            output=output_top_level_section,
+        ),
+    )
+
+    return result_sections

rdagent/components/coder/data_science/utils.py

@@ -0,0 +1,6 @@
+import re
+
+
+def remove_eda_part(stdout: str) -> str:
+    """Data Science scenario have a LLM-based EDA feature. We can remove it when current task does not involve EDA"""
+    return re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout, flags=re.DOTALL)

rdagent/components/coder/data_science/workflow/__init__.py

@@ -0,0 +1,132 @@
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.components.coder.data_science.workflow.eval import (
+    WorkflowGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: WorkflowTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        workflow_information_str = target_task.get_task_information()
+
+        # 1. query
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[workflow_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[workflow_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 2. code
+        system_prompt = T(".prompts:workflow_coder.system").r(
+            task_desc=workflow_information_str,
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+        )
+        user_prompt = T(".prompts:workflow_coder.user").r(
+            load_data_code=workspace.file_dict["load_data.py"],
+            feature_code=workspace.file_dict["feature.py"],
+            model_codes=workspace.get_codes(r"^model_(?!test)\w+\.py$"),
+            ensemble_code=workspace.file_dict["ensemble.py"],
+            latest_code=workspace.file_dict.get("main.py"),
+            code_spec=(
+                workspace.file_dict["spec/workflow.md"]
+                if DS_RD_SETTING.spec_enabled
+                else T("scenarios.data_science.share:component_spec.Workflow").r()
+            ),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            workflow_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if workflow_code != workspace.file_dict.get("main.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new workflow code.")
+
+        return {"main.py": workflow_code}
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class WorkflowCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            WorkflowGeneralCaseSpecEvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = WorkflowMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/workflow/eval.py

@@ -0,0 +1,158 @@
+import json
+import re
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERMultiFeedback,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+WorkflowSingleFeedback = CoSTEERSingleFeedback
+WorkflowMultiFeedback = CoSTEERMultiFeedback
+
+
+class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
+    """
+    Motivation case:
+    - Simplest case, we already split the data into train_data, valid_data, and test_data. We require the model to learn (optionally validate on valid data), and infer on test data.
+
+    Test workflow:
+    - Build train, valid, and test data to run it, and test the output (e.g., shape, etc.)
+    """
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> CoSTEERSingleFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return WorkflowSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        # # DockerEnv for MLEBench submission validation
+        # mle_de_conf = MLEBDockerConf()
+        # mle_de_conf.extra_volumes = {
+        #     f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
+        # }
+        # mde = DockerEnv(conf=mle_de_conf)
+        # mde.prepare()
+
+        # Clean the scores.csv & submission.csv.
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
+
+        stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py")
+
+        # remove EDA part
+        stdout = remove_eda_part(stdout)
+
+        # Check score file
+        score_fp = implementation.workspace_path / "scores.csv"
+        score_ret_code = 0
+        score_check_text = ""
+        if not score_fp.exists():
+            score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
+            score_ret_code = 1
+            implementation.execute(env=env, entry="python -m coverage json -o coverage.json")
+            coverage_report_path = implementation.workspace_path / "coverage.json"
+            if coverage_report_path.exists():
+                used_files = set(json.loads(coverage_report_path.read_text())["files"].keys())
+                coverage_report_path.unlink()
+                logger.info(f"All used scripts: {used_files}")
+                if len(used_files) == 1:
+                    score_check_text += f"\n[Error] The only used script is {used_files}.\nPlease check if you have implemented entry point in 'main.py'."
+        else:
+            try:
+                score_df = pd.read_csv(score_fp, index_col=0)
+                model_set_in_scores = set(score_df.index)
+                # We assume that model names in `score_df` are stored without the '.py' file extension.
+                model_set_in_folder = set(
+                    f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
+                )
+
+                # Check model names (index)
+                if model_set_in_scores != model_set_in_folder.union({"ensemble"}):
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {model_set_in_folder.union({'ensemble'})}\nscore_df is:\n{score_df}"
+                    score_ret_code = 1
+
+                # Check metric name (columns) - case insensitive
+                if [col.lower() for col in score_df.columns.tolist()] != [self.scen.metric_name.lower()]:
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
+                    score_ret_code = 1
+
+                # Check if scores contain NaN (values)
+                if score_df.isnull().values.any():
+                    nan_locations = score_df[score_df.isnull().any(axis=1)]
+                    score_check_text += f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+                    score_ret_code = 1
+
+            except Exception as e:
+                score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
+                score_ret_code = 1
+
+        # Check submission file
+        base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
+        implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+        # stdout += "----Submission Check 1-----\n"
+        submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+        submission_check_out = submission_result.stdout
+        submission_ret_code = submission_result.exit_code
+        stdout += "\n" + submission_check_out
+
+        system_prompt = T(".prompts:workflow_eval.system").r(
+            # here we pass `None` to `eda_output` because we do not have nor need EDA output for workflow.
+            scenario=self.scen.get_scenario_all_desc(eda_output=None),
+            task_desc=target_task.get_task_information(),
+            spec=(
+                implementation.file_dict["spec/workflow.md"]
+                if DS_RD_SETTING.spec_enabled
+                else T("scenarios.data_science.share:component_spec.Workflow").r()
+            ),
+        )
+        user_prompt = T(".prompts:workflow_eval.user").r(
+            stdout=stdout.strip(),
+            code=implementation.file_dict["main.py"],
+        )
+        wfb = build_cls_from_json_with_retry(
+            WorkflowSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=WorkflowSingleFeedback.val_and_update_init_dict,
+        )
+        if score_ret_code != 0:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + score_check_text
+        if submission_ret_code != 0:
+            wfb.final_decision = False
+            wfb.return_checking += "\nSubmission file check failed."
+        return wfb

rdagent/components/coder/data_science/workflow/eval_tests/submission_format_test.txt

@@ -0,0 +1,77 @@
+from pathlib import Path
+import pandas as pd
+import hashlib
+
+def calculate_md5(file_path):
+    with open(file_path, "rb") as f:
+        file_hash = hashlib.md5(f.read()).hexdigest()
+    return file_hash
+
+file_md5 = calculate_md5("scores.csv")
+
+"""
+find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
+./denoising-dirty-documents/sampleSubmission.csv
+./the-icml-2013-whale-challenge-right-whale-redux/sampleSubmission.csv
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv.zip
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv
+./random-acts-of-pizza/sampleSubmission.csv
+./text-normalization-challenge-english-language/en_sample_submission_2.csv.zip
+./text-normalization-challenge-english-language/en_sample_submission_2.csv
+./detecting-insults-in-social-commentary/sample_submission_null.csv
+"""
+
+# Find sample submission file dynamically
+input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}")
+# Look for common variations of sample submission filenames
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
+                         list(input_dir.glob("*sampleSubmission*.csv"))
+
+assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}"
+
+# Use first matching file
+sample_submission_name = sample_submission_files[0].name
+SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+print(f"Using sample submission file: {sample_submission_name}")
+
+# Check if the sample submission file exists
+assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
+
+# Check if our submission file exists
+assert Path('submission.csv').exists(), "Error: submission.csv not found"
+
+sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+our_submission = pd.read_csv('submission.csv')
+
+success = True
+# Print the columns of the sample submission file
+print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+print("Columns in our_submission.csv:", our_submission.columns)
+
+for col in sample_submission.columns:
+    if col not in our_submission.columns:
+        success = False
+        print(f'Column {col} not found in submission.csv')
+
+if success:
+    print(f'submission.csv\'s columns aligns with {sample_submission_name} .')
+
+
+# Print the first 5 rows of the two submission files, with columns separated by commas.
+def print_first_rows(file_path, file_name, num_rows=5):
+    print(f"\nFirst {num_rows} rows of {file_name}:")
+    try:
+        with open(file_path, 'r') as file:
+            for i, line in enumerate(file):
+                if i < num_rows:
+                    print(line.strip())
+                else:
+                    break
+    except FileNotFoundError:
+        print(f"Error: {file_name} not found.")
+
+print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+print_first_rows('submission.csv', 'submission.csv')
+
+assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
+print(f"\nPlease Checked the content of the submission file(submission.csv should align with {sample_submission_name}). ")

rdagent/components/coder/data_science/workflow/exp.py

@@ -0,0 +1,14 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.core.utils import cache_with_pickle
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class WorkflowTask(CoSTEERTask):
+    def __init__(self, name: str = "Workflow", *args, **kwargs) -> None:
+        super().__init__(name=name, *args, **kwargs)

rdagent/components/coder/data_science/workflow/prompts.yaml

@@ -0,0 +1,137 @@
+workflow_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+
+    Here is the competition information for this task:
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict["main.py"] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict["main.py"] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. Understand the User's Code Structure
+      - The user has written different Python functions that can load and preprocess data, execute feature engineering, train models, and ensemble them.
+      - Each functionality is in a separate Python file.
+    2. Your task is only to integrate the existing processes of load_data, feature, model, and ensemble into a complete workflow. Do not edit or modify the existing Python files. The final step should output the predictions in the required format.
+    3. The user may provide specific code organization rules and instructions. Ensure that the integration follows the given framework and structure.
+    4. After predicting the output, print the shape and other information of the output to stdout to help the evaluator assess the code.
+    5. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+  
+  user: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    --------- load data code ---------
+    file: load_data.py
+    {{ load_data_code }}
+
+    --------- feature engineering code ---------
+    file: feature.py
+    {{ feature_code }}
+
+    --------- model training code ---------
+    Attention: The input and output of the model function is flexible. Training dataset is necessary, but validation and test dateset might be optional. The hyperparameters can either be passed as arguments or be set as default values in the function. You need to use the function correctly.
+    All model files share the same function name. Please import the model files with their name like: from {file_name} import {function_name}
+    {{ model_codes }}
+
+    --------- ensemble code ---------
+    Note, we will check the index of the score.csv, so please use the model name as the index to feed into ensemble function.
+    file: ensemble.py
+    {{ ensemble_code }}
+
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %}
+
+workflow_eval:
+  system: |-
+    You are a data scientist responsible for evaluating workflow code generation.
+
+    ## Task Description
+    The user is trying to build a workflow in the following scenario:
+    {{ scenario }}
+
+    The main code generation task is as follows:
+    {{ task_desc }}
+
+    The user provides workflow information and its components.
+    The details on how to structure the workflow are given in the specification file:
+    ```markdown
+    {{ spec }}
+    ```
+
+    This workflow integrates multiple stages, including:
+    - Data loading
+    - Feature engineering
+    - Model training
+    - Ensembling
+
+    ## Evaluation Scope
+    Your focus is to check whether the workflow code:
+    1. Executes successfully, correctly organizing components and generating a final submission.
+    2. Generates predictions in the correct format, ensuring they align with the **sample submission** structure!
+
+    [Note] 
+    1. The individual components (data loading, feature engineering, model tuning, etc.) have already been evaluated by the user. You should only evaluate and improve the workflow code, unless there are critical issues in the components.
+    2. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
+    3. As long as the execution does not exceed the time limit, ensure that the code uses cross-validation to split the training data and train the model. If cross-validation is not used, mention it in the execution section and set `final_decision` to `false`.
+
+    ## Evaluation Criteria
+    You will be given the workflow execution output (`stdout`) to determine correctness.  
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the main workflow executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission, checking the index, column names, and CSV content.",
+        "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+  
+  user: |-
+    --------- Workflow test stdout ---------
+    {{ stdout }}
+    --------- Workflow code generated by user ---------
+    {{ code }}

rdagent/components/coder/data_science/workflow/test.py

@@ -0,0 +1,59 @@
+"""
+Generate dataset to test the workflow output
+"""
+
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
+from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
+from rdagent.components.coder.data_science.workflow.eval import (
+    WorkflowGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
+from rdagent.core.experiment import FBWorkspace
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+def develop_one_competition(competition: str):
+    scen = KaggleScen(competition=competition)
+    workflow_coder = WorkflowCoSTEER(scen)
+
+    wt = WorkflowTask(
+        name="WorkflowTask",
+        description="Integrate the existing processes of load_data, feature, model, and ensemble into a complete workflow.",
+        base_code="",
+    )
+
+    tpl_ex_path = Path(__file__).resolve() / Path("rdagent/scenarios/kaggle/tpl_ex").resolve() / competition
+    injected_file_names = ["spec/workflow.md", "load_data.py", "feature.py", "model01.py", "ensemble.py", "main.py"]
+
+    workflowexp = FBWorkspace()
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        workflowexp.inject_files(**{file_name: file_path.read_text()})
+
+    wt.base_code += workflowexp.file_dict["main.py"]
+    exp = DSExperiment(
+        sub_tasks=[wt],
+    )
+
+    """es = WorkflowMultiProcessEvolvingStrategy(scen=scen, settings=CoSTEER_SETTINGS)
+    new_code = es.implement_one_task(target_task=wt, queried_knowledge=None, workspace = workflowexp)
+    print(new_code)"""
+
+    """eva = WorkflowGeneralCaseSpecEvaluator(scen=scen)
+    exp.feedback = eva.evaluate(target_task=wt, queried_knowledge=None, implementation=workflowexp, gt_implementation=None)
+    print(exp.feedback)"""
+
+    # Run the experiment
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        exp.experiment_workspace.inject_files(**{file_name: file_path.read_text()})
+
+    exp = workflow_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")
+    # dotenv run -- python rdagent/components/coder/data_science/workflow/test.py

rdagent/components/coder/factor_coder/__init__.py

@@ -0,0 +1,32 @@
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator
+from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
+from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder
+from rdagent.components.coder.factor_coder.evolving_strategy import (
+    FactorMultiProcessEvolvingStrategy,
+)
+from rdagent.core.experiment import Experiment
+from rdagent.core.scenario import Scenario
+
+
+class FactorCoSTEER(CoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        setting = FACTOR_COSTEER_SETTINGS
+        eva = CoSTEERMultiEvaluator(FactorEvaluatorForCoder(scen=scen), scen=scen)
+        es = FactorMultiProcessEvolvingStrategy(scen=scen, settings=FACTOR_COSTEER_SETTINGS)
+
+        super().__init__(*args, settings=setting, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs)
+
+    def develop(self, exp: Experiment) -> Experiment:
+        try:
+            exp = super().develop(exp)
+        finally:
+            if hasattr(self, "evolve_agent") and self.evolve_agent.evolving_trace:
+                es = self.evolve_agent.evolving_trace[-1]
+                exp.prop_dev_feedback = es.feedback
+        return exp

rdagent/components/coder/factor_coder/config.py

@@ -0,0 +1,49 @@
+import os
+from typing import Optional
+
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.utils.env import CondaConf, Env, LocalEnv
+
+
+class FactorCoSTEERSettings(CoSTEERSettings):
+    model_config = SettingsConfigDict(env_prefix="FACTOR_CoSTEER_")
+
+    data_folder: str = "git_ignore_folder/factor_implementation_source_data"
+    """Path to the folder containing financial data (default is fundamental data in Qlib)"""
+
+    data_folder_debug: str = "git_ignore_folder/factor_implementation_source_data_debug"
+    """Path to the folder containing partial financial data (for debugging)"""
+
+    simple_background: bool = False
+    """Whether to use simple background information for code feedback"""
+
+    file_based_execution_timeout: int = 3600
+    """Timeout in seconds for each factor implementation execution"""
+
+    select_method: str = "random"
+    """Method for the selection of factors implementation"""
+
+    python_bin: str = "python"
+    """Path to the Python binary"""
+
+
+def get_factor_env(
+    conf_type: Optional[str] = None,
+    extra_volumes: dict = {},
+    running_timeout_period: int = 600,
+    enable_cache: Optional[bool] = None,
+) -> Env:
+    conf = FactorCoSTEERSettings()
+    if hasattr(conf, "python_bin"):
+        env = LocalEnv(conf=(CondaConf(conda_env_name=os.environ.get("CONDA_DEFAULT_ENV"))))
+    env.conf.extra_volumes = extra_volumes.copy()
+    env.conf.running_timeout_period = running_timeout_period
+    if enable_cache is not None:
+        env.conf.enable_cache = enable_cache
+    env.prepare()
+    return env
+
+
+FACTOR_COSTEER_SETTINGS = FactorCoSTEERSettings()

rdagent/components/coder/factor_coder/eva_utils.py

@@ -0,0 +1,550 @@
+import io
+import json
+from abc import abstractmethod
+from typing import Dict, Tuple
+
+import pandas as pd
+
+from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
+from rdagent.components.coder.factor_coder.factor import FactorTask
+from rdagent.core.experiment import Task, Workspace
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+
+class FactorEvaluator:
+    """Although the init method is same to Evaluator, but we want to emphasize they are different"""
+
+    def __init__(self, scen=None) -> None:
+        self.scen = scen
+
+    @abstractmethod
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        **kwargs,
+    ) -> Tuple[str, object]:
+        """You can get the dataframe by
+
+        .. code-block:: python
+
+            _, gen_df = implementation.execute()
+            _, gt_df = gt_implementation.execute()
+
+        Returns
+        -------
+        Tuple[str, object]
+            - str: the text-based description of the evaluation result
+            - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result
+
+        """
+        raise NotImplementedError("Please implement the `evaluator` method")
+
+    def _get_df(self, gt_implementation: Workspace, implementation: Workspace):
+        if gt_implementation is not None:
+            _, gt_df = gt_implementation.execute()
+            if isinstance(gt_df, pd.Series):
+                gt_df = gt_df.to_frame("gt_factor")
+            if isinstance(gt_df, pd.DataFrame):
+                gt_df = gt_df.sort_index()
+        else:
+            gt_df = None
+
+        _, gen_df = implementation.execute()
+        if isinstance(gen_df, pd.Series):
+            gen_df = gen_df.to_frame("source_factor")
+        if isinstance(gen_df, pd.DataFrame):
+            gen_df = gen_df.sort_index()
+        return gt_df, gen_df
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorCodeEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        target_task: FactorTask,
+        implementation: Workspace,
+        execution_feedback: str,
+        value_feedback: str = "",
+        gt_implementation: Workspace = None,
+        **kwargs,
+    ):
+        factor_information = target_task.get_task_information()
+        code = implementation.all_codes
+
+        system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(
+                    target_task,
+                    filtered_tag="feature",
+                    simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
+                )
+                if self.scen is not None
+                else "No scenario description."
+            )
+        )
+
+        execution_feedback_to_render = execution_feedback
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r(
+                factor_information=factor_information,
+                code=code,
+                execution_feedback=execution_feedback_to_render,
+                value_feedback=value_feedback,
+                gt_code=gt_implementation.code if gt_implementation else None,
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > APIBackend().chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+        critic_response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+        )
+
+        return critic_response, None
+
+
+class FactorInfEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        INF_count = gen_df.isin([float("inf"), -float("inf")]).sum().sum()
+        if INF_count == 0:
+            return "The source dataframe does not have any infinite values.", True
+        else:
+            return (
+                f"The source dataframe has {INF_count} infinite values. Please check the implementation.",
+                False,
+            )
+
+
+class FactorSingleColumnEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        if len(gen_df.columns) == 1:
+            return "The source dataframe has only one column which is correct.", True
+        else:
+            return (
+                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
+                False,
+            )
+
+
+class FactorOutputFormatEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Skip the evaluation of the output format.",
+                False,
+            )
+        buffer = io.StringIO()
+        gen_df.info(buf=buffer)
+        gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
+        system_prompt = T(".prompts:evaluator_output_format_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
+                if self.scen is not None
+                else "No scenario description."
+            )
+        )
+
+        # TODO: with retry_context(retry_n=3, except_list=[KeyError]):
+        max_attempts = 3
+        attempts = 0
+        final_evaluation_dict = None
+
+        while attempts < max_attempts:
+            try:
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
+                resp = api.build_messages_and_create_chat_completion(
+                    user_prompt=gen_df_info_str,
+                    system_prompt=system_prompt,
+                    json_mode=True,
+                    json_target_type=Dict[str, str | bool | int],
+                )
+                resp_dict = json.loads(resp)
+                resp_dict["output_format_decision"] = str(resp_dict["output_format_decision"]).lower() in ["true", "1"]
+
+                return (
+                    str(resp_dict["output_format_feedback"]),
+                    resp_dict["output_format_decision"],
+                )
+            except (KeyError, json.JSONDecodeError) as e:
+                attempts += 1
+                if attempts >= max_attempts:
+                    raise KeyError(
+                        "Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
+                    ) from e
+
+        return "Failed to evaluate output format after multiple attempts.", False
+
+
+class FactorDatetimeDailyEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str | object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return "The source dataframe is None. Skip the evaluation of the datetime format.", False
+
+        if "datetime" not in gen_df.index.names:
+            return "The source dataframe does not have a datetime index. Please check the implementation.", False
+
+        try:
+            pd.to_datetime(gen_df.index.get_level_values("datetime"))
+        except Exception:
+            return (
+                f"The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.\n The head of the output dataframe is: \n{gen_df.head()}",
+                False,
+            )
+
+        time_diff = pd.to_datetime(gen_df.index.get_level_values("datetime")).to_series().diff().dropna().unique()
+        if pd.Timedelta(minutes=1) in time_diff:
+            return (
+                "The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
+                False,
+            )
+        return "The generated dataframe is daily.", True
+
+
+class FactorRowCountEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
+        return (
+            (
+                f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+                + "Please verify the implementation. "
+                if ratio <= 0.99
+                else ""
+            ),
+            ratio,
+        )
+
+
+class FactorIndexEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index)
+        similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set))
+        return (
+            (
+                f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. "
+                + "Please check the implementation."
+                if similarity <= 0.99
+                else ""
+            ),
+            similarity,
+        )
+
+
+class FactorMissingValuesEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
+            return "Both dataframes have the same missing values.", True
+        else:
+            return (
+                f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
+                False,
+            )
+
+
+class FactorEqualValueRatioEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                -1,
+            )
+        try:
+            close_values = gen_df.sub(gt_df).abs().lt(1e-6)
+            result_int = close_values.astype(int)
+            pos_num = result_int.sum().sum()
+            acc_rate = pos_num / close_values.size
+        except:
+            close_values = gen_df
+        if close_values.all().iloc[0]:
+            return (
+                "All values in the dataframes are equal within the tolerance of 1e-6.",
+                acc_rate,
+            )
+        else:
+            return (
+                "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
+                acc_rate,
+            )
+
+
+class FactorCorrelationEvaluator(FactorEvaluator):
+    def __init__(self, hard_check: bool, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.hard_check = hard_check
+
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        concat_df = pd.concat([gen_df, gt_df], axis=1)
+        concat_df.columns = ["source", "gt"]
+        ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
+        ric = (
+            concat_df.groupby("datetime")
+            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+            .dropna()
+            .mean()
+        )
+
+        if self.hard_check:
+            if ic > 0.99 and ric > 0.99:
+                return (
+                    f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
+                    True,
+                )
+            else:
+                return (
+                    f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
+                    False,
+                )
+        else:
+            return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
+
+
+class FactorValueEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        version: int = 1,  # 1 for qlib factors and 2 for kaggle factors
+        **kwargs,
+    ) -> Tuple:
+        conclusions = []
+
+        # Initialize result variables
+        row_result = 0
+        index_result = 0
+        output_format_result = None
+        equal_value_ratio_result = 0
+        high_correlation_result = False
+        row_result = None
+
+        # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
+        if version == 1:
+            feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+        elif version == 2:
+            input_shape = self.scen.input_shape
+            _, gen_df = self._get_df(gt_implementation, implementation)
+            if gen_df.shape[-1] > input_shape[-1]:
+                conclusions.append(
+                    "Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure."
+                )
+
+        feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation)
+        conclusions.append(feedback_str)
+
+        # Check if the index of the dataframe is ("datetime", "instrument")
+        feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
+        conclusions.append(feedback_str)
+        if version == 1:
+            feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+        else:
+            daily_check_result = None
+
+        # Check dataframe format
+        if gt_implementation is not None:
+            feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+
+            feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+
+            feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+
+            feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+
+            if index_result > 0.99:
+                feedback_str, high_correlation_result = FactorCorrelationEvaluator(
+                    hard_check=True, scen=self.scen
+                ).evaluate(implementation, gt_implementation)
+            else:
+                high_correlation_result = False
+                feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless"
+            conclusions.append(feedback_str)
+
+        # Combine all conclusions into a single string
+        conclusion_str = "\n".join(conclusions)
+
+        if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
+            decision_from_value_check = True
+        elif (
+            row_result is not None
+            and row_result <= 0.99
+            or output_format_result is False
+            or daily_check_result is False
+            or inf_evaluate_res is False
+        ):
+            decision_from_value_check = False
+        else:
+            decision_from_value_check = None
+        return conclusion_str, decision_from_value_check
+
+
+class FactorFinalDecisionEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        target_task: FactorTask,
+        execution_feedback: str,
+        value_feedback: str,
+        code_feedback: str,
+        **kwargs,
+    ) -> Tuple:
+        system_prompt = T(".prompts:evaluator_final_decision_v1_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
+                if self.scen is not None
+                else "No scenario description."
+            )
+        )
+        execution_feedback_to_render = execution_feedback
+
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = T(".prompts:evaluator_final_decision_v1_user").r(
+                factor_information=target_task.get_task_information(),
+                execution_feedback=execution_feedback_to_render,
+                code_feedback=code_feedback,
+                value_feedback=(
+                    value_feedback
+                    if value_feedback is not None
+                    else "No Ground Truth Value provided, so no evaluation on value is performed."
+                ),
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > APIBackend().chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+
+        # TODO:  with retry_context(retry_n=3, except_list=[KeyError]):
+        final_evaluation_dict = None
+        attempts = 0
+        max_attempts = 3
+
+        while attempts < max_attempts:
+            try:
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
+                final_evaluation_dict = json.loads(
+                    api.build_messages_and_create_chat_completion(
+                        user_prompt=user_prompt,
+                        system_prompt=system_prompt,
+                        json_mode=True,
+                        seed=attempts,  # in case of useless retrying when cache enabled.
+                        json_target_type=Dict[str, str | bool | int],
+                    ),
+                )
+                final_decision = final_evaluation_dict["final_decision"]
+                final_feedback = final_evaluation_dict["final_feedback"]
+
+                final_decision = str(final_decision).lower() in ["true", "1"]
+                return final_decision, final_feedback
+
+            except json.JSONDecodeError as e:
+                raise ValueError("Failed to decode JSON response from API.") from e
+            except KeyError as e:
+                attempts += 1
+                if attempts >= max_attempts:
+                    raise KeyError(
+                        "Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts."
+                    ) from e
+
+        return None, None

rdagent/components/coder/factor_coder/evaluators.py

@@ -0,0 +1,130 @@
+import re
+
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERMultiFeedback,
+    CoSTEERSingleFeedbackDeprecated,
+)
+from rdagent.components.coder.factor_coder.eva_utils import (
+    FactorCodeEvaluator,
+    FactorFinalDecisionEvaluator,
+    FactorValueEvaluator,
+)
+from rdagent.components.coder.factor_coder.factor import FactorTask
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import Workspace
+
+FactorSingleFeedback = CoSTEERSingleFeedbackDeprecated
+
+
+class FactorEvaluatorForCoder(CoSTEEREvaluator):
+    """This class is the v1 version of evaluator for a single factor implementation.
+    It calls several evaluators in share modules to evaluate the factor implementation.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.value_evaluator = FactorValueEvaluator(self.scen)
+        self.code_evaluator = FactorCodeEvaluator(self.scen)
+        self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
+
+    def evaluate(
+        self,
+        target_task: FactorTask,
+        implementation: Workspace,
+        gt_implementation: Workspace = None,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> FactorSingleFeedback:
+        if implementation is None:
+            return None
+
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return FactorSingleFeedback(
+                execution_feedback="This task has failed too many times, skip implementation.",
+                value_generated_flag=False,
+                code_feedback="This task has failed too many times, skip code evaluation.",
+                value_feedback="This task has failed too many times, skip value evaluation.",
+                final_decision=False,
+                final_feedback="This task has failed too many times, skip final decision evaluation.",
+                final_decision_based_on_gt=False,
+            )
+        else:
+            factor_feedback = FactorSingleFeedback()
+
+            # 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
+            (
+                execution_feedback,
+                gen_df,
+            ) = implementation.execute()
+
+            execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
+            factor_feedback.execution_feedback = "\n".join(
+                [line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
+            )
+
+            # 2. Get factor value feedback
+            if gen_df is None:
+                factor_feedback.value_feedback = "No factor value generated, skip value evaluation."
+                factor_feedback.value_generated_flag = False
+                decision_from_value_check = None
+            else:
+                factor_feedback.value_generated_flag = True
+                (
+                    factor_feedback.value_feedback,
+                    decision_from_value_check,
+                ) = self.value_evaluator.evaluate(
+                    implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
+                )
+
+            factor_feedback.final_decision_based_on_gt = gt_implementation is not None
+
+            if decision_from_value_check is not None and decision_from_value_check is True:
+                # To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
+                factor_feedback.code_feedback = "Final decision is True and there are no code critics."
+                factor_feedback.final_decision = decision_from_value_check
+                factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
+            elif decision_from_value_check is not None and decision_from_value_check is False:
+                factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
+                    target_task=target_task,
+                    implementation=implementation,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    value_feedback=factor_feedback.value_feedback,
+                    gt_implementation=gt_implementation,
+                )
+                factor_feedback.final_decision = decision_from_value_check
+                factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
+            else:
+                factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
+                    target_task=target_task,
+                    implementation=implementation,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    value_feedback=factor_feedback.value_feedback,
+                    gt_implementation=gt_implementation,
+                )
+                (
+                    factor_feedback.final_decision,
+                    factor_feedback.final_feedback,
+                ) = self.final_decision_evaluator.evaluate(
+                    target_task=target_task,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    value_feedback=factor_feedback.value_feedback,
+                    code_feedback=factor_feedback.code_feedback,
+                )
+            return factor_feedback
+
+
+# TODO:
+def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
+    """When the prompt is too long. We have to shorten it.
+    But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
+    """
+    # TODO: this should replace most of code in
+    # - FactorFinalDecisionEvaluator.evaluate
+    # - FactorCodeEvaluator.evaluate

rdagent/components/coder/factor_coder/evolving_strategy.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Dict
+
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
+from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
+from rdagent.core.experiment import FBWorkspace
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+
+class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.num_loop = 0
+        self.haveSelected = False
+
+    def error_summary(
+        self,
+        target_task: FactorTask,
+        queried_former_failed_knowledge_to_render: list,
+        queried_similar_error_knowledge_to_render: list,
+    ) -> str:
+        error_summary_system_prompt = T(".prompts:evolving_strategy_error_summary_v2_system").r(
+            scenario=self.scen.get_scenario_all_desc(target_task),
+            factor_information_str=target_task.get_task_information(),
+            code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
+        )
+        for _ in range(10):  # max attempt to reduce the length of error_summary_user_prompt
+            error_summary_user_prompt = T(".prompts:evolving_strategy_error_summary_v2_user").r(
+                queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt
+                )
+                < APIBackend().chat_token_limit
+            ):
+                break
+            elif len(queried_similar_error_knowledge_to_render) > 0:
+                queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
+        error_summary_critics = APIBackend(
+            use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
+        ).build_messages_and_create_chat_completion(
+            user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt, json_mode=False
+        )
+        return error_summary_critics
+
+    def implement_one_task(
+        self,
+        target_task: FactorTask,
+        queried_knowledge: CoSTEERQueriedKnowledge,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> str:
+        target_factor_task_information = target_task.get_task_information()
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[target_factor_task_information]
+            if queried_knowledge is not None
+            else []
+        )  # A list, [success task implement knowledge]
+
+        if isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2):
+            queried_similar_error_knowledge = (
+                queried_knowledge.task_to_similar_error_successful_knowledge[target_factor_task_information]
+                if queried_knowledge is not None
+                else {}
+            )  # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
+        else:
+            queried_similar_error_knowledge = {}
+
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[target_factor_task_information][0]
+            if queried_knowledge is not None
+            else []
+        )
+
+        queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
+
+        latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[
+            target_factor_task_information
+        ][1]
+        system_prompt = T(".prompts:evolving_strategy_factor_implementation_v1_system").r(
+            scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
+            queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+        )
+        queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
+        queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
+        # 动态地防止prompt超长
+        for _ in range(10):  # max attempt to reduce the length of user_prompt
+            # 总结error（可选）
+            if (
+                isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2)
+                and FACTOR_COSTEER_SETTINGS.v2_error_summary
+                and len(queried_similar_error_knowledge_to_render) != 0
+                and len(queried_former_failed_knowledge_to_render) != 0
+            ):
+                error_summary_critics = self.error_summary(
+                    target_task,
+                    queried_former_failed_knowledge_to_render,
+                    queried_similar_error_knowledge_to_render,
+                )
+            else:
+                error_summary_critics = None
+            # 构建user_prompt。开始写代码
+            user_prompt = T(".prompts:evolving_strategy_factor_implementation_v2_user").r(
+                factor_information_str=target_factor_task_information,
+                queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
+                error_summary_critics=error_summary_critics,
+                latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt)
+                < APIBackend().chat_token_limit
+            ):
+                break
+            elif len(queried_former_failed_knowledge_to_render) > 1:
+                queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
+            elif len(queried_similar_successful_knowledge_to_render) > len(
+                queried_similar_error_knowledge_to_render,
+            ):
+                queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[:-1]
+            elif len(queried_similar_error_knowledge_to_render) > 0:
+                queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
+        for _ in range(10):
+            try:
+                response = APIBackend(
+                    use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
+                ).build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                    json_mode=True,
+                    json_target_type=Dict[str, str],
+                )
+
+                try:
+                    code = json.loads(response)["code"]
+                except json.decoder.JSONDecodeError:
+                    # extract python code block
+                    match = re.search(r"```python(.*?)```", response, re.DOTALL)
+                    if match:
+                        code = match.group(1).strip()
+                    else:
+                        raise  # continue to retry
+
+                return code
+
+            except (json.decoder.JSONDecodeError, KeyError):
+                pass
+        else:
+            return ""  # return empty code if failed to get code after 10 attempts
+
+    def assign_code_list_to_evo(self, code_list, evo):
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                evo.sub_workspace_list[index] = FactorFBWorkspace(target_task=evo.sub_tasks[index])
+            # Since the `implement_one_task` method is not standardized and the `code_list` has both `str` and `dict` data types,
+            # we ended up getting an `TypeError` here, so we chose to fix the problem temporarily with this dirty method.
+            if isinstance(code_list[index], dict):
+                evo.sub_workspace_list[index].inject_files(**code_list[index])
+            else:
+                evo.sub_workspace_list[index].inject_files(**{"factor.py": code_list[index]})
+        return evo

rdagent/components/coder/factor_coder/factor.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import subprocess
+import uuid
+from pathlib import Path
+from typing import Tuple, Union
+
+import pandas as pd
+from filelock import FileLock
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
+from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
+from rdagent.core.experiment import Experiment, FBWorkspace
+from rdagent.core.utils import cache_with_pickle
+from rdagent.oai.llm_utils import md5_hash
+
+
+class FactorTask(CoSTEERTask):
+    # TODO:  generalized the attributes into the Task
+    # - factor_* -> *
+    def __init__(
+        self,
+        factor_name,
+        factor_description,
+        factor_formulation,
+        *args,
+        variables: dict = {},
+        resource: str = None,
+        factor_implementation: bool = False,
+        **kwargs,
+    ) -> None:
+        self.factor_name = (
+            factor_name  # TODO: remove it in the later version. Keep it only for pickle version compatibility
+        )
+        self.factor_formulation = factor_formulation
+        self.variables = variables
+        self.factor_resources = resource
+        self.factor_implementation = factor_implementation
+        super().__init__(name=factor_name, description=factor_description, *args, **kwargs)
+
+    @property
+    def factor_description(self):
+        """for compatibility"""
+        return self.description
+
+    def get_task_information(self):
+        return f"""factor_name: {self.factor_name}
+factor_description: {self.factor_description}
+factor_formulation: {self.factor_formulation}
+variables: {str(self.variables)}"""
+
+    def get_task_brief_information(self):
+        return f"""factor_name: {self.factor_name}
+factor_description: {self.factor_description}
+factor_formulation: {self.factor_formulation}
+variables: {str(self.variables)}"""
+
+    def get_task_information_and_implementation_result(self):
+        return {
+            "factor_name": self.factor_name,
+            "factor_description": self.factor_description,
+            "factor_formulation": self.factor_formulation,
+            "variables": str(self.variables),
+            "factor_implementation": str(self.factor_implementation),
+        }
+
+    @staticmethod
+    def from_dict(dict):
+        return FactorTask(**dict)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.factor_name}]>"
+
+
+class FactorFBWorkspace(FBWorkspace):
+    """
+    This class is used to implement a factor by writing the code to a file.
+    Input data and output factor value are also written to files.
+    """
+
+    # TODO: (Xiao) think raising errors may get better information for processing
+    FB_EXEC_SUCCESS = "Execution succeeded without error."
+    FB_CODE_NOT_SET = "code is not set."
+    FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
+    FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
+    FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
+
+    def __init__(
+        self,
+        *args,
+        raise_exception: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.raise_exception = raise_exception
+
+    def hash_func(self, data_type: str = "Debug") -> str:
+        return (
+            md5_hash(data_type + self.file_dict["factor.py"])
+            if ("factor.py" in self.file_dict and not self.raise_exception)
+            else None
+        )
+
+    @cache_with_pickle(hash_func)
+    def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
+        """
+        execute the implementation and get the factor value by the following steps:
+        1. make the directory in workspace path
+        2. write the code to the file in the workspace path
+        3. link all the source data to the workspace path folder
+        if call_factor_py is True:
+            4. execute the code
+        else:
+            4. generate a script from template to import the factor.py dump get the factor value to result.h5
+        5. read the factor value from the output file in the workspace path folder
+        returns the execution feedback as a string and the factor value as a pandas dataframe
+
+
+        Regarding the cache mechanism:
+        1. We will store the function's return value to ensure it behaves as expected.
+        - The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception])
+
+        """
+        self.before_execute()
+        if self.file_dict is None or "factor.py" not in self.file_dict:
+            if self.raise_exception:
+                raise CodeFormatError(self.FB_CODE_NOT_SET)
+            else:
+                return self.FB_CODE_NOT_SET, None
+        with FileLock(self.workspace_path / "execution.lock"):
+            if self.target_task.version == 1:
+                source_data_path = (
+                    Path(
+                        FACTOR_COSTEER_SETTINGS.data_folder_debug,
+                    )
+                    if data_type == "Debug"  # FIXME: (yx) don't think we should use a debug tag for this.
+                    else Path(
+                        FACTOR_COSTEER_SETTINGS.data_folder,
+                    )
+                )
+            elif self.target_task.version == 2:
+                # TODO you can change the name of the data folder for a better understanding
+                source_data_path = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition
+
+            source_data_path.mkdir(exist_ok=True, parents=True)
+            code_path = self.workspace_path / f"factor.py"
+
+            self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path)
+
+            execution_feedback = self.FB_EXECUTION_SUCCEEDED
+            execution_success = False
+            execution_error = None
+
+            if self.target_task.version == 1:
+                execution_code_path = code_path
+            elif self.target_task.version == 2:
+                execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
+                execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
+
+            try:
+                subprocess.check_output(
+                    f"{FACTOR_COSTEER_SETTINGS.python_bin} {execution_code_path}",
+                    shell=True,
+                    cwd=self.workspace_path,
+                    stderr=subprocess.STDOUT,
+                    timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout,
+                )
+                execution_success = True
+            except subprocess.CalledProcessError as e:
+                import site
+
+                execution_feedback = (
+                    e.output.decode()
+                    .replace(str(execution_code_path.parent.absolute()), r"/path/to")
+                    .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
+                )
+                if len(execution_feedback) > 2000:
+                    execution_feedback = (
+                        execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
+                    )
+                if self.raise_exception:
+                    raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
+            except subprocess.TimeoutExpired:
+                execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds."
+                if self.raise_exception:
+                    raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
+
+            workspace_output_file_path = self.workspace_path / "result.h5"
+            if workspace_output_file_path.exists() and execution_success:
+                try:
+                    executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
+                    execution_feedback += self.FB_OUTPUT_FILE_FOUND
+                except Exception as e:
+                    execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
+                    executed_factor_value_dataframe = None
+            else:
+                execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
+                executed_factor_value_dataframe = None
+                if self.raise_exception:
+                    raise NoOutputError(execution_feedback)
+                else:
+                    execution_error = NoOutputError(execution_feedback)
+
+        return execution_feedback, executed_factor_value_dataframe
+
+    def __str__(self) -> str:
+        # NOTE:
+        # If the code cache works, the workspace will be None.
+        return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    @staticmethod
+    def from_folder(task: FactorTask, path: Union[str, Path], **kwargs):
+        path = Path(path)
+        code_dict = {}
+        for file_path in path.iterdir():
+            if file_path.suffix == ".py":
+                code_dict[file_path.name] = file_path.read_text()
+        return FactorFBWorkspace(target_task=task, code_dict=code_dict, **kwargs)
+
+
+FactorExperiment = Experiment
+FeatureExperiment = Experiment

rdagent/components/coder/factor_coder/factor_execution_template.txt

@@ -0,0 +1,15 @@
+import os
+
+import numpy as np
+import pandas as pd
+from factor import feature_engineering_cls
+
+if os.path.exists("X_valid.pkl"):
+    valid_df = pd.read_pickle("X_valid.pkl").head(1000)
+else:
+    raise FileNotFoundError("No valid data found.")
+
+cls = feature_engineering_cls()
+cls.fit(valid_df)
+new_feat = cls.transform(valid_df)
+new_feat.to_hdf("result.h5", key="data", mode="w")

rdagent/components/coder/factor_coder/prompts.yaml

@@ -0,0 +1,209 @@
+
+evaluator_code_feedback_v1_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User will provide you the information of the factor.
+
+  Your job is to check whether user's code is align with the factor and the scenario.
+  The user will provide the source python code and the execution error message if execution failed.
+  The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic.
+
+  User has also compared the factor values calculated by the user's code and the ground truth code. The user will provide you some analyze result comparing two output. You may find some error in the code which caused the difference between the two output.
+
+  If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
+  If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct.
+
+  Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX".
+
+  You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
+  
+  You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output:
+  critic 1: The critic message to critic 1
+  critic 2: The critic message to critic 2
+
+evaluator_code_feedback_v1_user: |-
+  --------------Factor information:---------------
+  {{ factor_information }}
+  --------------Python code:---------------
+  {{ code }}
+  --------------Execution feedback:---------------
+  {{ execution_feedback }}
+  {% if value_feedback is not none %}
+  --------------Factor value feedback:---------------
+  {{ value_feedback }}
+  {% endif %}
+  {% if gt_code is not none %}
+  --------------Ground truth Python code:---------------
+  {{ gt_code }}
+  {% endif %}
+
+evolving_strategy_factor_implementation_v1_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  Your code is expected to align the scenario in any form which means The user needs to get the exact factor values with your code as expected.
+
+  To help you write the correct code, the user might provide multiple information that helps you write the correct code:
+  1. The user might provide you the correct code to similar factors. Your should learn from these code to write the correct code.
+  2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code.
+  3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code.
+  
+  Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
+
+  Notice that you should not add any other text before or after the json format.
+
+  {% if queried_former_failed_knowledge|length != 0 %}
+  --------------Your former latest attempt:---------------
+  =====Code to the former implementation=====
+  {{ queried_former_failed_knowledge[-1].implementation.all_codes }}
+  =====Feedback to the former implementation=====
+  {{ queried_former_failed_knowledge[-1].feedback }}
+  {% endif %}
+
+  Please response the code in the following json format. Here is an example structure for the JSON output:
+  {
+      "code": "The Python code as a string."
+  }
+
+evolving_strategy_factor_implementation_v2_user: |-
+  --------------Target factor information:---------------
+  {{ factor_information_str }}
+
+  {% if queried_similar_error_knowledge|length != 0 %}
+  {% if error_summary_critics is none %}
+  Recall your last failure, your implementation met some errors.
+  When doing other tasks, you met some similar errors but you finally solve them. Here are some examples:
+  {% for error_content, similar_error_knowledge in queried_similar_error_knowledge %} 
+  --------------Factor information to similar error ({{error_content}}):---------------
+  {{ similar_error_knowledge[0].target_task.get_task_information() }}
+  =====Code with similar error ({{error_content}}):=====
+  {{ similar_error_knowledge[0].implementation.all_codes }}
+  =====Success code to former code with similar error ({{error_content}}):=====
+  {{ similar_error_knowledge[1].implementation.all_codes }}
+  {% endfor %}
+  {% else %}
+  Recall your last failure, your implementation met some errors.
+  After reviewing some similar errors and their solutions, here are some suggestions for you to correct your code:
+  {{error_summary_critics}}
+  {% endif %}
+  {% endif %}
+  {% if queried_similar_successful_knowledge|length != 0 %}
+  Here are some success implements of similar component tasks, take them as references:
+  --------------Correct code to similar factors:---------------
+  {% for similar_successful_knowledge in queried_similar_successful_knowledge %}
+  =====Factor {{loop.index}}:=====
+  {{ similar_successful_knowledge.target_task.get_task_information() }}
+  =====Code:=====
+  {{ similar_successful_knowledge.implementation.all_codes }}
+  {% endfor %}
+  {% endif %}
+  {% if latest_attempt_to_latest_successful_execution is not none %}
+  You have tried to correct your former failed code but still met some errors. Here is the latest attempt to the latest successful execution, try not to get the same error to your new code:
+  =====Your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.implementation.all_codes }}
+  =====Feedback to your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.feedback }}
+  {% endif %}
+
+evolving_strategy_error_summary_v2_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User is doing the following task: 
+  {{factor_information_str}}
+
+  You have written some code but it meets errors like the following:
+  {{code_and_feedback}}
+
+  The user has found some tasks that met similar errors, and their final correct solutions.
+  Please refer to these similar errors and their solutions, provide some clear, short and accurate critics that might help you solve the issues in your code.
+
+  You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
+
+  [NOTE]
+  1. When processing data, avoid time leakage.
+
+  Please response the critic in the following format. Here is an example structure for the output:
+  critic 1: The critic message to critic 1
+  critic 2: The critic message to critic 2
+  
+evolving_strategy_error_summary_v2_user: |-
+  {% if queried_similar_error_knowledge|length != 0 %}
+  {% for error_content, similar_error_knowledge in queried_similar_error_knowledge %} 
+  --------------Factor information to similar error ({{error_content}}):---------------
+  {{ similar_error_knowledge[0].target_task.get_task_information() }}
+  =====Code with similar error ({{error_content}}):=====
+  {{ similar_error_knowledge[0].implementation.all_codes }}
+  =====Success code to former code with similar error ({{error_content}}):=====
+  {{ similar_error_knowledge[1].implementation.all_codes }}
+  {% endfor %}
+  {% endif %}
+
+
+select_implementable_factor_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  Your job is to help the user select the easiest-to-implement factors. Some factors may be difficult to implement due to a lack of information or excessive complexity. The user will provide the number of factors you should pick and information about the factors, including their descriptions, formulas, and variable explanations.
+  User will provide you the former attempt to implement the factor and the feedback to the implementation. You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors.
+  Please analyze the difficulties of the each factors and provide the reason and response the indices of selected implementable factor in the json format. Here is an example structure for the JSON output:
+  {
+      "Analysis": "Analyze the difficulties of the each factors and provide the reason why the factor can be implemented or not."
+      "selected_factor": "The indices of selected factor index in the list, like [0, 2, 3].The length should be the number of factor left after filtering.",
+  }
+
+select_implementable_factor_user: |-
+  Number of factor you should pick: {{ factor_num }}
+  {% for factor_info in sub_tasks %} 
+  =============Factor index:{{factor_info[0]}}:=============
+  =====Factor name:=====
+  {{ factor_info[1].factor_name }}
+  =====Factor description:=====
+  {{ factor_info[1].factor_description }}
+  =====Factor formulation:=====
+  {{ factor_info[1].factor_formulation }}
+  {% if factor_info[2]|length != 0 %}
+  --------------Your former attempt:---------------
+  {% for former_attempt in factor_info[2] %}
+  =====Code to attempt {{ loop.index }}=====
+  {{ former_attempt.implementation.all_codes }}
+  =====Feedback to attempt {{ loop.index }}=====
+  {{ former_attempt.feedback }}
+  {% endfor %}
+  {% endif %}
+  {% endfor %}
+
+evaluator_output_format_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User will provide you the format of the output. Please help to check whether the output is align with the format.
+  Please respond in the JSON format. Here is an example structure for the JSON output:
+  {
+      "output_format_decision": True,
+      "output_format_feedback": "The output format is correct."
+  }
+
+
+evaluator_final_decision_v1_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User has finished evaluation and got some feedback from the evaluator.
+  The evaluator run the code and get the factor value dataframe and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the scenario and factor description to give a final decision about the evaluation result. The final decision concludes whether the factor is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False.
+
+  The implementation final decision is considered in the following logic:
+  1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
+  2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
+  3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description. The implementation cannot be considered correct if the code execution failed, no matter what the reason is.
+
+  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+  {
+      "final_decision": True,
+      "final_feedback": "The final feedback message",
+  }
+
+evaluator_final_decision_v1_user: |-
+  --------------Factor information:---------------
+  {{ factor_information }}
+  --------------Execution feedback:---------------
+  {{ execution_feedback }}
+  --------------Code feedback:---------------
+  {{ code_feedback }}
+  --------------Factor value feedback:---------------
+  {{ value_feedback }}

rdagent/components/coder/finetune/__init__.py

@@ -0,0 +1,391 @@
+"""
+LLM Fine-tuning CoSTEER Implementation
+
+This module provides fine-tuning specific components for the CoSTEER framework,
+including evaluators and evolving strategies.
+"""
+
+import json
+from pathlib import Path
+from typing import Callable
+
+import yaml
+
+from rdagent.app.finetune.llm.conf import FT_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.finetune.conf import (
+    FT_DATA_SCRIPT_NAME,
+    FT_PATHS,
+    FT_TEST_PARAMS_FILE_NAME,
+    FT_YAML_FILE_NAME,
+    FTCoderCoSTEERSettings,
+)
+from rdagent.components.coder.finetune.eval import FTCoderEvaluator, FTDataEvaluator
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.finetune.scen.llama_factory_manager import LLaMAFactory_manager
+from rdagent.scenarios.finetune.scen.utils import FinetuneDatasetDescriptor
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class LLMFinetuneEvolvingStrategy(MultiProcessEvolvingStrategy):
+    """LLM Fine-tuning specific evolving strategy"""
+
+    def __init__(self, scen: Scenario, settings, *args, **kwargs):
+        super().__init__(scen, settings)
+        self.llama_factory_manager = LLaMAFactory_manager
+
+    def implement_func_list(self) -> list[Callable]:
+        return [self.implement_data, self.implement_lf_config]
+
+    def implement_data(
+        self,
+        target_task: Task,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        """Generate data processing script based on task.
+
+        This method generates a Python script that processes seed datasets
+        and outputs a data.json file in Alpaca format.
+
+        Returns:
+            dict with "process_data.py" key containing the script code,
+            or empty dict if data already exists.
+        """
+        # Check if proposal decided to skip data processing (reuse SOTA's data processing script)
+        if getattr(target_task, "skip_data_processing", False):
+            # Defensive check: ensure data script actually exists before skipping
+            script_exists = False
+            if workspace is not None:
+                script_exists = FT_DATA_SCRIPT_NAME in workspace.file_dict
+
+            if script_exists:
+                logger.info("Proposal decided to skip data processing, reusing SOTA's data script")
+                return {}
+            else:
+                logger.warning(
+                    "skip_data_processing=True but process_data.py not found in workspace, "
+                    "this indicates SOTA injection failed - system design issue"
+                )
+                # Don't fallback silently, let it fail early to expose the issue
+
+        # check whether the current code passes evaluation
+        if (
+            prev_task_feedback is not None
+            and "FTDataEvaluator" in prev_task_feedback.source_feedback
+            and prev_task_feedback.source_feedback["FTDataEvaluator"]
+        ):
+            logger.info("Previous data processing code passed evaluation, skipping regeneration")
+            return {}
+
+        # build former failed trace
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get(FT_YAML_FILE_NAME)
+                != workspace.file_dict.get(FT_YAML_FILE_NAME)
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # Get dataset information for the task
+        involving_datasets = getattr(target_task, "involving_datasets", [])
+        dataset_info = self._get_dataset_info(involving_datasets, datasets_path=FT_PATHS.datasets)
+
+        # Generate data processing script using LLM
+        system_prompt = T(".prompts:data_coder.system").r(
+            scenario=self.scen.get_scenario_all_desc(),
+            task_desc=target_task.get_task_information(),
+            dataset_info=dataset_info,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            api_max_workers=FT_RD_SETTING.api_max_workers,
+            datasets_path=FT_PATHS.datasets,
+            workspace_path=FT_PATHS.workspace,
+            force_think_token=FT_RD_SETTING.force_think_token,
+        )
+
+        user_prompt = T(".prompts:data_coder.user").r(
+            datasets_path=FT_PATHS.datasets,
+            workspace_path=FT_PATHS.workspace,
+            latest_code=workspace.file_dict.get(FT_DATA_SCRIPT_NAME, "") if workspace else "",
+            latest_feedback=prev_task_feedback,
+            involved_dataset_folder_desc={
+                ds_name: FinetuneDatasetDescriptor().describe_dataset_folder(
+                    Path(FT_RD_SETTING.file_path) / "datasets" / ds_name, include_dataset_readme=True
+                )
+                for ds_name in involving_datasets
+            },
+        )
+
+        script_code = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+            code_block_language="python",
+            code_block_fallback=False,
+        )
+        logger.info(f"Generated data processing script ({len(script_code)} chars)")
+
+        return {FT_DATA_SCRIPT_NAME: script_code}
+
+    def _get_dataset_info(self, involving_datasets: list[str], datasets_path: str = None) -> str:
+        """Read dataset_info.json and return information for specified datasets.
+
+        Handles unified tasks structure:
+        - readme: Dataset README content
+        - file_tree: Directory structure
+        - total_samples: Total sample count
+        - tasks: Dict of task info (use "_root" for root-level data files)
+
+        Args:
+            involving_datasets: List of dataset names to include
+            datasets_path: Base path for datasets (e.g., "/assets/datasets/")
+        """
+        datasets_dir = Path(FT_RD_SETTING.file_path) / "datasets"
+        dataset_info_path = datasets_dir / "dataset_info.json"
+
+        # Use provided path or get from config
+        if datasets_path is None:
+            datasets_path = FT_PATHS.datasets
+
+        if not dataset_info_path.exists():
+            logger.warning(f"dataset_info.json not found at {dataset_info_path}")
+            return "No dataset information available."
+
+        try:
+            with open(dataset_info_path, "r", encoding="utf-8") as f:
+                all_dataset_info = json.load(f)
+        except Exception as e:
+            logger.error(f"Failed to read dataset_info.json: {e}")
+            return f"Error reading dataset info: {e}"
+
+        # Filter to only involved datasets, or use all if none specified
+        if involving_datasets:
+            filtered_info = {name: info for name, info in all_dataset_info.items() if name in involving_datasets}
+        else:
+            filtered_info = all_dataset_info
+
+        if not filtered_info:
+            return "No matching datasets found in dataset_info.json."
+
+        # Format dataset info for the prompt
+        info_parts = []
+        for name, info in filtered_info.items():
+            info_text = f"### Dataset: {name}\n"
+            # IMPORTANT: Tell LLM the full path to dataset directory
+            dataset_full_path = f"{datasets_path}{name}/"
+            info_text += f"- **Dataset path**: `{dataset_full_path}` (each dataset has its own subdirectory)\n"
+            info_text += f"- Total samples: {info.get('total_samples', 'N/A')}\n"
+            info_text += f"- Size: {info.get('total_size_mb', 'N/A')} MB\n"
+
+            # File tree for understanding directory structure
+            if info.get("file_tree"):
+                file_tree = info["file_tree"]
+                # Truncate if too long
+                if len(file_tree) > 1000:
+                    file_tree = file_tree[:1000] + "\n..."
+                info_text += f"\n**File Structure** (relative to `{dataset_full_path}`):\n```\n{file_tree}\n```\n"
+
+            # Handle unified tasks structure
+            tasks = info.get("tasks", {})
+            if tasks:
+                info_text += "\n**Tasks:**\n"
+                for task_name, task_info in tasks.items():
+                    # "_root" indicates data files are in root directory
+                    display_name = "(root)" if task_name == "_root" else task_name
+                    info_text += f"\n#### {display_name}\n"
+                    # Show full paths for data files
+                    files = task_info.get("files", [])
+                    info_text += f"- Files: {files}\n"
+                    if files:
+                        info_text += f"  - Full path example: `{dataset_full_path}{files[0]}`\n"
+                    info_text += f"- Sample count: {task_info.get('sample_count', 'N/A')}\n"
+                    if task_info.get("column_stats"):
+                        # Show key token stats
+                        stats_summary = []
+                        for col, stats in task_info["column_stats"].items():
+                            if stats.get("p50_tokens", 0) > 0:
+                                stats_summary.append(f"{col}: p50={stats['p50_tokens']}, p99={stats['p99_tokens']}")
+                        if stats_summary:
+                            info_text += f"- Token stats: {'; '.join(stats_summary[:5])}\n"
+
+            # README excerpt
+            if info.get("readme"):
+                readme = info["readme"]
+                if len(readme) > 500:
+                    readme = readme[:500] + "..."
+                info_text += f"\n**README:**\n{readme}\n"
+
+            info_parts.append(info_text)
+
+        return "\n".join(info_parts)
+
+    def implement_lf_config(
+        self,
+        target_task: Task,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        """Implement a single fine-tuning task by generating LlamaFactory config"""
+        if prev_task_feedback is not None and prev_task_feedback.source_feedback.get("FTCoderEvaluator", False):
+            logger.info("Previous training code passed evaluation, skipping regeneration")
+            return {}
+
+        task_info = target_task.get_task_information()
+
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[task_info] if queried_knowledge is not None else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get(FT_YAML_FILE_NAME)
+                != workspace.file_dict.get(FT_YAML_FILE_NAME)
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # Get task parameters from the task object
+        base_model = getattr(target_task, "base_model")
+
+        # Use LLM to generate LlamaFactory config YAML
+        # Coder will decide method based on hypothesis and available parameters
+        config_files = self._generate_llamafactory_config_with_llm(
+            base_model=base_model,
+            task_info=task_info,
+            queried_former_failed_knowledge=queried_former_failed_knowledge,
+            prev_feedback=prev_task_feedback,
+            workspace=workspace,
+        )
+
+        # Return generated config files directly - validation happens in evaluator
+        return config_files
+
+    def _generate_llamafactory_config_with_llm(
+        self,
+        base_model: str,
+        task_info: str = "",
+        queried_former_failed_knowledge: tuple = None,
+        prev_feedback=None,
+        workspace=None,
+    ) -> dict[str, str]:
+        """Generate LlamaFactory configuration YAML using LLM"""
+
+        # Query LLaMA Factory parameters: shared params once + method-specific params
+        available_methods = self.llama_factory_manager.methods
+        shared_params = self.llama_factory_manager.format_shared_params()
+
+        # Format method-specific parameters only (no duplication of shared params)
+        methods_specific_params = {}
+        for method in available_methods:
+            methods_specific_params[method] = self.llama_factory_manager.format_method_specific_params(method)
+
+        # Use environment-aware paths (Docker vs Conda)
+        # Note: datasets_path in finetune_coder uses workspace path where processed
+        # data.json and dataset_info.json are located (generated by FTDataEvaluator)
+
+        # Generate prompts using templates with all required parameters
+        system_prompt = T(".prompts:finetune_coder.system").r(
+            scenario=self.scen.get_scenario_all_desc(),
+            task_desc=task_info,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            available_methods=", ".join(available_methods),
+            shared_params=shared_params,
+            methods_specific_params=methods_specific_params,
+        )
+
+        # Read data_stats.json from workspace (injected by FTDataEvaluator)
+        data_stats = workspace.file_dict.get("data_stats.json", "")
+
+        user_prompt = T(".prompts:finetune_coder.user").r(
+            latest_code=workspace.file_dict.get(FT_YAML_FILE_NAME, ""),
+            latest_feedback=prev_feedback,
+            base_model=base_model,
+            models_path=FT_PATHS.models,
+            datasets_path=FT_PATHS.workspace,  # Training config uses workspace path for processed data
+            workspace_path=FT_PATHS.workspace,
+            deepspeed_path=FT_PATHS.deepspeed,
+            data_stats=data_stats,
+            has_think_token=self.scen.model_info.get("has_think_token", False),
+            force_think_token=FT_RD_SETTING.force_think_token,
+        )
+
+        # Call LLM to generate config (multi-turn)
+        session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
+
+        # Turn 1: Generate main training config
+        train_config_yaml = session.build_chat_completion(
+            user_prompt=user_prompt,
+            json_mode=False,
+            code_block_language="yaml",
+            code_block_fallback=False,
+        )
+
+        # Validate main config YAML syntax
+        yaml.safe_load(train_config_yaml)
+        logger.info("Extracted main YAML config successfully")
+
+        # Turn 2: Generate test parameters (test_params.yaml)
+        test_params_prompt = T(".prompts:finetune_coder.user_test_params").r(workspace_path=FT_PATHS.workspace)
+        test_params_yaml = session.build_chat_completion(
+            user_prompt=test_params_prompt,
+            json_mode=False,
+            code_block_language="yaml",
+            code_block_fallback=False,
+        )
+
+        # Validate test params YAML syntax
+        yaml.safe_load(test_params_yaml)
+        logger.info("Extracted test params YAML successfully")
+
+        return {FT_YAML_FILE_NAME: train_config_yaml, FT_TEST_PARAMS_FILE_NAME: test_params_yaml}
+
+
+class LLMFinetuneCoSTEER(CoSTEER):
+    """LLM Fine-tuning CoSTEER implementation"""
+
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = FTCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator([FTDataEvaluator(scen=scen), FTCoderEvaluator(scen=scen)], scen=scen)
+        es = LLMFinetuneEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=FT_RD_SETTING.coder_max_loop if hasattr(FT_RD_SETTING, "coder_max_loop") else 5,
+            stop_eval_chain_on_fail=True,  # finetune involve partial implementation.
+            **kwargs,
+        )

rdagent/components/coder/finetune/conf.py

@@ -0,0 +1,387 @@
+import json
+import os
+import re
+import shutil
+from pathlib import Path
+from typing import Any, Literal
+
+from rdagent.app.finetune.llm.conf import FT_RD_SETTING
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.core.experiment import FBWorkspace
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.finetune.scen.utils import _compute_column_stats
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.env import (
+    BenchmarkCondaConf,
+    BenchmarkCondaEnv,
+    BenchmarkDockerConf,
+    BenchmarkDockerEnv,
+    DockerEnv,
+    Env,
+    FTCondaConf,
+    FTCondaEnv,
+    FTDockerEnv,
+)
+
+
+def is_docker_env(env: Env) -> bool:
+    """Check if the environment is Docker-based."""
+    return isinstance(env, DockerEnv)
+
+
+def get_workspace_prefix(env: Env) -> str:
+    """Return workspace path prefix based on env type.
+
+    Docker uses /workspace as mount point, conda uses current directory.
+    """
+    return "/workspace" if is_docker_env(env) else "."
+
+
+FT_YAML_FILE_NAME = "train.yaml"
+FT_DATA_PROC_FILE_NAME = "data_process.py"
+FT_DEBUG_YAML_FILE_NAME = "debug_train.yaml"
+FT_TEST_PARAMS_FILE_NAME = "test_params.yaml"
+FT_DATA_FILE_NAME = "data.json"
+FT_DATA_SCRIPT_NAME = "process_data.py"
+
+# ENV Info:  the path of the model and dataset in the container/environment
+FT_MODEL_PATH = "/assets/models"
+FT_DATASET_PATH = "/assets/datasets"
+
+
+def get_data_processing_cache_key(local_path: str | Path) -> list[list[str]]:
+    """Generate cache key based only on data processing script and dataset info.
+
+    This ensures that data processing results are reused as long as the script
+    and dataset configuration remain unchanged, even if other files in the
+    workspace (like training config) have been modified.
+    """
+    content = []
+    local_path = Path(local_path)
+    # We only care about the script that generates data and the dataset configuration
+    for filename in [FT_DATA_SCRIPT_NAME, "dataset_info.json"]:
+        file_path = local_path / filename
+        if file_path.exists():
+            content.append([filename, file_path.read_text()])
+    content.sort(key=lambda x: x[0])
+    return content
+
+
+class FTPathConfig:
+    """Centralized path configuration for FT scenario.
+
+    Provides environment-aware paths for Docker vs Conda modes.
+    Uses lazy evaluation (properties) to avoid import-time errors.
+
+    Usage:
+        from rdagent.components.coder.finetune.conf import FT_PATHS
+
+        models_path = FT_PATHS.models      # e.g., "/assets/models/" or "/path/to/finetune/models/"
+        datasets_path = FT_PATHS.datasets  # e.g., "/assets/datasets/" or "/path/to/finetune/datasets/"
+        workspace_path = FT_PATHS.workspace  # e.g., "/workspace/" or "./"
+    """
+
+    @property
+    def is_docker(self) -> bool:
+        """Check if current environment is Docker-based."""
+        # FIXME: the env should work in same way for docker and conda env.
+        # We should not expose the env type everywhere.
+        return FTCoderCoSTEERSettings().env_type == "docker"
+
+    @property
+    def models(self) -> str:
+        """Model directory path (with trailing slash)."""
+        if self.is_docker:
+            return FT_MODEL_PATH + "/"
+        return str(FT_RD_SETTING.file_path / "models") + "/"
+
+    @property
+    def datasets(self) -> str:
+        """Dataset directory path for raw datasets (with trailing slash)."""
+        if self.is_docker:
+            return FT_DATASET_PATH + "/"
+        return str(FT_RD_SETTING.file_path / "datasets") + "/"
+
+    @property
+    def workspace(self) -> str:
+        """Workspace path prefix for prompts (with trailing slash)."""
+        return "/workspace/" if self.is_docker else "./"
+
+    @property
+    def deepspeed(self) -> str:
+        """DeepSpeed config directory."""
+        if self.is_docker:
+            return "/app/examples/deepspeed/"
+        # Conda mode: use bundled deepspeed configs in project
+        # Path: conf.py -> finetune -> coder -> components -> rdagent -> scenarios/finetune/env/conda/deepspeed
+        rdagent_root = Path(__file__).parent.parent.parent.parent
+        deepspeed_path = rdagent_root / "scenarios" / "finetune" / "env" / "conda" / "deepspeed"
+        return str(deepspeed_path) + "/" if deepspeed_path.exists() else ""
+
+
+# Singleton instance for path configuration
+FT_PATHS = FTPathConfig()
+
+
+class FTCoderCoSTEERSettings(CoSTEERSettings):
+    """LLM Fine-tuning CoSTEER settings"""
+
+    class Config:
+        env_prefix = "FT_Coder_CoSTEER_"
+
+    max_seconds_multiplier: int = 8
+    """LLM training takes longer, use higher multiplier"""
+
+    env_type: str = "docker"
+    """Environment type for LLM fine-tuning (docker/conda)"""
+
+    extra_eval: list[str] = []
+    """Extra evaluators"""
+
+
+def _get_standard_ft_volumes() -> dict:
+    """Get standard mount volume configuration for LLM finetune environments.
+
+    Creates standard directory mappings:
+    - models -> /assets/models (ro)
+    - datasets -> /assets/datasets (ro)
+
+    Returns:
+        Dictionary of local_path -> docker_mount_config mappings
+    """
+    base_path = Path(FT_RD_SETTING.file_path)
+    volumes = {}
+
+    # Read-only mounts for data and models
+    readonly_mounts = [
+        ("models", FT_MODEL_PATH),
+        ("datasets", FT_DATASET_PATH),
+    ]
+
+    for local_dir, docker_path in readonly_mounts:
+        local_path = base_path / local_dir
+        volumes[str(local_path)] = {"bind": docker_path, "mode": "ro"}
+
+    return volumes
+
+
+def get_ft_env(
+    extra_volumes: dict = {},
+    operation: str = "full_training",
+    enable_cache: bool | None = None,
+) -> Env:
+    """LLM finetune dedicated environment construction function.
+
+    Automatically includes standard finetune volume mounts:
+    - models -> /assets/models (ro)
+    - datasets -> /assets/datasets (ro)
+    - output -> /workspace/output (rw, auto-created)
+
+    Note: .llama_factory_info is no longer automatically mounted.
+    Pass llama_factory_info volume via extra_volumes when needed.
+
+    Args:
+        extra_volumes: Additional volume mounts beyond standard ones
+        operation: Operation type for timeout selection.
+            - "data_processing": Data processing (data_processing_timeout)
+            - "micro_batch": Micro-batch test (micro_batch_timeout)
+            - "full_training": Full training (full_timeout)
+        enable_cache: Whether to enable caching (None means use config value)
+
+    Returns:
+        Configured environment ready for use
+    """
+
+    conf = FTCoderCoSTEERSettings()
+
+    # Select timeout based on operation type
+    timeout_map = {
+        "data_processing": FT_RD_SETTING.data_processing_timeout,
+        "debug_data_processing": FT_RD_SETTING.debug_data_processing_timeout,
+        "micro_batch": FT_RD_SETTING.micro_batch_timeout,
+        "full_training": FT_RD_SETTING.full_timeout,
+    }
+    running_timeout_period = timeout_map.get(operation, FT_RD_SETTING.full_timeout)
+
+    # Use config value if enable_cache is not explicitly provided
+    if enable_cache is None:
+        enable_cache = FT_RD_SETTING.docker_enable_cache
+
+    # Use dedicated LLM docker or conda env based on config
+    if conf.env_type == "docker":
+        env = FTDockerEnv()
+        # Docker mode: setup volume mounts for models/datasets
+        standard_volumes = _get_standard_ft_volumes()
+        combined_volumes = standard_volumes.copy()
+        combined_volumes.update(extra_volumes)
+        env.conf.extra_volumes = combined_volumes
+    elif conf.env_type == "conda":
+        env = FTCondaEnv(conf=FTCondaConf())  # Auto-installs dependencies if env doesn't exist
+        # Conda mode: no volume mounts needed, use local paths directly
+        # extra_volumes are ignored in conda mode
+    else:
+        raise ValueError(f"Unknown env type: {conf.env_type}")
+
+    env.conf.running_timeout_period = running_timeout_period
+    env.conf.enable_cache = enable_cache
+    env.prepare()
+    return env
+
+
+def get_data_processing_env(
+    enable_cache: bool | None = None,
+    is_debug: bool = False,
+) -> tuple[Env, dict]:
+    """Get environment for data processing scripts with LLM API access.
+
+    This environment is configured for running data processing scripts that may
+    need to call LLM APIs. It includes:
+    - Standard finetune volume mounts (datasets, models)
+    - LLM API environment variables (OPENAI_API_KEY, OPENAI_BASE_URL, etc.)
+
+    Args:
+        enable_cache: Whether to enable Docker caching
+        is_debug: Whether running in debug mode (shorter timeout, default 20 min vs 1 hour)
+
+    Returns:
+        Tuple of (env, env_vars) where env_vars contains LLM API keys
+        to be passed to env.run() as the env parameter
+    """
+    env = get_ft_env(
+        operation="debug_data_processing" if is_debug else "data_processing",
+        enable_cache=enable_cache,
+    )
+
+    # Collect LLM API environment variables to pass to env.run()
+    llm_env_vars = {"PYTHONPATH": "./"}  # Base env var
+
+    # Pass OPENAI_API_KEY directly
+    if api_key := os.getenv("OPENAI_API_KEY"):
+        llm_env_vars["OPENAI_API_KEY"] = api_key
+
+    # Read OPENAI_API_BASE from env, but pass as OPENAI_BASE_URL (OpenAI SDK expects this name)
+    if api_base := os.getenv("OPENAI_API_BASE"):
+        llm_env_vars["OPENAI_BASE_URL"] = api_base
+
+    # Pass model pools as JSON environment variables for load balancing
+    llm_env_vars["STRONG_MODEL_POOL"] = json.dumps(FT_RD_SETTING.strong_models)
+    llm_env_vars["WEAK_MODEL_POOL"] = json.dumps(FT_RD_SETTING.weak_models)
+
+    return env, llm_env_vars
+
+
+def clear_workspace(workspace: FBWorkspace, env: Env) -> None:
+    """
+    Clean the files in LLM finetune workspace.
+    Only keeps the files that are injected by the coder (in workspace.file_dict) and `logs`.
+
+    Args:
+        workspace: The workspace object containing the file dictionary.
+        env: The environment to execute the clean command in.
+    """
+    target_path = workspace.workspace_path
+    if not target_path.exists():
+        return
+
+    # The cache_path is created when mounting, so the permissions changes does not work.
+    keep_items = {"logs", T("scenarios.data_science.share:scen.cache_path").r()}
+
+    for file_path in workspace.file_dict.keys():
+        top_level = Path(file_path).parts[0]
+        keep_items.add(top_level)
+
+    remove_items = []
+    for item in target_path.iterdir():
+        if item.name in keep_items:
+            continue
+        remove_items.append(item.name)
+
+    if remove_items:
+        ws_prefix = get_workspace_prefix(env)
+        # Construct rm command with all items to remove
+        # Items are relative to workspace root inside the env
+        items_str = " ".join([f"'{ws_prefix}/{item}'" for item in remove_items])
+        cmd = f"rm -rf {items_str}"
+        workspace.execute(env=env, entry=cmd)
+
+
+def get_benchmark_env(
+    extra_volumes: dict = {},
+    timeout: int | None = None,
+) -> Env:
+    """OpenCompass benchmark environment construction function.
+
+    Supports both Docker and conda environments based on FT_Coder_CoSTEER_env_type.
+
+    Args:
+        extra_volumes: Additional volume mounts (only used in Docker mode)
+        timeout: Running timeout in seconds (None uses config default)
+
+    Returns:
+        Configured environment ready for benchmark evaluation
+    """
+    conf = FTCoderCoSTEERSettings()
+
+    # Use benchmark-specific timeout or config default
+    if timeout is None:
+        # 0 means no timeout, use 7 days as practical "infinite"
+        timeout = FT_RD_SETTING.benchmark_timeout if FT_RD_SETTING.benchmark_timeout > 0 else 86400 * 7
+
+    benchmark_volumes = {}
+    # Setup finetune share folder mount for models
+    (FT_RD_SETTING.file_path / "benchmarks").mkdir(parents=True, exist_ok=True)
+    # NOTE: we choose a folder in the workspace as the mount point due to we may run multiple instances in same
+    # host machine. If conda env is used, the mount point will conflict with each other.
+    benchmark_volumes[str((FT_RD_SETTING.file_path / "benchmarks").resolve())] = {
+        "bind": "./benchmarks",
+        "mode": "rw",
+    }
+    env_dict = {"COMPASS_DATA_CACHE": "./benchmarks/opencompass_data"}
+    # Mount models directory for LoRA base model access (vLLM needs base model config)
+    models_path = FT_RD_SETTING.file_path / "models"
+    if models_path.exists():
+        benchmark_volumes[str(models_path.resolve())] = {"bind": FT_MODEL_PATH, "mode": "ro"}
+    benchmark_volumes.update(extra_volumes)
+
+    if conf.env_type == "docker":
+        docker_conf = BenchmarkDockerConf()
+        docker_conf.running_timeout_period = timeout
+        docker_conf.extra_volumes = benchmark_volumes
+        docker_conf.env_dict = env_dict
+        env = BenchmarkDockerEnv(conf=docker_conf)
+    elif conf.env_type == "conda":
+        # NOTE:
+        # We assume user has the permissions to create the softlink in the target directory.
+        # If we have requirements in the future, we suggest make the target directory configurable in BenchmarkCondaConf.
+        conda_conf = BenchmarkCondaConf()
+        conda_conf.running_timeout_period = timeout
+        conda_conf.extra_volumes = benchmark_volumes
+        conda_conf.env_dict = env_dict
+        env = BenchmarkCondaEnv(conf=conda_conf)  # Auto-installs dependencies if env doesn't exist
+    else:
+        raise ValueError(f"Unknown env type: {conf.env_type}")
+
+    env.prepare()
+    return env
+
+
+def inject_data_stats(implementation: FBWorkspace, data: list, stdout: str) -> None:
+    """Compute token statistics and inject data_stats.json.
+
+    Used by both FTDataEvaluator (coding stage) and FTRunnerEvaluator (running stage).
+
+    Args:
+        implementation: The workspace to inject data_stats.json into
+        data: The data list from data.json
+        stdout: The stdout from process_data.py execution
+    """
+    token_stats = _compute_column_stats(data)
+
+    data_stats = {
+        "total_samples": len(data),
+        "token_stats": token_stats,
+        "stdout_summary": stdout,
+    }
+
+    implementation.inject_files(**{"data_stats.json": json.dumps(data_stats, indent=2)})
+    logger.info(f"Injected data_stats.json with {len(data)} samples")

rdagent/components/coder/finetune/eval.py

@@ -0,0 +1,399 @@
+"""
+LLM Fine-tuning Evaluation Components
+
+Provides simplified evaluation: parameter filtering + micro-batch testing.
+No redundant LLM feedback generation - test results speak for themselves.
+"""
+
+import json
+import random
+from pathlib import Path
+from typing import Optional
+
+from rdagent.app.finetune.llm.conf import FT_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.finetune.conf import (
+    FT_DATA_FILE_NAME,
+    FT_DATA_SCRIPT_NAME,
+    FT_YAML_FILE_NAME,
+    clear_workspace,
+    get_data_processing_cache_key,
+    get_data_processing_env,
+    get_ft_env,
+    get_workspace_prefix,
+    inject_data_stats,
+)
+from rdagent.components.coder.finetune.unified_validator import (
+    SYSTEM_MANAGED_PARAMS,
+    LLMConfigValidator,
+)
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class FTDataEvaluator(CoSTEEREvaluator):
+    """Evaluator for data processing results.
+
+    This evaluator:
+    1. Executes the process_data.py script in Docker
+    2. Validates the output data.json file
+    3. Generates dataset_info.json for LlamaFactory
+    """
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: Optional[QueriedKnowledge] = None,
+        **kwargs,
+    ) -> CoSTEERSingleFeedback:
+        """Evaluate data processing implementation with LLM feedback."""
+
+        script_code = implementation.file_dict.get(FT_DATA_SCRIPT_NAME, "")
+        data_json_path = implementation.workspace_path / FT_DATA_FILE_NAME
+        execution_output = ""
+        exit_code = 0
+        data = None
+        error_msg = None
+
+        # Step 1: Check script exists
+        if not script_code:
+            feedback = CoSTEERSingleFeedback(
+                execution=f"No {FT_DATA_SCRIPT_NAME} found",
+                return_checking="Data processing script missing",
+                code="Please generate a data processing script first.",
+                final_decision=False,
+            )
+            logger.log_object(feedback, tag="evaluator_feedback.FTDataEvaluator")
+            return feedback
+
+        # NOTE: we depends cache for speeding up the process of data generation.
+        # So we clear the workspace every time.
+
+        # Step 3: Execute script in DEBUG mode (generates ~10 samples for fast validation)
+        env, env_vars = get_data_processing_env(is_debug=True)
+
+        # Clear workspace (except logs and file_dict items) before data processing
+        clear_workspace(implementation, env=env)
+        ws_prefix = get_workspace_prefix(env)
+
+        # Use FTWorkspace.run() for unified Docker logging
+        # --debug flag tells the script to generate only ~10 samples
+        result = implementation.run(
+            env=env,
+            entry=f"python {ws_prefix}/{FT_DATA_SCRIPT_NAME} --debug",
+            env_vars=env_vars,
+            cache_key_extra_func=get_data_processing_cache_key,
+            cache_files_to_extract=[FT_DATA_FILE_NAME],
+        )
+        execution_output = result.stdout if hasattr(result, "stdout") else str(result)
+        exit_code = result.exit_code if hasattr(result, "exit_code") else -1
+
+        # Step 4: Validate output
+        if not data_json_path.exists():
+            error_msg = f"{FT_DATA_FILE_NAME} not generated"
+        else:
+            validation_result = self._validate_data_json(data_json_path)
+            if not validation_result["valid"]:
+                error_msg = validation_result["error"]
+            else:
+                self._update_dataset_info(implementation, validation_result["sample_count"])
+
+        # Step 5: Load data if valid
+        if error_msg is None and data_json_path.exists():
+            with open(data_json_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+
+        # Step 5.5: Compute token stats and inject data_stats for yaml coder
+        if data is not None and error_msg is None:
+            inject_data_stats(implementation, data, execution_output)
+
+        # Step 6: Generate LLM feedback
+        # Truncate stdout from end for LLM (summary at the end is more useful)
+        stdout_summary = execution_output[-1500:] if execution_output else ""
+        return self._generate_llm_feedback(
+            target_task=target_task,
+            script_code=script_code if error_msg else "",  # Only show script on error
+            stdout=stdout_summary,  # Always show summary (truncated from end)
+            exit_code=exit_code,
+            data=data,
+            error_msg=error_msg,
+            queried_knowledge=queried_knowledge,
+            raw_stdout=execution_output,  # Full log for UI
+        )
+
+    def _generate_llm_feedback(
+        self,
+        target_task: Task,
+        script_code: str,
+        stdout: str,
+        exit_code: int,
+        data: Optional[list],
+        error_msg: Optional[str],
+        queried_knowledge: Optional[QueriedKnowledge],
+        raw_stdout: str = "",
+    ) -> CoSTEERSingleFeedback:
+        """Generate LLM-based feedback for data processing evaluation."""
+
+        # Prepare data statistics and samples
+        if data:
+            stats = self._analyze_data_quality(data)
+            data_stats = json.dumps(stats, indent=2)
+            sampled_data = self._sample_data(data)
+            data_samples = json.dumps(sampled_data, indent=2, ensure_ascii=False)
+            sample_count = len(sampled_data)
+            total_samples = len(data)
+        else:
+            data_stats = json.dumps({"error": error_msg or "No data generated"})
+            data_samples = "[]"
+            sample_count = 0
+            total_samples = 0
+
+        # Extract similar successful knowledge
+        queried_similar_successful_knowledge = []
+        if queried_knowledge is not None:
+            task_info = target_task.get_task_information()
+            queried_similar_successful_knowledge = queried_knowledge.task_to_similar_task_successful_knowledge.get(
+                task_info, []
+            )
+
+        # Build prompts
+        system_prompt = T(".prompts:data_eval.system").r(
+            scenario=self.scen.get_scenario_all_desc(),
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            upper_data_size_limit=FT_RD_SETTING.upper_data_size_limit,
+            force_think_token=FT_RD_SETTING.force_think_token,
+        )
+        user_prompt = T(".prompts:data_eval.user").r(
+            task_desc=target_task.get_task_information(),
+            script_code=script_code,
+            exit_code=exit_code,
+            stdout=stdout[:3000] if stdout else "",  # Empty string triggers {% if stdout %} = false
+            data_stats=data_stats,
+            sample_count=sample_count,
+            total_samples=total_samples,
+            data_samples=data_samples,
+        )
+
+        logger.info(
+            f"Generating LLM feedback for data evaluation (samples: {total_samples}, has_error: {bool(error_msg)})"
+        )
+
+        feedback = build_cls_from_json_with_retry(
+            CoSTEERSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=CoSTEERSingleFeedback.val_and_update_init_dict,
+        )
+
+        # NOTE: 0 exit code is a hard criteria for success
+        if exit_code != 0:
+            feedback.final_decision = False
+
+        feedback.raw_execution = raw_stdout
+        feedback.source_feedback[self.__class__.__name__] = feedback.final_decision
+        logger.log_object(feedback, tag="evaluator_feedback.FTDataEvaluator")
+        return feedback
+
+    def _validate_data_json(self, data_json_path: Path) -> dict:
+        """Validate data.json file format and content."""
+        try:
+            with open(data_json_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+
+            # Must be a non-empty list
+            if not isinstance(data, list):
+                return {"valid": False, "error": "data.json must be a JSON array", "sample_count": 0}
+
+            if len(data) == 0:
+                return {"valid": False, "error": "data.json is empty", "sample_count": 0}
+
+            # Check required fields in samples
+            required_fields = ["instruction", "output"]
+            for i, sample in enumerate(data[:10]):  # Check first 10 samples
+                if not isinstance(sample, dict):
+                    return {"valid": False, "error": f"Sample {i} is not a dict", "sample_count": 0}
+
+                missing = [f for f in required_fields if f not in sample]
+                if missing:
+                    return {"valid": False, "error": f"Sample {i} missing fields: {missing}", "sample_count": 0}
+
+                # Check for empty required fields
+                for field in required_fields:
+                    if not sample.get(field):
+                        return {
+                            "valid": False,
+                            "error": f"Sample {i} has empty '{field}' field",
+                            "sample_count": 0,
+                        }
+
+            return {"valid": True, "error": None, "sample_count": len(data)}
+
+        except json.JSONDecodeError as e:
+            return {"valid": False, "error": f"Invalid JSON: {e}", "sample_count": 0}
+        except Exception as e:
+            return {"valid": False, "error": f"Error reading file: {e}", "sample_count": 0}
+
+    def _update_dataset_info(self, implementation: FBWorkspace, sample_count: int):
+        """Generate dataset_info.json for LlamaFactory to use the processed data.
+
+        Note: LlamaFactory's columns mapping uses internal names (prompt, query, response)
+        that map to the actual column names in the data file (instruction, input, output).
+        See: https://github.com/hiyouga/LLaMA-Factory/blob/main/src/llamafactory/data/parser.py
+        """
+        dataset_info = {
+            "processed_data": {
+                "file_name": FT_DATA_FILE_NAME,
+                "formatting": "alpaca",
+                "columns": {
+                    "prompt": "instruction",
+                    "query": "input",
+                    "response": "output",
+                },
+            }
+        }
+
+        try:
+            implementation.inject_files(**{"dataset_info.json": json.dumps(dataset_info, indent=2)})
+            logger.info(f"Updated dataset_info.json with processed_data ({sample_count} samples)")
+        except Exception as e:
+            logger.warning(f"Failed to update dataset_info.json: {e}")
+
+    def _sample_data(self, data: list, n: int = 5) -> list:
+        """Random sampling for LLM evaluation."""
+        if len(data) <= n:
+            return data
+        return random.sample(data, n)
+
+    def _analyze_data_quality(self, data: list) -> dict:
+        """Analyze data quality statistics for all fields."""
+        if not data:
+            return {"total_samples": 0, "error": "Empty data"}
+
+        # Analyze length stats for all standard fields
+        fields = ["instruction", "input", "output"]
+        stats = {"total_samples": len(data)}
+
+        for field in fields:
+            lens = [len(str(d.get(field, ""))) for d in data]
+            empty_count = sum(1 for d in data if not d.get(field))
+            stats[f"{field}_len"] = {
+                "min": min(lens),
+                "max": max(lens),
+                "avg": round(sum(lens) / len(lens), 1),
+            }
+            stats[f"{field}_empty_ratio"] = round(empty_count / len(data) * 100, 1)
+
+        # Detect duplicates by full record (instruction + input + output)
+        record_set = set(
+            (str(d.get("instruction", "")), str(d.get("input", "")), str(d.get("output", ""))) for d in data
+        )
+        duplicate_count = len(data) - len(record_set)
+        stats["duplicate_count"] = duplicate_count
+        stats["duplicate_ratio"] = round(duplicate_count / len(data) * 100, 1)
+
+        return stats
+
+
+class FTCoderEvaluator(CoSTEEREvaluator):
+    """Evaluator for LLM fine-tuning implementations with simplified validation"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> CoSTEERSingleFeedback:
+        """Evaluate LLM fine-tuning implementation with two-step validation"""
+
+        task_info = target_task.get_task_information()
+
+        # Check task history
+        if queried_knowledge is not None:
+            if task_info in queried_knowledge.success_task_to_knowledge_dict:
+                return queried_knowledge.success_task_to_knowledge_dict[task_info].feedback
+            elif task_info in queried_knowledge.failed_task_info_set:
+                feedback = CoSTEERSingleFeedback(
+                    execution="Task failed too many times, skipping.",
+                    return_checking="Task failed too many times, skipping.",
+                    code="Task failed too many times, skipping.",
+                    final_decision=False,
+                )
+                logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator")
+                return feedback
+
+        env = get_ft_env(operation="micro_batch")
+        config_yaml = implementation.file_dict.get(FT_YAML_FILE_NAME, "")
+        if not config_yaml:
+            feedback = CoSTEERSingleFeedback(
+                execution=f"No {FT_YAML_FILE_NAME} found",
+                return_checking="Configuration file missing",
+                code="No valid configuration file",
+                final_decision=False,
+            )
+            logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator")
+            return feedback
+
+        # Two-step validation: parameter filtering + micro-batch test
+        validation_result = LLMConfigValidator().validate_and_test(
+            config_yaml=config_yaml, workspace=implementation, env=env
+        )
+        # NOTE: Docker execution is logged by FTWorkspace.run() automatically
+
+        # Update config with filtered version
+        if validation_result.filtered_config != config_yaml:
+            implementation.inject_files(**{FT_YAML_FILE_NAME: validation_result.filtered_config})
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[target_task.get_task_information()]
+            if queried_knowledge is not None
+            else []
+        )
+
+        system_prompt = T(".prompts:finetune_eval.system").r(
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            system_managed_params=SYSTEM_MANAGED_PARAMS,
+        )
+        user_prompt = T(".prompts:finetune_eval.user").r(
+            scenario=self.scen.get_scenario_all_desc(),
+            task_desc=target_task.get_task_information(),
+            stdout=validation_result.execution_output or "No output",
+            code_yaml=implementation.file_dict[FT_YAML_FILE_NAME],
+            workspace_files="\n".join(
+                [
+                    f"- {file.name} ({file.stat().st_size} bytes)"
+                    for file in implementation.workspace_path.rglob("*")
+                    if file.is_file() and "checkpoint" not in file.absolute().as_posix()
+                ]
+            ),
+        )
+        feedback = build_cls_from_json_with_retry(
+            CoSTEERSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=CoSTEERSingleFeedback.val_and_update_init_dict,
+        )
+
+        # Force failure if validation failed programmatically
+        if not validation_result.success:
+            feedback.final_decision = False
+            logger.warning("FTCoderEvaluator: Forced final_decision=False due to validation failure")
+
+        feedback.raw_execution = validation_result.raw_stdout or ""
+        feedback.source_feedback[self.__class__.__name__] = feedback.final_decision
+        logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator")
+        return feedback

rdagent/components/coder/finetune/exp.py

@@ -0,0 +1,39 @@
+"""
+LLM Fine-tuning Experiment Components
+
+Defines tasks for LLM fine-tuning following data science pattern.
+"""
+
+from typing import List, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+class FTTask(CoSTEERTask):
+    """Training task class for LLM fine-tuning operations - follows data science pattern"""
+
+    def __init__(
+        self,
+        base_model: str,
+        description: str,
+        benchmark: str,
+        involving_datasets: Optional[List[str]] = None,
+        skip_data_processing: bool = False,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(name="LLM-Fine-Tuning", description=description, *args, **kwargs)
+        self.base_model = base_model
+        self.benchmark = benchmark
+        self.involving_datasets = involving_datasets or []
+        self.skip_data_processing = skip_data_processing  # If True, reuse SOTA's data processing script
+
+    def get_task_information(self) -> str:
+        """Get task information for coder prompt generation"""
+        task_desc = f"""name: {self.name}
+description: {self.description}
+base_model: {self.base_model}
+"""
+        if self.involving_datasets:
+            task_desc += f"involving_datasets: {self.involving_datasets}\n"
+        return task_desc

rdagent/components/coder/finetune/prompts.yaml

@@ -0,0 +1,742 @@
+data_coder:
+  system: |-
+    You are a world-class data engineer specializing in preparing training data for large language model fine-tuning.
+    Your expertise includes processing various data formats and converting them to the Alpaca format required by LlamaFactory.
+
+    # Part 1: Context
+
+    ## 1.1 Scenario Description
+    {{ scenario }}
+
+    ## 1.2 Task Description
+    {{ task_desc }}
+
+    ## 1.3 Available Datasets
+    The following datasets are available for processing:
+    {{ dataset_info }}
+
+    ## 1.4 Priority Rules (CRITICAL)
+    **Task Description requirements are MANDATORY.** You MUST implement all data processing requirements specified in the Task Description exactly as described.
+
+    # Part 2: Output Specification
+
+    ## 2.1 Alpaca Format Definition
+    Your script must output a JSON file named `data.json` in the current working directory (`{{ workspace_path }}`).
+    The output must be in Alpaca format: a JSON array where each element has:
+    - `instruction`: The instruction or prompt for the model (required, non-empty)
+    - `input`: Optional additional context (can be empty string)
+    - `output`: The expected response from the model (required, non-empty)
+
+    ## 2.2 Output Example
+    ```json
+    [
+      {
+        "instruction": "Translate the following English text to French.",
+        "input": "Hello, how are you?",
+        "output": "Bonjour, comment allez-vous?"
+      },
+      {
+        "instruction": "Summarize the following article.",
+        "input": "Article content here...",
+        "output": "Summary of the article..."
+      }
+    ]
+    ```
+
+    ## 2.3 Data Quality Awareness (IMPORTANT)
+    - Raw datasets may contain low-quality, noisy, or incorrect samples
+    - It is better to DISCARD questionable samples than to include them in training data
+    - When encountering samples that are ambiguous, malformed, or have inconsistent answers, prefer filtering them out
+    - A smaller but high-quality dataset is more valuable than a larger noisy one
+    - High filtering rate is acceptable and expected - it means the script is doing quality control properly
+
+    ## 2.4 Data Validation Rules
+    Before writing the final data.json, implement these validations:
+
+    ### 2.4.1 Answer Consistency Check (CRITICAL)
+    - Verify generated answer matches expected answer
+    - Prefer string normalization over LLM when feasible
+    - Answer format varies by task (e.g., `\boxed{}` for math, JSON for structured, code output for programming)
+    - Filter samples with mismatched answers
+
+    ### 2.4.2 Over-length Filtering (MANDATORY)
+    - Filter out samples where `total_tokens > max_position_embeddings`
+    - Do NOT truncate - filter instead
+    - See Part 6 for COT-specific validation requirements
+
+    # Part 3: Script Implementation Requirements
+
+    ## 3.1 Basic Conventions
+    1. Read data from `{{ datasets_path }}` directory (mounted read-only)
+    2. Use standard Python libraries (json, csv, os, pathlib) when possible
+    3. Handle file encoding properly (use utf-8)
+    4. Include error handling for file operations
+    5. Print progress information to stdout for debugging
+    6. **IMPORTANT**: Your script MUST support the `--debug` command-line argument (see 3.2). Other than `--debug`, do NOT expect any other command-line arguments.
+
+    ## 3.2 Debug Mode (CRITICAL)
+    Your script MUST support `--debug` for fast validation:
+    - Sampling/filtering is pure code operation (no LLM), so it runs completely in both modes
+    - `--debug`: Process ~100 samples through LLM pipeline, print actual sampled total
+    - No flag: Process ALL sampled data through LLM pipeline
+
+    ### Debug Mode Example
+    ```python
+    import random
+
+    # Step 1: Run complete sampling/filtering (fast, no LLM) - runs in BOTH modes
+    sampled_data = apply_sampling_strategy(raw_data)  # e.g., 50000 → 2000
+
+    # Step 2: Limit LLM processing in debug mode only
+    if args.debug:
+        samples_to_process = random.sample(sampled_data, min(100, len(sampled_data)))
+    else:
+        samples_to_process = sampled_data
+
+    # Step 3: Show the actual number of sampled items (Do not estimate; count the exact number of samples that will be processed when not in debug mode.)
+    print(f"Sampled data size from raw: {len(sampled_data)} / {len(raw_data)}")  # Actual training data size
+    ```
+
+    ## 3.3 Logging Convention
+    Only print progress at 20%, 40%, 60%, 80%, 100%. No per-item logs.
+
+    ## 3.4 Output Statistics Format
+    Your script should print statistics at the end of execution:
+
+    ### Script Execution Summary (REQUIRED)
+    ```
+    # Debug mode (--debug):
+    ========== SUMMARY ==========
+    Total output samples: {actual_output}
+    Sampled data size from raw: {sampled_count} / {raw_count}
+    Debug samples processed: {debug_processed_count}
+    Estimated full output: ~{int(actual_output / debug_processed_count * sampled_count)}
+    Output file: {{ workspace_path }}data.json
+    =============================
+
+    # Full mode (no --debug):
+    ========== SUMMARY ==========
+    Total output samples: {actual_output}
+    Sampled data size from raw: {sampled_count} / {raw_count}
+    Output file: {{ workspace_path }}data.json
+    =============================
+    ```
+
+    ### CoT Quality Statistics (REQUIRED for COT tasks)
+    ```
+    ========== COT QUALITY STATS ==========
+    COT format check: {with_think_tags}/{total} have <think> tags
+    Over-length filtered: {count} ({percentage}%)
+    Answer consistency check: {passed}/{total} passed
+    Length distribution: p25={}, p50={}, p75={}, p99={}
+    =======================================
+    ```
+
+    # Part 4: Scope Clarification (IMPORTANT)
+    **Your script should ONLY handle data processing and output data.json.**
+    - DO NOT generate training configuration files (e.g., train.yaml, training_config.json)
+    - DO NOT include training scripts or fine-tuning code
+    - DO NOT save any files other than data.json
+    - Training configuration will be handled separately by another component
+
+    # Part 5: LLM API Usage Guide
+
+    ## 5.1 Model Pool - Load Balancing
+    **All models have INDEPENDENT quotas** - distribute load evenly across models!
+
+    ```python
+    import os, json
+    import litellm; litellm.suppress_debug_info = True
+    from litellm import completion
+
+    STRONG_MODELS = json.loads(os.getenv("STRONG_MODEL_POOL", "[]"))  # CoT generation
+    WEAK_MODELS = json.loads(os.getenv("WEAK_MODEL_POOL", "[]"))      # simple/fast tasks
+
+    # Default timeout for API calls (in seconds)
+    API_TIMEOUT = 120
+
+    def call_llm(messages, models, start_idx=0, timeout=API_TIMEOUT):
+        """Load-balanced LLM call with timeout. Use start_idx to distribute across models."""
+        if not models:
+            raise RuntimeError("Model pool is empty. Set STRONG_MODEL_POOL/WEAK_MODEL_POOL env vars.")
+        last_err = None
+        for i in range(len(models)):
+            model = models[(start_idx + i) % len(models)]
+            try:
+                resp = completion(model=model, messages=messages, drop_params=True, timeout=timeout)
+                return resp.choices[0].message.content
+            except Exception as e:
+                last_err = e
+                continue
+        raise RuntimeError(f"All models failed. Last error: {last_err}")
+    ```
+
+    ## 5.2 Timeout & Efficiency (CRITICAL)
+    - Set `timeout=120` for API calls to prevent blocking on complex problems
+    - If timeout after retries, skip sample and continue
+    - Prefer string/regex over LLM for validation (answer check, structure check) when possible
+
+    ## 5.3 Concurrency - CRITICAL
+    **MANDATORY**: Use `ThreadPoolExecutor(max_workers={{ api_max_workers }})` for parallel sample processing.
+    - DO NOT use `os.cpu_count()` - API concurrency should follow endpoint quota, not local CPU count
+    - The value {{ api_max_workers }} comes from `FT_API_MAX_WORKERS`; do not hard-code a larger value
+    - Pass `start_idx=sample_index % len(models)` to distribute load evenly
+
+    ```python
+    with ThreadPoolExecutor(max_workers={{ api_max_workers }}) as executor:
+        futures = {executor.submit(process_sample, i, sample, i % len(STRONG_MODELS)): i
+                   for i, sample in enumerate(samples)}
+    ```
+
+    # Part 6: CoT Processing Guide (CRITICAL)
+    ## 6.1 CoT Output Requirement (MANDATORY)
+    **CRITICAL: ALL training data MUST include Chain-of-Thought reasoning in output field.**
+
+    ### Why This Matters
+    - Models learn to reason by seeing reasoning examples
+    - Direct answers (A/B/C/D, True/False) provide NO training signal for reasoning
+
+    ### Generation Process
+    - Ask LLM to provide step-by-step reasoning before the final answer
+    - Good: "Explain your reasoning step by step, then give the final answer"
+    - Bad: "Output with <think> tags" (models will refuse)
+    - Let LLM generate reasoning naturally
+
+    ### Output Format
+    {% if force_think_token %}
+    - Your script MUST wrap LLM output into `<think>...</think>` format
+    - Format: `<think>{reasoning}</think>{answer}`
+    - The **answer** (content AFTER `</think>`) must follow **Benchmark Description**
+    - DO NOT ask for `<think>` tags in prompts (models refuse this)
+    {% else %}
+    - If base model is NOT a thinking model (no native `<think>` token), DO NOT add `<think>` tags
+    - Output must contain step-by-step reasoning (CoT)
+    {% endif %}
+    - **Answer format must follow Benchmark Description**
+
+    ## 6.2 Post-Processing Validation
+    {% if force_think_token %}
+    - **Structure check**: `"<think>" in output and "</think>" in output`
+    {% endif %}
+    - **Content check**: Output must contain reasoning (not just direct answer)
+    - **Answer check**: Answer format must match Benchmark Description
+
+    # Part 7: Previous Failed Attempts
+    {% if queried_former_failed_knowledge|length != 0 %}
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    # Part 8: Response Format
+    Provide ONLY the Python script in a markdown code block:
+    ```python
+    # Your complete Python script here
+    ```
+
+    Do NOT add explanations before or after the code block.
+
+  user: |-
+    Please generate a Python script that processes the available datasets and outputs a `data.json` file in Alpaca format.
+
+    The script will be executed in two modes:
+    1. **Debug mode (coding phase):** `python {{ workspace_path }}process_data.py --debug` - process 100 samples for fast validation
+    2. **Full mode (running phase):** `python {{ workspace_path }}process_data.py` - generates all samples for training
+
+    Dataset files are located at: {{ datasets_path }}
+
+    ## Detailed Dataset Descriptions
+    {% for ds_name, ds_desc in involved_dataset_folder_desc.items() %}
+    ### Dataset: {{ ds_name }}
+    (Note: All file paths for this dataset are relative to `{{ datasets_path }}{{ ds_name }}/`)
+    {{ ds_desc }}
+    {% endfor %}
+
+    Output file should be: {{ workspace_path }}data.json
+
+    {% if latest_code %}
+    ## Previous Data Processing Script
+    ```python
+    {{ latest_code }}
+    ```
+
+    {% if latest_feedback is not none %}
+    ## Feedback on Previous Script
+    {{ latest_feedback }}
+
+    Please improve the 'Previous Data Processing Script' based on the feedback above. Do not create a new script. Consider the feedback carefully and make necessary corrections. If the feedback asks for more information or logging, make sure to include that in your revised script to help the evaluator to better assess your implementation.
+    {% endif %}
+    {% else %}
+    Please create a new Data Processing Script based on the task description.
+    {% endif %}
+
+    **IMPORTANT**: Make sure your script supports the `--debug` argument as described in the system prompt.
+
+finetune_coder:
+  system: |-
+    You are a world-class machine learning engineer specializing in large language model fine-tuning using LlamaFactory.
+    Your expertise includes creating optimal LlamaFactory configuration files for various fine-tuning scenarios.
+
+    # Scenario Description
+    {{ scenario }}
+
+    # Task Description
+    {{ task_desc }}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    ## Previous Failed Attempts
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Available Fine-tuning Methods
+    {{ available_methods }}
+
+    ## Shared Parameters
+    These parameters apply to all fine-tuning methods:
+    {{ shared_params }}
+
+    ## Method-Specific Parameters
+    {% for method, params_desc in methods_specific_params.items() %}
+    {{ params_desc }}
+    {% endfor %}
+
+    ## Priority Rules (CRITICAL)
+    **Task Description parameters are MANDATORY.** You MUST use exactly the hyperparameter values specified in the Task Description. Guidelines below are defaults only - they apply ONLY when task description does not specify a value.
+
+    ## Requirements
+    1. Create a LlamaFactory configuration file named `train.yaml`
+    2. Based on the hypothesis provided by the user, select the most appropriate fine-tuning method
+    3. Generate full training configuration (no sample limit)
+    4. Ensure all parameters are valid for LlamaFactory
+    5. **Adaptive Logging Configuration (CRITICAL)**:
+       - Set `logging_strategy` to 'steps' for consistent monitoring
+       - Calculate `logging_steps` adaptively:
+         * Check `stdout_summary` in data_stats for `Estimated full output` (NOT `total_samples` which is debug mode count)
+         * total_steps = estimated_full × num_epochs / (batch_size × gradient_accumulation_steps × num_gpus)
+         * Target 20-50 log entries total
+    6. **Validation and Checkpoint Strategy (CRITICAL for best model selection)**:
+       - **Validation Split**: Set `val_size` to split a portion of training data for validation. Choose ratio based on dataset size and task needs.
+       - **Save Strategy**: Choose `save_strategy` ('steps' or 'epoch') based on training duration. MUST ensure `eval_strategy` == `save_strategy`.
+        - If using 'steps', set `save_steps` based on estimated full output appropriately, DON'T set it very low or high.
+        - set 'per_device_eval_batch_size' appropriately to speed up eval without OOM.
+       - **Best Model Selection**: Use `load_best_model_at_end: true` with `save_total_limit: 1` to automatically keep and load the best checkpoint based on eval_loss. Note: `save_total_limit` will be force-injected to 1.
+    7. If the former configuration faces error, please make sure to fix the error while aligning with the task. If these two goals conflict, please prioritize fixing the error.
+
+    ## Configuration Principle
+    **ONLY include parameters you want to change from defaults**
+    If a parameter's default value matches your intention, OMIT it entirely
+    This prevents unnecessary dependencies and keeps configuration clean
+    Example: if `mixture_of_depths` defaults to `false` and you don't need it, DO NOT include it
+
+    ## Output Format
+    You MUST output the YAML configuration in a standard markdown code block:
+    ```yaml
+    model_name_or_path: /path/to/model
+    stage: sft
+    ...
+    ```
+
+    Do NOT add explanations before or after the YAML block.
+
+  user: |-
+    ## Path Configuration
+    - dataset_dir: "{{ datasets_path }}"
+    - output_dir: "./output" (auto-injected, you can omit this)
+    - model_name_or_path: "{{ models_path }}{{ base_model }}"
+    - tokenized_path: "{{ workspace_path }}tokenized_cache"
+
+    ## Critical Configuration Rules
+    - dataset: MUST be "processed_data" (this is the dataset name in dataset_info.json)
+    - model_name_or_path: use local model path instead of HuggingFace model identifier
+    - dataset_info.json is located at: "{{ datasets_path }}dataset_info.json" (contains the "processed_data" entry)
+    - template: NEVER set to "auto" or "none" - these are invalid values.
+      - For Qwen series model, set to "qwen", and for Qwen3 series model especially, set to "qwen3".
+      - For other models, DO NOT include this field (LlamaFactory auto-detects from tokenizer).
+    - tokenized_path: MUST set to "{{ workspace_path }}tokenized_cache" (datasets directory is read-only mounted)
+    - batch_size: Be aware that `auto_find_batch_size` can cause synchronization issues in multi-GPU (DDP) training. Consider setting `per_device_train_batch_size` explicitly if training hangs
+    - flash_attn: For models supporting flash attention2 (e.g., Qwen series, llama series), set to "fa2" to enhance training speed and reduce memory usage
+    {% if deepspeed_path %}- deepspeed: If number of GPUs > 1, use DeepSpeed with ZeRO Stage 2 or 3 for memory optimization. specifically, set to "{{ deepspeed_path }}ds_z3_config.json" for ZeRO Stage 3, otherwise use "{{ deepspeed_path }}ds_z2_config.json" for ZeRO Stage 2{% endif %}
+    - **IMPORTANT Compatibility Rules**:
+      - `pissa_init: true` is NOT compatible with DeepSpeed ZeRO-3. If using ZeRO-3, do NOT set pissa_init to true
+        - If you need PiSSA initialization, use ZeRO Stage 2 instead of ZeRO Stage 3
+      - `load_best_model_at_end: true` requires `eval_strategy` == `save_strategy` (both "steps" or both "epoch"). Always set both to the same value.
+
+    {% if force_think_token %}
+    {% if has_think_token is defined and not has_think_token %}
+    ## Special Token Configuration for CoT Training
+    The base model does NOT have `<think>` token in its vocabulary.
+    To train with Chain-of-Thought reasoning format (output like `<think>reasoning</think>answer`), you MUST add special tokens AND train the new embeddings:
+    ```yaml
+    new_special_tokens: ["<think>", "</think>"]
+    resize_vocab: true
+    additional_target: embed_tokens,lm_head  # MANDATORY for LoRA/QLoRA when resize_vocab=true! And Full Training does not need this field.
+    ```
+    This ensures `<think>` and `</think>` are tokenized as single tokens, not split into subwords.
+    {% elif has_think_token is defined and has_think_token %}
+    ## Special Token Note
+    The base model already supports `<think>` token natively. No need to add special tokens for CoT training.
+    {% endif %}
+    {% endif %}
+    {# When force_think_token=false, no special token configuration needed #}
+
+    {% if data_stats %}
+    ## Processed Data Statistics (from debug mode)
+    {{ data_stats }}
+
+    **Your Task**: Implement the training configuration specified in the task description.
+    
+    - Follow task requirements first (method, batch size, epochs, cutoff_len, etc.)
+    - Apply technical constraints only when task doesn't specify:
+      - `cutoff_len`: ≤ min(max_position_embeddings, memory limit, data p99)
+      - `per_device_train_batch_size`: Choose based on Memory Estimates table
+      - `gradient_accumulation_steps`: Adjust for stable training (effective_batch = batch × accum × gpus)
+    - Validation setup: `val_size`, `eval_strategy` == `save_strategy`, `load_best_model_at_end: true`
+    {% endif %}
+
+    {% if latest_code %}
+    ## Previous Configuration
+    ```yaml
+    {{ latest_code }}
+    ```
+
+    {% if latest_feedback is not none %}
+    ## Feedback on Previous Configuration
+    {{ latest_feedback }}
+
+    Please improve the configuration based on the feedback above and the hypothesis.
+    {% endif %}
+    {% else %}
+    Please create a new configuration for the model {{ base_model }} based on the hypothesis above.
+
+    **Remember to include ALL required fields:**
+    - stage: sft
+    - finetuning_type: [select appropriate method based on hypothesis]
+    - do_train: true
+    - model_name_or_path: {{ models_path }}{{ base_model }}
+    - dataset: processed_data
+    - dataset_dir: {{ datasets_path }}
+    - tokenized_path: {{ workspace_path }}tokenized_cache
+    {% endif %}
+
+  user_test_params: |-
+    Now, please provide a set of "test parameters" that will be merged into the above configuration specifically for the DEBUG/MICRO-BATCH test phase.
+    
+    The debug phase runs on a very small subset (~10 samples).
+    You need to override parameters that adapt to the dataset for quick debugging the yaml config.
+
+    **Example for Test Parameters:**
+    - Set `num_train_epochs` to 1.
+    - Set `max_samples` to a very small number.
+
+    **Output Format:**
+    Output ONLY the YAML block for these test parameters:
+    ```yaml
+    num_train_epochs: 1
+    ...
+    ```
+
+finetune_eval:
+  system: |-
+    You are a world-class machine learning engineer specializing in evaluating fine-tuning configurations for large language models using LlamaFactory.
+    Your expertise includes validating LlamaFactory configuration files to ensure they meet all necessary requirements for successful fine-tuning.
+    
+    You will be provided with:
+    1. A detailed scenario description which requires a fine-tuning LLM.
+    2. A yaml configuration file named `train.yaml` created for LlamaFactory fine-tuning.
+    3. A structured execution summary (JSON format) containing: status, exit_code, errors, training metrics, and warnings.
+    4. The files generated during the execution.
+    5. Some other yaml configuration for similar tasks which might help you better provide feedback and possible corrections.
+
+    Your task is to:
+    1. Check the execution summary to determine if the run succeeded.
+    2. validate the provided `train.yaml` configuration file to ensure it adheres to the required standards for LlamaFactory fine-tuning using the specified method.
+    3. Provide clear and concise feedback on any issues found in the configuration file or execution logs.
+    4. Suggest specific corrections or improvements if any issues are identified.
+
+    You must give a false final decision only if:
+    - The execution fails with non-zero exit code.
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    ### Similar Successful Implementations to help training config Improvement
+    The user has done several similar tasks and get some successful implementations. These yaml configurations might not be implemented to the same task, but they are similar to your task and they might work well on your task.
+    Please refer to these successful implementation and provide your suggestions in your response on how to correct your current code based on these successful implementations.
+    ## Successful Implementations for Similar Tasks
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Similar Task {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Yaml configurations:=====
+    {{ similar_successful_knowledge.implementation.all_codes }}
+    {% endfor %} 
+    {% endif %}
+
+    # Important Notice
+    - You may find that the execution is short with limited data and iterations. This is expected as we are only validating the configuration file's correctness and not performing full-scale training. Don't treat this as a failure. Also do not put this information in your feedback.
+
+    ## Output Format
+    Please respond with your feedback in the following JSON format without anything else.
+    ```json
+    {
+        "execution": "State if run succeeded. If errors, include all messages verbatim. Classify cause: algorithm, implementation, or environment."
+        "return_checking": "Plain text. Examine the generated files from the user input. Does the output contains a fine-tuned model or expected artifacts? If not, specify what is missing or incorrect.",
+        "code": "Plain text. Use short simple sentences: say if approach fits task, what works, main issues, brief improvement suggestions."
+        "final_decision": <true/false>, # Final decision on whether the configuration is acceptable for full data fine-tuning
+    }
+    ```
+
+  user: |-
+    # Scenario Information
+    {{ scenario }}
+
+    # Task Description
+    {{ task_desc }}
+
+    # Yaml Configuration File
+    ```yaml
+    {{ code_yaml }}
+
+    ## Execution Summary (Structured)
+    ```json
+    {{ stdout }}
+    ```
+
+    ## Workspace Files
+    {{ workspace_files }}
+
+data_eval:
+  system: |-
+    You are a data quality expert for LLM fine-tuning using LlamaFactory.
+    Your expertise includes evaluating training data quality and validating data processing scripts.
+
+    You will evaluate:
+    1. **Data format correctness**: Alpaca format requires instruction, input (optional), output fields
+    2. **Data quality**: length distribution, duplicates, semantic reasonableness
+    3. **Alignment with task objectives**: whether the data matches what the task requires
+    4. **Code logic correctness**: whether the processing script is well-designed
+
+    ## The Main Scenario Description
+    {{ scenario }}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    ## Similar Successful Data Processing Examples
+    The following are successful data processing implementations for similar tasks:
+    {% for knowledge in queried_similar_successful_knowledge %}
+    ### Example {{ loop.index }}:
+    **Task:** {{ knowledge.target_task.get_task_information() }}
+    **Code:**
+    ```python
+    {{ knowledge.implementation.file_dict.get("process_data.py", "N/A") }}
+    ```
+    {% endfor %}
+    {% endif %}
+
+    ## Debug Mode Context (IMPORTANT)
+    This evaluation runs during the CODING phase in DEBUG MODE.
+    - The script is executed with `--debug` flag to process only ~100 samples for fast validation
+    - Sample count less than 100 is EXPECTED and should NOT be considered a quality issue
+    - Focus on evaluating:
+      1. Data format correctness (Alpaca format)
+      2. Data quality of the generated samples
+      3. Script logic correctness (will it work in full mode?)
+    - Do NOT fail the evaluation just because sample count is low
+
+    ## Evaluation Criteria
+    - **Format**: All samples must have non-empty instruction and output fields
+    - **Length**: instruction/output should be reasonable length (not too short or excessively long)
+    - **Duplicates**: High duplicate ratio indicates data quality issues
+    - **Semantic**: instruction should be a question/task, output should be an answer/response
+    - **Alignment**: Data should match the task's training objective
+
+    ## CoT Quality Evaluation (Task-Adaptive)
+    **IMPORTANT: CoT quality ≠ CoT length. Adapt criteria based on task type from README metadata.**
+
+    **Check README's `CoT Quality Assessment` section for `task_type` and `quality_ready` fields.**
+
+    1. **Over-length Check** (Report only):
+       - Report percentage of samples exceeding `max_position_embeddings`
+       - High over-length ratio is a warning sign, but NOT an automatic failure if the script handles it correctly
+
+    2. **Answer Consistency Check** (Informational):
+       - Note: The data processing script already filters for answer consistency
+       - If the script implements answer verification, trust its filtering logic
+       - Only flag as issue if the SCRIPT lacks answer verification logic entirely
+
+    3. **Structure Quality Check** (Task-adaptive):
+       - **Math/Code**: Look for step-by-step markers, verification, backtracking
+       - **Chemistry/Structured**: Look for JSON structure or "Step N:" format (short but structured is OK)
+       - **General**: No strict structure requirement
+
+    4. **Length Assessment** (Informational only):
+       - Report length distribution for reference
+       - Length alone should NOT determine pass/fail
+       - Different tasks have different natural length distributions
+
+    5. **Polish Quality Assessment**:
+       - All data must be polished before use
+       - If README shows `baseline_quality: high`: verify enrichment was applied
+       - If README shows `baseline_quality: low`: verify full generation/rewrite was done
+       - Check polish met the requirements in `polish_strategy`
+
+    **Include in return_checking:**
+    - "Task type: {type}, Quality ready: {ready}"
+    - "CoT stats: p50={}, over-length={X}%, structure quality={Y}%"
+    - Assessment based on task-appropriate criteria
+
+    ## Hard Check Criteria (AUTOMATIC FAIL if not met)
+    {% if force_think_token %}
+    ### 1. COT Format Verification (HARD FAIL)
+    - EVERY sample MUST contain `<think>` and `</think>` tags
+    - Content AFTER `</think>` must be non-empty
+
+    **Rejection:** "FAIL: {X} samples missing <think> tags."
+    {% else %}
+    ### 1. COT Format Verification (HARD FAIL)
+    - Output must contain reasoning content (not just a direct answer)
+    - Answer format must match **Benchmark Description**
+    - Do NOT reject for reasoning quality or answer correctness
+
+    **Rejection:** "FAIL: {X}% of samples are direct answers without reasoning."
+    {% endif %}
+
+    ### 2. Sample Count Check
+    - Debug mode should generate ~100 samples
+    - Estimated full run samples should be at most {{ upper_data_size_limit }}
+    - Reject if either criteria is not met
+
+    ## Final Decision Guidelines
+    **Core Principle: Strict on COT format, lenient on reasoning quality and answer correctness.**
+
+    - **Approve (true)** if:
+      - Script runs successfully (exit_code == 0)
+      - At least 1 sample is generated
+      {% if force_think_token %}- ALL samples have `<think>` and `</think>` tags (MANDATORY){% else %}- ALL samples contain reasoning content (not just direct answers){% endif %}
+      - Data format is correct (Alpaca format with instruction/output)
+
+    - **Reject (false)** if ANY of these:
+      - Script fails to run (exit_code != 0)
+      - Zero samples are generated
+      {% if force_think_token %}- **ANY sample missing `<think>` or `</think>` tags (HARD FAIL)**{% else %}- **ANY sample missing reasoning content (just direct answer)**{% endif %}
+      - Data format is fundamentally broken
+      - **Data does NOT match task description requirements**
+
+    - **Do NOT reject** for:
+      - Low sample count in debug mode (expected)
+      - Moderate quality variations in individual samples
+      - Length distribution not matching ideal patterns
+      - High filtering rate (script doing its job)
+  
+    ## Important Note
+    - Do not summarize the code into your feedback and DO NOT copy the task description also. Only provide new insights based on your evaluation.
+    - If you think the current logging information is not sufficient to find out the issues, please specify what additional logging information is needed in your feedback and put this information in 'code' block. The user will add further provide you the additional logging information in the next iteration.
+    - Do not write any code in your response, use plain text only.
+
+    ## Output Format
+    Respond with JSON only (no markdown code block):
+    {
+        "execution": "Script execution status and data generation result. Include exit code and any errors.",
+    "return_checking": "Data quality analysis: format validation, length distribution assessment, duplicate ratio, semantic issues found; Hard check criteria: does the solution meet the hard check criteria",
+        "code": "Code issues and specific improvement suggestions. What works well, what needs fixing.",
+        "final_decision": true/false
+    }
+
+  user: |-
+    # Task Description
+    {{ task_desc }}
+    {% if script_code %}
+
+    # Data Processing Script (for debugging)
+    ```python
+    {{ script_code }}
+    ```
+    {% endif %}
+    {% if stdout %}
+
+    # Execution Output ({% if exit_code != 0 %}error logs{% else %}summary{% endif %})
+    ```
+    Exit code: {{ exit_code }}
+    {{ stdout }}
+    ```
+    {% endif %}
+
+    # Data Statistics
+    ```json
+    {{ data_stats }}
+    ```
+
+    # Sample Data ({{ sample_count }} samples from total {{ total_samples }}) [DEBUG MODE]
+    ```json
+    {{ data_samples }}
+    ```
+
+runner_eval:
+  system: |-
+    You are a world-class ML engineer evaluating LLM fine-tuning results.
+
+    ## Your Task
+    Analyze the training run information and determine if the experiment succeeded.
+
+    ## Evaluation Criteria (for final_decision)
+    1. **Execution Success**: Did training complete without errors? Check exit_code and model outputs.
+    2. **Benchmark Execution**: Did benchmark run successfully? Check benchmark results availability.
+
+    ## Loss Analysis (for improvement suggestions ONLY - does NOT affect final_decision)
+    - Analyze loss trajectory: Is loss decreasing steadily? Any signs of overfitting?
+    - Use this information ONLY to provide suggestions in the "code" field
+    - Loss patterns should NEVER cause final_decision to be false
+
+    ## Error Categories (if failed)
+    - **Timeout (exit_code=124)**: Process was killed due to timeout. Check "failed_stage" and "timeout" fields in stdout:
+      - If failed_stage is "data_processing": Data processing script timed out. This is often due to LLM API calls for CoT data generation taking too long.
+      - If failed_stage is "training": Training timed out. 
+    - **OOM**: GPU memory exhaustion - suggest batch size/model changes
+    - **CUDA**: Driver/device issues - suggest environment checks
+    - **Config**: Invalid parameters - suggest specific fixes
+    - **Data**: Dataset issues - suggest data pipeline fixes
+
+    ## Output Format
+    Respond with JSON only:
+    {
+        "execution": "Execution status: SUCCESS or FAILED with category [OOM/CUDA/Config/Data]. Include key metrics or error details.",
+        "return_checking": "If success: benchmark analysis. If failed: what failed and expected behavior.",
+        "code": "Configuration assessment and improvement suggestions",
+        "final_decision": true/false  // Set to true as long as training succeeded (exit_code=0) and benchmark ran successfully
+    }
+
+  user: |-
+    # Task Description
+    {{ task_desc }}
+
+    # Training Configuration
+    ```yaml
+    {{ config_yaml }}
+    ```
+
+    # Execution Info
+    - Exit Code: {{ exit_code }}
+    - Model Output Files: {{ model_files_status }}
+    {% if failed_stage %}- Failed Stage: {{ failed_stage }}
+    - Stage Timeout Config: {{ timeout_seconds }} seconds
+    {% endif %}
+
+    # Benchmark Results
+    ```json
+    {{ benchmark_result }}
+    ```
+
+    # Loss History (train loss and eval_loss if validation enabled)
+    ```json
+    {{ loss_history }}
+    ```
+    {% include "components.coder.finetune.prompts:runner_eval.train_output" %}
+
+  train_output: |-
+    # Training Output (key information extracted from stdout)
+    ```
+    {{ stdout }}
+    ```

rdagent/components/coder/finetune/unified_validator.py

@@ -0,0 +1,304 @@
+"""
+Simplified LLM Fine-tuning Configuration Validator
+
+Two-step validation:
+1. Parameter filtering - Remove unsupported parameters
+2. Micro-batch testing - Runtime validation with small dataset
+"""
+
+import json
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+import yaml
+
+from rdagent.components.coder.finetune.conf import (
+    FT_DEBUG_YAML_FILE_NAME,
+    FT_TEST_PARAMS_FILE_NAME,
+    get_ft_env,
+    get_workspace_prefix,
+)
+from rdagent.core.experiment import FBWorkspace
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.finetune.scen.llama_factory_manager import LLaMAFactory_manager
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+# System-managed parameters that are automatically injected during validation.
+# These should NOT be checked for alignment in eval prompts.
+# Single source of truth: modify here to change injected parameters.
+SYSTEM_MANAGED_PARAMS = {
+    "overwrite_cache": True,  # Avoid HF datasets cache lock contention
+    "save_only_model": True,  # Save disk space
+    # "save_total_limit": 1,  # Limit checkpoint count to save disk space
+    "output_dir": "./output",  # Standardize model output location
+    "per_device_eval_batch_size": 1,  # Prevent OOM during evaluation
+}
+
+
+@dataclass
+class ValidationResult:
+    """Configuration validation result"""
+
+    success: bool
+    filtered_config: str
+    execution_output: str = ""  # Parsed/summarized output for LLM
+    raw_stdout: str = ""  # Full raw stdout for UI display
+    errors: List[str] = field(default_factory=list)
+    execution_time: float = 0.0
+
+
+class LLMConfigValidator:
+    """LLM configuration validator with two-step validation:
+
+    1. Parameter filtering - Remove unsupported parameters
+    2. Micro-batch test - Runtime validation with small dataset
+
+    The micro-batch test inherently validates completeness, so no separate completeness check is needed.
+    """
+
+    def __init__(self):
+        self._supported_params_cache: Optional[Set[str]] = None
+
+    def validate_and_test(self, config_yaml: str, workspace: FBWorkspace, env) -> ValidationResult:
+        """Three-step validation: parameter filtering + injection + micro-batch testing"""
+        start_time = time.time()
+
+        # Step 1: Parameter filtering
+        filtered_config, removed_params = self._filter_parameters(config_yaml)
+
+        # Step 2: Inject required parameters for multi-task environments
+        injected_config = self._inject_required_parameters(filtered_config)
+
+        # Step 3: Micro-batch testing (validates everything at runtime)
+        result = self._run_micro_batch_test(injected_config, workspace, env)
+        result.execution_time = time.time() - start_time
+
+        # Add filtered params info to execution_output for agent learning
+        if removed_params:
+            filter_info = (
+                f"\n\n[Filtered Parameters] {len(removed_params)} unsupported params removed: {removed_params}"
+            )
+            result.execution_output += filter_info
+
+        return result
+
+    def _filter_parameters(self, config_yaml: str) -> tuple[str, List[str]]:
+        """Filter configuration parameters to only include supported ones.
+
+        Returns:
+            tuple: (filtered_yaml, removed_params_list)
+        """
+        config_dict = yaml.safe_load(config_yaml)
+        if not isinstance(config_dict, dict):
+            return config_yaml, []
+
+        supported_params = self._get_supported_parameters()
+
+        filtered_config = {}
+        removed_params = []
+        for k, v in config_dict.items():
+            if k in supported_params:
+                filtered_config[k] = v
+            else:
+                removed_params.append(k)
+
+        if removed_params:
+            logger.info(f"Filtered out {len(removed_params)} unsupported parameters: {removed_params}")
+
+        return yaml.dump(filtered_config, default_flow_style=False, sort_keys=False), removed_params
+
+    def _inject_required_parameters(self, config_yaml: str) -> str:
+        """Inject required parameters for multi-task environments.
+
+        Uses SYSTEM_MANAGED_PARAMS as the single source of truth.
+        """
+        config = yaml.safe_load(config_yaml)
+        if not isinstance(config, dict):
+            return config_yaml
+
+        config.update(SYSTEM_MANAGED_PARAMS)
+
+        logger.info(f"Injected required parameters: {SYSTEM_MANAGED_PARAMS}")
+        return yaml.dump(config, default_flow_style=False, sort_keys=False)
+
+    def _get_supported_parameters(self) -> Set[str]:
+        """Get supported parameters from LlamaFactory Manager"""
+        if self._supported_params_cache is not None:
+            return self._supported_params_cache
+
+        all_params = LLaMAFactory_manager.get_parameters()
+
+        # Extract all parameter names from all parameter types (including nested structures)
+        supported_params = set()
+        for param_type, params_dict in all_params.items():
+            if isinstance(params_dict, dict):
+                # Recursively extract parameter names from nested dictionaries
+                for key, value in params_dict.items():
+                    if isinstance(value, dict) and "name" in value:
+                        # This is a parameter definition with metadata
+                        supported_params.add(key)
+                    elif isinstance(value, dict):
+                        # This is a nested category (e.g., BaseModelArguments, LoraArguments)
+                        # Extract parameter names from the nested structure
+                        for nested_key, nested_value in value.items():
+                            if isinstance(nested_value, dict) and "name" in nested_value:
+                                supported_params.add(nested_key)
+
+        if not supported_params:
+            raise RuntimeError("No parameters found in LlamaFactory Manager")
+
+        logger.info(f"Loaded {len(supported_params)} parameters from LlamaFactory Manager")
+        self._supported_params_cache = supported_params
+        return supported_params
+
+    def _parse_execution_log(self, stdout: str, exit_code: int, failed_stage: str = None) -> str:
+        """Parse execution log and extract key information for LLM evaluation.
+
+        Reduces log from ~36k tokens to ~500 tokens by extracting only:
+        - Status and exit code
+        - Error messages (if any)
+        - Training metrics (if successful)
+        - Warnings (limited)
+        - Timeout and stage information (if applicable)
+
+        Args:
+            stdout: The execution output
+            exit_code: The process exit code
+            failed_stage: Which stage failed - "data_processing" or "training"
+        """
+        result = {
+            "status": "success" if exit_code == 0 else "failed",
+            "exit_code": exit_code,
+        }
+
+        # Handle timeout (exit_code 124)
+        if exit_code == 124:
+            result["timeout"] = True
+            if failed_stage:
+                result["failed_stage"] = failed_stage
+
+        # 1. Extract error information (highest priority)
+        # Strategy: extract rank0's error block (each line prefixed with [rank0]:)
+        error_text = None
+
+        # Method A: Extract [rank0]: prefixed lines and reconstruct traceback
+        rank0_lines = re.findall(r"\[rank0\]:[^\n]+", stdout)
+        if rank0_lines:
+            rank0_block = "\n".join(line.replace("[rank0]: ", "").replace("[rank0]:", "") for line in rank0_lines)
+            # Find traceback in rank0 block
+            tb_match = re.search(
+                r"Traceback \(most recent call last\):.*?(?:Error|Exception):[^\n]+", rank0_block, re.DOTALL
+            )
+            if tb_match:
+                error_text = tb_match.group(0)
+
+        # Method B: Fallback to generic traceback (no rank prefix)
+        # Use findall to get ALL tracebacks, then keep the first one (root cause)
+        if not error_text:
+            all_tracebacks = re.findall(
+                r"Traceback \(most recent call last\):.*?(?:Error|Exception):[^\n]+", stdout, re.DOTALL
+            )
+            if all_tracebacks:
+                # First traceback is usually the root cause
+                error_text = all_tracebacks[0]
+                if len(all_tracebacks) > 1:
+                    error_text += f"\n\n[Note: {len(all_tracebacks)} total errors, showing root cause]"
+
+        if error_text:
+            # Limit length but keep from the END (actual error type/message is at the end of traceback)
+            result["error"] = error_text[-4000:] if len(error_text) > 4000 else error_text
+
+        # 2. Extract training information
+        if "Running training" in stdout:
+            result["training_started"] = True
+
+            # Extract training config
+            # NOTE: we may have log like "Num examples = 1,000,000" and "Num Epochs = 1,000"; So we need to handle ","
+            num_examples = re.search(r"Num examples\s*=\s*([\d,]+)", stdout)
+            num_epochs = re.search(r"Num Epochs\s*=\s*([\d,]+)", stdout)
+            if num_examples:
+                result["num_examples"] = int(num_examples.group(1).replace(",", ""))
+            if num_epochs:
+                result["num_epochs"] = int(num_epochs.group(1).replace(",", ""))
+
+            # Extract final metrics (JSON format from trainer output)
+            final_metrics = re.search(r"\{'train_runtime':[^}]+\}", stdout)
+            if final_metrics:
+                try:
+                    metrics = eval(final_metrics.group(0))  # Safe: only numbers and strings
+                    result["final_metrics"] = {
+                        "train_loss": metrics.get("train_loss"),
+                        "train_runtime": metrics.get("train_runtime"),
+                        "train_samples_per_second": metrics.get("train_samples_per_second"),
+                    }
+                except Exception:
+                    pass
+
+            # Check completion
+            if "Training completed" in stdout:
+                result["completed"] = True
+
+        # 3. Extract warnings (limit to 20)
+        warnings = re.findall(r"\[WARNING[^\]]*\][^\n]+", stdout)
+        if warnings:
+            result["warnings"] = list(set(warnings))[:20]
+
+        # 4. Fallback: if parsing failed, include truncated raw log
+        if not result.get("error") and not result.get("training_started"):
+            result["raw_log_tail"] = stdout[-2000:] if len(stdout) > 2000 else stdout
+
+        return json.dumps(result, indent=2, ensure_ascii=False)
+
+    def _run_micro_batch_test(self, config_yaml: str, workspace: FBWorkspace, env) -> ValidationResult:
+        """Run micro-batch training test for runtime validation"""
+        result = ValidationResult(success=True, filtered_config=config_yaml)
+        ws_prefix = get_workspace_prefix(env)
+
+        # Create micro-batch test configuration
+        config = yaml.safe_load(config_yaml)
+        if not isinstance(config, dict):
+            result.success = False
+            result.execution_output = "Invalid YAML configuration"
+            result.errors.append("Invalid configuration for micro-batch test")
+            return result
+
+        test_config = config.copy()
+
+        # Load extra test parameters from workspace (generated by coder in 2nd turn)
+        extra_test_params = yaml.safe_load(workspace.file_dict[FT_TEST_PARAMS_FILE_NAME])
+
+        # Merge extra test parameters (overrides previous settings)
+        if extra_test_params:
+            test_config.update(extra_test_params)
+
+        # Run micro-batch training
+        workspace.inject_files(**{FT_DEBUG_YAML_FILE_NAME: yaml.dump(test_config, default_flow_style=False)})
+        training_result = workspace.run(
+            env=env,
+            entry=f"llamafactory-cli train {FT_DEBUG_YAML_FILE_NAME}",
+        )
+
+        # Remove micro-batch test files
+        workspace.remove_files([FT_DEBUG_YAML_FILE_NAME, FT_TEST_PARAMS_FILE_NAME])
+
+        # Parse and store structured execution output (reduces ~36k tokens to ~500)
+        raw_stdout = training_result.stdout if training_result.stdout else ""
+        result.raw_stdout = raw_stdout  # Keep full log for UI
+        result.execution_output = self._parse_execution_log(raw_stdout, training_result.exit_code)
+
+        # Check results
+        progress_indicators = ["train_loss", "Training:", "Epoch", "loss:", "step"]
+        has_progress = any(ind.lower() in training_result.stdout.lower() for ind in progress_indicators)
+
+        if training_result.exit_code == 0 and has_progress:
+            logger.info("Micro-batch test passed")
+            result.success = True
+        else:
+            result.success = False
+            result.errors.append(f"Micro-batch test failed (exit_code={training_result.exit_code})")
+
+        return result

rdagent/components/coder/model_coder/__init__.py

@@ -0,0 +1,21 @@
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator
+from rdagent.components.coder.model_coder.evaluators import ModelCoSTEEREvaluator
+from rdagent.components.coder.model_coder.evolving_strategy import (
+    ModelMultiProcessEvolvingStrategy,
+)
+from rdagent.core.scenario import Scenario
+
+
+class ModelCoSTEER(CoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        eva = CoSTEERMultiEvaluator(ModelCoSTEEREvaluator(scen=scen), scen=scen)
+        es = ModelMultiProcessEvolvingStrategy(scen=scen, settings=CoSTEER_SETTINGS)
+
+        super().__init__(*args, settings=CoSTEER_SETTINGS, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs)

rdagent/components/coder/model_coder/benchmark/eval.py

@@ -0,0 +1,71 @@
+# TODO: inherent from the benchmark base class
+import torch
+
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace
+
+
+def get_data_conf(init_val):
+    # TODO: design this step in the workflow
+    in_dim = 1000
+    in_channels = 128
+    exec_config = {"model_eval_param_init": init_val}
+    node_feature = torch.randn(in_dim, in_channels)
+    edge_index = torch.randint(0, in_dim, (2, 2000))
+    return (node_feature, edge_index), exec_config
+
+
+class ModelImpValEval:
+    """
+    Evaluate the similarity of the model structure by changing the input and observe the output.
+
+    Assumption:
+    - If the model structure is similar, the output will change in similar way when we change the input.
+
+    Challenge:
+    - The key difference between it and implementing models is that we have parameters in the layers (Model operators often have no parameters or are given parameters).
+    - we try to initialize the model param in similar value. So only the model structure is different.
+
+    Comparing the correlation of following sequences
+    - modelA[init1](input1).hidden_out1, modelA[init1](input2).hidden_out1, ...
+    - modelB[init1](input1).hidden_out1, modelB[init1](input2).hidden_out1, ...
+
+    For each hidden output, we can calculate a correlation. The average correlation will be the metrics.
+    """
+
+    def evaluate(self, gt: ModelFBWorkspace, gen: ModelFBWorkspace):
+        round_n = 10
+
+        eval_pairs: list[tuple] = []
+
+        # run different input value
+        for _ in range(round_n):
+            # run different model initial parameters.
+            for init_val in [-0.2, -0.1, 0.1, 0.2]:
+                _, gt_res = gt.execute(input_value=init_val, param_init_value=init_val)
+                _, res = gen.execute(input_value=init_val, param_init_value=init_val)
+                eval_pairs.append((res, gt_res))
+
+        # flat and concat the output
+        res_batch, gt_res_batch = [], []
+        for res, gt_res in eval_pairs:
+            res_batch.append(res.reshape(-1))
+            gt_res_batch.append(gt_res.reshape(-1))
+        res_batch = torch.stack(res_batch)
+        gt_res_batch = torch.stack(gt_res_batch)
+
+        res_batch = res_batch.detach().numpy()
+        gt_res_batch = gt_res_batch.detach().numpy()
+
+        # pearson correlation of each hidden output
+        def norm(x):
+            return (x - x.mean(axis=0)) / x.std(axis=0)
+
+        dim_corr = (norm(res_batch) * norm(gt_res_batch)).mean(axis=0)  # the correlation of each hidden output
+
+        # aggregate all the correlation
+        avr_corr = dim_corr.mean()
+        # FIXME:
+        # It is too high(e.g. 0.944) .
+        # Check if it is not a good evaluation!!
+        # Maybe all the same initial params will results in extreamly high correlation without regard to the model structure.
+        return avr_corr

rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py

@@ -0,0 +1,134 @@
+import math
+from typing import Any, Callable, Dict, Optional, Union
+
+import torch
+from torch import Tensor
+from torch.nn import Parameter
+from torch_geometric.nn.conv import GCNConv, MessagePassing
+from torch_geometric.nn.inits import zeros
+from torch_geometric.nn.resolver import activation_resolver
+from torch_geometric.typing import Adj
+
+
+class AntiSymmetricConv(torch.nn.Module):
+    r"""The anti-symmetric graph convolutional operator from the
+    `"Anti-Symmetric DGN: a stable architecture for Deep Graph Networks"
+    <https://openreview.net/forum?id=J3Y7cgZOOS>`_ paper.
+
+    .. math::
+        \mathbf{x}^{\prime}_i = \mathbf{x}_i + \epsilon \cdot \sigma \left(
+            (\mathbf{W}-\mathbf{W}^T-\gamma \mathbf{I}) \mathbf{x}_i +
+            \Phi(\mathbf{X}, \mathcal{N}_i) + \mathbf{b}\right),
+
+    where :math:`\Phi(\mathbf{X}, \mathcal{N}_i)` denotes a
+    :class:`~torch.nn.conv.MessagePassing` layer.
+
+    Args:
+        in_channels (int): Size of each input sample.
+        phi (MessagePassing, optional): The message passing module
+            :math:`\Phi`. If set to :obj:`None`, will use a
+            :class:`~torch_geometric.nn.conv.GCNConv` layer as default.
+            (default: :obj:`None`)
+        num_iters (int, optional): The number of times the anti-symmetric deep
+            graph network operator is called. (default: :obj:`1`)
+        epsilon (float, optional): The discretization step size
+            :math:`\epsilon`. (default: :obj:`0.1`)
+        gamma (float, optional): The strength of the diffusion :math:`\gamma`.
+            It regulates the stability of the method. (default: :obj:`0.1`)
+        act (str, optional): The non-linear activation function :math:`\sigma`,
+            *e.g.*, :obj:`"tanh"` or :obj:`"relu"`. (default: :class:`"tanh"`)
+        act_kwargs (Dict[str, Any], optional): Arguments passed to the
+            respective activation function defined by :obj:`act`.
+            (default: :obj:`None`)
+        bias (bool, optional): If set to :obj:`False`, the layer will not learn
+            an additive bias. (default: :obj:`True`)
+
+    Shapes:
+        - **input:**
+          node features :math:`(|\mathcal{V}|, F_{in})`,
+          edge indices :math:`(2, |\mathcal{E}|)`,
+          edge weights :math:`(|\mathcal{E}|)` *(optional)*
+        - **output:** node features :math:`(|\mathcal{V}|, F_{in})`
+    """
+
+    def __init__(
+        self,
+        in_channels: int,
+        phi: Optional[MessagePassing] = None,
+        num_iters: int = 1,
+        epsilon: float = 0.1,
+        gamma: float = 0.1,
+        act: Union[str, Callable, None] = "tanh",
+        act_kwargs: Optional[Dict[str, Any]] = None,
+        bias: bool = True,
+    ):
+        super().__init__()
+
+        self.in_channels = in_channels
+        self.num_iters = num_iters
+        self.gamma = gamma
+        self.epsilon = epsilon
+        self.act = activation_resolver(act, **(act_kwargs or {}))
+
+        if phi is None:
+            phi = GCNConv(in_channels, in_channels, bias=False)
+
+        self.W = Parameter(torch.empty(in_channels, in_channels))
+        self.register_buffer("eye", torch.eye(in_channels))
+        self.phi = phi
+
+        if bias:
+            self.bias = Parameter(torch.empty(in_channels))
+        else:
+            self.register_parameter("bias", None)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        r"""Resets all learnable parameters of the module."""
+        torch.nn.init.kaiming_uniform_(self.W, a=math.sqrt(5))
+        self.phi.reset_parameters()
+        zeros(self.bias)
+
+    def forward(self, x: Tensor, edge_index: Adj, *args, **kwargs) -> Tensor:
+        r"""Runs the forward pass of the module."""
+        antisymmetric_W = self.W - self.W.t() - self.gamma * self.eye
+
+        for _ in range(self.num_iters):
+            h = self.phi(x, edge_index, *args, **kwargs)
+            h = x @ antisymmetric_W.t() + h
+
+            if self.bias is not None:
+                h += self.bias
+
+            if self.act is not None:
+                h = self.act(h)
+
+            x = x + self.epsilon * h
+
+        return x
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}("
+            f"{self.in_channels}, "
+            f"phi={self.phi}, "
+            f"num_iters={self.num_iters}, "
+            f"epsilon={self.epsilon}, "
+            f"gamma={self.gamma})"
+        )
+
+
+model_cls = AntiSymmetricConv
+
+
+if __name__ == "__main__":
+    node_features = torch.load("node_features.pt")
+    edge_index = torch.load("edge_index.pt")
+
+    # Model instantiation and forward pass
+    model = AntiSymmetricConv(in_channels=node_features.size(-1))
+    output = model(node_features, edge_index)
+
+    # Save output to a file
+    torch.save(output, "gt_output.pt")

rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py

@@ -0,0 +1,89 @@
+import copy
+
+import torch
+from torch import Tensor
+from torch_geometric.nn.conv import MessagePassing
+
+
+class DirGNNConv(torch.nn.Module):
+    r"""A generic wrapper for computing graph convolution on directed
+    graphs as described in the `"Edge Directionality Improves Learning on
+    Heterophilic Graphs" <https://arxiv.org/abs/2305.10498>`_ paper.
+    :class:`DirGNNConv` will pass messages both from source nodes to target
+    nodes and from target nodes to source nodes.
+
+    Args:
+        conv (MessagePassing): The underlying
+            :class:`~torch_geometric.nn.conv.MessagePassing` layer to use.
+        alpha (float, optional): The alpha coefficient used to weight the
+            aggregations of in- and out-edges as part of a convex combination.
+            (default: :obj:`0.5`)
+        root_weight (bool, optional): If set to :obj:`True`, the layer will add
+            transformed root node features to the output.
+            (default: :obj:`True`)
+    """
+
+    def __init__(
+        self,
+        conv: MessagePassing,
+        alpha: float = 0.5,
+        root_weight: bool = True,
+    ):
+        super().__init__()
+
+        self.alpha = alpha
+        self.root_weight = root_weight
+
+        self.conv_in = copy.deepcopy(conv)
+        self.conv_out = copy.deepcopy(conv)
+
+        if hasattr(conv, "add_self_loops"):
+            self.conv_in.add_self_loops = False
+            self.conv_out.add_self_loops = False
+        if hasattr(conv, "root_weight"):
+            self.conv_in.root_weight = False
+            self.conv_out.root_weight = False
+
+        if root_weight:
+            self.lin = torch.nn.Linear(conv.in_channels, conv.out_channels)
+        else:
+            self.lin = None
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        r"""Resets all learnable parameters of the module."""
+        self.conv_in.reset_parameters()
+        self.conv_out.reset_parameters()
+        if self.lin is not None:
+            self.lin.reset_parameters()
+
+    def forward(self, x: Tensor, edge_index: Tensor) -> Tensor:
+        """"""  # noqa: D419
+        x_in = self.conv_in(x, edge_index)
+        x_out = self.conv_out(x, edge_index.flip([0]))
+
+        out = self.alpha * x_out + (1 - self.alpha) * x_in
+
+        if self.root_weight:
+            out = out + self.lin(x)
+
+        return out
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.conv_in}, alpha={self.alpha})"
+
+
+model_cls = DirGNNConv
+
+
+if __name__ == "__main__":
+    node_features = torch.load("node_features.pt")
+    edge_index = torch.load("edge_index.pt")
+
+    # Model instantiation and forward pass
+    model = DirGNNConv(MessagePassing())
+    output = model(node_features, edge_index)
+
+    # Save output to a file
+    torch.save(output, "gt_output.pt")

rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py

@@ -0,0 +1,198 @@
+import inspect
+from typing import Any, Dict, Optional
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+from torch.nn import Dropout, Linear, Sequential
+from torch_geometric.nn.attention import PerformerAttention
+from torch_geometric.nn.conv import MessagePassing
+from torch_geometric.nn.inits import reset
+from torch_geometric.nn.resolver import activation_resolver, normalization_resolver
+from torch_geometric.typing import Adj
+from torch_geometric.utils import to_dense_batch
+
+
+class GPSConv(torch.nn.Module):
+    r"""The general, powerful, scalable (GPS) graph transformer layer from the
+    `"Recipe for a General, Powerful, Scalable Graph Transformer"
+    <https://arxiv.org/abs/2205.12454>`_ paper.
+
+    The GPS layer is based on a 3-part recipe:
+
+    1. Inclusion of positional (PE) and structural encodings (SE) to the input
+       features (done in a pre-processing step via
+       :class:`torch_geometric.transforms`).
+    2. A local message passing layer (MPNN) that operates on the input graph.
+    3. A global attention layer that operates on the entire graph.
+
+    .. note::
+
+        For an example of using :class:`GPSConv`, see
+        `examples/graph_gps.py
+        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/
+        graph_gps.py>`_.
+
+    Args:
+        channels (int): Size of each input sample.
+        conv (MessagePassing, optional): The local message passing layer.
+        heads (int, optional): Number of multi-head-attentions.
+            (default: :obj:`1`)
+        dropout (float, optional): Dropout probability of intermediate
+            embeddings. (default: :obj:`0.`)
+        act (str or Callable, optional): The non-linear activation function to
+            use. (default: :obj:`"relu"`)
+        act_kwargs (Dict[str, Any], optional): Arguments passed to the
+            respective activation function defined by :obj:`act`.
+            (default: :obj:`None`)
+        norm (str or Callable, optional): The normalization function to
+            use. (default: :obj:`"batch_norm"`)
+        norm_kwargs (Dict[str, Any], optional): Arguments passed to the
+            respective normalization function defined by :obj:`norm`.
+            (default: :obj:`None`)
+        attn_type (str): Global attention type, :obj:`multihead` or
+            :obj:`performer`. (default: :obj:`multihead`)
+        attn_kwargs (Dict[str, Any], optional): Arguments passed to the
+            attention layer. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        channels: int,
+        conv: Optional[MessagePassing],
+        heads: int = 1,
+        dropout: float = 0.0,
+        act: str = "relu",
+        act_kwargs: Optional[Dict[str, Any]] = None,
+        norm: Optional[str] = "batch_norm",
+        norm_kwargs: Optional[Dict[str, Any]] = None,
+        attn_type: str = "multihead",
+        attn_kwargs: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__()
+
+        self.channels = channels
+        self.conv = conv
+        self.heads = heads
+        self.dropout = dropout
+        self.attn_type = attn_type
+
+        attn_kwargs = attn_kwargs or {}
+        if attn_type == "multihead":
+            self.attn = torch.nn.MultiheadAttention(
+                channels,
+                heads,
+                batch_first=True,
+                **attn_kwargs,
+            )
+        elif attn_type == "performer":
+            self.attn = PerformerAttention(
+                channels=channels,
+                heads=heads,
+                **attn_kwargs,
+            )
+        else:
+            # TODO: Support BigBird
+            raise ValueError(f"{attn_type} is not supported")
+
+        self.mlp = Sequential(
+            Linear(channels, channels * 2),
+            activation_resolver(act, **(act_kwargs or {})),
+            Dropout(dropout),
+            Linear(channels * 2, channels),
+            Dropout(dropout),
+        )
+
+        norm_kwargs = norm_kwargs or {}
+        self.norm1 = normalization_resolver(norm, channels, **norm_kwargs)
+        self.norm2 = normalization_resolver(norm, channels, **norm_kwargs)
+        self.norm3 = normalization_resolver(norm, channels, **norm_kwargs)
+
+        self.norm_with_batch = False
+        if self.norm1 is not None:
+            signature = inspect.signature(self.norm1.forward)
+            self.norm_with_batch = "batch" in signature.parameters
+
+    def reset_parameters(self):
+        r"""Resets all learnable parameters of the module."""
+        if self.conv is not None:
+            self.conv.reset_parameters()
+        self.attn._reset_parameters()
+        reset(self.mlp)
+        if self.norm1 is not None:
+            self.norm1.reset_parameters()
+        if self.norm2 is not None:
+            self.norm2.reset_parameters()
+        if self.norm3 is not None:
+            self.norm3.reset_parameters()
+
+    def forward(
+        self,
+        x: Tensor,
+        edge_index: Adj,
+        batch: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Tensor:
+        r"""Runs the forward pass of the module."""
+        hs = []
+        if self.conv is not None:  # Local MPNN.
+            h = self.conv(x, edge_index, **kwargs)
+            h = F.dropout(h, p=self.dropout, training=self.training)
+            h = h + x
+            if self.norm1 is not None:
+                if self.norm_with_batch:
+                    h = self.norm1(h, batch=batch)
+                else:
+                    h = self.norm1(h)
+            hs.append(h)
+
+        # Global attention transformer-style model.
+        h, mask = to_dense_batch(x, batch)
+
+        if isinstance(self.attn, torch.nn.MultiheadAttention):
+            h, _ = self.attn(h, h, h, key_padding_mask=~mask, need_weights=False)
+        elif isinstance(self.attn, PerformerAttention):
+            h = self.attn(h, mask=mask)
+
+        h = h[mask]
+        h = F.dropout(h, p=self.dropout, training=self.training)
+        h = h + x  # Residual connection.
+        if self.norm2 is not None:
+            if self.norm_with_batch:
+                h = self.norm2(h, batch=batch)
+            else:
+                h = self.norm2(h)
+        hs.append(h)
+
+        out = sum(hs)  # Combine local and global outputs.
+
+        out = out + self.mlp(out)
+        if self.norm3 is not None:
+            if self.norm_with_batch:
+                out = self.norm3(out, batch=batch)
+            else:
+                out = self.norm3(out)
+
+        return out
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}({self.channels}, "
+            f"conv={self.conv}, heads={self.heads}, "
+            f"attn_type={self.attn_type})"
+        )
+
+
+model_cls = GPSConv
+
+
+if __name__ == "__main__":
+    node_features = torch.load("node_features.pt")
+    edge_index = torch.load("edge_index.pt")
+
+    # Model instantiation and forward pass
+    model = GPSConv(channels=node_features.size(-1), conv=MessagePassing())
+    output = model(node_features, edge_index)
+
+    # Save output to a file
+    torch.save(output, "gt_output.pt")

rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py

@@ -0,0 +1,187 @@
+import math
+
+import torch
+from torch import Tensor
+from torch.nn import BatchNorm1d, Parameter
+from torch_geometric.nn import inits
+from torch_geometric.nn.conv import MessagePassing
+from torch_geometric.nn.models import MLP
+from torch_geometric.typing import Adj, OptTensor
+from torch_geometric.utils import spmm
+
+
+class SparseLinear(MessagePassing):
+    def __init__(self, in_channels: int, out_channels: int, bias: bool = True):
+        super().__init__(aggr="add")
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+
+        self.weight = Parameter(torch.empty(in_channels, out_channels))
+        if bias:
+            self.bias = Parameter(torch.empty(out_channels))
+        else:
+            self.register_parameter("bias", None)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        inits.kaiming_uniform(self.weight, fan=self.in_channels, a=math.sqrt(5))
+        inits.uniform(self.in_channels, self.bias)
+
+    def forward(
+        self,
+        edge_index: Adj,
+        edge_weight: OptTensor = None,
+    ) -> Tensor:
+        # propagate_type: (weight: Tensor, edge_weight: OptTensor)
+        out = self.propagate(edge_index, weight=self.weight, edge_weight=edge_weight)
+
+        if self.bias is not None:
+            out = out + self.bias
+
+        return out
+
+    def message(self, weight_j: Tensor, edge_weight: OptTensor) -> Tensor:
+        if edge_weight is None:
+            return weight_j
+        else:
+            return edge_weight.view(-1, 1) * weight_j
+
+    def message_and_aggregate(self, adj_t: Adj, weight: Tensor) -> Tensor:
+        return spmm(adj_t, weight, reduce=self.aggr)
+
+
+class LINKX(torch.nn.Module):
+    r"""The LINKX model from the `"Large Scale Learning on Non-Homophilous
+    Graphs: New Benchmarks and Strong Simple Methods"
+    <https://arxiv.org/abs/2110.14446>`_ paper.
+
+    .. math::
+        \mathbf{H}_{\mathbf{A}} &= \textrm{MLP}_{\mathbf{A}}(\mathbf{A})
+
+        \mathbf{H}_{\mathbf{X}} &= \textrm{MLP}_{\mathbf{X}}(\mathbf{X})
+
+        \mathbf{Y} &= \textrm{MLP}_{f} \left( \sigma \left( \mathbf{W}
+        [\mathbf{H}_{\mathbf{A}}, \mathbf{H}_{\mathbf{X}}] +
+        \mathbf{H}_{\mathbf{A}} + \mathbf{H}_{\mathbf{X}} \right) \right)
+
+    .. note::
+
+        For an example of using LINKX, see `examples/linkx.py <https://
+        github.com/pyg-team/pytorch_geometric/blob/master/examples/linkx.py>`_.
+
+    Args:
+        num_nodes (int): The number of nodes in the graph.
+        in_channels (int): Size of each input sample, or :obj:`-1` to derive
+            the size from the first input(s) to the forward method.
+        hidden_channels (int): Size of each hidden sample.
+        out_channels (int): Size of each output sample.
+        num_layers (int): Number of layers of :math:`\textrm{MLP}_{f}`.
+        num_edge_layers (int, optional): Number of layers of
+            :math:`\textrm{MLP}_{\mathbf{A}}`. (default: :obj:`1`)
+        num_node_layers (int, optional): Number of layers of
+            :math:`\textrm{MLP}_{\mathbf{X}}`. (default: :obj:`1`)
+        dropout (float, optional): Dropout probability of each hidden
+            embedding. (default: :obj:`0.0`)
+    """
+
+    def __init__(
+        self,
+        num_nodes: int,
+        in_channels: int,
+        hidden_channels: int,
+        out_channels: int,
+        num_layers: int,
+        num_edge_layers: int = 1,
+        num_node_layers: int = 1,
+        dropout: float = 0.0,
+    ):
+        super().__init__()
+
+        self.num_nodes = num_nodes
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.num_edge_layers = num_edge_layers
+
+        self.edge_lin = SparseLinear(num_nodes, hidden_channels)
+
+        if self.num_edge_layers > 1:
+            self.edge_norm = BatchNorm1d(hidden_channels)
+            channels = [hidden_channels] * num_edge_layers
+            self.edge_mlp = MLP(channels, dropout=0.0, act_first=True)
+        else:
+            self.edge_norm = None
+            self.edge_mlp = None
+
+        channels = [in_channels] + [hidden_channels] * num_node_layers
+        self.node_mlp = MLP(channels, dropout=0.0, act_first=True)
+
+        self.cat_lin1 = torch.nn.Linear(hidden_channels, hidden_channels)
+        self.cat_lin2 = torch.nn.Linear(hidden_channels, hidden_channels)
+
+        channels = [hidden_channels] * num_layers + [out_channels]
+        self.final_mlp = MLP(channels, dropout=dropout, act_first=True)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        r"""Resets all learnable parameters of the module."""
+        self.edge_lin.reset_parameters()
+        if self.edge_norm is not None:
+            self.edge_norm.reset_parameters()
+        if self.edge_mlp is not None:
+            self.edge_mlp.reset_parameters()
+        self.node_mlp.reset_parameters()
+        self.cat_lin1.reset_parameters()
+        self.cat_lin2.reset_parameters()
+        self.final_mlp.reset_parameters()
+
+    def forward(
+        self,
+        x: OptTensor,
+        edge_index: Adj,
+        edge_weight: OptTensor = None,
+    ) -> Tensor:
+        """"""  # noqa: D419
+        out = self.edge_lin(edge_index, edge_weight)
+
+        if self.edge_norm is not None and self.edge_mlp is not None:
+            out = out.relu_()
+            out = self.edge_norm(out)
+            out = self.edge_mlp(out)
+
+        out = out + self.cat_lin1(out)
+
+        if x is not None:
+            x = self.node_mlp(x)
+            out = out + x
+            out = out + self.cat_lin2(x)
+
+        return self.final_mlp(out.relu_())
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(num_nodes={self.num_nodes}, "
+            f"in_channels={self.in_channels}, "
+            f"out_channels={self.out_channels})"
+        )
+
+
+model_cls = LINKX
+
+if __name__ == "__main__":
+    node_features = torch.load("node_features.pt")
+    edge_index = torch.load("edge_index.pt")
+
+    # Model instantiation and forward pass
+    model = LINKX(
+        num_nodes=node_features.size(0),
+        in_channels=node_features.size(1),
+        hidden_channels=node_features.size(1),
+        out_channels=node_features.size(1),
+        num_layers=1,
+    )
+    output = model(node_features, edge_index)
+
+    # Save output to a file
+    torch.save(output, "gt_output.pt")

rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py

@@ -0,0 +1,118 @@
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+from torch_geometric.nn import SimpleConv
+from torch_geometric.nn.dense.linear import Linear
+
+
+class PMLP(torch.nn.Module):
+    r"""The P(ropagational)MLP model from the `"Graph Neural Networks are
+    Inherently Good Generalizers: Insights by Bridging GNNs and MLPs"
+    <https://arxiv.org/abs/2212.09034>`_ paper.
+    :class:`PMLP` is identical to a standard MLP during training, but then
+    adopts a GNN architecture during testing.
+
+    Args:
+        in_channels (int): Size of each input sample.
+        hidden_channels (int): Size of each hidden sample.
+        out_channels (int): Size of each output sample.
+        num_layers (int): The number of layers.
+        dropout (float, optional): Dropout probability of each hidden
+            embedding. (default: :obj:`0.`)
+        norm (bool, optional): If set to :obj:`False`, will not apply batch
+            normalization. (default: :obj:`True`)
+        bias (bool, optional): If set to :obj:`False`, the module
+            will not learn additive biases. (default: :obj:`True`)
+    """
+
+    def __init__(
+        self,
+        in_channels: int,
+        hidden_channels: int,
+        out_channels: int,
+        num_layers: int,
+        dropout: float = 0.0,
+        norm: bool = True,
+        bias: bool = True,
+    ):
+        super().__init__()
+
+        self.in_channels = in_channels
+        self.hidden_channels = hidden_channels
+        self.out_channels = out_channels
+        self.num_layers = num_layers
+        self.dropout = dropout
+        self.bias = bias
+
+        self.lins = torch.nn.ModuleList()
+        self.lins.append(Linear(in_channels, hidden_channels, self.bias))
+        for _ in range(self.num_layers - 2):
+            lin = Linear(hidden_channels, hidden_channels, self.bias)
+            self.lins.append(lin)
+        self.lins.append(Linear(hidden_channels, out_channels, self.bias))
+
+        self.norm = None
+        if norm:
+            self.norm = torch.nn.BatchNorm1d(
+                hidden_channels,
+                affine=False,
+                track_running_stats=False,
+            )
+
+        self.conv = SimpleConv(aggr="mean", combine_root="self_loop")
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        r"""Resets all learnable parameters of the module."""
+        for lin in self.lins:
+            torch.nn.init.xavier_uniform_(lin.weight, gain=1.414)
+            if self.bias:
+                torch.nn.init.zeros_(lin.bias)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        edge_index: Optional[Tensor] = None,
+    ) -> torch.Tensor:
+        """"""  # noqa: D419
+        if not self.training and edge_index is None:
+            raise ValueError(f"'edge_index' needs to be present during " f"inference in '{self.__class__.__name__}'")
+
+        for i in range(self.num_layers):
+            x = x @ self.lins[i].weight.t()
+            if not self.training:
+                x = self.conv(x, edge_index)
+            if self.bias:
+                x = x + self.lins[i].bias
+            if i != self.num_layers - 1:
+                if self.norm is not None:
+                    x = self.norm(x)
+                x = x.relu()
+                x = F.dropout(x, p=self.dropout, training=self.training)
+
+        return x
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.in_channels}, " f"{self.out_channels}, num_layers={self.num_layers})"
+
+
+model_cls = PMLP
+
+if __name__ == "__main__":
+    node_features = torch.load("node_features.pt")
+    edge_index = torch.load("edge_index.pt")
+
+    # Model instantiation and forward pass
+    model = PMLP(
+        in_channels=node_features.size(-1),
+        hidden_channels=node_features.size(-1),
+        out_channels=node_features.size(-1),
+        num_layers=1,
+    )
+    output = model(node_features, edge_index)
+
+    # Save output to a file
+    torch.save(output, "gt_output.pt")

rdagent/components/coder/model_coder/benchmark/model_dict.json

@@ -0,0 +1,80 @@
+{
+    "PMLP": {
+        "description": "`PMLP` is identical to a standard MLP during training, but then adopts a GNN architecture (add message passing) during testing.",
+        "formulation": "\\hat{y}_u = \\psi(\\text{MP}(\\{h^{(l-1)}_v\\}_{v \\in N_u \\cup \\{u\\}}))",
+        "variables": {
+            "\\hat{y}_u": "The predicted output for node u",
+            "\\psi": "A function representing the feed-forward process, consisting of a linear feature transformation followed by a non-linear activation",
+            "\\text{MP}": "Message Passing operation that aggregates neighbored information",
+            "h^{(l-1)}_v": "The feature representation of node v at layer (l-1)",
+            "N_u": "The set of neighbored nodes centered at node u"
+        },
+        "key": "pmlp",
+        "model_type": "TimeSeries"
+    },
+    "LINKX": {
+        "description": "A scalable model for node classification that separately embeds adjacency and node features, combines them with MLPs, and applies simple transformations.",
+        "formulation": "Y = MLP_f(\\sigma(W[h_A; h_X] + h_A + h_X))",
+        "variables": {
+            "Y": "The output predictions",
+            "\\sigma": "Non-linear activation function",
+            "W": "Learned weight matrix",
+            "h_A": "Embedding of the adjacency matrix",
+            "h_X": "Embedding of the node features",
+            "MLP_f": "Final multilayer perceptron for prediction"
+        },
+        "key": "linkx",
+        "model_type": "TimeSeries"
+    },
+    "GPSConv": {
+        "description": "A scalable and powerful graph transformer with linear complexity, capable of handling large graphs with state-of-the-art results across diverse benchmarks.",
+        "formulation": "X^{(l+1)} = \\text{MPNN}^{(l)}(X^{(l)}, A) + \\text{GlobalAttn}^{(l)}(X^{(l)})",
+        "variables": {
+            "X^{(l)}": "The node features at layer l",
+            "A": "The adjacency matrix of the graph",
+            "X^{(l+1)}": "The updated node features at layer l+1",
+            "MPNN^{(l)}": "The message-passing neural network function at layer l",
+            "GlobalAttn^{(l)}": "The global attention function at layer l"
+        },
+        "key": "gpsconv",
+        "model_type": "TimeSeries"
+    },
+    "ViSNet": {
+        "description": "ViSNet is an equivariant geometry-enhanced graph neural network designed for efficient molecular modeling[^1^][1][^2^][2]. It utilizes a Vector-Scalar interactive message passing mechanism to extract and utilize geometric features with low computational costs, achieving state-of-the-art performance on multiple molecular dynamics benchmarks.",
+        "formulation": "\\text{ViSNet}(G) = \\sum_{u \\in G} f(\\mathbf{h}_u, \\mathbf{e}_u, \\mathbf{v}_u)",
+        "variables": {
+            "\\mathbf{h}_u": "Node embedding for atom u",
+            "\\mathbf{e}_u": "Edge embedding associated with atom u",
+            "\\mathbf{v}_u": "Direction unit vector for atom u"
+        },
+        "key": "visnet",
+        "model_type": "TimeSeries"
+    },
+    "Dir-GNN": {
+        "description": "A framework for deep learning on directed graphs that extends MPNNs to incorporate edge directionality.",
+        "formulation": "x^{(k)}_i = COM^{(k)}\\left(x^{(k-1)}_i, m^{(k)}_{i,\\leftarrow}, m^{(k)}_{i,\\rightarrow}\\right)",
+        "variables": {
+            "x^{(k)}_i": "The feature representation of node i at layer k",
+            "m^{(k)}_{i,\\leftarrow}": "The aggregated incoming messages to node i at layer k",
+            "m^{(k)}_{i,\\rightarrow}": "The aggregated outgoing messages from node i at layer k"
+        },
+        "key": "dirgnn",
+        "model_type": "TimeSeries"
+    },
+    "A-DGN": {
+        "description": "A framework for stable and non-dissipative DGN design, conceived through the lens of ordinary differential equations (ODEs). It ensures long-range information preservation between nodes and prevents gradient vanishing or explosion during training.",
+        "formulation": "\\frac{\\partial x_u(t)}{\\partial t} = \\sigma(W^T x_u(t) + \\Phi(X(t), N_u) + b)",
+        "variables": {
+            "x_u(t)": "The state of node u at time t",
+            "\\frac{\\partial x_u(t)}{\\partial t}": "The rate of change of the state of node u at time t",
+            "\\sigma": "A monotonically non-decreasing activation function",
+            "W": "A weight matrix",
+            "b": "A bias vector",
+            "\\Phi(X(t), N_u)": "The aggregation function for the states of the nodes in the neighborhood of u",
+            "X(t)": "The node feature matrix of the whole graph at time t",
+            "N_u": "The set of neighboring nodes of u"
+        },
+        "key": "A-DGN",
+        "model_type": "TimeSeries"
+    }
+}

rdagent/components/coder/model_coder/conf.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.utils.env import Env, QlibCondaConf, QlibCondaEnv, QTDockerEnv
+
+
+class ModelCoSTEERSettings(CoSTEERSettings):
+    model_config = SettingsConfigDict(env_prefix="MODEL_CoSTEER_")
+
+    env_type: str = "conda"  # or "docker"
+    """Environment to run model code in coder and runner: 'conda' for local conda env, 'docker' for Docker container"""
+
+
+def get_model_env(
+    conf_type: Optional[str] = None,
+    extra_volumes: dict = {},
+    running_timeout_period: int = 600,
+    enable_cache: Optional[bool] = None,
+) -> Env:
+    conf = ModelCoSTEERSettings()
+    if conf.env_type == "docker":
+        env = QTDockerEnv()
+    elif conf.env_type == "conda":
+        env = QlibCondaEnv(conf=QlibCondaConf())
+    else:
+        raise ValueError(f"Unknown env type: {conf.env_type}")
+
+    env.conf.extra_volumes = extra_volumes.copy()
+    env.conf.running_timeout_period = running_timeout_period
+    if enable_cache is not None:
+        env.conf.enable_cache = enable_cache
+    env.prepare()
+    return env
+
+
+MODEL_COSTEER_SETTINGS = ModelCoSTEERSettings()

rdagent/components/coder/model_coder/eva_utils.py

@@ -0,0 +1,166 @@
+import json
+from typing import Dict, Tuple
+
+import numpy as np
+
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEEREvaluator
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
+from rdagent.core.experiment import Task, Workspace
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+
+# This shape evaluator is also used in data_science
+def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]:
+    if target_shape is None or prediction is None:
+        return (
+            "No output generated from the model. No shape evaluation conducted.",
+            False,
+        )
+    pre_shape = prediction.shape
+
+    if pre_shape == target_shape:
+        return "The shape of the output is correct.", True
+    else:
+        return (
+            f"The shape of the output is incorrect. Expected {target_shape}, but got {pre_shape}.",
+            False,
+        )
+
+
+def value_evaluator(
+    prediction: np.ndarray,
+    target: np.ndarray,
+) -> Tuple[np.ndarray, bool]:
+    if prediction is None:
+        return "No output generated from the model. Skip value evaluation", False
+    elif target is None:
+        return (
+            "No ground truth output provided. Value evaluation not impractical",
+            False,
+        )
+    else:
+        # Calculate the mean absolute difference
+        diff = np.mean(np.abs(target - prediction))
+        return (
+            f"The value of the output is correct. The mean absolute difference is {diff}.",
+            diff < 0.1,
+        )
+
+
+class ModelCodeEvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        model_execution_feedback: str = "",
+        model_value_feedback: str = "",
+    ):
+        assert isinstance(target_task, ModelTask)
+        assert isinstance(implementation, ModelFBWorkspace)
+        if gt_implementation is not None:
+            assert isinstance(gt_implementation, ModelFBWorkspace)
+
+        model_task_information = target_task.get_task_information()
+        code = implementation.all_codes
+
+        system_prompt = T(".prompts:evaluator_code_feedback.system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
+                if self.scen is not None
+                else "No scenario description."
+            )
+        )
+        execution_feedback_to_render = model_execution_feedback
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = T(".prompts:evaluator_code_feedback.user").r(
+                model_information=model_task_information,
+                code=code,
+                model_execution_feedback=execution_feedback_to_render,
+                model_value_feedback=model_value_feedback,
+                gt_code=gt_implementation.all_codes if gt_implementation else None,
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > APIBackend().chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+
+        critic_response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+        )
+
+        return critic_response, None
+
+
+class ModelFinalEvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        model_execution_feedback: str,
+        model_shape_feedback: str,
+        model_value_feedback: str,
+        model_code_feedback: str,
+    ):
+        assert isinstance(target_task, ModelTask)
+        assert isinstance(implementation, ModelFBWorkspace)
+        if gt_implementation is not None:
+            assert isinstance(gt_implementation, ModelFBWorkspace)
+
+        system_prompt = T(".prompts:evaluator_final_feedback.system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
+                if self.scen is not None
+                else "No scenario description."
+            )
+        )
+
+        execution_feedback_to_render = model_execution_feedback
+
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = T(".prompts:evaluator_final_feedback.user").r(
+                model_information=target_task.get_task_information(),
+                model_execution_feedback=execution_feedback_to_render,
+                model_shape_feedback=model_shape_feedback,
+                model_code_feedback=model_code_feedback,
+                model_value_feedback=model_value_feedback,
+            )
+
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > APIBackend().chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+
+        final_evaluation_dict = json.loads(
+            APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                json_mode=True,
+                json_target_type=Dict[str, str | bool | int],
+            ),
+        )
+        if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[
+            "final_decision"
+        ].lower() in ("true", "false"):
+            final_evaluation_dict["final_decision"] = bool(final_evaluation_dict["final_decision"])
+        return (
+            final_evaluation_dict["final_feedback"],
+            final_evaluation_dict["final_decision"],
+        )

rdagent/components/coder/model_coder/evaluators.py

@@ -0,0 +1,104 @@
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERMultiFeedback,
+    CoSTEERSingleFeedbackDeprecated,
+)
+from rdagent.components.coder.model_coder.eva_utils import (
+    ModelCodeEvaluator,
+    ModelFinalEvaluator,
+    shape_evaluator,
+    value_evaluator,
+)
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import Task, Workspace
+
+ModelSingleFeedback = CoSTEERSingleFeedbackDeprecated
+ModelMultiFeedback = CoSTEERMultiFeedback
+
+
+class ModelCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> ModelSingleFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return ModelSingleFeedback(
+                execution_feedback="This task has failed too many times, skip implementation.",
+                shape_feedback="This task has failed too many times, skip implementation.",
+                value_feedback="This task has failed too many times, skip implementation.",
+                code_feedback="This task has failed too many times, skip implementation.",
+                final_feedback="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+        assert isinstance(target_task, ModelTask)
+
+        # NOTE: Use fixed input to test the model to avoid randomness
+        batch_size = 8
+        num_features = 30
+        num_timesteps = 40
+        input_value = 0.4
+        param_init_value = 0.6
+
+        assert isinstance(implementation, ModelFBWorkspace)
+        model_execution_feedback, gen_np_array = implementation.execute(
+            batch_size=batch_size,
+            num_features=num_features,
+            num_timesteps=num_timesteps,
+            input_value=input_value,
+            param_init_value=param_init_value,
+        )
+        if gt_implementation is not None:
+            assert isinstance(gt_implementation, ModelFBWorkspace)
+            _, gt_np_array = gt_implementation.execute(
+                batch_size=batch_size,
+                num_features=num_features,
+                num_timesteps=num_timesteps,
+                input_value=input_value,
+                param_init_value=param_init_value,
+            )
+        else:
+            gt_np_array = None
+
+        shape_feedback, shape_decision = shape_evaluator(
+            gen_np_array,
+            (batch_size, self.scen.model_output_channel if hasattr(self.scen, "model_output_channel") else 1),
+        )
+        value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array)
+        code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
+            target_task=target_task,
+            implementation=implementation,
+            gt_implementation=gt_implementation,
+            model_execution_feedback=model_execution_feedback,
+            model_value_feedback="\n".join([shape_feedback, value_feedback]),
+        )
+        final_feedback, final_decision = ModelFinalEvaluator(scen=self.scen).evaluate(
+            target_task=target_task,
+            implementation=implementation,
+            gt_implementation=gt_implementation,
+            model_execution_feedback=model_execution_feedback,
+            model_shape_feedback=shape_feedback,
+            model_value_feedback=value_feedback,
+            model_code_feedback=code_feedback,
+        )
+
+        return ModelSingleFeedback(
+            execution_feedback=model_execution_feedback,
+            shape_feedback=shape_feedback,
+            value_feedback=value_feedback,
+            code_feedback=code_feedback,
+            final_feedback=final_feedback,
+            final_decision=final_decision,
+            value_generated_flag=(gen_np_array is not None),
+            final_decision_based_on_gt=(gt_implementation is not None),
+        )

rdagent/components/coder/model_coder/evolving_strategy.py

@@ -0,0 +1,89 @@
+import json
+from typing import Dict
+
+from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
+from rdagent.core.experiment import FBWorkspace
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+
+class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: ModelTask,
+        queried_knowledge: CoSTEERQueriedKnowledge = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> str:
+        model_information_str = target_task.get_task_information()
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[model_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[model_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+
+        queried_former_failed_knowledge_to_render = (
+            queried_former_failed_knowledge[0]
+            if isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2)
+            else queried_former_failed_knowledge
+        )
+        system_prompt = T(".prompts:evolving_strategy_model_coder.system").r(
+            scenario=self.scen.get_scenario_all_desc(filtered_tag="model"),
+            queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            current_code=workspace.file_dict.get("model.py"),
+        )
+
+        queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
+        for _ in range(10):  # max attempt to reduce the length of user_prompt
+            user_prompt = T(".prompts:evolving_strategy_model_coder.user").r(
+                model_information_str=model_information_str,
+                queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                < APIBackend().chat_token_limit
+            ):
+                break
+            elif len(queried_former_failed_knowledge_to_render) > 1:
+                queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
+            elif len(queried_similar_successful_knowledge_to_render) > 1:
+                queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
+
+        code = json.loads(
+            APIBackend(use_chat_cache=CoSTEER_SETTINGS.coder_use_cache).build_messages_and_create_chat_completion(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                json_mode=True,
+                json_target_type=Dict[str, str],
+            ),
+        )["code"]
+        return code
+
+    def assign_code_list_to_evo(self, code_list, evo):
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                evo.sub_workspace_list[index] = ModelFBWorkspace(target_task=evo.sub_tasks[index])
+            evo.sub_workspace_list[index].inject_files(**{"model.py": code_list[index]})
+        return evo

rdagent/components/coder/model_coder/gt_code.py

@@ -0,0 +1,136 @@
+"""
+This is just an exmaple.
+It will be replaced wtih a list of ground truth tasks.
+"""
+
+import math
+from typing import Any, Callable, Dict, Optional, Union
+
+import torch
+from torch import Tensor
+from torch.nn import Parameter
+from torch_geometric.nn.conv import GCNConv, MessagePassing
+from torch_geometric.nn.inits import zeros
+from torch_geometric.nn.resolver import activation_resolver
+from torch_geometric.typing import Adj
+
+
+class AntiSymmetricConv(torch.nn.Module):
+    r"""The anti-symmetric graph convolutional operator from the
+    `"Anti-Symmetric DGN: a stable architecture for Deep Graph Networks"
+    <https://openreview.net/forum?id=J3Y7cgZOOS>`_ paper.
+
+    .. math::
+        \mathbf{x}^{\prime}_i = \mathbf{x}_i + \epsilon \cdot \sigma \left(
+            (\mathbf{W}-\mathbf{W}^T-\gamma \mathbf{I}) \mathbf{x}_i +
+            \Phi(\mathbf{X}, \mathcal{N}_i) + \mathbf{b}\right),
+
+    where :math:`\Phi(\mathbf{X}, \mathcal{N}_i)` denotes a
+    :class:`~torch.nn.conv.MessagePassing` layer.
+
+    Args:
+        in_channels (int): Size of each input sample.
+        phi (MessagePassing, optional): The message passing module
+            :math:`\Phi`. If set to :obj:`None`, will use a
+            :class:`~torch_geometric.nn.conv.GCNConv` layer as default.
+            (default: :obj:`None`)
+        num_iters (int, optional): The number of times the anti-symmetric deep
+            graph network operator is called. (default: :obj:`1`)
+        epsilon (float, optional): The discretization step size
+            :math:`\epsilon`. (default: :obj:`0.1`)
+        gamma (float, optional): The strength of the diffusion :math:`\gamma`.
+            It regulates the stability of the method. (default: :obj:`0.1`)
+        act (str, optional): The non-linear activation function :math:`\sigma`,
+            *e.g.*, :obj:`"tanh"` or :obj:`"relu"`. (default: :class:`"tanh"`)
+        act_kwargs (Dict[str, Any], optional): Arguments passed to the
+            respective activation function defined by :obj:`act`.
+            (default: :obj:`None`)
+        bias (bool, optional): If set to :obj:`False`, the layer will not learn
+            an additive bias. (default: :obj:`True`)
+
+    Shapes:
+        - **input:**
+          node features :math:`(|\mathcal{V}|, F_{in})`,
+          edge indices :math:`(2, |\mathcal{E}|)`,
+          edge weights :math:`(|\mathcal{E}|)` *(optional)*
+        - **output:** node features :math:`(|\mathcal{V}|, F_{in})`
+    """
+
+    def __init__(
+        self,
+        in_channels: int,
+        phi: Optional[MessagePassing] = None,
+        num_iters: int = 1,
+        epsilon: float = 0.1,
+        gamma: float = 0.1,
+        act: Union[str, Callable, None] = "tanh",
+        act_kwargs: Optional[Dict[str, Any]] = None,
+        bias: bool = True,
+    ):
+        super().__init__()
+
+        self.in_channels = in_channels
+        self.num_iters = num_iters
+        self.gamma = gamma
+        self.epsilon = epsilon
+        self.act = activation_resolver(act, **(act_kwargs or {}))
+
+        if phi is None:
+            phi = GCNConv(in_channels, in_channels, bias=False)
+
+        self.W = Parameter(torch.empty(in_channels, in_channels))
+        self.register_buffer("eye", torch.eye(in_channels))
+        self.phi = phi
+
+        if bias:
+            self.bias = Parameter(torch.empty(in_channels))
+        else:
+            self.register_parameter("bias", None)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        r"""Resets all learnable parameters of the module."""
+        torch.nn.init.kaiming_uniform_(self.W, a=math.sqrt(5))
+        self.phi.reset_parameters()
+        zeros(self.bias)
+
+    def forward(self, x: Tensor, edge_index: Adj, *args, **kwargs) -> Tensor:
+        r"""Runs the forward pass of the module."""
+        antisymmetric_W = self.W - self.W.t() - self.gamma * self.eye
+
+        for _ in range(self.num_iters):
+            h = self.phi(x, edge_index, *args, **kwargs)
+            h = x @ antisymmetric_W.t() + h
+
+            if self.bias is not None:
+                h += self.bias
+
+            if self.act is not None:
+                h = self.act(h)
+
+            x = x + self.epsilon * h
+
+        return x
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}("
+            f"{self.in_channels}, "
+            f"phi={self.phi}, "
+            f"num_iters={self.num_iters}, "
+            f"epsilon={self.epsilon}, "
+            f"gamma={self.gamma})"
+        )
+
+
+if __name__ == "__main__":
+    node_features = torch.load("node_features.pt")
+    edge_index = torch.load("edge_index.pt")
+
+    # Model instantiation and forward pass
+    model = AntiSymmetricConv(in_channels=node_features.size(-1))
+    output = model(node_features, edge_index)
+
+    # Save output to a file
+    torch.save(output, "gt_output.pt")

rdagent/components/coder/model_coder/model.py

@@ -0,0 +1,163 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.components.coder.model_coder.conf import MODEL_COSTEER_SETTINGS
+from rdagent.core.experiment import Experiment, FBWorkspace
+from rdagent.core.utils import cache_with_pickle
+from rdagent.oai.llm_utils import md5_hash
+from rdagent.utils.env import KGDockerEnv, QlibCondaConf, QlibCondaEnv, QTDockerEnv
+
+
+class ModelTask(CoSTEERTask):
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        architecture: str,
+        *args,
+        hyperparameters: Dict[str, str],
+        training_hyperparameters: Dict[str, str],
+        formulation: str = None,
+        variables: Dict[str, str] = None,
+        model_type: Optional[str] = None,
+        **kwargs,
+    ) -> None:
+        self.formulation: str = formulation
+        self.architecture: str = architecture
+        self.variables: str = variables
+        self.hyperparameters: str = hyperparameters
+        self.training_hyperparameters: str = training_hyperparameters
+        self.model_type: str = (
+            model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
+        )
+        super().__init__(name=name, description=description, *args, **kwargs)
+
+    def get_task_information(self):
+        task_desc = f"""name: {self.name}
+description: {self.description}
+"""
+        task_desc += f"formulation: {self.formulation}\n" if self.formulation else ""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"variables: {self.variables}\n" if self.variables else ""
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"training_hyperparameters: {self.training_hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc
+
+    def get_task_brief_information(self):
+        task_desc = f"""name: {self.name}
+description: {self.description}
+"""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"training_hyperparameters: {self.training_hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc
+
+    @staticmethod
+    def from_dict(dict):
+        return ModelTask(**dict)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.name}>"
+
+
+class ModelFBWorkspace(FBWorkspace):
+    """
+    It is a Pytorch model implementation task;
+    All the things are placed in a folder.
+
+    Folder
+    - data source and documents prepared by `prepare`
+        - Please note that new data may be passed in dynamically in `execute`
+    - code (file `model.py` ) injected by `inject_code`
+        - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure
+            - `model_cls` is a instance of `torch.nn.Module`;
+
+    We support two ways of interface:
+        (version 1) for qlib we'll make a script to import the model in the implementation in file `model.py` after setting the cwd into the directory
+            - from model import model_cls
+            - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
+            - And then verify the model.
+
+        (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory
+    """
+
+    def hash_func(
+        self,
+        batch_size: int = 8,
+        num_features: int = 10,
+        num_timesteps: int = 4,
+        num_edges: int = 20,
+        input_value: float = 1.0,
+        param_init_value: float = 1.0,
+    ) -> str:
+        target_file_name = f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}"
+        for code_file_name in sorted(list(self.file_dict.keys())):
+            target_file_name = f"{target_file_name}_{self.file_dict[code_file_name]}"
+        return md5_hash(target_file_name)
+
+    @cache_with_pickle(hash_func)
+    def execute(
+        self,
+        batch_size: int = 8,
+        num_features: int = 10,
+        num_timesteps: int = 4,
+        num_edges: int = 20,
+        input_value: float = 1.0,
+        param_init_value: float = 1.0,
+    ):
+        self.before_execute()
+        try:
+            if self.target_task.version == 1:
+                if MODEL_COSTEER_SETTINGS.env_type == "docker":
+                    qtde = QTDockerEnv()
+                elif MODEL_COSTEER_SETTINGS.env_type == "conda":
+                    qtde = QlibCondaEnv(conf=QlibCondaConf())
+                else:
+                    raise ValueError(f"Unknown env_type: {MODEL_COSTEER_SETTINGS.env_type}")
+            else:
+                qtde = KGDockerEnv()
+            qtde.prepare()
+
+            if self.target_task.version == 1:
+                dump_code = f"""
+MODEL_TYPE = "{self.target_task.model_type}"
+BATCH_SIZE = {batch_size}
+NUM_FEATURES = {num_features}
+NUM_TIMESTEPS = {num_timesteps}
+NUM_EDGES = {num_edges}
+INPUT_VALUE = {input_value}
+PARAM_INIT_VALUE = {param_init_value}
+{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()}
+"""
+            elif self.target_task.version == 2:
+                dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text()
+
+            log, results = qtde.dump_python_code_run_and_get_results(
+                code=dump_code,
+                dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"],
+                local_path=str(self.workspace_path),
+                env={},
+                code_dump_file_py_name="model_test",
+            )
+            if len(results) == 0:
+                raise RuntimeError(f"Error in running the model code: {log}")
+            [execution_feedback_str, execution_model_output] = results
+
+        except Exception as e:
+            execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
+            execution_model_output = None
+
+        if len(execution_feedback_str) > 2000:
+            execution_feedback_str = (
+                execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:]
+            )
+        return execution_feedback_str, execution_model_output
+
+
+ModelExperiment = Experiment

rdagent/components/coder/model_coder/model_execute_template_v1.txt

@@ -0,0 +1,44 @@
+# MODEL_TYPE = "Tabular"
+# BATCH_SIZE = 32
+# NUM_FEATURES = 10
+# NUM_TIMESTEPS = 4
+# NUM_EDGES = 20
+# INPUT_VALUE = 1.0
+# PARAM_INIT_VALUE = 1.0
+
+import pickle
+
+import torch
+from model import model_cls
+
+if MODEL_TYPE == "Tabular":
+    input_shape = (BATCH_SIZE, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[1])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "TimeSeries":
+    input_shape = (BATCH_SIZE, NUM_TIMESTEPS, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[2], num_timesteps=input_shape[1])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "Graph":
+    node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES)
+    edge_index = torch.randint(0, BATCH_SIZE, (2, NUM_EDGES))
+    m = model_cls(num_features=NUM_FEATURES)
+    data = (node_feature, edge_index)
+else:
+    raise ValueError(f"Unsupported model type: {MODEL_TYPE}")
+
+# Initialize all parameters of `m` to `param_init_value`
+for _, param in m.named_parameters():
+    param.data.fill_(PARAM_INIT_VALUE)
+
+# Execute the model
+if MODEL_TYPE == "Graph":
+    out = m(*data)
+else:
+    out = m(data)
+
+execution_model_output = out.cpu().detach().numpy()
+execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/model_execute_template_v2.txt

@@ -0,0 +1,24 @@
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+import torch
+from model import fit, predict
+
+train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+train_y = pd.Series(np.random.randint(0, 2, 8))
+valid_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+valid_y = pd.Series(np.random.randint(0, 2, 8))
+
+model = fit(train_X, train_y, valid_X, valid_y)
+execution_model_output = predict(model, valid_X)
+
+if isinstance(execution_model_output, torch.Tensor):
+    execution_model_output = execution_model_output.cpu().detach().numpy()
+
+
+execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/one_shot/__init__.py

@@ -0,0 +1,35 @@
+import re
+from pathlib import Path
+
+from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
+from rdagent.core.developer import Developer
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class ModelCodeWriter(Developer[ModelExperiment]):
+    def develop(self, exp: ModelExperiment) -> ModelExperiment:
+        mti_l = []
+        for t in exp.sub_tasks:
+            mti = ModelFBWorkspace(t)
+            mti.prepare()
+
+            user_prompt = T(".prompts:code_implement_user").r(
+                name=t.name,
+                description=t.description,
+                formulation=t.formulation,
+                variables=t.variables,
+            )
+            system_prompt = T(".prompts:code_implement_sys").r()
+
+            resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt)
+
+            # Extract the code part from the response
+            match = re.search(r".*```[Pp]ython\n(.*)\n```.*", resp, re.DOTALL)
+            code = match.group(1)
+            mti.inject_files(**{"model.py": code})
+            mti_l.append(mti)
+        exp.sub_workspace_list = mti_l
+        return exp

rdagent/components/coder/model_coder/one_shot/prompt.yaml

@@ -0,0 +1,27 @@
+
+
+code_implement_sys: |-
+  You are an assistant whose job is to answer user's question.
+code_implement_user: |-
+  With the following given information, write a python code using pytorch and torch_geometric to implement the model.
+  This model is in the graph learning field, only have one layer.
+  The input will be node_feature [num_nodes, dim_feature] and edge_index [2, num_edges]  (It would be the input of the forward model)
+  There is not edge attribute or edge weight as input. The model should detect the node_feature and edge_index shape, if there is Linear transformation layer in the model, the input and output shape should be consistent. The in_channels is the dimension of the node features.
+  Implement the model forward function based on the following information:model formula information.
+  1. model name:{{name}}
+  2. model description:{{description}}
+  3. model formulation:{{formulation}}
+  4. model variables:{{variables}}.
+  You must complete the forward function as far as you can do.
+  Execution Your implemented code will be executed in the follow way:
+  The the implemented code will be placed in a file like [uuid]/model.py
+  We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
+  - from model import model_cls (So you must have a variable named `model_cls` in the file)
+    - So your implemented code could follow the following pattern
+      ```Python
+      class XXXLayer(torch.nn.Module):
+          ...
+      model_cls = XXXLayer
+      ```
+  - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
+  - And then verify the model by comparing the output tensors by feeding specific input tensor.

rdagent/components/coder/model_coder/prompts.yaml

@@ -0,0 +1,156 @@
+extract_model_formulation_system: |-
+    offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like 
+    {
+        "model_name (The name of the model)": {
+            "description": "A detailed description of the model",
+            "formulation": "A LaTeX formula representing the model's formulation",
+            "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
+            "variables": {
+                "\\hat{y}_u": "The predicted output for node u",
+                "variable_name_2": "Description of variable 2",
+                "variable_name_3": "Description of variable 3"
+            },
+            "hyperparameters": {
+                "hyperparameter_name_1": "value of hyperparameter 1",
+                "hyperparameter_name_2": "value of hyperparameter 2",
+                "hyperparameter_name_3": "value of hyperparameter 3"
+            },
+            "training_hyperparameters" {  # All values are for reference; you can set them yourself
+                "n_epochs": "100",
+                "lr": "1e-3",
+                "early_stop": 10,
+                "batch_size": 256,
+                "weight_decay": 1e-4,
+            }
+            "model_type": "Tabular or TimeSeries or Graph or XGBoost"  # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost"
+        }
+    }
+    such format content should be begin with ```json and end with ``` and the content should be in json format.
+
+evolving_strategy_model_coder:
+    system: |-
+        User is trying to implement some pytorch models in the following scenario:
+        {{ scenario }}
+        Your code is expected to align the scenario in any form which means The user needs to get the prediction of the model based on the input data.
+
+        To help you write the correct code, the user might provide multiple information that helps you write the correct code:
+        1. The user might provide you the correct code to similar models. Your should learn from these code to write the correct code.
+        2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the model output value. You should analyze the feedback and try to correct the latest code.
+        3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code.
+
+        Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
+
+        {% if current_code is not none %}
+        User has write some code before. You should write the new code based on this code. Here is the latest code:
+        ```python
+        {{ current_code }}
+        ```
+        Your code should be very similar to the former code which means your code should be ninety more percent same as the former code! You should not modify the right part of the code.
+        {% else %}
+        User has not write any code before. You should write the new code from scratch.
+        {% endif %}
+
+        {% if queried_former_failed_knowledge|length != 0 %}
+        --------------Your former latest attempt:---------------
+        =====Code to the former implementation=====
+        {{ queried_former_failed_knowledge[-1].implementation.all_codes }}
+        =====Feedback to the former implementation=====
+        {{ queried_former_failed_knowledge[-1].feedback }}
+        {% endif %}
+        
+        Please response the code in the following json format. Here is an example structure for the JSON output:
+        {
+            "code": "The Python code as a string."
+        }
+
+    user: |-
+        --------------Target model information:---------------
+        {{ model_information_str }}
+
+        {% if queried_similar_successful_knowledge|length != 0 %}
+        --------------Correct code to similar models:---------------
+        {% for similar_successful_knowledge in queried_similar_successful_knowledge %}
+        =====Model {{loop.index}}:=====
+        {{ similar_successful_knowledge.target_task.get_task_information() }}
+        =====Code:=====
+        {{ similar_successful_knowledge.implementation.all_codes }}
+        {% endfor %}
+        {% endif %}
+
+        {% if queried_former_failed_knowledge|length != 0 %}
+        --------------Former failed code:---------------
+        {% for former_failed_knowledge in queried_former_failed_knowledge %}
+        =====Code to implementation {{ loop.index }}=====
+        {{ former_failed_knowledge.implementation.all_codes }}
+        =====Feedback to implementation {{ loop.index }}=====
+        {{ former_failed_knowledge.feedback }}
+        {% endfor %}
+        {% endif %}
+
+evaluator_code_feedback:
+    system: |-
+        User is trying to implement some models in the following scenario:
+        {{ scenario }}
+        User will provide you the information of the model.
+
+        Your job is to check whether user's code is align with the model information and the scenario.
+        The user will provide the source python code and the execution error message if execution failed.
+        The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic.
+
+        User has also compared the output generated by the user's code and the ground truth code. The user will provide you some analysis results comparing two output. You may find some error in the code which caused the difference between the two output.
+
+        If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
+        If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct to the description and to the scenario.
+
+        Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX".
+
+        You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
+
+        You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output:
+        critic 1: The critic message to critic 1
+        critic 2: The critic message to critic 2
+    
+    user: |-
+        --------------Model information:---------------
+        {{ model_information }}
+        --------------Python code:---------------
+        {{ code }}
+        --------------Execution feedback:---------------
+        {{ model_execution_feedback }}
+        {% if model_value_feedback is not none %}
+        --------------Model value feedback:---------------
+        {{ model_value_feedback }}
+        {% endif %}
+        {% if gt_code is not none %}
+        --------------Ground truth Python code:---------------
+        {{ gt_code }}
+        {% endif %}
+
+
+evaluator_final_feedback:
+    system: |-
+        User is trying to implement a model in the following scenario:
+        {{ scenario }}
+        User has finished evaluation and got some feedback from the evaluator.
+        The evaluator run the code and get the output and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the scenario and model description to give a final decision about the evaluation result. The final decision concludes whether the model is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False.
+
+        The implementation final decision is considered in the following logic:
+        1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
+        2. If no ground truth value is not provided, the implementation is considered correct if the code execution is successful and the code feedback is align with the scenario and model description.
+
+        Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+        {
+            "final_decision": True,
+            "final_feedback": "The final feedback message",
+        }
+    user: |-
+        --------------Model information:---------------
+        {{ model_information }}
+        --------------Model Execution feedback:---------------
+        {{ model_execution_feedback }}
+        --------------Model shape feedback:---------------
+        {{ model_shape_feedback }}
+        --------------Model Code feedback:---------------
+        {{ model_code_feedback }}
+        --------------Model value feedback:---------------
+        {{ model_value_feedback }}

rdagent/components/coder/model_coder/task_loader.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import json
+import re
+
+from pydantic import BaseModel, Field
+
+from rdagent.components.coder.model_coder.model import ModelTask
+from rdagent.components.document_reader.document_reader import (
+    load_and_process_pdfs_by_langchain,
+)
+from rdagent.components.loader.task_loader import ModelTaskLoader
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.workflow import wait_retry
+
+
+def extract_model_from_doc(doc_content: str) -> dict:
+    """
+    Extract model information from document content.
+
+    Parameters
+    ----------
+    doc_content : str
+        Document content.
+
+    Returns
+    -------
+    dict
+        {model_name: dict{description, formulation, variables}}
+    """
+    session = APIBackend().build_chat_session(
+        session_system_prompt=T(".prompts:extract_model_formulation_system").r(),
+    )
+    current_user_prompt = doc_content
+
+    # Extract model information from document content.
+    model_dict = {}
+
+    for _ in range(10):
+        # try to extract model information from the document content, retry at most 10 times.
+        extract_result_resp = session.build_chat_completion(
+            user_prompt=current_user_prompt,
+            json_mode=False,
+        )
+        re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
+        ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
+        try:
+            ret_dict = json.loads(ret_json_str)
+            parse_success = bool(isinstance(ret_dict, dict))
+        except json.JSONDecodeError:
+            parse_success = False
+        if ret_json_str is None or not parse_success:
+            current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
+        else:
+            for name, formulation_and_description in ret_dict.items():
+                if name not in model_dict:
+                    model_dict[name] = formulation_and_description
+            if len(model_dict) == 0:
+                current_user_prompt = "No model extracted. Please try again."
+            else:
+                break
+
+    logger.info(f"已经完成{len(model_dict)}个模型的提取")
+
+    return model_dict
+
+
+def merge_file_to_model_dict_to_model_dict(
+    file_to_model_dict: dict[str, dict],
+) -> dict:
+    model_dict = {}
+    for file_name in file_to_model_dict:
+        for model_name in file_to_model_dict[file_name]:
+            model_dict.setdefault(model_name, [])
+            model_dict[model_name].append(file_to_model_dict[file_name][model_name])
+
+    model_dict_simple_deduplication = {}
+    for model_name in model_dict:
+        if len(model_dict[model_name]) > 1:
+            model_dict_simple_deduplication[model_name] = max(
+                model_dict[model_name],
+                key=lambda x: len(x["formulation"]),
+            )
+        else:
+            model_dict_simple_deduplication[model_name] = model_dict[model_name][0]
+    return model_dict_simple_deduplication
+
+
+def extract_model_from_docs(docs_dict):
+    model_dict = {}
+    for doc_name, doc_content in docs_dict.items():
+        model_dict[doc_name] = extract_model_from_doc(doc_content)
+    return model_dict
+
+
+class ModelExperimentLoaderFromDict(ModelTaskLoader):
+    def load(self, model_dict: dict) -> QlibModelExperiment:
+        """Load data from a dict."""
+        task_l = []
+        for model_name, model_data in model_dict.items():
+            task = ModelTask(
+                name=model_name,
+                description=model_data["description"],
+                formulation=model_data["formulation"],
+                architecture=model_data["architecture"],
+                variables=model_data["variables"],
+                hyperparameters=model_data["hyperparameters"],
+                training_hyperparameters=model_data["training_hyperparameters"],
+                model_type=model_data["model_type"],
+            )
+            task_l.append(task)
+        return QlibModelExperiment(sub_tasks=task_l)
+
+
+class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader):
+    @wait_retry(retry_n=5)
+    def load(self, file_or_folder_path: str) -> QlibModelExperiment:
+        docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path)  # dict{file_path:content}
+        model_dict = extract_model_from_docs(
+            docs_dict
+        )  # dict{file_name: dict{model_name: dict{description, formulation, variables}}}
+        model_dict = merge_file_to_model_dict_to_model_dict(
+            model_dict
+        )  # dict {model_name: dict{description, formulation, variables}}
+        return ModelExperimentLoaderFromDict().load(model_dict)

rdagent/components/coder/rl/__init__.py

@@ -0,0 +1 @@
+from rdagent.components.coder.rl.costeer import RLCoSTEER

rdagent/components/coder/rl/costeer.py

@@ -0,0 +1,140 @@
+"""RL CoSTEER - Code generation component for RL post-training"""
+
+from typing import Generator
+
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.core.evolving_agent import EvolvingStrategy, EvoStep
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+
+class RLCoderCoSTEERSettings(CoSTEERSettings):
+    """RL Coder settings."""
+
+    pass
+
+
+class RLEvolvingStrategy(EvolvingStrategy):
+    """RL code generation strategy using LLM."""
+
+    def __init__(self, scen: Scenario, settings: CoSTEERSettings):
+        self.scen = scen
+        self.settings = settings
+
+    def evolve_iter(
+        self,
+        *,
+        evo: EvolvingItem,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        evolving_trace: list[EvoStep] = [],
+        **kwargs,
+    ) -> Generator[EvolvingItem, EvolvingItem, None]:
+        """Generate code for all tasks using LLM."""
+        for index, target_task in enumerate(evo.sub_tasks):
+            code = self._generate_code(target_task, evolving_trace)
+            if evo.sub_workspace_list[index] is None:
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code)
+
+        evo = yield evo
+        return
+
+    def _generate_code(self, task: Task, evolving_trace: list[EvoStep] = []) -> dict[str, str]:
+        """Generate RL training code using LLM."""
+        from rdagent.app.rl.conf import RL_RD_SETTING
+
+        # 获取上轮反馈
+        feedback = None
+        if evolving_trace:
+            last_step = evolving_trace[-1]
+            if hasattr(last_step, "feedback") and last_step.feedback:
+                feedback = str(last_step.feedback)
+
+        # 构造 prompt
+        system_prompt = T(".prompts:rl_coder.system").r()
+        user_prompt = T(".prompts:rl_coder.user").r(
+            task_description=task.description if hasattr(task, "description") else str(task),
+            base_model=RL_RD_SETTING.base_model or "",
+            benchmark=RL_RD_SETTING.benchmark or "",
+            hypothesis=str(task.name) if hasattr(task, "name") else "Train RL model",
+            feedback=feedback,
+        )
+
+        # 调用 LLM
+        session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
+        code = session.build_chat_completion(
+            user_prompt=user_prompt,
+            json_mode=False,
+            code_block_language="python",
+        )
+        logger.info(f"LLM generated code:\n{code[:200]}...")
+        return {"main.py": code}
+
+    def _mock_code(self) -> dict[str, str]:
+        """Fallback mock code."""
+        return {"main.py": """import gymnasium as gym
+from stable_baselines3 import PPO
+
+env = gym.make("CartPole-v1")
+model = PPO("MlpPolicy", env, verbose=1)
+model.learn(total_timesteps=1000)
+model.save("ppo_cartpole")
+print("Training completed!")
+"""}
+
+
+class RLCoderEvaluator:
+    """RL code evaluator (mock implementation)."""
+
+    def __init__(self, scen: Scenario) -> None:
+        self.scen = scen
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace | None,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+    ) -> CoSTEERSingleFeedback:
+        """Evaluate RL code. Currently returns mock success."""
+        # TODO: 实现真正的评估逻辑
+        return CoSTEERSingleFeedback(
+            execution="Mock: executed successfully",
+            return_checking=None,
+            code="Mock: code looks good",
+            final_decision=True,
+        )
+
+
+class RLCoSTEER(CoSTEER):
+    """RL CoSTEER - orchestrates code generation and evaluation."""
+
+    def __init__(self, scen: Scenario, *args, **kwargs) -> None:
+        settings = RLCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator([RLCoderEvaluator(scen=scen)], scen=scen)
+        es = RLEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            scen=scen,
+            max_loop=1,
+            stop_eval_chain_on_fail=False,
+            with_knowledge=False,
+            knowledge_self_gen=False,
+            **kwargs,
+        )

rdagent/components/coder/rl/prompts.yaml

@@ -0,0 +1,94 @@
+rl_coder:
+  system: |-
+    你是 RL post-training 专家，负责生成训练代码。
+
+    ## 运行环境
+    代码会被部署到 `$WORKSPACE/code/main.py` 并在该目录下执行。
+    以下环境变量已由框架设置，代码中用 `os.environ["..."]` 读取：
+    - `MODEL_PATH`: 基础模型绝对路径（只读）
+    - `DATA_PATH`: 训练数据目录绝对路径（只读）
+    - `OUTPUT_DIR`: 模型输出目录绝对路径（`$WORKSPACE/output/`）
+    - `GRADING_SERVER_URL`: 评测服务地址（训练完后系统自动提交，代码不需要调用）
+
+    ## 框架: trl (版本 0.27+)
+
+    ## 可用算法
+    - **GRPO**: 推荐，只需 reward function，不需要预构建偏好对
+    - **DPO**: 需要 (prompt, chosen, rejected) 偏好对
+
+    ## API 要点
+
+    ### GRPOTrainer
+    ```python
+    from trl import GRPOConfig, GRPOTrainer
+
+    trainer = GRPOTrainer(
+        model=MODEL_PATH,               # 模型路径
+        reward_funcs=reward_fn,          # reward 函数
+        args=GRPOConfig(
+            output_dir=OUTPUT_DIR,       # 输出目录
+            ...
+        ),
+        train_dataset=dataset,           # 必须有 "prompt" 列
+        processing_class=tokenizer,
+    )
+    ```
+
+    ### reward function 签名（重要！）
+    ```python
+    def reward_fn(completions, answer, **kwargs):
+        # completions: list[str] - 模型生成的回复
+        # answer: list[str] - 数据集中的 answer 列（自动传入）
+        # kwargs: 数据集其他列（如 question）
+        return [float(...) for ...]  # 返回 reward 列表
+    ```
+
+    ### GRPOConfig 关键参数
+    - `num_generations`: 每个 prompt 采样次数，必须 >= 2
+    - `max_completion_length`: 生成最大长度
+    - `per_device_train_batch_size`: 批次大小
+
+    ## 输出要求
+    - 生成完整的 `main.py`，可直接运行
+    - 路径全部通过 `os.environ` 获取，**不要硬编码路径**
+    - 数据从 `$DATA_PATH` 下的 jsonl 文件加载
+    - 模型保存到 `$OUTPUT_DIR`（可用子目录如 `$OUTPUT_DIR/v1`）
+
+    ## 评测机制
+    训练完成后，系统自动将 `$OUTPUT_DIR` 下最新的模型提交到 Grading Server。
+    - 有模型 → 自动评测，返回 score
+    - 为空 → 跳过评测
+    代码只需负责训练和保存模型，**不需要**自行调用评测 API。
+
+    ## 代码模板
+    ```python
+    import os
+    MODEL_PATH = os.environ["MODEL_PATH"]
+    DATA_PATH = os.environ["DATA_PATH"]
+    OUTPUT_DIR = os.environ["OUTPUT_DIR"]
+    # ... 训练逻辑 ...
+    trainer.save_model(OUTPUT_DIR)
+    ```
+
+  user: |-
+    ## 任务
+    {{ task_description }}
+
+    ## 基础模型
+    - 名称: {{ base_model }}
+    - 路径: 通过 $MODEL_PATH 环境变量获取
+
+    ## 训练数据
+    - 数据集: {{ benchmark }}
+    - 路径: 通过 $DATA_PATH 环境变量获取
+
+    ## 假设
+    {{ hypothesis }}
+
+    {% if feedback %}
+    ## 上轮反馈
+    {{ feedback }}
+    {% endif %}
+
+    请根据数据格式和假设，生成完整的训练代码（main.py）。
+    注意：路径全部通过 os.environ 获取，不要硬编码。