chore: import upstream snapshot with attribution

rdagent/components/coder/data_science/conf.py

@@ -0,0 +1,87 @@
+from typing import Literal
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.utils.env import (
+    CondaConf,
+    DockerEnv,
+    DSDockerConf,
+    Env,
+    LocalEnv,
+    MLEBDockerConf,
+    MLECondaConf,
+)
+
+
+class DSCoderCoSTEERSettings(CoSTEERSettings):
+    """Data Science CoSTEER settings"""
+
+    class Config:
+        env_prefix = "DS_Coder_CoSTEER_"
+
+    max_seconds_multiplier: int = 4
+    env_type: str = "docker"
+    # TODO: extract a function for env and conf.
+    extra_evaluator: list[str] = []
+    """Extra evaluators to use"""
+
+    extra_eval: list[str] = []
+    """
+    Extra evaluators
+
+    The evaluator follows the following assumptions:
+    - It runs after previous evaluator (So the running results are already there)
+
+    It is not a complete feature due to it is only implemented in DS Pipeline & Coder.
+
+    TODO: The complete version should be implemented in the CoSTEERSettings.
+    """
+
+
+def get_ds_env(
+    conf_type: Literal["kaggle", "mlebench"] = "kaggle",
+    extra_volumes: dict = {},
+    running_timeout_period: int | None = DS_RD_SETTING.debug_timeout,
+    enable_cache: bool | None = None,
+) -> Env:
+    """
+    Retrieve the appropriate environment configuration based on the env_type setting.
+
+    Returns:
+        Env: An instance of the environment configured either as DockerEnv or LocalEnv.
+
+    Raises:
+        ValueError: If the env_type is not recognized.
+    """
+    conf = DSCoderCoSTEERSettings()
+    assert conf_type in ["kaggle", "mlebench"], f"Unknown conf_type: {conf_type}"
+
+    if conf.env_type == "docker":
+        env_conf = DSDockerConf() if conf_type == "kaggle" else MLEBDockerConf()
+        env = DockerEnv(conf=env_conf)
+    elif conf.env_type == "conda":
+        env = LocalEnv(
+            conf=(
+                CondaConf(conda_env_name=conf_type) if conf_type == "kaggle" else MLECondaConf(conda_env_name=conf_type)
+            )
+        )
+    else:
+        raise ValueError(f"Unknown env type: {conf.env_type}")
+    env.conf.extra_volumes = extra_volumes.copy()
+    env.conf.running_timeout_period = running_timeout_period
+    if enable_cache is not None:
+        env.conf.enable_cache = enable_cache
+    env.prepare()
+    return env
+
+
+def get_clear_ws_cmd(stage: Literal["before_training", "before_inference"] = "before_training") -> str:
+    """
+    Clean the files in workspace to a specific stage
+    """
+    assert stage in ["before_training", "before_inference"], f"Unknown stage: {stage}"
+    if DS_RD_SETTING.enable_model_dump and stage == "before_training":
+        cmd = "rm -r submission.csv scores.csv models trace.log"
+    else:
+        cmd = "rm submission.csv scores.csv trace.log"
+    return cmd

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -0,0 +1,164 @@
+"""
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.ensemble.eval import EnsembleCoSTEEREvaluator
+from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: EnsembleTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # Get task information for knowledge querying
+        ensemble_information_str = target_task.get_task_information()
+
+        # Query knowledge
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[ensemble_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[ensemble_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("ensemble.py") != workspace.file_dict.get("ensemble.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # Generate code with knowledge integration
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        system_prompt = T(".prompts:ensemble_coder.system").r(
+            task_desc=ensemble_information_str,
+            competition_info=competition_info,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=(
+                queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None
+            ),
+            all_code=workspace.all_codes,
+            out_spec=PythonAgentOut.get_spec(),
+        )
+
+        if DS_RD_SETTING.spec_enabled:
+            code_spec = workspace.file_dict["spec/ensemble.md"]
+        else:
+            test_code = (
+                Environment(undefined=StrictUndefined)
+                .from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text())
+                .render(
+                    model_names=[
+                        fn[:-3] for fn in workspace.file_dict.keys() if fn.startswith("model_") and "test" not in fn
+                    ],
+                    metric_name=self.scen.metric_name,
+                )
+            )
+            code_spec = T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.Ensemble").r(), test_code=test_code
+            )
+        user_prompt = T(".prompts:ensemble_coder.user").r(
+            code_spec=code_spec,
+            latest_code=workspace.file_dict.get("ensemble.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            ensemble_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if ensemble_code != workspace.file_dict.get("ensemble.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new ensemble code.")
+
+        return {
+            "ensemble.py": ensemble_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class EnsembleCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(EnsembleCoSTEEREvaluator(scen=scen), scen=scen)
+        es = EnsembleMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/ensemble/conf.py

@@ -0,0 +1,2 @@
+# Configuration file for ensemble component
+# Currently empty as no specific configuration is needed

rdagent/components/coder/data_science/ensemble/eval.py

@@ -0,0 +1,100 @@
+import json
+import re
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+EnsembleEvalFeedback = CoSTEERSingleFeedback
+
+
+class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> EnsembleEvalFeedback:
+
+        target_task_information = target_task.get_task_information()
+        metric_name = self.scen.metric_name
+
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return EnsembleEvalFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        fname = "test/ensemble_test.txt"
+        test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()
+        test_code = (
+            Environment(undefined=StrictUndefined)
+            .from_string(test_code)
+            .render(
+                model_names=[
+                    fn[:-3] for fn in implementation.file_dict.keys() if fn.startswith("model_") and "test" not in fn
+                ],
+                metric_name=metric_name,
+            )
+        )
+
+        implementation.inject_files(**{fname: test_code})
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.exit_code
+
+        stdout += f"\nNOTE: the above scripts run with return code {ret_code}"
+
+        if "main.py" in implementation.file_dict and ret_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        system_prompt = T(".prompts:ensemble_eval.system").r(
+            task_desc=target_task_information,
+            test_code=test_code,
+            metric_name=metric_name,
+            code=implementation.file_dict["ensemble.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:ensemble_eval.user").r(
+            stdout=stdout,
+            workflow_stdout=workflow_stdout,
+        )
+        efb = build_cls_from_json_with_retry(
+            EnsembleEvalFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=EnsembleEvalFeedback.val_and_update_init_dict,
+        )
+        efb.final_decision = efb.final_decision and ret_code == 0
+        return efb

rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt

@@ -0,0 +1,137 @@
+"""
+Tests for `ensemble_workflow` in ensemble.py
+
+A qualified ensemble_workflow implementation should:
+- Return predictions
+- Have correct shapes for inputs and outputs
+- Use validation data appropriately
+- Generate a scores.csv file
+"""
+
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from sklearn.model_selection import train_test_split
+import torch
+import tensorflow as tf
+from load_data import load_data
+from feature import feat_eng
+from ensemble import ensemble_workflow
+
+def print_preds_info(model_name, data_type, preds):
+    if preds is None:
+        print(f"Model {model_name} {data_type} predictions: None")
+    else:
+        print(f"Model {model_name} {data_type} predictions shape: {preds.shape}")
+
+        print("Showing a preview of the predictions (first few entries only):")
+        if isinstance(preds, (pd.DataFrame, pd.Series)):
+            print(preds.head())
+        elif isinstance(preds, (np.ndarray, torch.Tensor, tf.Tensor)):
+            print(preds[:2])
+        elif isinstance(preds, list):
+            print(pd.DataFrame(preds[:5]))
+        else:
+            print(f"Unknown prediction type: {type(preds)}")
+
+def get_length(data):
+    return data.shape[0] if hasattr(data, 'shape') else len(data)
+
+X, y, test_X, test_ids = load_data()
+X, y, test_X = feat_eng(X, y, test_X)
+train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# Print the types of train_y and val_y
+print(f"train_y type: {type(train_y)}, val_y type: {type(val_y)}")
+
+test_preds_dict = {}
+val_preds_dict = {}
+{% for mn in model_names %}
+from {{mn}} import model_workflow as {{mn}}_workflow
+val_preds_dict["{{mn}}"], test_preds_dict["{{mn}}"], _ = {{mn}}_workflow(
+    X=train_X,
+    y=train_y,
+    val_X=val_X,
+    val_y=val_y,
+    test_X=test_X
+)
+
+print_preds_info("{{mn}}", "test", test_preds_dict["{{mn}}"])
+{% endfor %}
+
+for key in val_preds_dict.keys():
+    if val_preds_dict[key] is None: 
+        print(f"Model {key} validation predictions (val_preds_dict[key]) is None.")
+    elif isinstance(val_preds_dict[key], list):
+        print(f"Model {key} validation predictions (val_preds_dict[key]) (list type) length: {len(val_preds_dict[key])}")
+    else:
+        print(f"Model {key} validation predictions (val_preds_dict[key]) shape: {val_preds_dict[key].shape}")
+
+    if test_preds_dict[key] is None: 
+        print(f"Model {key} test predictions (test_preds_dict[key]) is None.")
+    elif isinstance(test_preds_dict[key], list):
+        print(f"Model {key} test predictions (test_preds_dict[key]) (list type) length: {len(test_preds_dict[key])}")
+    else:
+        print(f"Model {key} test predictions (test_preds_dict[key]) shape: {test_preds_dict[key].shape}")
+
+print(f"val_y.shape: {val_y.shape}" if not isinstance(val_y, list) else f"val_y(list)'s length: {len(val_y)}")
+
+import sys
+import reprlib
+def debug_info_print(func):
+    aRepr = reprlib.Repr()
+    aRepr.maxother=300
+    def wrapper(*args, **kwargs):
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == func.__code__:
+                print("\n" + "="*20 + "Running ensemble code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+
+
+# Run ensemble
+final_pred = debug_info_print(ensemble_workflow)(test_preds_dict, val_preds_dict, val_y)
+
+print_preds_info("ensemble", "test", final_pred)
+
+# Check type
+pred_type = type(next(iter(test_preds_dict.values())))
+assert isinstance(final_pred, pred_type), (
+    f"Type mismatch: 'final_pred' is of type {type(final_pred)}, but expected {pred_type} "
+)
+
+# Check shape
+if isinstance(final_pred, (list, np.ndarray, pd.DataFrame, torch.Tensor, tf.Tensor)):
+    assert get_length(final_pred) == get_length(test_X), (
+        f"Wrong output sample size: get_length(final_pred)={get_length(final_pred)} "
+        f"vs. get_length(test_X)={get_length(test_X)}"
+    )
+
+# check scores.csv
+assert Path("scores.csv").exists(), "scores.csv is not generated"
+score_df = pd.read_csv("scores.csv", index_col=0)
+model_set_in_scores = set(score_df.index)
+
+assert model_set_in_scores == set({{model_names}}).union({"ensemble"}), (
+    f"The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {{model_names}} + ['ensemble']\nscore_df is:\n{score_df}"
+)
+assert score_df.index.is_unique, "The scores dataframe has duplicate model names."
+assert score_df.columns.tolist() == ["{{metric_name}}"], f"The column names of the scores dataframe should be ['{{metric_name}}'], but is '{score_df.columns.tolist()}'"
+
+# Check for NaN values in score_df
+assert not score_df.isnull().values.any(), (
+    f"The scores dataframe contains NaN values at the following locations:\n{score_df[score_df.isnull().any(axis=1)]}"
+)
+
+
+print("Ensemble test end.")

rdagent/components/coder/data_science/ensemble/exp.py

@@ -0,0 +1,13 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.core.utils import cache_with_pickle
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class EnsembleTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/ensemble/prompts.yaml

@@ -0,0 +1,124 @@
+ensemble_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    Currently, you are working on model ensemble implementation. Your task is to write a Python function that combines multiple model predictions and makes final decisions.
+
+    Your specific task as follows:
+    {{ task_desc }}
+
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict["ensemble.py"] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict["ensemble.py"] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. The function's code is associated with several other functions including a data loader, feature engineering, and model training. all codes are as follows:
+    {{ all_code }}
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %}
+
+
+ensemble_eval:
+  system: |-
+    You are a data scientist responsible for evaluating ensemble implementation code generation.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## Ensemble Code
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The ensemble code is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+    You will analyze the execution results based on the test output provided.
+    
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The ensemble code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the ensemble test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+
+    The metric used for scoring the predictions:
+    **{{ metric_name }}**
+
+    ## Evaluation Criteria
+    - You will be given the standard output (`stdout`) from the ensemble test and, if applicable, the workflow test.
+    - Code should have no try-except blocks because they can hide errors.
+    - Check whether the code implement the scoring process using the given metric.
+    - The stdout includes the local variable values from the ensemble code execution. Check whether the validation score is calculated correctly.
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the ensemble executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Detail the checks performed on the ensemble results, including shape and value validation.",
+        "code": "Assess code quality, readability, and adherence to specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+  user: |-    
+    --------- Ensemble test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/ensemble/test.py

@@ -0,0 +1,58 @@
+"""
+Helper functions for testing the ensemble coder(CoSTEER-based) component.
+"""
+
+import sys
+from pathlib import Path
+
+from rdagent.components.coder.data_science.ensemble import EnsembleCoSTEER
+from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+# Add the competition folder to path
+COMPETITION_PATH = (
+    Path(__file__).parent.parent.parent.parent.parent
+    / "scenarios"
+    / "kaggle"
+    / "tpl_ex"
+    / "aerial-cactus-identification"
+)
+sys.path.append(str(COMPETITION_PATH))
+
+EnsembleExperiment = DSExperiment
+
+
+def load_ensemble_spec():
+    spec_path = COMPETITION_PATH / "spec" / "ensemble.md"
+    with open(spec_path, "r") as f:
+        return f.read()
+
+
+def develop_one_competition(competition: str):
+    # Initialize scenario and coder
+    scen = KaggleScen(competition=competition)
+    ensemble_coder = EnsembleCoSTEER(scen)
+    # Load ensemble specification
+    ensemble_spec = load_ensemble_spec()
+
+    # Create the ensemble task with actual data context and specification
+    task = EnsembleTask(
+        name="EnsembleTask",
+        description="""
+        Implement ensemble and decision making for model predictions.
+        """,
+    )
+
+    exp = EnsembleExperiment(pending_tasks_list=[task])
+
+    # Injecting the corresponding specification
+    exp.experiment_workspace.inject_files(**{"spec/ensemble.md": ensemble_spec})
+
+    # Develop the experiment
+    exp = ensemble_coder.develop(exp)
+    return exp
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")

rdagent/components/coder/data_science/feature/__init__.py

@@ -0,0 +1,140 @@
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.feature.eval import FeatureCoSTEEREvaluator
+from rdagent.components.coder.data_science.feature.exp import FeatureTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: FeatureTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # return a workspace with "load_data.py", "spec/load_data.md" inside
+        # assign the implemented code to the new workspace.
+        feature_information_str = target_task.get_task_information()
+
+        # 1. query
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[feature_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[feature_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("feature.py") != workspace.file_dict.get("feature.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 2. code
+        system_prompt = T(".prompts:feature_coder.system").r(
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
+            task_desc=feature_information_str,
+            data_loader_code=workspace.file_dict.get("load_data.py"),
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+        )
+        code_spec = (
+            workspace.file_dict["spec/feature.md"]
+            if DS_RD_SETTING.spec_enabled
+            else T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.FeatureEng").r(),
+                test_code=(DIRNAME / "eval_tests" / "feature_test.txt").read_text(),
+            )
+        )
+        user_prompt = T(".prompts:feature_coder.user").r(
+            code_spec=code_spec,
+            latest_code=workspace.file_dict.get("feature.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            feature_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if feature_code != workspace.file_dict.get("feature.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new feature code.")
+
+        return {
+            "feature.py": feature_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class FeatureCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            FeatureCoSTEEREvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = FeatureMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/feature/eval.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+FeatureEvalFeedback = CoSTEERSingleFeedback
+
+
+class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> FeatureEvalFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return FeatureEvalFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        # TODO: do we need to clean the generated temporary content?
+        fname = "test/feature_test.py"
+        test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text()
+        implementation.inject_files(**{fname: test_code})
+
+        result = implementation.run(env=env, entry=f"python {fname}")
+
+        if "main.py" in implementation.file_dict and result.exit_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        system_prompt = T(".prompts:feature_eval.system").r(
+            task_desc=target_task.get_task_information(),
+            test_code=test_code,
+            code=implementation.file_dict["feature.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:feature_eval.user").r(
+            stdout=result.stdout,
+            workflow_stdout=workflow_stdout,
+        )
+
+        fb = build_cls_from_json_with_retry(
+            FeatureEvalFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=FeatureEvalFeedback.val_and_update_init_dict,
+        )
+        fb.final_decision = fb.final_decision and result.exit_code == 0
+
+        return fb

rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt

@@ -0,0 +1,114 @@
+"""
+Tests for `feat_eng` in feature.py
+"""
+
+
+from copy import deepcopy
+import sys
+import numpy as np
+import pandas as pd
+from feature import feat_eng
+from load_data import load_data
+import reprlib
+aRepr = reprlib.Repr()
+aRepr.maxother=300
+
+X, y, X_test, test_ids = load_data()
+print("X:", aRepr.repr(X))
+print("y:", aRepr.repr(y))
+print("X_test:", aRepr.repr(X_test))
+print("test_ids", aRepr.repr(test_ids))
+
+print(f"X.shape: {X.shape}" if hasattr(X, 'shape') else f"X length: {len(X)}")
+print(f"y.shape: {y.shape}" if hasattr(y, 'shape') else f"y length: {len(y)}")
+print(f"X_test.shape: {X_test.shape}" if hasattr(X_test, 'shape') else f"X_test length: {len(X_test)}")
+print(f"test_ids length: {len(test_ids)}")
+
+X_loaded = deepcopy(X)
+y_loaded = deepcopy(y)
+X_test_loaded = deepcopy(X_test)
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+
+def debug_info_print(func):
+    def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == original_code:
+                print("\n" + "="*20 + "Running feat_eng code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+X, y, X_test = debug_info_print(feat_eng)(X, y, X_test)
+
+
+def get_length(data):
+    return data.shape[0] if hasattr(data, 'shape') else len(data)
+
+
+def get_width(data):
+    return 1 if isinstance(data, list) else data.shape[1:]
+
+
+def get_column_list(data):
+    return data.columns.tolist() if isinstance(data, pd.DataFrame) else None
+
+
+assert X is not None, "The feature engineering function returned None for X."
+assert y is not None, "The feature engineering function returned None for y."
+assert X_test is not None, "The feature engineering function returned None for X_test."
+
+assert get_length(X_test) == get_length(
+    test_ids
+), f"Mismatch in length of test images and test IDs: X_test ({get_length(X_test)}) and test_ids ({get_length(test_ids)})"
+assert get_length(X) == get_length(
+    y
+), f"Mismatch in length of training images and labels: X ({get_length(X)}) and y ({get_length(y)})"
+
+assert get_length(X) != 0, f"Training data is empty."
+assert get_length(y) != 0, f"Training labels are empty."
+assert get_length(X_test) != 0, f"Test data is empty."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+if isinstance(X, pd.DataFrame) and isinstance(X_test, pd.DataFrame):
+    assert get_column_list(X) == get_column_list(X_test), "Mismatch in column names of training and test data."
+
+if isinstance(X, pd.DataFrame):
+    def normalize_dtype(dtype):
+        return "numeric" if np.issubdtype(dtype, np.number) else str(dtype)
+
+    X_dtypes_unique_sorted = sorted(set(normalize_dtype(dt) for dt in X.dtypes.unique()))
+    X_loaded_dtypes_unique_sorted = sorted(set(normalize_dtype(dt) for dt in X_loaded.dtypes.unique()))
+
+    X_dtypes_unique_sorted_new = [
+        dt for dt in X_dtypes_unique_sorted if dt not in X_loaded_dtypes_unique_sorted and dt != "object"
+    ]
+    assert (
+        np.dtypes.ObjectDType in X_loaded_dtypes_unique_sorted or len(X_dtypes_unique_sorted_new) == 0
+    ), f"feature engineering has produced new data types which is not allowed, data loader data types are {X_loaded_dtypes_unique_sorted} and feature engineering data types are {X_dtypes_unique_sorted}"
+
+
+print(
+    "Feature Engineering test passed successfully. All checks including length, width, and data types have been validated."
+)

rdagent/components/coder/data_science/feature/exp.py

@@ -0,0 +1,13 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.core.utils import cache_with_pickle
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class FeatureTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/feature/prompts.yaml

@@ -0,0 +1,131 @@
+feature_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict["feature.py"] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict["feature.py"] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. If feature engineering is unnecessary or should be combined with model training, you may skip this step.
+    2. Be cautious of any column drop in the code. Dropping a column easily without any more attempts, it may not be a good practice.
+    3. The function input is the output of the following data loader:
+    ```python
+    {{ data_loader_code }}
+    ```
+    4. **Additional Guidance:**
+      - If a previous attempt exists, improve upon it without repeating mistakes.
+      - If errors indicate a missing file, find a way to download it or implement an alternative solution.
+      - You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    5. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache```
+    6. Coding tricks:
+      - If the input consists of a batch of file paths and you need to modify the file contents to complete your feature engineering task, you can accomplish your feature engineering task by modifying these files and creating new files in a subfolder within "{% include "scenarios.data_science.share:scen.cache_path" %}" (this path is persistent, otherwise you may lose your created file). Then the new file paths are returned.
+
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+  
+  user: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %}
+
+
+feature_eval:
+  system: |-
+    You are a data scientist responsible for evaluating feature engineering code generation.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## Feature Engineering Code
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The feature engineering code is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+    You will analyze the execution results based on the test output provided.
+
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The feature engineering code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the feature engineering test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the feature engineering test and, if applicable, the workflow test.
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the feature engineering executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Evaluate the correctness and integrity of processed data, checking for missing values, incorrect transformations, and data consistency.",
+        "code": "Assess code quality, readability, and adherence to specifications. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for optimization.",
+        "final_decision": <true/false>
+    }
+    ```
+  
+  user: |-
+    --------- Feature engineering test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/feature/test.py

@@ -0,0 +1,37 @@
+"""
+Helper functions for testing the feature coder(CoSTEER-based) component.
+- Does the developer loop work correctly
+
+It is NOT:
+- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop)
+"""
+
+from rdagent.components.coder.data_science.feature import FeatureCoSTEER
+from rdagent.components.coder.data_science.feature.exp import FeatureTask
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+def develop_one_competition(competition: str):  # -> experiment
+    scen = KaggleScen(competition=competition)
+    feature_coder = FeatureCoSTEER(scen)
+
+    with open("./rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/feature.md", "r") as file:
+        feat_spec = file.read()
+
+    # Create the experiment
+    ft = FeatureTask(name="FeatureTask", description=scen.get_competition_full_desc())
+    exp = DSExperiment(
+        sub_tasks=[ft],
+    )
+
+    with open("./rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/load_data.py", "r") as file:
+        load_data_code = file.read()
+    exp.experiment_workspace.inject_files(**{"load_data.py": load_data_code, "spec/feature.md": feat_spec})
+
+    # Develop the experiment
+    exp = feature_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")

rdagent/components/coder/data_science/model/__init__.py

@@ -0,0 +1,173 @@
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.model.eval import (
+    ModelGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.model.exp import ModelTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonBatchEditOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: ModelTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        model_information_str = target_task.get_task_information()
+
+        # 1. query
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[model_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[model_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get(f"{target_task.name}.py")
+                != workspace.file_dict.get(f"{target_task.name}.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 2. code
+        system_prompt = T(".prompts:model_coder.system").r(
+            task_desc=model_information_str,
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
+            data_loader_code=workspace.file_dict.get("load_data.py"),
+            feature_code=workspace.file_dict["feature.py"],
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonBatchEditOut.get_spec(),
+        )
+        # user_prompt = T(".prompts:model_coder.user").r(
+        #     model_spec=workspace.file_dict["spec/model.md"],
+        #     feature_code=workspace.file_dict["feature.py"],
+        #     latest_code=workspace.file_dict.get(f"{target_task.name}.py", None),
+        # )
+        # We want to use a simpler way to
+        code_spec = (
+            workspace.file_dict["spec/model.md"]
+            if DS_RD_SETTING.spec_enabled
+            else T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.Model").r(),
+                test_code=(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name),
+            )
+        )
+        user_prompt = T(".prompts:model_coder.user_general").r(
+            code_spec=code_spec,
+            latest_model_code=workspace.get_codes(
+                r"^model_(?!test)\w+\.py$"
+            ),  # TODO: If we have high failure rate here, we should clean this step with less information.
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            batch_edit = PythonBatchEditOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+
+            if not all(i.startswith("model_") for i in batch_edit.keys()):
+                user_prompt += "\nYou should only update model codes!"
+                continue
+
+            # 3. post process to align file name to the task name
+            # we assumpt batch_edit only contains one model file update.
+            batch_edit = {
+                (f"{target_task.name}.py" if value != "__DEL__" and key != f"{target_task.name}.py" else key): value
+                for key, value in batch_edit.items()
+            }
+
+            user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+            # TODO: besides same code problem, we should also consider other problems lead to retry.
+            if f"{target_task.name}.py" not in batch_edit:
+                continue
+
+            if batch_edit and max(len(i.encode("utf-8")) for i in batch_edit.keys()) > 255:
+                continue
+
+            if batch_edit[f"{target_task.name}.py"] != "__DEL__" and batch_edit[
+                f"{target_task.name}.py"
+            ] != workspace.file_dict.get(f"{target_task.name}.py"):
+                break
+
+            # If the task involves model removal, assume it can only process one model at a time.
+            if len(batch_edit) == 1 and batch_edit[f"{target_task.name}.py"] == "__DEL__":
+                break
+        else:
+            raise CoderError("Failed to generate a new model code.")
+
+        return batch_edit
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class ModelCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            ModelGeneralCaseSpecEvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        # eva = ModelGeneralCaseSpecEvaluator(scen=scen)
+        es = ModelMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/model/eval.py

@@ -0,0 +1,123 @@
+"""
+Beyond previous tests
+-
+"""
+
+import json
+import re
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+ModelSingleFeedback = CoSTEERSingleFeedback
+
+
+# Below are unit tests for testing the specification of the implemented model ------------------
+class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
+    """
+    Motivation case:
+    - Simplest case, we already split the data into train_data, valid_data, and test_data. We require the model to learn (optionally validate on valid data), and infer on test data.
+
+    Test workflow:
+    - Build train, valid, and test data to run it, and test the output (e.g., shape, etc.)
+    """
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> ModelSingleFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return ModelSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        if_model_removed = False
+
+        if f"{target_task.name}.py" in implementation.file_dict:
+            fname = "test/model_test.py"
+            test_code = (
+                (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
+            )  # only check the model changed this time
+            implementation.inject_files(**{fname: test_code})
+            result = implementation.run(env=env, entry=f"python {fname}")
+            stdout = result.stdout
+            ret_code = result.exit_code
+
+            if stdout is None:
+                raise CoderError(
+                    "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit."
+                )
+        else:
+            ret_code = 0
+            if_model_removed = True
+            stdout = f"Model {target_task.name} removal succeeded."
+
+        if "main.py" in implementation.file_dict and ret_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        if if_model_removed:
+            system_prompt = T(".prompts:model_eval_rm.system").r(
+                task_desc=target_task.get_task_information(),
+                workflow_stdout=workflow_stdout,
+                workflow_code=implementation.all_codes,
+            )
+            user_prompt = T(".prompts:model_eval_rm.user").r(
+                stdout=stdout,
+                workflow_stdout=workflow_stdout,
+            )
+        else:
+            system_prompt = T(".prompts:model_eval.system").r(
+                task_desc=target_task.get_task_information(),
+                test_code=test_code,
+                code=implementation.file_dict[f"{target_task.name}.py"],
+                workflow_stdout=workflow_stdout,
+                workflow_code=implementation.all_codes,
+            )
+            user_prompt = T(".prompts:model_eval.user").r(
+                stdout=stdout,
+                workflow_stdout=workflow_stdout,
+            )
+
+        fb = build_cls_from_json_with_retry(
+            ModelSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict,
+        )
+        fb.final_decision = fb.final_decision and ret_code == 0
+
+        return fb

rdagent/components/coder/data_science/model/eval_tests/model_test.txt

@@ -0,0 +1,105 @@
+"""
+Tests for `model_workflow` in model01.py
+"""
+import sys
+import time
+
+from feature import feat_eng
+from load_data import load_data
+from model01 import model_workflow
+from sklearn.model_selection import train_test_split
+
+
+def log_execution_results(start_time, val_pred, test_pred, hypers, execution_label):
+    """Log the results of a single model execution."""
+    feedback_str = f"{execution_label} end.\n"
+    feedback_str += f"Validation predictions shape: {val_pred.shape if val_pred is not None else 'None'}\n"
+    feedback_str += f"Test predictions shape: {test_pred.shape if test_pred is not None else 'None'}\n"
+    feedback_str += f"Hyperparameters: {hypers if hypers is not None else 'None'}\n"
+    feedback_str += f"Execution time: {time.time() - start_time:.2f} seconds.\n"
+    print(feedback_str)
+
+
+import reprlib
+aRepr = reprlib.Repr()
+aRepr.maxother=300
+
+# Load and preprocess data
+X, y, test_X, test_ids = load_data()
+X, y, test_X = feat_eng(X, y, test_X)
+
+print(f"X.shape: {X.shape}" if hasattr(X, 'shape') else f"X length: {len(X)}")
+print(f"y.shape: {y.shape}" if hasattr(y, 'shape') else f"y length: {len(y)}")
+print(f"test_X.shape: {test_X.shape}" if hasattr(test_X, 'shape') else f"test_X length: {len(test_X)}")
+print(f"test_ids length: {len(test_ids)}")
+
+train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.8, random_state=42)
+
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+print("train_X:", aRepr.repr(train_X))
+print("train_y:", aRepr.repr(train_y))
+print("val_X:", aRepr.repr(val_X))
+print("val_y:", aRepr.repr(val_y))
+
+print(f"train_X.shape: {train_X.shape}" if hasattr(train_X, 'shape') else f"train_X length: {len(train_X)}")
+print(f"train_y.shape: {train_y.shape}" if hasattr(train_y, 'shape') else f"train_y length: {len(train_y)}")
+print(f"val_X.shape: {val_X.shape}" if hasattr(val_X, 'shape') else f"val_X length: {len(val_X)}")
+print(f"val_y.shape: {val_y.shape}" if hasattr(val_y, 'shape') else f"val_y length: {len(val_y)}")
+
+
+
+def debug_info_print(func):
+    def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == original_code:
+                print("\n" + "="*20 + "Running model training code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+
+# First execution
+print("The first execution begins.\n")
+start_time = time.time()
+val_pred, test_pred, hypers = debug_info_print(model_workflow)(
+    X=train_X,
+    y=train_y,
+    val_X=val_X,
+    val_y=val_y,
+    test_X=None,
+)
+log_execution_results(start_time, val_pred, test_pred, hypers, "The first execution")
+
+# Second execution
+print("The second execution begins.\n")
+start_time = time.time()
+val_pred, test_pred, final_hypers = debug_info_print(model_workflow)(
+    X=train_X,
+    y=train_y,
+    val_X=None,
+    val_y=None,
+    test_X=test_X,
+    hyper_params=hypers,
+)
+log_execution_results(start_time, val_pred, test_pred, final_hypers, "The second execution")
+
+print("Model code test end.")

rdagent/components/coder/data_science/model/exp.py

@@ -0,0 +1,21 @@
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class ModelTask(CoSTEERTask):
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(name=name, description=description, *args, **kwargs)
+
+    def get_task_information(self):
+        task_desc = f"""name: {self.name}
+description: {self.description}
+"""
+        return task_desc

rdagent/components/coder/data_science/model/prompts.yaml

@@ -0,0 +1,186 @@
+model_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict[similar_successful_knowledge.target_task.name ~ '.py'] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict[former_failed_knowledge.target_task.name ~ '.py'] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. The function's input is from the output of a feature engineering function whose input is the output of a data loading function. The data loader function and feature engineering function code is as follows:
+    --------- Data Loader Code ---------
+    {{ data_loader_code }}
+    --------- Feature Engineering Code ---------
+    {{ feature_code }}
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. If the model can both be implemented by PyTorch and Tensorflow, please use pytorch for broader compatibility.
+    4. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache``
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    The file name should be the model name described in the model task in the format "{task_name}.py". You should always follow this name format.
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user_general: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    --------- Former model code ---------
+    {% if latest_model_code|length == 0 %}
+    So far the workspace is empty. No model code has been implemented yet.
+    {% else %}
+    {{ latest_model_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    {% endif %}
+
+model_eval:
+  system: |-
+    You are a data scientist responsible for evaluating model building code generation.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## Model Building Code
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The model building code is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+
+    ### Execution Phases
+    The model is tested in two phases:
+
+    1. Initial Training Phase:
+       - The model receives **train and valid inputs** with **empty hyperparameters**.
+       - The focus is on verifying whether the model successfully trains and produces **valid outputs and hyperparameter outputs**.
+
+    2. Retraining Phase:
+       - The model receives **train and test inputs** (without valid inputs).
+       - The hyperparameters generated from the first phase are passed back for **retraining**.
+
+
+    ### Key Requirements for Approval
+    A model can only be approved if it meets all of the following conditions:
+    1. Hyperparameter Handling
+      - If hyperparameters are returned, they must include an early stop round.
+      - The hyperparameters must be correctly utilized in the model for retraining.
+      - If the early stop round is provided, it must be used in the model implementation.
+    2. The model output shape must strictly match the specifications in `spec.md`.
+
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The model building code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the model building test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the model building test and, if applicable, the workflow test.
+    [Note] If no stdout for model buidling test is provided, the model failed due to a timeout or out-of-memory error. You should analyze potential optimizations.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the model building executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in spec.md. You also need to check whether the hyperparameters used for retraining are correctly returned during the test execution of the model.",
+        "code": "Assess code quality, readability, and adherence to specifications. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for optimization.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Model building test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}
+
+model_eval_rm:
+  system: |-
+    You are a data scientist responsible for evaluating model removal process.
+
+    ## Task Description
+    {{ task_desc }}
+
+    {% if workflow_stdout is not none %}
+    ## Whole Workflow Consideration
+    The model building code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the model removal test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the model removal test and, if applicable, the workflow test.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the model removal executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in spec.md.",
+        "code": "Assess code quality, readability, and adherence to specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Model removal test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/model/test.py

@@ -0,0 +1,67 @@
+"""
+Generate dataset to test the model workflow output
+"""
+
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
+from rdagent.components.coder.data_science.model import ModelCoSTEER
+from rdagent.components.coder.data_science.model.eval import (
+    ModelGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.model.exp import ModelTask
+from rdagent.core.experiment import FBWorkspace
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+# Take tasks, spec.md and feat as input, generate a feedback as output
+def develop_one_competition(competition: str):
+    scen = KaggleScen(competition=competition)
+    model_coder = ModelCoSTEER(scen)
+
+    # Create the task
+    mt = ModelTask(
+        name="ModelTask",
+        description="A CNN Model",
+        model_type="CNN",
+        architecture="\hat{y}_u = CNN(X_u)",
+        # variables="variables: {'\\hat{y}_u': 'The predicted output for node u', 'X_u': 'The input features for node u'}",
+        hyperparameters="...",
+        base_code="",
+    )
+
+    tpl_ex_path = Path(__file__).resolve() / Path("rdagent/scenarios/kaggle/tpl_ex").resolve() / competition
+    injected_file_names = ["spec/model.md", "load_data.py", "feature.py", "model01.py"]
+
+    modelexp = FBWorkspace()
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        modelexp.inject_files(**{file_name: file_path.read_text()})
+
+    mt.base_code += modelexp.file_dict["model01.py"]
+    exp = DSExperiment(
+        sub_tasks=[mt],
+    )
+
+    # Test the evaluator:
+    """eva = ModelGeneralCaseSpecEvaluator(scen=scen)
+    exp.feedback = eva.evaluate(target_task=mt, queried_knowledge=None, implementation=modelexp, gt_implementation=None)
+    print(exp.feedback)"""
+
+    # Test the evolving strategy:
+    """es = ModelMultiProcessEvolvingStrategy(scen=scen, settings=CoSTEER_SETTINGS)
+    new_code = es.implement_one_task(target_task=mt, queried_knowledge=None, workspace=modelexp)
+    print(new_code)"""
+
+    # Run the experiment
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        exp.experiment_workspace.inject_files(**{file_name: file_path.read_text()})
+
+    exp = model_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")
+    # dotenv run -- python rdagent/components/coder/data_science/model/test.py

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -0,0 +1,165 @@
+"""
+
+Loop should not large change exclude
+- Action Choice[current data loader & spec]
+- other should share
+    - Propose[choice] => Task[Choice] => CoSTEER =>
+        -
+
+Extra feature:
+- cache
+
+
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator
+from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: PipelineTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        data_folder_info = self.scen.processed_data_folder_description
+        pipeline_task_info = target_task.get_task_information()
+
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[pipeline_task_info] if queried_knowledge is not None else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        system_prompt = T(".prompts:pipeline_coder.system").r(
+            task_desc=pipeline_task_info,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+            runtime_environment=self.scen.get_runtime_environment(),
+            package_info=target_task.package_info,
+            enable_model_dump=DS_RD_SETTING.enable_model_dump,
+            enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM,
+            spec=T("scenarios.data_science.share:component_spec.Pipeline").r(
+                metric_name=self.scen.metric_name,
+                enable_notebook_conversion=DS_RD_SETTING.enable_notebook_conversion,
+            ),
+        )
+        user_prompt = T(".prompts:pipeline_coder.user").r(
+            competition_info=competition_info,
+            folder_spec=data_folder_info,
+            latest_code=workspace.file_dict.get("main.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            pipeline_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if pipeline_code != workspace.file_dict.get("main.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new pipeline code.")
+
+        return {
+            "main.py": pipeline_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class PipelineCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eval_l = [PipelineCoSTEEREvaluator(scen=scen)]
+        if DS_RD_SETTING.enable_model_dump:
+            eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample"))
+        for evaluator in settings.extra_evaluator:
+            eval_l.append(import_class(evaluator)(scen=scen))
+
+        for extra_eval in DSCoderCoSTEERSettings().extra_eval:
+            kls = import_class(extra_eval)
+            eval_l.append(kls(scen=scen))
+
+        eva = CoSTEERMultiEvaluator(
+            single_evaluator=eval_l, scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/pipeline/eval.py

@@ -0,0 +1,348 @@
+# tess successfully running.
+# (GPT) if it aligns with the spec & rationality of the spec.
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.agent.context7 import Agent as DocAgent
+from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.share.notebook import NotebookConverter
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.data_science.test_eval import get_test_eval
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+@dataclass
+class DSCoderFeedback(CoSTEERSingleFeedback):
+    """
+    Feedback for Data Science CoSTEER evaluation.
+    This feedback is used to evaluate the code and execution of the Data Science CoSTEER task.
+    """
+
+    requires_documentation_search: bool | None = None  # Keep None means the feature is disabled
+    error_message: str | None = None
+
+    @staticmethod
+    def val_and_update_init_dict(data: dict) -> dict:
+        # First call parent class validation method to handle base fields
+        data = CoSTEERSingleFeedback.val_and_update_init_dict(data)
+
+        # Validate new fields
+        if "requires_documentation_search" in data:
+            if isinstance(data["requires_documentation_search"], str):
+                if data["requires_documentation_search"] == "false" or data["requires_documentation_search"] == "False":
+                    data["requires_documentation_search"] = False
+                elif data["requires_documentation_search"] == "true" or data["requires_documentation_search"] == "True":
+                    data["requires_documentation_search"] = True
+                else:
+                    raise ValueError(
+                        f"'requires_documentation_search' string value must be 'true', 'True', 'false', or 'False', not '{data['requires_documentation_search']}'"
+                    )
+            elif data["requires_documentation_search"] is not None and not isinstance(
+                data["requires_documentation_search"], bool
+            ):
+                raise ValueError(
+                    f"'requires_documentation_search' must be a boolean, string, or None, not {type(data['requires_documentation_search'])}"
+                )
+
+        if "error_message" in data:
+            if data["error_message"] is not None and not isinstance(data["error_message"], str):
+                raise ValueError(f"'error_message' must be a string or None, not {type(data['error_message'])}")
+
+        return data
+
+    def __str__(self) -> str:
+        base_str = super().__str__()
+
+        if self.requires_documentation_search is not None:
+            base_str += f"-------------------Documentation Search Required------------------\n{self.requires_documentation_search}\n"
+
+        if self.error_message is not None:
+            # Check if error_message contains Context7 documentation results
+            if "### API Documentation Reference:" in self.error_message:
+                base_str += f"-------------------Error Analysis & Documentation Search Results ------------------\n{self.error_message}\n"
+            else:
+                base_str += f"-------------------Error Message------------------\n{self.error_message}\n"
+
+        return base_str
+
+    @classmethod
+    def merge(cls, feedback_li: list[CoSTEERSingleFeedback]) -> "DSCoderFeedback":
+        # Call parent class merge method to handle base fields
+        merged_fb = super().merge(feedback_li)
+
+        # Convert to DSCoderFeedback type if needed
+        if not isinstance(merged_fb, DSCoderFeedback):
+            merged_fb = DSCoderFeedback(
+                execution=merged_fb.execution,
+                return_checking=merged_fb.return_checking,
+                code=merged_fb.code,
+                final_decision=merged_fb.final_decision,
+            )
+
+        # Merge error_message fields
+        error_messages = [
+            fb.error_message for fb in feedback_li if isinstance(fb, DSCoderFeedback) and fb.error_message is not None
+        ]
+        if error_messages:
+            merged_fb.error_message = "\n\n".join(error_messages)
+
+        # Merge requires_documentation_search fields (True if any is True)
+        requires_search = [
+            fb.requires_documentation_search
+            for fb in feedback_li
+            if isinstance(fb, DSCoderFeedback) and fb.requires_documentation_search is not None
+        ]
+        if requires_search:
+            merged_fb.requires_documentation_search = any(requires_search)
+
+        return merged_fb
+
+
+PipelineSingleFeedback = DSCoderFeedback  # Only for compatible
+PipelineMultiFeedback = CoSTEERMultiFeedback
+
+
+class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
+        **kwargs,
+    ) -> PipelineSingleFeedback:
+
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return PipelineSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                error_message="This task has failed too many times, skip implementation.",
+                requires_documentation_search=None,
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        stdout = ""
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
+        if DS_RD_SETTING.sample_data_by_LLM:
+            # Because coder runs on full data, we need to run debug mode in advance to save time
+            result = implementation.run(
+                env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py --debug"
+            )
+        else:
+            result = implementation.run(
+                env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py"
+            )
+        result_stdout = result.stdout
+
+        nb_conversion_ret_code = 0
+        nb_conversion_check_text = ""
+        if DS_RD_SETTING.enable_notebook_conversion:
+            notebook_converter = NotebookConverter()
+            code = implementation.file_dict["main.py"]
+            error_msg = notebook_converter.validate_code_format(code)
+            if error_msg is not None:
+                nb_conversion_check_text = error_msg
+                nb_conversion_ret_code = 1
+            else:
+                notebook_converter.convert(
+                    task=target_task,
+                    code=code,
+                    stdout=result_stdout,
+                    outfile=implementation.workspace_path / "main.ipynb",
+                    use_debug_flag=DS_RD_SETTING.sample_data_by_LLM,
+                )
+
+        sample_submission_check = True
+        test_eval = get_test_eval()
+        if (sample_submission_file_name := test_eval.get_sample_submission_name(self.scen.competition)) is not None:
+            # check whether code ever opens the sample submission file
+            if (implementation.workspace_path / "trace.log").exists():
+                opened_trace_lines = [
+                    line
+                    for line in (implementation.workspace_path / "trace.log").read_text().splitlines()
+                    if "openat" in line and sample_submission_file_name in line
+                ]
+                if len(opened_trace_lines) > 0:
+                    stdout += f"Code opened the sample submission file '{sample_submission_file_name}' during execution.\n Reject the implementation!\n"
+                    sample_submission_check = False
+
+        result_stdout = remove_eda_part(result_stdout)
+        if result.exit_code != 0:
+            stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result_stdout.strip()}\n"
+        else:
+            stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result_stdout.strip()}\n"
+        if DS_RD_SETTING.sample_data_by_LLM:
+            debug_time, full_estimated_time = None, None
+            if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result_stdout, re.DOTALL):
+                debug_time = float(match.group(1))
+            if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result_stdout, re.DOTALL):
+                full_estimated_time = float(match.group(1))
+            if debug_time is not None and full_estimated_time is not None:
+                stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds. The estimated time is {full_estimated_time / env.conf.running_timeout_period * 100:.2f}% the debug time."
+            else:
+                stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n"
+
+        score_fp = implementation.workspace_path / "scores.csv"
+        score_ret_code = 0
+        score_check_text = ""
+        if not score_fp.exists():
+            score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
+            score_ret_code = 1
+        else:
+            try:
+                score_df = pd.read_csv(score_fp, index_col=0)
+                model_set_in_scores = set(score_df.index)
+
+                # Check model names (index)
+                if not score_df.index.is_unique:
+                    score_check_text += "\n[Error] The file 'scores.csv' contains duplicate model names."
+                    score_ret_code = 1
+                if "ensemble" not in model_set_in_scores:
+                    score_check_text += "\n[Error] The file 'scores.csv' doesn't contain the ensemble model."
+                    score_ret_code = 1
+                if score_ret_code != 0:
+                    score_check_text += f"The dataframe in file 'scores.csv' is:\n{score_df}"
+
+                # Check metric name (columns) - case insensitive
+                if [col.lower() for col in score_df.columns.tolist()] != [self.scen.metric_name.lower()]:
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
+                    score_ret_code = 1
+
+                # Check if scores contain NaN (values)
+                if score_df.isnull().values.any():
+                    nan_locations = score_df[score_df.isnull().any(axis=1)]
+                    score_check_text += f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+                    score_ret_code = 1
+
+            except Exception as e:
+                score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
+                score_ret_code = 1
+
+        test_eval = get_test_eval()
+        if DS_RD_SETTING.sample_data_by_LLM and test_eval.enabled(self.scen.competition):
+            submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation)
+            stdout += f"\n### Submission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
+        elif not test_eval.is_sub_enabled(self.scen.competition):
+            submission_ret_code = 0
+        else:
+            # Check submission file
+            base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
+            implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+            # stdout += "----Submission Check 1-----\n"
+            submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+            submission_check_out = submission_result.stdout
+            submission_ret_code = submission_result.exit_code
+            stdout += "\n" + submission_check_out
+
+        if not isinstance(implementation, FBWorkspace):
+            eda_output = None
+        else:
+            eda_output = implementation.file_dict.get("EDA.md", None)
+
+        # extract enable_mcp_documentation_search from data science configuration
+        enable_mcp_documentation_search = DS_RD_SETTING.enable_mcp_documentation_search
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[target_task.get_task_information()]
+            if queried_knowledge is not None
+            else []
+        )
+
+        system_prompt = T(".prompts:pipeline_eval.system").r(
+            is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
+            debug_mode=DS_RD_SETTING.sample_data_by_LLM,
+            enable_mcp_documentation_search=enable_mcp_documentation_search,
+            mle_check=DS_RD_SETTING.sample_data_by_LLM,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+        )
+        user_prompt = T(".prompts:pipeline_eval.user").r(
+            scenario=self.scen.get_scenario_all_desc(eda_output=eda_output),
+            task_desc=target_task.get_task_information(),
+            stdout=stdout.strip(),
+            spec=T("scenarios.data_science.share:component_spec.Pipeline").r(
+                metric_name=self.scen.metric_name,
+                enable_notebook_conversion=DS_RD_SETTING.enable_notebook_conversion,
+            ),
+            code=implementation.file_dict["main.py"],
+        )
+        wfb = build_cls_from_json_with_retry(
+            PipelineSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=PipelineSingleFeedback.val_and_update_init_dict,
+        )
+
+        # judge whether we should perform documentation search
+        do_documentation_search = enable_mcp_documentation_search and wfb.requires_documentation_search
+
+        if do_documentation_search:
+            # Use MCPAgent for clean, user-friendly interface
+            try:
+                # Create agent targeting Context7 service - model config comes from mcp_config.json
+                doc_agent = DocAgent()
+
+                # Synchronous query - perfect for evaluation context
+                if wfb.error_message:  # Type safety check
+                    context7_result = doc_agent.query(query=wfb.error_message)
+
+                    if context7_result:
+                        logger.info("Context7: Documentation search completed successfully")
+                        wfb.error_message += f"\n\n### API Documentation Reference:\nThe following API documentation was retrieved based on the error. This provides factual information about API changes or parameter specifications only:\n\n{context7_result}"
+                    else:
+                        logger.warning("Context7: Documentation search failed or no results found")
+                else:
+                    logger.warning("Context7: No error message to search for")
+
+            # TODO: confirm what exception will be raised when timeout
+            # except concurrent.futures.TimeoutError:
+            #     logger.error("Context7: Query timed out after 180 seconds")
+            except Exception as e:
+                error_msg = str(e) if str(e) else type(e).__name__
+                logger.error(f"Context7: Query failed - {error_msg}")
+
+        if score_ret_code != 0 and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + score_check_text
+        if submission_ret_code != 0 and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += "\nSubmission file check failed."
+        if sample_submission_check is False and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += (
+                "\nSample submission file check failed. Code should not open the sample submission file."
+            )
+        if nb_conversion_ret_code != 0 and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + nb_conversion_check_text
+        return wfb

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -0,0 +1,94 @@
+import hashlib
+from pathlib import Path
+import pandas as pd
+
+
+def calculate_md5(file_path):
+    with open(file_path, "rb") as f:
+        file_hash = hashlib.md5(f.read()).hexdigest()
+    return file_hash
+
+
+if Path("scores.csv").exists():
+    file_md5 = calculate_md5("scores.csv")
+else:
+    print("Warning: scores.csv does not exist. MD5 check will be skipped.")
+    file_md5 = None
+    
+"""
+find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
+./denoising-dirty-documents/sampleSubmission.csv
+./the-icml-2013-whale-challenge-right-whale-redux/sampleSubmission.csv
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv.zip
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv
+./random-acts-of-pizza/sampleSubmission.csv
+./text-normalization-challenge-english-language/en_sample_submission_2.csv.zip
+./text-normalization-challenge-english-language/en_sample_submission_2.csv
+./detecting-insults-in-social-commentary/sample_submission_null.csv
+"""
+
+# Find sample submission file dynamically
+input_dir = Path('{% include "scenarios.data_science.share:scen.input_path" %}')
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
+    input_dir.glob("*sampleSubmission*.csv")
+) + list(input_dir.glob("*randomPredictions*.tsv"))
+
+if not sample_submission_files:
+    print(f'Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}')
+    sample_submission_name = None
+    SAMPLE_SUBMISSION_PATH = None
+else:
+    sample_submission_name = sample_submission_files[0].name
+    SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+    print(f"Using sample submission file: {sample_submission_name}")
+
+if SAMPLE_SUBMISSION_PATH is not None and not Path(SAMPLE_SUBMISSION_PATH).exists():
+    print(f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}")
+
+if not Path("submission.csv").exists():
+    print("Error: submission.csv not found")
+
+if SAMPLE_SUBMISSION_PATH is not None and Path(SAMPLE_SUBMISSION_PATH).exists() and Path("submission.csv").exists():
+    sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+    our_submission = pd.read_csv("submission.csv")
+
+    success = True
+    print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+    print("Columns in our_submission.csv:", our_submission.columns)
+
+    for col in sample_submission.columns:
+        if col not in our_submission.columns:
+            success = False
+            print(f"Column {col} not found in submission.csv")
+
+    if success:
+        print(f"submission.csv's columns aligns with {sample_submission_name} .")
+    else:
+        print(f"submission.csv's columns does not align with {sample_submission_name} .")
+
+
+    def print_first_rows(file_path, file_name, num_rows=5):
+        print(f"\nFirst {num_rows} rows of {file_name}:")
+        try:
+            with open(file_path, "r") as file:
+                for i, line in enumerate(file):
+                    if i < num_rows:
+                        print(line.strip())
+                    else:
+                        break
+        except FileNotFoundError:
+            print(f"Error: {file_name} not found.")
+
+
+    print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+    print_first_rows("submission.csv", "submission.csv")
+
+    if file_md5 is not None:
+        if calculate_md5("scores.csv") != file_md5:
+            print("Warning: scores.csv has been rewritten in the test script!")
+else:
+    print("Skipping comparison and preview due to missing files.")
+
+print(
+    f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). "
+)

rdagent/components/coder/data_science/pipeline/exp.py

@@ -0,0 +1,8 @@
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class PipelineTask(CoSTEERTask):
+    def __init__(self, name: str = "Pipeline", package_info: str | None = None, *args, **kwargs) -> None:
+        super().__init__(name=name, *args, **kwargs)
+        self.package_info = package_info

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -0,0 +1,347 @@
+pipeline_coder:
+  system: |-
+    You are a grandmaster-level data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+    Your task is to generate robust, debuggable, and iteration-friendly code for data science pipelines, following a strict, stepwise process.
+
+    **Important Context**: You are working on sample datasets and your code will go through automated iterations. Design your code to be iteration-friendly with comprehensive print statements and clear debugging information to facilitate the automatic improvement process.
+
+    # Task Description
+    {{ task_desc }}
+    
+    ## The runtime environment your code will running on
+    {{ runtime_environment }}
+
+    {% if package_info is not none %}
+    To help you write the runnable code, the user has provided the package information which contains the package names and versions.
+    You should be careful about the package versions, as the code will be executed in the environment with the specified version and the api might be different from the latest version.
+    The user might provide the packages the environment doesn't have, you should avoid using any of them.
+    ## Package Information
+    {{ package_info }}
+    {% endif %}
+    
+    ## Hyperparameters Specification
+    Follow the hyperparameter choices if they are specified in the task description, unless they are unreasonable or incorrect.
+    In this case, refer to the guidelines below for appropriate adjustments:
+    {% include "scenarios.data_science.share:spec.hyperparameter" %}
+    
+    # Specification your code should follow
+    {{ spec }}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    ## Previous Failed Attempts
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    # Workflow Overview
+    You must complete the following stages in order. 
+
+    ## Data Loading
+    - Load the dataset strictly from `{% include "scenarios.data_science.share:scen.input_path" %}` as described in the **Data Folder Description**. DO NOT attempt to load data from the current directory (`./`).
+    - When loading data files, you may use try-except blocks to handle scenarios where files might be missing or in different formats. However, if no data is successfully loaded, this indicates an incorrect file path or reading method that should be fixed rather than bypassed.
+    - **Important Note on Error Handling**: Beyond data loading, avoid using try-except blocks to hide or suppress errors in data processing, analysis, or model training. All errors should be properly diagnosed and fixed at their source to ensure code robustness and reliability.
+
+    ## Exploratory Data Analysis (EDA) (Required)
+    Please follow this systematic methodology (in the required schema) for your analysis.
+    1. Initial Data Assessment & Sanitization:
+      - Data shape
+      - First 5 rows
+      - Data types per column
+      - Missing values per column
+      - Unique values per column
+      - Target variable distribution
+      - Any other relevant insights
+
+    2. Detailed Feature Analysis (A Non-Exhaustive Guide):
+    For Numerical & Categorical Features:
+      - Central Tendency & Dispersion
+      - Distribution Shape & Imbalance
+      - Outliers & Anomalies
+      - Cardinality & Granularity
+    For Text Features:
+      - Text Granularity & Scale
+      - Core Content & Topicality
+      - Linguistic Structure & Style
+      - Vocabulary Richness & Redundancy
+
+    3. The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      {EDA content}
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - An evaluation agent will help to check whether the EDA part is added correctly.
+    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    {% if enable_model_dump %}
+    ## Model Dumping
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+    {% endif %}
+
+    {% if enable_debug_mode %}
+    ## Debug Mode
+    Your code will be executed in a debug mode with following command: 
+    ```bash
+    python main.py --debug
+    ```
+    Please simulate the following code to check whether the code is running in debug mode:
+    ```python
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--debug', action='store_true', help='Run in debug mode')
+    args = parser.parse_args()
+    DEBUG = False
+    if args.debug:
+      DEBUG = True
+    ```
+    In debug mode, you should only sample ten percent of the training data and run the minimum epochs to quickly test the correctness of the code.
+    In debug mode, you should implement a timer to measure the time taken for your debug configuration and estimate the time required for the full run. Your timer should only measure the time taken for the training part, not the data loading or feature engineering part.
+    For example:
+    ```python
+    # Read data, feature engineering, etc.
+    start_time = time.time()
+    # Train your model
+    end_time = time.time()
+    debug_time = end_time - start_time
+    # post processing, saving model, etc.
+    ```
+    In debug mode, your code should run faster, so the environment will set a shorter time limit than the standard time limit for your code.
+    For example, you can sample ten percent of the training data and run for one epoch, then the full run with ten epochs will take one hundred times the time taken for the debug run. The scale is calculated by yourself depending on the data sampling and epoch number you choose. If your full run enables early stopping, the scale should be smaller considering the early stopping will stop the training earlier than the full epochs.
+    Be careful about the train-valid split strategy. Stratified related split is highly risk since the data has some categories with only one sample. If you use Stratified related split, you should consider using a try-except block to catch the error and use a different split strategy if the error occurs. Example code:
+    ```python
+    try:
+      fold_indices = StratifiedKFold(...).split(train_X, train_y) or StratifiedShuffleSplit or StratifiedSubsetSampler etc.
+    except Exception as e:
+        fold_indices = KFold(...).split(train_X, train_y) or other split strategy
+    ```
+    You should sample the data after train valid split. When you split the data after sampling, you might get a class with only one sample which might cause the split strategy to fail. 
+    Your debug code should run exactly the same as the full run, except for the data sampling and epoch number, to ensure the correctness of the code.
+    You should print total time and estimated time in standard output using print function in the following schema:
+    === Start of Debug Information ===
+    debug_time: time_taken_for_debug_run_in_seconds (e.g., 'debug_time: 10.0')
+    estimated_time: estimated_time_for_full_run_in_seconds (e.g., 'estimated_time: 100.0')
+    === End of Debug Information ===
+    User will use the following code to match: re.search(r"(.*?)=== Start of Debug Information ===(.*)=== End of Debug Information ===", stdout, re.DOTALL).groups()[1]
+    Notice, data sampling should only be applied in debug mode. Always use the full data in the full run!
+    Example code:
+    ```python
+    if args.debug:
+      sample_size = int(0.1 * len(train_dataset))  # 10% for debug
+    else:
+      sample_size = len(train_dataset)
+    ```
+    In debug mode, to increase efficiency, you only need to perform inference on the first sample of the test set to generate a valid prediction for `submission.csv`. For all other samples in the test set, you should use a placeholder value (e.g., 0 or a default value) to fill the prediction column. This ensures that the generated `submission.csv` has the same number of rows as the full run and passes the format check.
+    Example code:
+    ```python
+    all_preds = []
+    for i, batch in enumerate(test_loader):
+        # In debug mode, use placeholders for all batches after the first one to improve efficiency.
+        if args.debug and i > 0:
+            # The shape and data type of the placeholder must match the model's actual output.
+            # Here, we assume `predictions` is a NumPy array.
+            placeholder = np.zeros_like(predictions)
+            all_preds.append(placeholder)
+            continue
+
+        # In full mode, or for the first batch in debug mode, perform actual model inference.
+        predictions = model.predict(batch)
+        all_preds.append(predictions)
+
+    # final_predictions = np.concatenate(all_preds)
+    # ... then create and save submission.csv
+    ```
+    You should be very careful about the label classes number in the debug mode. The label classes should be the same as the full run even when you are in the debug mode. The label classes number is often used to build the model.
+    {% endif %}
+
+    ## General Guidelines
+    1. Code correctness is the top priority. Ensure your code is runnable and produces the expected output even if some task requirements are not fully met because the task itself might contain some errors like the wrong package name or wrong package function names.
+    2. Use the print() function for all output; do not use the logging module.
+    3. **Avoid all hard-coded values (e.g., fixed dataset sizes)**. Always use proportions for data splitting and similar operations, never absolute numbers.
+    4. Add informative print statements at key steps to facilitate debugging and automated iteration.
+    5. For model training, use reasonable epoch numbers. ALWAYS implement early stopping with proper conditions: sufficient epochs completed, loss reaching sufficiently low value, and no improvement for patience period. Save best model checkpoints based on validation performance.
+    6. Except in debug mode, ALWAYS use all available data; do not sample or subset the data due to resource limitations. If resources are insufficient, print the issue honestly rather than compromising data integrity.
+    7. Do not use tqdm or similar progress bar tools.
+    8. **Try-except blocks are ONLY allowed when reading files. If no files are successfully read, it indicates incorrect file paths or reading methods, not a try-except issue. Try-except is PROHIBITED elsewhere in the code. Assert statements are PROHIBITED throughout the entire code.**
+    9. ATTENTION: ALWAYS use the best saved model (not necessarily final epoch) for predictions. **NEVER create dummy/placeholder submissions (e.g., all 1s, random values)**. If training fails, report failure honestly rather than generating fake submission files.
+    10. You should ALWAYS generate the complete code rather than partial code.
+    11. If the task contains any user instructions, you must strictly follow them. User instructions have the highest priority and should be followed even if they conflict with other specifications or guidelines.
+    12. Strictly follow all specifications and general guidelines described above.
+
+    ### Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    # Competition Information
+    {{ competition_info }}
+
+    # Data Folder Description (All path are relative to the data folder, i.e. "{% include "scenarios.data_science.share:scen.input_path" %}")
+    {{ folder_spec }}
+    
+    {% if latest_code %}
+    # Former code
+    ```
+    {{ latest_code }}
+    ```
+    {% if latest_code_feedback is not none %}
+    ## Feedback to former code
+    {{ latest_code_feedback }}
+    
+    ## Improvement Planning
+    Before modifying the code, carefully analyze the feedback and identify no more than three key areas requiring changes. Plan your modifications strategically:
+    1. Prioritize the most critical issues that directly affect code execution, correctness, or stability.
+    2. Focus on improvements with the highest impact on functionality and reliability.
+    3. Preserve existing working components. Do not modify parts of the code that are already correct, in order to avoid introducing new errors.
+    
+    The previous version of the code contained errors. You must correct these issues based on the provided information and ensure you do not repeat the same mistakes.
+    
+    {% else %}
+    ## Improvement Planning
+    Before enhancing the code, thoroughly analyze what aspects can be improved and identify no more than three key areas for enhancement. Plan your improvements strategically:
+    1. Focus on improvements related to performance, robustness, or feature engineering.
+    2. Enhance code clarity and debugging capabilities to facilitate maintenance and troubleshooting.
+    3. Optimize model configuration or validation strategy to improve overall effectiveness.
+    
+    The previous version of the code is correct. You should improve the code based on the provided task while ensuring that unrelated parts remain unchanged.
+    {% endif %}
+    {% endif %}
+
+pipeline_eval:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %}
+    You will be provided with:
+    1. A detailed competition scenario description.
+    2. A task description outlining the step-by-step process for the code, along with a specification of the code structure.
+    3. A code implementation and its execution output.
+    Your task is to rigorously evaluate the code implementation against the provided scenario and task description, ensuring it meets all requirements, adheres to the specified structure, and executes successfully.
+
+    ## Evaluation Aspects
+    
+    ### Execution Success
+    - Goal: Ensure the code executes successfully without any errors.
+    - Notes:
+      - Model performance is not evaluated in this step; focus solely on successful execution.
+      - Warnings are acceptable if they do not interfere with successful code execution.
+    - If the code execute successfully:
+      - Proceed to Step 2.
+    - If the code does not execute successfully:
+      - Set the "final_decision" to false.
+      {% if enable_mcp_documentation_search %}
+      - Given that my package/environment is fixed and unchangeable, first you should go through the code and the execution output,if the problem could be solved by looking up the official documentation to confirm feature/API availability, compatible usage, or official alternatives in the fixed environment, set the "requires_documentation_search" to true.
+      {% endif %}
+      - Write complete analysis in the "execution" field.
+
+    ### Competition Alignment
+    - Goal: Confirm strict adherence to the competition's evaluation rules and experimental setup.
+    - Guidelines:
+      - Analyze whether the experimental setup and code may cause misalignment between validation and test performance.
+      - Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
+        - The metric implementation must exactly match scenario requirements (metric value itself is not the focus).
+        - Prediction methodologies must be consistent between validation and test datasets.
+        - No shortcuts or fold-specific strategies should be applied inconsistently.
+      - Check for corner-case consistency.
+      - Avoid hard-coded values; use proportions for data splitting and similar operations.
+    - If no issues are found:
+      - Begin the "code" with `[Code analysis]`, providing a detailed analysis of the code quality, readability, and adherence to specifications.
+    - If discrepancies or risks are found:
+      - Set the "final_decision" to false.
+      - Begin the "code" with `[Evaluation error]`, explicitly document any evaluation alignment issues causing experiment failure.
+
+    {% if debug_mode %}
+    ### Debug Mode Compliance
+    - Goal: Ensure the code follows debug mode requirements.
+    - Guidelines:
+      - Sufficient debugging information (print statements, clear error messages) should be included to facilitate automatic improvement processes.
+      - The code should be executed in debug mode with the command `python main.py --debug`.
+      - In debug mode, the code should sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code.
+      - Check whether the code follows these requirements. If not, emphasize it in your feedback and reject this implementation.
+      - Execution time and estimated time for the full run should be checked. Estimated time should not be too large to finish in the given time limit.
+      - Consider the early stopping mechanism in the code. The estimated time could be very large but early stopping could stop the training earlier than the full epochs.
+      - Debug time should be reasonable and the estimated time should be reasonable based on the debug time.
+      - Data sampling should only be applied in debug mode. Always use the full data in the full run.
+      - The label classes number should be the same as the full run even in debug mode.
+    - If the code passes this step: Proceed to Next Aspects.
+    - If the code does not pass this step: Clearly document the debug mode compliance issues and reject the implementation.{% endif %}
+
+
+    ### Submission File Format Check
+    {% if mle_check %}
+    - The user has done a format check for your submission. Since you didn't sample any test data, your debug mode output should be the same format as the full run.
+    - The user will put the check result in the "Submission check" section of the execution output.
+    - If the submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should give the conclusion that the code executed successfully. If no other code related issues are found, set the "final_decision" to true.
+    - If the submission check returns an error message, you should set the "final_decision" to false and clearly document the issues in the "return_checking" field.
+    {% elif is_sub_enabled %}
+    - Goal: Verify that the code correctly generates the final submission in the expected format and that the submission is authentic.
+    - Guidelines:
+      - The submission file must strictly match the required structure (correct columns, index format, data types). The index names and column names must be identical to the format specified in the Competition Information's '====== Submission Format ======' section.
+      - Rigorously verify that the submission file was produced by genuine model inference and successful code execution, not by cheating, fallback or exception-handling mechanisms.
+        - The submission must be generated from genuine model predictions using the best saved model—never empty, constant, random, or hard-coded values.
+        - Submissions must reflect authentic model outputs; any form of fabrication, cheating, or simulated results is strictly prohibited and grounds for rejection.
+        - Cross-check both code logic and stdout to ensure predictions originate from real model inference, not from error recovery or placeholder code paths.
+      - Only check the format of the submission since only part of the data is provided; the submission might have a different index than expected due to data sampling.
+      - Verify honest failure reporting if training issues occur.
+    - If the code passes this step, Finalize evaluation.
+    - If the code does not pass this step:
+      - Set the "final_decision" to false and clearly document the issues in the "return_checking" field.
+    {% else %}
+      Submission File Format Check is not conducted since no target submission format is provided. You should consider this submission file is valid.
+    {% endif %}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    ### Step 6: Similar Successful Implementations to help Code Improvement
+    The user has done several similar tasks and get some successful implementations. These code might not be implemented to the same task, but they are similar to your task and they might work well on your dataset.
+    Please refer to these successful implementation and provide your suggestions in your response on how to correct your current code based on these successful implementations.
+    ## Successful Implementations for Similar Tasks
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Similar Task {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.all_codes }}
+    {% endfor %} 
+    {% endif %}
+
+    ## Output Format
+    Please respond with your feedback in the following JSON format without anything else.
+    ```json
+    {
+        {% if enable_mcp_documentation_search %}
+        "requires_documentation_search": <true/false>,
+        {% endif %}"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. If errors occurred, analyze the root causes: (1) Are they fundamental algorithmic/approach issues, or (2) Implementation details that can be easily fixed, or (3) Environment/dependency problems?",
+        "return_checking": "Examine the generated files by cross-referencing the code logic and stdout output. Verify: (1) Format matches required submission format (index, column names, CSV content); (2) **File generation authenticity**: Is the file genuinely produced by successful model execution, or is it a result of exception handling/fallback mechanisms? Cite specific code sections and stdout evidence.",
+        "code": "Begin explicitly with [Code analysis] or [Evaluation error]. Provide structured analysis: (1) **Technical Appropriateness**: Does the chosen approach (algorithms, data processing, validation strategy) match this problem's data characteristics and competition requirements? (2) **Effective Components**: What specific parts work well and why are they effective for this problem type? (3) **Issues & Improvements**: Identify concrete problems and suggest actionable improvement directions (without providing actual code). (4) **Code Quality**: Assess readability, structure, and adherence to specifications.",
+        {% if enable_mcp_documentation_search %}
+        "error_message": "If the code execution has problems, extract the error information in the following format, otherwise set to empty string: ### TRACEBACK: <full relevant traceback extracted from execution output> ### SUPPLEMENTARY_INFO: <only if TRACEBACK is unclear - copy exact code fragments: import statements, variable=value assignments, function calls with parameters as they appear in code>",
+        {% endif %}"final_decision": <true/false>
+    }
+    ```
+
+
+  user: |-
+    # Competition Information
+    {{ scenario }}
+
+    # Task Description
+    {{ task_desc }}
+
+    ## Task Specification for Code Structure
+    {{ spec }}
+
+    # Code
+    ```
+    {{ code }}
+    ```
+
+    ## Execution Output
+    ```
+    {{ stdout }}
+    ```

rdagent/components/coder/data_science/raw_data_loader/README.md

@@ -0,0 +1,15 @@
+# CoSTEER
+
+- subworkspace使用主experiment_workspace `RD-Agent/rdagent/scenarios/data_science/experiment/experiment.py`
+
+## evolving_strategy ( implement_one_task() )
+
+1. xxxTask (in exp.py)
+    - spec
+    - description
+2. 
+
+## evaluator
+
+1. queried_knowledge部分 共用
+2. eval_test脚本

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -0,0 +1,242 @@
+"""
+
+Loop should not large change exclude
+- Action Choice[current data loader & spec]
+- other should share
+    - Propose[choice] => Task[Choice] => CoSTEER =>
+        -
+
+Extra feature:
+- cache
+
+
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+import re
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import (
+    DSCoderCoSTEERSettings,
+    get_ds_env,
+)
+from rdagent.components.coder.data_science.raw_data_loader.eval import (
+    DataLoaderCoSTEEREvaluator,
+)
+from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: DataLoaderTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # return a workspace with "load_data.py", "spec/load_data.md" inside
+        # assign the implemented code to the new workspace.
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        data_folder_info = self.scen.processed_data_folder_description
+        data_loader_task_info = target_task.get_task_information()
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[data_loader_task_info]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[data_loader_task_info]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("load_data.py") != workspace.file_dict.get("load_data.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 1. specifications
+        # TODO: We may move spec into a separated COSTEER task
+        if DS_RD_SETTING.spec_enabled:
+            if "spec/data_loader.md" not in workspace.file_dict:  # Only generate the spec once
+                system_prompt = T(".prompts:spec.system").r(
+                    runtime_environment=self.scen.get_runtime_environment(),
+                    task_desc=data_loader_task_info,
+                    competition_info=competition_info,
+                    folder_spec=data_folder_info,
+                )
+                data_loader_prompt = T(".prompts:spec.user.data_loader").r(
+                    latest_spec=workspace.file_dict.get("spec/data_loader.md")
+                )
+                feature_prompt = T(".prompts:spec.user.feature").r(
+                    latest_spec=workspace.file_dict.get("spec/feature.md")
+                )
+                model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md"))
+                ensemble_prompt = T(".prompts:spec.user.ensemble").r(
+                    latest_spec=workspace.file_dict.get("spec/ensemble.md")
+                )
+                workflow_prompt = T(".prompts:spec.user.workflow").r(
+                    latest_spec=workspace.file_dict.get("spec/workflow.md")
+                )
+
+                spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
+
+                data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
+                feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
+                model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
+                ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
+                workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
+            else:
+                data_loader_spec = workspace.file_dict["spec/data_loader.md"]
+                feature_spec = workspace.file_dict["spec/feature.md"]
+                model_spec = workspace.file_dict["spec/model.md"]
+                ensemble_spec = workspace.file_dict["spec/ensemble.md"]
+                workflow_spec = workspace.file_dict["spec/workflow.md"]
+
+        # 2. code
+        system_prompt = T(".prompts:data_loader_coder.system").r(
+            task_desc=data_loader_task_info,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+        )
+        code_spec = (
+            data_loader_spec
+            if DS_RD_SETTING.spec_enabled
+            else T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.DataLoadSpec").r(),
+                test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(),
+            )
+        )
+        user_prompt = T(".prompts:data_loader_coder.user").r(
+            competition_info=competition_info,
+            code_spec=code_spec,
+            folder_spec=data_folder_info,
+            latest_code=workspace.file_dict.get("load_data.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            data_loader_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if data_loader_code != workspace.file_dict.get("load_data.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new data loader code.")
+
+        return (
+            {
+                "spec/data_loader.md": data_loader_spec,
+                "spec/feature.md": feature_spec,
+                "spec/model.md": model_spec,
+                "spec/ensemble.md": ensemble_spec,
+                "spec/workflow.md": workflow_spec,
+                "load_data.py": data_loader_code,
+            }
+            if DS_RD_SETTING.spec_enabled
+            else {
+                "load_data.py": data_loader_code,
+            }
+        )
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class DataLoaderCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            DataLoaderCoSTEEREvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = DataLoaderMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )
+
+    def develop(self, exp):
+        new_exp = super().develop(exp)
+
+        env = get_ds_env(
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            },
+            running_timeout_period=self.scen.real_full_timeout(),
+        )
+
+        stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py")
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
+        eda_output = match.groups()[1] if match else None
+        if eda_output is not None:
+            new_exp.experiment_workspace.inject_files(**{"EDA.md": eda_output})
+        else:
+            eda_output = "No EDA output."
+            new_exp.experiment_workspace.inject_files(**{"EDA.md": eda_output})
+        return new_exp

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -0,0 +1,94 @@
+# tess successfully running.
+# (GPT) if it aligns with the spec & rationality of the spec.
+import json
+import re
+from pathlib import Path
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+DataLoaderEvalFeedback = CoSTEERSingleFeedback
+
+
+class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
+        **kwargs,
+    ) -> DataLoaderEvalFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return DataLoaderEvalFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        # TODO: do we need to clean the generated temporary content?
+        fname = "test/data_loader_test.py"
+        test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text()
+        implementation.inject_files(**{fname: test_code})
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.exit_code
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL)
+        stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "")
+        stdout = stdout_part_1 + stdout_part_2
+        if eda_output is not None and len(eda_output.split(" ")) > 10000:
+            eda_output += "Length of EDA output is too long, truncated. Please reject this implementation and motivate it to reduce the length of EDA output."
+
+        if "main.py" in implementation.file_dict and ret_code == 0:
+            workflow_stdout = implementation.execute(env=env, entry="python main.py")
+            workflow_stdout = remove_eda_part(workflow_stdout)
+        else:
+            workflow_stdout = None
+
+        system_prompt = T(".prompts:data_loader_eval.system").r(
+            task_desc=target_task.get_task_information(),
+            test_code=test_code,
+            code=implementation.file_dict["load_data.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:data_loader_eval.user").r(
+            stdout=stdout,
+            eda_output=eda_output,
+            workflow_stdout=workflow_stdout,
+        )
+
+        fb = build_cls_from_json_with_retry(
+            DataLoaderEvalFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=DataLoaderEvalFeedback.val_and_update_init_dict,
+        )
+        fb.final_decision = fb.final_decision and ret_code == 0
+
+        return fb

rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt

@@ -0,0 +1,83 @@
+"""
+Tests for `load_data` in load_data.py
+"""
+
+import pickle
+
+import pandas as pd
+from load_data import load_data
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+
+def debug_info_print(func):
+    aRepr = reprlib.Repr()
+    aRepr.maxother=300
+    def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
+        def local_trace(frame, event, arg):
+            if event == "return" and frame.f_code == original_code:
+                print("\n" + "="*20 + "Running data_load code, local variable values:" + "="*20)
+                for k, v in frame.f_locals.items():
+                    printed = aRepr.repr(v)
+                    print(f"{k}:\n {printed}")
+                print("="*20 + "Local variable values end" + "="*20)
+            return local_trace
+        
+        sys.settrace(local_trace)
+        try:
+            return func(*args, **kwargs)
+        finally:
+            sys.settrace(None)
+    return wrapper
+
+X, y, X_test, test_ids = debug_info_print(load_data)()
+
+
+def get_length(data):
+    return data.shape[0] if hasattr(data, 'shape') else len(data)
+
+
+def get_width(data):
+    return data.shape[1:] if hasattr(data, 'shape') else 1
+
+
+def get_column_list(data):
+    return data.columns.tolist() if isinstance(data, pd.DataFrame) else None
+
+assert X is not None, "Training data (X) is None."
+assert y is not None, "Training labels (y) are None."
+assert X_test is not None, "Test data (X_test) is None."
+assert test_ids is not None, "Test IDs (test_ids) are None."
+
+assert get_length(X_test) == get_length(
+    test_ids
+), f"Mismatch in length of test images and test IDs: X_test ({get_length(X_test)}) and test_ids ({get_length(test_ids)})"
+assert get_length(X) == get_length(
+    y
+), f"Mismatch in length of training images and labels: X ({get_length(X)}) and y ({get_length(y)})"
+
+assert get_length(X) != 0, f"Training data is empty."
+assert get_length(y) != 0, f"Training labels are empty."
+assert get_length(X_test) != 0, f"Test data is empty."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+if isinstance(X, pd.DataFrame) and isinstance(X_test, pd.DataFrame):
+    assert get_column_list(X) == get_column_list(X_test), "Mismatch in column names of training and test data."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+print("Data loader test passed successfully. Length of test images matches length of test IDs.")

rdagent/components/coder/data_science/raw_data_loader/exp.py

@@ -0,0 +1,6 @@
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class DataLoaderTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/raw_data_loader/prompts.yaml

@@ -0,0 +1,402 @@
+
+spec:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    Currently, you are working on a Kaggle competition project. 
+    This project involves analyzing data and building models to beat other competitors, with the code being generated by large language models.
+
+    The runtime environment you are working in includes the following libraries and their respective versions:
+    {{ runtime_environment }}
+
+    Your overall task is provided below:
+    {{ task_desc }}
+    
+    Your task is to write five specification texts (in markdown format) for the following tasks, based on the competition information provided
+    - Data loading (and preprocessing)
+    - Feature Engineering
+    - Model Building
+    - Ensemble
+    - The overall workflow
+
+    The specifications for each step should be tailored to the competition information provided. 
+    
+    Your specification should consists two parts:
+    1. The function definition in code format, including type annotations and a clear, complete docstring that describes the function's purpose, input parameters, return value, and any relevant exceptions.
+    2. Additional information or notes that the coder should consider while implementing the function.
+    
+    Your specifications should include only the function definition and docstring, without any code implementation or inline comments.
+
+    ## Competition Information for This Task
+    {{ competition_info }}
+
+    ----------- Folder Description (All path are relative to the data folder) ---------
+    - Ensure that all columns in sample_submission can be generated.
+    {{ folder_spec }}
+
+  user:
+    data_loader: |-
+      Data loader specification text should follow these detailed requirements:
+      1. Function Interface:
+        - Function Name: `load_data`
+        - Input: No input arguments.
+        - Output:
+          - `X` (DT, define based on competition information): Feature matrix for training data.
+          - `y` (DT): Target vector for training data.
+          - `X_test` (DT): Feature matrix for test data.
+          - `test_ids` (DT): Identifiers for the test data.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Specify the data source location (`{% include "scenarios.data_science.share:scen.input_path" %}`).
+          - Clearly define the structure and type of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+      2. Notes:
+        - Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+
+      Responsibilities and notes of an implemented data loader that aligns with the generated specification.
+      {% include "scenarios.data_science.share:component_spec.DataLoadSpec" %}
+
+      {% if latest_spec %}
+      6. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    feature: |-
+      Feature engineering specification text should adhere to the following requirements:
+      1. Function Interface:
+        - Function Name: `feat_eng`
+        - Parameters:
+          - `X` (DT): Train data to be transformed.
+          - `y` (DT): Train label data.
+          - `X_test` (DT): Test data.
+        - Output:
+          - `X_transformed` (DT): Transformed train data.
+          - `y_transformed` (DT): Transformed train label data.
+          - `X_test_transformed` (DT): Transformed test data.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Clarify the input parameters and their data types.
+          - Define the structure and format of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+
+      2. Precautions for Feature Engineering:
+        - Well handle the shape of the data:
+          - The sample size of the train data and the test data should be the same in all scenarios.
+          - To some tabular or time-series data, you may add or remove some columns so your inferred column number may be unsure.
+          - For scenarios where each dimension does not have a special meaning (like image, audio, and so on), the input shape and the output shape should be exactly the same in most cases unless there is a compelling reason to change them.
+        - Integration with the Model Pipeline:
+          - If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage.
+            - Model-related operations should not be implemented in this step. (e.g., it uses tools combined with models like torch.Dataset with rich data transformation/augmentation)
+          - Otherwise, ensure this function applies all required transformations while avoiding data leakage.
+        - General Considerations:
+          - Ensure scalability for large datasets.
+          - Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
+          - Ensure consistency between feature data types and transformations.
+          - Prevent data leakage: Do not use information derived from the test set when transforming training data.
+        - Domain-Specific Features:
+          - Apply logic for competition-specific features (e.g., text vectorization, image augmentations, categorical encoding).
+
+      3. Code Standards:
+        - Avoid using progress bars (e.g., `tqdm`) in the implementation.          
+
+      4. Notes:
+        - Align `DT` (data type) definitions with those in the Data Loader specification.
+        - GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+      
+      {% if latest_spec %}
+      5. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    model: |-
+      Model building specification text should adhere to the following requirements:
+
+      1. Function Interface:
+        - Function Name: `model_workflow`
+        - Parameters:
+          - `X` (DT): Training feature data.
+          - `y` (DT): Training label data.
+          - `val_X` (Optional[DT]): Validation feature data.
+          - `val_y` (Optional[DT]): Validation label data.
+          - `test_X` (Optional[DT]): Test feature data.
+          - `hyper_params` (dict): Dictionary of hyperparameters for model configuration.
+        - Output:
+          - `pred_val` (Optional[DT]): Predictions on validation data.
+          - `pred_test` (Optional[DT]): Predictions on test data.
+          - `hyper_params` (dict): Updated dictionary of hyperparameters after training.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Clarify the input parameters and their data types.
+          - Define the structure and format of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+
+      2. Code Standards:
+        - Do not use progress bars (e.g., `tqdm`) in the implementation.
+
+      3. Precautions:
+        - Ensure input arrays (`X`, `y`, `val_X`, `val_y`, `test_X`) have consistent dimensions and shapes.
+        - Use default values for hyperparameters if `hyper_params` is not provided.
+        - Train the model on `X` and `y`.
+        - Evaluate the model using `val_X` and `val_y` if validation data is available.
+        - If `test_X` is provided, generate predictions for it.
+
+      4. Notes:
+        - Align `DT` (data type) with the definitions used in Feature Engineering specifications.
+        - The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process.
+        - Some data transformations/augmentations can be included in this step (e.g., data tools provided by TensorFlow and Torch)
+
+      {% if latest_spec %}
+      5. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    ensemble: |-
+      Ensemble specification text adhere to the following requirements:
+      1. Function Interface:
+        - Function Name: `ensemble_workflow`
+        - Parameters:
+          - `test_preds_dict` (Dict[str, DT]): A dictionary of test predictions from different models. The key is the model file name.
+          - `val_preds_dict` (Dict[str, DT]): A dictionary of validation predictions from different models. The key is the model file name.
+          - `val_label` (DT): Validation label.
+        - Output:
+          - `final_pred` (DT): Ensemble prediction for the test data.
+        - Docstring Requirements:
+          - Describe the purpose of the function.
+          - Clarify the input parameters and their data types.
+          - Define the structure and format of the output.
+          - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
+
+      2. Precautions:
+        - Input Validation:
+          - Ensure all predictions in `test_preds_dict` and `val_preds_dict` have consistent shapes and dimensions.
+          - Verify that `val_label` is provided and matches the length of `val_preds_dict` predictions.
+          - Handle empty or invalid inputs gracefully with appropriate error messages.
+        - Metric Calculation and Storage:
+          - Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`, e.g.:
+            ```python
+            scores = {}
+            for model_name, val_pred in val_preds_dict.items():
+                scores[model_name] = calculate_metric(val_label, val_pred)
+            
+            ...
+            some code about ensemble strategy
+            ...
+            ensemble_val_pred = ...
+
+            ensemble_score = calculate_metric(val_label, ensemble_val_pred)
+            scores["ensemble"] = ensemble_score  # Ensure "ensemble" is explicitly stored
+            
+            scores_df = pd.DataFrame(scores.items(), columns=["Model", <metric_name>])
+            scores_df.to_csv("scores.csv", index=False)
+            ```
+          - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
+        
+      3. Code Standards:
+        - Do not use progress bars (e.g., tqdm) in the code.
+
+      4. Notes:
+        - Align `DT` (data type) definitions with those used in model specifications.
+        - Ensure flexibility to handle multiple ensemble strategies based on competition requirements.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+
+      {% if latest_spec %}
+      5. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
+
+    workflow: |-
+      {% include "scenarios.data_science.share:component_spec.Workflow" %}
+
+      {% if latest_spec %}
+      7. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}
+
+      ## Output Format
+      You should return the specification in markdown format directly.
+      You should create the rules based on the competition information instead of copying the requirements.
+
+data_loader_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementation Examples for Similar Task ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Example {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.all_codes }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache```
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+    
+    ## Exploratory Data Analysis (EDA) part(Required):
+    - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - An evaluation agent will help to check whether the EDA part is added correctly.
+    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    --------- Competition Information ---------
+    {{ competition_info }}
+
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    --------- Data Folder Description (All path are relative to the data folder, i.e. "{% include "scenarios.data_science.share:scen.input_path" %}") ---------
+    {{ folder_spec }}
+    
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %} 
+
+    You should strictly follow the code specifications provided by the specification to implement the function.
+
+
+data_loader_eval:
+  system: |-
+    You are a data scientist responsible for evaluating data loader code for a Kaggle-style machine learning competition project.
+    
+    ## Task Description
+    {{ task_desc }}
+
+    ## Data Loader Code
+    The data loader code is located in `load_data.py`:
+    ```python
+    {{ code }}
+    ```
+
+    ## Testing Process
+    The data loader is tested using the following script:
+    ```python
+    {{ test_code }}
+    ```
+
+    {% if workflow_stdout is not none %}
+    ### Whole Workflow Consideration
+    The data loader is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    {{ workflow_code }}
+
+    You should evaluate both the data loader test results and the overall workflow execution. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the data loader test and, if applicable, the workflow test.
+
+    ## Exploratory Data Analysis (EDA) Part evaluation
+    - The code has also generated some EDA output to help understand the data better. 
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+    You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
+    
+    Your response must follow this structured JSON format:
+    ```json
+    {
+        "execution": "Describe how well the data loader executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Evaluate the correctness and integrity of the loaded data. Check for issues like missing values, incorrect data types, outliers, or formatting inconsistencies.",
+        "code": "Assess code quality, readability, and adherence to best practices. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for faster data loading.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Data loader test stdout ---------
+    {{ stdout }}   
+    --------- Data loader EDA stdout ---------
+    {% if eda_output is not none %}
+    {{ eda_output }}
+    {% else %}
+    No EDA output is provided.
+    {% endif %}
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}

rdagent/components/coder/data_science/raw_data_loader/test.py

@@ -0,0 +1,30 @@
+"""
+Helper functions for testing the raw_data_loader coder(CoSTEER-based) component.
+- Does the developer loop work correctly
+
+It is NOT:
+- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop)
+"""
+
+from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER
+from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+def develop_one_competition(competition: str):  # -> experiment
+    scen = KaggleScen(competition=competition)
+    data_loader_coder = DataLoaderCoSTEER(scen)
+
+    # Create the experiment
+    dlt = DataLoaderTask(name="DataLoaderTask", description="")
+    exp = DSExperiment(
+        sub_tasks=[dlt],
+    )
+
+    # Develop the experiment
+    exp = data_loader_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")

rdagent/components/coder/data_science/share/doc.py

@@ -0,0 +1,37 @@
+"""
+Developers concentrating on writing documents for a workspace
+"""
+
+from rdagent.core.developer import Developer
+from rdagent.core.experiment import Experiment, FBWorkspace
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import MarkdownAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class DocDev(Developer[Experiment]):
+    """
+    The developer is responsible for writing documents for a workspace.
+    """
+
+    def develop(self, exp: Experiment) -> None:
+        """
+        Write documents for the workspace.
+        """
+        ws: FBWorkspace = exp.experiment_workspace
+
+        file_li = [str(file.relative_to(ws.workspace_path)) for file in ws.workspace_path.rglob("*") if file.is_file()]
+
+        key_file_list = ["main.py", "scores.csv"]
+
+        system_prompt = T(".prompts:docdev.system").r()
+        user_prompt = T(".prompts:docdev.user").r(
+            file_li=file_li,
+            key_files={f: (ws.workspace_path / f).read_text() for f in key_file_list},
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt, system_prompt=system_prompt
+        )
+        markdown = MarkdownAgentOut.extract_output(resp)
+        ws.inject_files(**{"README.md": markdown})

rdagent/components/coder/data_science/share/ds_costeer.py

@@ -0,0 +1,9 @@
+from rdagent.components.coder.CoSTEER import CoSTEER
+
+
+class DSCoSTEER(CoSTEER):
+    def get_develop_max_seconds(self) -> int | None:
+        """
+        The coder uses the scenario's real debug timeout as the maximum seconds for development.
+        """
+        return int(self.scen.real_debug_timeout() * self.settings.max_seconds_multiplier)

rdagent/components/coder/data_science/share/eval.py

@@ -0,0 +1,176 @@
+import re
+from pathlib import Path
+from typing import Literal
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+PipelineSingleFeedback = CoSTEERSingleFeedback
+PipelineMultiFeedback = CoSTEERMultiFeedback
+
+NO_SUB = "<No submission.csv file found.>"
+NO_SCORE = "<No scores.csv file found.>"
+
+
+class ModelDumpEvaluator(CoSTEEREvaluator):
+    """This evaluator assumes that it runs after the model"""
+
+    def __init__(self, scen: Scenario, data_type: Literal["sample", "full"]):
+        super().__init__(scen)
+        self.data_type = data_type
+
+    def evaluate(
+        self, target_task: Task, implementation: FBWorkspace, gt_implementation: FBWorkspace, *kargs, **kwargs
+    ) -> CoSTEERSingleFeedback:
+
+        model_folder = implementation.workspace_path / "models"
+        # 1) Check if the model_folder is not empty
+        if not model_folder.exists() or not any(model_folder.iterdir()):
+            err_msg = "Model folder (`models` sub folder) is empty or does not exist. The model is not dumped."
+            return CoSTEERSingleFeedback(
+                execution=err_msg,
+                return_checking=err_msg,
+                code=err_msg,
+                final_decision=False,
+            )
+
+        data_source_path = (
+            f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}"
+            if self.data_type == "full"
+            else self.scen.debug_path
+        )
+        env = get_ds_env(
+            extra_volumes={data_source_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=(
+                self.scen.real_full_timeout() if self.data_type == "full" else self.scen.real_debug_timeout()
+            ),
+        )
+
+        # 2) check the result and stdout after reruning the model.
+
+        # Read the content of files submission.csv and scores.csv before execution
+        submission_content_before = (
+            (implementation.workspace_path / "submission.csv").read_text()
+            if (implementation.workspace_path / "submission.csv").exists()
+            else NO_SUB
+        )
+        scores_content_before = (
+            (implementation.workspace_path / "scores.csv").read_text()
+            if (implementation.workspace_path / "scores.csv").exists()
+            else NO_SCORE
+        )
+
+        # Remove the files submission.csv and scores.csv
+        implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference"))
+
+        # Execute the main script
+        stdout = remove_eda_part(
+            implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference")
+        )
+
+        # walk model_folder and list the files
+        model_folder_files = [
+            str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
+        ]
+
+        opened_trace_lines = None
+        if (implementation.workspace_path / "trace.log").exists():
+            input_path = T("scenarios.data_science.share:scen.input_path").r()
+            abs_input_path = str(Path(input_path).resolve())
+            # matching path in string like `openat(AT_FDCWD, "/home/user/project/main.py", O_RDONLY) = 5`
+            path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"')
+            log_content = (implementation.workspace_path / "trace.log").read_text()
+
+            opened_files = set()
+            for line in log_content.splitlines():
+                if "openat" not in line or (abs_input_path not in line and input_path not in line):
+                    continue
+
+                match = path_regex.search(line)
+                if match:
+                    full_path = Path(match.group(1)).resolve()
+                    if str(full_path).startswith(abs_input_path):
+                        opened_files.add(Path(data_source_path).resolve() / full_path.relative_to(abs_input_path))
+
+            from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator
+
+            tree_gen = FileTreeGenerator(allowed_paths=opened_files)  # pass opened files filter
+            opened_trace_lines = tree_gen.generate_tree(Path(data_source_path).resolve())
+            # Limitation: training and test are expected to be different files.
+
+        # this will assert the generation of necessary files
+        for f in ["submission.csv", "scores.csv"]:
+            if not (implementation.workspace_path / f).exists():
+                err_msg = f"{f} does not exist. The model is not dumped. Make sure that the required files, like submission.csv and scores.csv, are created even if you bypass the model training step by loading the saved model file directly."
+                return CoSTEERSingleFeedback(
+                    execution=err_msg,
+                    return_checking=err_msg,
+                    code=err_msg,
+                    final_decision=False,
+                )
+
+        # Check if scores contain NaN (values)
+        score_df = pd.read_csv((implementation.workspace_path / "scores.csv"), index_col=0)
+        if score_df.isnull().values.any():
+            nan_locations = score_df[score_df.isnull().any(axis=1)]
+            err_msg = f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+            return CoSTEERSingleFeedback(
+                execution=err_msg,
+                return_checking=err_msg,
+                code=err_msg,
+                final_decision=False,
+            )
+
+        submission_content_after = (
+            (implementation.workspace_path / "submission.csv").read_text()
+            if (implementation.workspace_path / "submission.csv").exists()
+            else NO_SUB
+        )
+        scores_content_after = (
+            (implementation.workspace_path / "scores.csv").read_text()
+            if (implementation.workspace_path / "scores.csv").exists()
+            else NO_SCORE
+        )
+
+        system_prompt = T(".prompts:dump_model_eval.system").r()
+        user_prompt = T(".prompts:dump_model_eval.user").r(
+            stdout=stdout.strip(),
+            code=implementation.all_codes,
+            model_folder_files=model_folder_files,
+            scores_content_before=scores_content_before,
+            scores_content_after=scores_content_after,
+            opened_trace_lines=opened_trace_lines,
+        )
+
+        csfb = build_cls_from_json_with_retry(
+            CoSTEERSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+        )
+
+        if DS_RD_SETTING.model_dump_check_level == "high":
+            # Read the content of files submission.csv and scores.csv after execution
+            # Check if the content has changed
+            # excactly same checking. But it will take more user's time
+            if scores_content_before != scores_content_after:
+                return_msg = "\n[Error] The content of scores.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it."
+                return_msg += f"\nBefore:\n{scores_content_before}\nAfter:\n{scores_content_after}"
+                if submission_content_before != submission_content_after:
+                    # If the scores file changes, display the two contents and append it into the return_checking
+                    return_msg = "[Error] The content of submission.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it."
+                csfb.return_checking = (csfb.return_checking or "") + return_msg
+        return csfb

rdagent/components/coder/data_science/share/notebook.py

@@ -0,0 +1,135 @@
+"""
+Handles conversion from a Python file to a Jupyter notebook.
+"""
+
+import argparse
+from typing import Optional
+
+import nbformat
+
+from rdagent.components.coder.data_science.share.util import (
+    extract_first_section_name_from_code,
+    extract_function_body,
+    split_code_and_output_into_sections,
+)
+from rdagent.core.experiment import Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import MarkdownAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class NotebookConverter:
+    """
+    Builder responsible for writing a Jupyter notebook for a workspace.
+    """
+
+    def validate_code_format(self, code: str) -> str | None:
+        """
+        Returns None if the code format is valid, otherwise returns an error message.
+        """
+        main_function_body = extract_function_body(code, "main")
+        if not main_function_body:
+            return "[Error] No main function found in the code. Please ensure that the main function is defined and contains the necessary print statements to divide sections."
+
+        found_section_name = extract_first_section_name_from_code(main_function_body)
+        if not found_section_name:
+            return "[Error] No sections found in the code. Expected to see 'print(\"Section: <section name>\")' as section dividers. Also make sure that they are actually run and not just comments."
+
+        return None
+
+    def convert(
+        self,
+        task: Optional[Task],
+        code: str,
+        stdout: str,
+        outfile: Optional[str] = None,
+        use_debug_flag: bool = False,
+    ) -> str:
+        """
+        Build a notebook based on the current progression.
+        """
+        # Handle argparse in the code to ensure it works in a notebook environment
+        should_handle_argparse = "argparse" in code
+        sections = split_code_and_output_into_sections(code=code, stdout=stdout)
+        notebook = nbformat.v4.new_notebook()
+
+        # Use LLM to generate an intro cell for the notebook
+        if task:
+            system_prompt = T(".prompts:notebookconverter.system").r()
+            user_prompt = T(".prompts:notebookconverter.user").r(
+                plan=task.get_task_information(),
+                code=code,
+            )
+            resp = APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt, system_prompt=system_prompt
+            )
+            intro_content = MarkdownAgentOut.extract_output(resp)
+            notebook.cells.append(nbformat.v4.new_markdown_cell(intro_content))
+
+        if should_handle_argparse:
+            # Remove extra `import sys` since it will be added for argparse handling
+            if "import sys\n" in sections[0]["code"]:
+                sections[0]["code"] = sections[0]["code"].replace("import sys\n", "")
+
+            # Add sys.argv modification for argparse handling
+            sections[0]["code"] = (
+                "\n".join(
+                    [
+                        "import sys",
+                        "# hack to allow argparse to work in notebook",
+                        ('sys.argv = ["main.py", "--debug"]' if use_debug_flag else 'sys.argv = ["main.py"]'),
+                    ]
+                )
+                + "\n\n"
+                + sections[0]["code"].lstrip()
+            )
+
+        for section in sections:
+            # Create a markdown cell for the section name and comments
+            markdown_content = ""
+            if section["name"]:
+                markdown_content += f"## {section['name']}\n"
+            if section["comments"]:
+                markdown_content += f"{section['comments']}\n"
+            if markdown_content:
+                notebook.cells.append(nbformat.v4.new_markdown_cell(markdown_content))
+
+            # Create a code cell for the section code and output
+            if section["code"]:
+                cell = nbformat.v4.new_code_cell(section["code"])
+                if section["output"]:
+                    # For simplicity, treat all output as coming from stdout
+                    # TODO: support Jupyter kernel execution and handle outputs appropriately here
+                    cell.outputs = [nbformat.v4.new_output("stream", name="stdout", text=section["output"])]
+                notebook.cells.append(cell)
+
+        # Save the notebook or return it as a string
+        if outfile:
+            with open((outfile), "w", encoding="utf-8") as f:
+                nbformat.write(notebook, f)
+                logger.info(f"Notebook written to {outfile}")
+
+        return nbformat.writes(notebook)
+
+
+if __name__ == "__main__":
+    converter = NotebookConverter()
+    parser = argparse.ArgumentParser(description="Convert Python code to Jupyter notebook.")
+    parser.add_argument("inputfile", type=str, help="Path to the input Python file.")
+    parser.add_argument("outfile", type=str, help="Path to the output Notebook file.")
+    parser.add_argument(
+        "--stdout",
+        type=str,
+        default="",
+        help="Standard output from the code execution.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Use debug flag to modify sys.argv.")
+    args = parser.parse_args()
+    converter.convert(
+        task=None,
+        code=open(args.inputfile, "r").read(),
+        stdout=args.stdout,
+        outfile=args.outfile,
+        use_debug_flag=False,
+    )

rdagent/components/coder/data_science/share/prompts.yaml

@@ -0,0 +1,123 @@
+dump_model_coder:
+  guideline: |-
+    Your code will be executed in a inference mode with following command: 
+    ```bash
+    python main.py --inference
+    ```
+    Please dump the model in a "models/" subfolder in the first running, and the script rerun performs inference without needing to retrain the model when running the code again.
+    In inference Mode, the script MUST NOT load any training data. 
+    If there are parameters generated from the training data that might be needed for inference on test data, please save them in the "models/" subfolder as well.
+    If no test set is provided, reserve a portion of the data as your test set and save the generated test files in the models/ subfolder for use in submission and inference.
+    Make sure that the required files, like submission.csv and scores.csv, are created without model training step through loading the saved model and test data file directly.
+    
+
+dump_model_eval:
+  system: |-
+    You are a data scientist tasked with evaluating code generation. You've developed a Kaggle competition code that can produce a submission file.
+    The code should follow the guideline below:
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+
+    You will receive the following information:
+    - The implemented code
+    - The stdout from running the code
+    - The file list in "models/" subfolder
+    - The scores.csv file generated during both training and inference (if it exists)
+
+    Focus on these aspects:
+    - Check if the code saves the model in the "models/" subfolder.
+    - Check if the code saves the test data in the "models/" subfolder when there is no test data specified.
+    - Ensure that when the code is rerun in inference mode, it skips the training process and loads the model from the "models/" subfolder for direct inference.
+      - Verify that there is no training activity in the output.
+      - Verify that the script does not load the original training data.
+    - Ensure that even if you skip the model training by loading saved models, the files like scores.csv and submission.csv are still correctly created.
+    - The model's performance should remain consistent and not vary unreasonably between training and inference.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. Carefully check the stdout to ensure that when the code is rerun, it skips the training process and loads the model from the 'models/' subfolder for direct inference. Append the information that makes you think that the model is still being retrained when rerunning the code."
+        "return_checking": "Verify the generated files include necessary files. Make sure scores.csv file does not change unreasonably between training and inference",
+        "code": "The code has explicity dump the model into 'models/' subfolder; When the modes files are already in 'models/' subfolder, the code will explicity skip the training process.",
+        "final_decision": <true or false in boolean type; only return true when ensuring that the code saves the model in a 'models/' subfolder, and the script rerun performs inference without needing to retrain the model.>
+    }
+    ```
+
+  user: |-
+    ------------ The implemented code ------------ 
+    {{code}}
+
+    ------------ The stdout from running the code ------------ 
+    {{stdout}}
+
+    ------------ File opened by the code ------------
+    {{opened_trace_lines}}
+
+    ------------ The file list in "models/" subfolder ------------
+    {% for f in model_folder_files %}
+    - {{ f }}
+    {% endfor %}
+
+    ------------ The scores.csv file generated ------------
+    # Training:
+    {{scores_content_before}}
+
+    # Inference:
+    {{scores_content_after}}
+
+
+docdev:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %}  Your task is to create documentation for a data science solution.
+
+    You will be given:
+    - a list of files in the folder.
+    - content from some important files.
+
+    Please explain the trained models in the "models/" folder. The training and inference processes are detailed in the `main.py` file. The models' evaluation results are in `scores.csv`. Please respond with a markdown file that includes the following information:
+    - Explain the purpose of each model. If some models are part of a group (like those from cross-validation), describe them together.
+    - Provide key details for each model group:
+      - Important training parameters
+      - Model details
+      - Performance of each model
+
+    Be brief. Mention the file path when you introduce files.
+    Don't introduce anything other than models.
+
+    {% include "utils.agent.tpl:MarkdownOut" %}
+
+  user: |-
+    --------------- The file list in the workspace ---------------
+    {% for f in file_li %}
+    - {{ f }}
+    {% endfor %}
+
+    --------------- File content of each file ---------------
+    {% for fname, content in key_files.items() %}
+    File Path: {{fname}}
+    ```
+    {{content}}
+    ```
+    {% endfor %}
+
+notebookconverter:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %} Your task is to provide a summary for a data science solution.
+
+    You will be given:
+    - The original implementation plan for the script.
+    - A Python script that contains code and output.
+
+    Your task is to generate markdown content that includes a title and a short paragraph summarizing the technique in model training, the type of model produced and any other noteworthy details in the solution.
+
+    The return content should be like the format below(Please note that "````" is used to avoid confliction of "```" in markdown file)
+    ````markdown
+    # <The title of the notebook>
+    <the content of markdown file>
+    ````
+
+  user: |-
+    --------------- The implementation plan ---------------
+    {{plan}}
+
+    --------------- The Python script content ---------------
+    {{code}}

rdagent/components/coder/data_science/share/util.py

@@ -0,0 +1,365 @@
+import ast
+import io
+import re
+import tokenize
+from itertools import zip_longest
+from typing import List, Optional, Set, Tuple, TypedDict
+
+
+class CodeSection(TypedDict):
+    """
+    Represents a section of the original Python source code, to be converted to a notebook cell.
+    """
+
+    name: Optional[str]
+    code: Optional[str]
+    comments: Optional[str]
+    output: Optional[str]
+
+
+def extract_function_body(source_code: str, function_name: str) -> Optional[str]:
+    """
+    Extracts the body of a function from the source code.
+    Returns None if the function is not found.
+
+    Assumption: The function is multiline and defined at the top level.
+    """
+    tree = ast.parse(source_code)
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef) and node.name == function_name:
+            lines = source_code.splitlines()
+            start = node.body[0].lineno
+            end = node.body[-1].end_lineno
+            body_lines = lines[start - 1 : end]
+            indent_level = len(body_lines[0]) - len(body_lines[0].lstrip())
+            return "\n".join(line[indent_level:] for line in body_lines)
+    return None
+
+
+def split_sections(
+    text: str, section_header_regex: str, known_sections: Optional[list[str]] = None
+) -> tuple[Optional[str], list[str], list[str]]:
+    """
+    Split text into sections based on the section headers.
+    """
+    sections = []
+    section_names = []
+    current_section = []
+    next_section_name_index = 0
+    for line in text.splitlines():
+        match = re.match(section_header_regex, line)
+        extracted_section_name = match.group(1).strip() if match else None
+        if extracted_section_name and (
+            not known_sections
+            or (
+                next_section_name_index < len(known_sections)
+                and extracted_section_name == known_sections[next_section_name_index]
+            )
+        ):
+            if current_section:
+                sections.append("\n".join(current_section))
+                current_section = []
+            current_section.append(line)
+            section_names.append(extracted_section_name)
+            next_section_name_index += 1
+        else:
+            current_section.append(line)
+    if current_section:
+        sections.append("\n".join(current_section))
+
+    # If the first section does not match the header regex, treat it as a header section.
+    header_section = None
+    if sections and not re.search(section_header_regex, sections[0]):
+        header_section = sections[0]
+        sections = sections[1:]
+
+    return header_section, sections, section_names
+
+
+def split_code_sections(source_code: str) -> tuple[Optional[str], list[str]]:
+    """
+    Split code into sections based on the section headers.
+    """
+    return split_sections(source_code, r'^print\(["\']Section: (.+)["\']\)')
+
+
+def split_output_sections(stdout: str, known_sections: list[str]) -> tuple[Optional[str], list[str]]:
+    """
+    Split output into sections based on the section headers.
+    """
+    header_section, sections, _ = split_sections(stdout, r"^Section: (.+)", known_sections=known_sections)
+    return header_section, sections
+
+
+def extract_comment_under_first_print(source_code) -> tuple[Optional[str], str]:
+    """
+    Extract comments from the source code after the first print statement.
+    """
+    lines = source_code.splitlines()
+    lines_to_remove = set()
+    all_comments = []
+
+    parsed = ast.parse(source_code)
+    # Find the first print statement only
+    first_print_lineno = None
+    for node in ast.walk(parsed):
+        if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
+            if getattr(node.value.func, "id", None) == "print":
+                first_print_lineno = node.lineno
+                break
+
+    if first_print_lineno is None:
+        # No print statement found, return empty comments and original code
+        return None, source_code
+
+    for i in range(first_print_lineno, len(lines)):
+        stripped = lines[i].strip()
+        if stripped.startswith("#"):
+            comment_text = stripped.lstrip("# ").strip()
+            all_comments.append(comment_text)
+            lines_to_remove.add(i)
+        elif stripped == "":
+            continue
+        elif i > first_print_lineno:
+            break  # stop after hitting actual code line
+
+    cleaned_lines = [line for idx, line in enumerate(lines) if idx not in lines_to_remove]
+    cleaned_code = "\n".join(cleaned_lines)
+    comments_str = "\n".join(all_comments) if all_comments else None
+
+    return comments_str, cleaned_code
+
+
+def extract_first_section_name_from_code(source_code):
+    """
+    Extract the first section name from the source code.
+    """
+    parsed = ast.parse(source_code)
+    for node in ast.walk(parsed):
+        if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
+            call = node.value
+            if getattr(call.func, "id", None) == "print" and call.args:
+                arg0 = call.args[0]
+                if isinstance(arg0, ast.Constant) and isinstance(arg0.value, str):
+                    # Match "Section: ..." pattern
+                    m = re.match(r"Section:\s*(.+)", arg0.value)
+                    if m:
+                        return m.group(1).strip()
+    return None
+
+
+def extract_first_section_name_from_output(stdout: str) -> Optional[str]:
+    """
+    Extract the first section name from the output string.
+    """
+    match = re.search(r"Section:\s*(.+)", stdout)
+    if match:
+        return match.group(1).strip()
+    return None
+
+
+def is_function_called(source_code: str, func_name: str) -> bool:
+    """
+    Returns True if the function named `func_name` is called in `source_code`.
+    """
+    tree = ast.parse(source_code)
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Call):
+            # For simple function calls like func()
+            if isinstance(node.func, ast.Name) and node.func.id == func_name:
+                return True
+
+            # For calls like module.func()
+            elif isinstance(node.func, ast.Attribute) and node.func.attr == func_name:
+                return True
+    return False
+
+
+def remove_function(source_code: str, function_name: str) -> str:
+    """
+    Remove a function definition from the source code.
+    """
+    tree = ast.parse(source_code)
+    lines = source_code.splitlines()
+
+    for node in tree.body:
+        if isinstance(node, ast.FunctionDef) and node.name == function_name:
+            start_lineno = node.lineno - 1
+            end_lineno = node.end_lineno
+            return "\n".join(lines[:start_lineno] + lines[end_lineno:])
+
+    return source_code
+
+
+def remove_main_block(source_code: str) -> str:
+    """
+    Remove the if __name__ == "__main__": block from the source code.
+    """
+    tree = ast.parse(source_code)
+    lines = source_code.splitlines()
+
+    # Find the main block and note its line numbers
+    for node in tree.body:
+        if isinstance(node, ast.If):
+            test = node.test
+            if (
+                isinstance(test, ast.Compare)
+                and isinstance(test.left, ast.Name)
+                and test.left.id == "__name__"
+                and len(test.ops) == 1
+                and isinstance(test.ops[0], ast.Eq)
+                and len(test.comparators) == 1
+                and isinstance(test.comparators[0], ast.Constant)
+                and test.comparators[0].value == "__main__"
+            ):
+
+                # Remove lines corresponding to this block
+                start_lineno = node.lineno - 1
+                end_lineno = node.end_lineno
+                return "\n".join(lines[:start_lineno] + lines[end_lineno:])
+
+    return source_code
+
+
+def extract_top_level_functions_with_decorators_and_comments(
+    code: str,
+) -> List[Tuple[str, str]]:
+    """
+    Returns list of (function_name, source_segment) for top-level functions (excluding "main"),
+    including decorators and contiguous preceding comments.
+    """
+    # Parse AST to get function nodes
+    tree = ast.parse(code)
+    lines = code.splitlines(keepends=True)
+
+    # Precompute which line numbers have comment tokens
+    comment_lines: Set[int] = set()
+    lines = code.splitlines(keepends=True)  # preserve exact line content for prefix checks
+
+    tokgen = tokenize.generate_tokens(io.StringIO(code).readline)  # yields (type, string, start, end, line)
+    for tok_type, _, (srow, scol), _, _ in tokgen:
+        if tok_type == tokenize.COMMENT:
+            # everything before the comment on that line must be whitespace
+            prefix = lines[srow - 1][:scol]
+            if prefix.strip() == "":
+                comment_lines.add(srow)
+
+    functions = []
+
+    for node in tree.body:  # only top-level
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        if node.name == "main":
+            continue
+
+        # Determine the starting line: earliest decorator if present, else the def/async line
+        if node.decorator_list:
+            start_lineno = min(d.lineno for d in node.decorator_list)
+        else:
+            start_lineno = node.lineno
+
+        # Extend upward to include contiguous comment lines (no intervening non-blank/non-comment)
+        span_start = start_lineno
+        curr = span_start - 1  # check line above; lines are 1-based
+        while curr > 0:
+            line_text = lines[curr - 1]
+            if curr in comment_lines:
+                span_start = curr
+                curr -= 1
+                continue
+            if line_text.strip() == "":
+                # blank line: include it and keep scanning upward
+                span_start = curr
+                curr -= 1
+                continue
+            break  # encountered code or something else; stop
+
+        # Determine end line of the function definition including its body
+        # Prefer end_lineno if available (Python 3.8+)
+        if hasattr(node, "end_lineno") and node.end_lineno is not None:
+            span_end = node.end_lineno
+        else:
+            # Fallback: get last lineno from the deepest child in body
+            def _max_lineno(n):
+                max_ln = getattr(n, "lineno", 0)
+                for child in ast.iter_child_nodes(n):
+                    ln = _max_lineno(child)
+                    if ln > max_ln:
+                        max_ln = ln
+                return max_ln
+
+            span_end = _max_lineno(node)
+
+        # Slice the original source lines
+        segment = "".join(lines[span_start - 1 : span_end])
+        functions.append((node.name, segment))
+
+    return functions
+
+
+def split_code_and_output_into_sections(code: str, stdout: str) -> list[CodeSection]:
+    """
+    Converts a Python script and its output into a list of CodeSections.
+    Pre-condition: The code in the main() function contains print statements that indicate section names, e.g., `print("Section: <section name>")`.
+    """
+    # This will hold all top-level code and by default all function definitions.
+    # Functions will later be moved to more relevant sections if needed.
+    # The first step is to remove both the if __name__ == "__main__": block and the main function
+    top_level_code = remove_main_block(remove_function(code, "main"))
+
+    main_function_body = extract_function_body(code, "main")
+    functions = extract_top_level_functions_with_decorators_and_comments(top_level_code)
+
+    # Split the main function body into sections based on print("Section: <section name>") code
+    main_fn_top_level_section, main_fn_sections, known_section_names = (
+        split_code_sections(main_function_body) if main_function_body else (None, [], [])
+    )
+
+    # Split the output into sections based on "Section: " headers
+    output_top_level_section, output_sections = split_output_sections(stdout, known_section_names)
+
+    # Merge code and outputs into code sections
+    result_sections: list[CodeSection] = []
+    for output_section, code_section in zip_longest(output_sections, main_fn_sections):
+        name = None
+        if code_section is not None:
+            # If code section is available, extract the section name from it
+            name = extract_first_section_name_from_code(code_section)
+        elif output_section:
+            # If only output section is available, extract the section name from it
+            name = extract_first_section_name_from_output(output_section)
+        comments, cleaned_code = (
+            extract_comment_under_first_print(code_section) if code_section is not None else (None, None)
+        )
+        # Strip whitespaces for the cell
+        if cleaned_code is not None:
+            cleaned_code = cleaned_code.strip()
+        result_sections.append(CodeSection(name=name, code=cleaned_code, comments=comments, output=output_section))
+
+    # Small optimization: move function definitions to the sections where they are first called
+    # TODO: this doesn't handle nested function references, e.g., fn A calls fn B which calls fn C
+    # currently will not move C to the section where A is called
+    for name, segment in functions:
+        for section in result_sections:
+            if section["code"] and is_function_called(section["code"], name):
+                section["code"] = segment.strip() + "\n\n" + section["code"].lstrip()
+                top_level_code = top_level_code.replace(segment, "")
+                break
+
+    # Inject the top-level code at the beginning of the sections
+    top_level_code = (
+        top_level_code.rstrip() + "\n\n" + main_fn_top_level_section.lstrip()
+        if main_fn_top_level_section
+        else top_level_code
+    )
+    result_sections.insert(
+        0,
+        CodeSection(
+            name=None,
+            code=top_level_code,
+            comments=None,
+            output=output_top_level_section,
+        ),
+    )
+
+    return result_sections

rdagent/components/coder/data_science/utils.py

@@ -0,0 +1,6 @@
+import re
+
+
+def remove_eda_part(stdout: str) -> str:
+    """Data Science scenario have a LLM-based EDA feature. We can remove it when current task does not involve EDA"""
+    return re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout, flags=re.DOTALL)

rdagent/components/coder/data_science/workflow/__init__.py

@@ -0,0 +1,132 @@
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER
+from rdagent.components.coder.data_science.workflow.eval import (
+    WorkflowGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: WorkflowTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        workflow_information_str = target_task.get_task_information()
+
+        # 1. query
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[workflow_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[workflow_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # 2. code
+        system_prompt = T(".prompts:workflow_coder.system").r(
+            task_desc=workflow_information_str,
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+        )
+        user_prompt = T(".prompts:workflow_coder.user").r(
+            load_data_code=workspace.file_dict["load_data.py"],
+            feature_code=workspace.file_dict["feature.py"],
+            model_codes=workspace.get_codes(r"^model_(?!test)\w+\.py$"),
+            ensemble_code=workspace.file_dict["ensemble.py"],
+            latest_code=workspace.file_dict.get("main.py"),
+            code_spec=(
+                workspace.file_dict["spec/workflow.md"]
+                if DS_RD_SETTING.spec_enabled
+                else T("scenarios.data_science.share:component_spec.Workflow").r()
+            ),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            workflow_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if workflow_code != workspace.file_dict.get("main.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new workflow code.")
+
+        return {"main.py": workflow_code}
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class WorkflowCoSTEER(DSCoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            WorkflowGeneralCaseSpecEvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = WorkflowMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/workflow/eval.py

@@ -0,0 +1,158 @@
+import json
+import re
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERMultiFeedback,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+WorkflowSingleFeedback = CoSTEERSingleFeedback
+WorkflowMultiFeedback = CoSTEERMultiFeedback
+
+
+class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
+    """
+    Motivation case:
+    - Simplest case, we already split the data into train_data, valid_data, and test_data. We require the model to learn (optionally validate on valid data), and infer on test data.
+
+    Test workflow:
+    - Build train, valid, and test data to run it, and test the output (e.g., shape, etc.)
+    """
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> CoSTEERSingleFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return WorkflowSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
+
+        # # DockerEnv for MLEBench submission validation
+        # mle_de_conf = MLEBDockerConf()
+        # mle_de_conf.extra_volumes = {
+        #     f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
+        # }
+        # mde = DockerEnv(conf=mle_de_conf)
+        # mde.prepare()
+
+        # Clean the scores.csv & submission.csv.
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
+
+        stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py")
+
+        # remove EDA part
+        stdout = remove_eda_part(stdout)
+
+        # Check score file
+        score_fp = implementation.workspace_path / "scores.csv"
+        score_ret_code = 0
+        score_check_text = ""
+        if not score_fp.exists():
+            score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
+            score_ret_code = 1
+            implementation.execute(env=env, entry="python -m coverage json -o coverage.json")
+            coverage_report_path = implementation.workspace_path / "coverage.json"
+            if coverage_report_path.exists():
+                used_files = set(json.loads(coverage_report_path.read_text())["files"].keys())
+                coverage_report_path.unlink()
+                logger.info(f"All used scripts: {used_files}")
+                if len(used_files) == 1:
+                    score_check_text += f"\n[Error] The only used script is {used_files}.\nPlease check if you have implemented entry point in 'main.py'."
+        else:
+            try:
+                score_df = pd.read_csv(score_fp, index_col=0)
+                model_set_in_scores = set(score_df.index)
+                # We assume that model names in `score_df` are stored without the '.py' file extension.
+                model_set_in_folder = set(
+                    f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
+                )
+
+                # Check model names (index)
+                if model_set_in_scores != model_set_in_folder.union({"ensemble"}):
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {model_set_in_folder.union({'ensemble'})}\nscore_df is:\n{score_df}"
+                    score_ret_code = 1
+
+                # Check metric name (columns) - case insensitive
+                if [col.lower() for col in score_df.columns.tolist()] != [self.scen.metric_name.lower()]:
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
+                    score_ret_code = 1
+
+                # Check if scores contain NaN (values)
+                if score_df.isnull().values.any():
+                    nan_locations = score_df[score_df.isnull().any(axis=1)]
+                    score_check_text += f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+                    score_ret_code = 1
+
+            except Exception as e:
+                score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
+                score_ret_code = 1
+
+        # Check submission file
+        base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
+        implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+        # stdout += "----Submission Check 1-----\n"
+        submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+        submission_check_out = submission_result.stdout
+        submission_ret_code = submission_result.exit_code
+        stdout += "\n" + submission_check_out
+
+        system_prompt = T(".prompts:workflow_eval.system").r(
+            # here we pass `None` to `eda_output` because we do not have nor need EDA output for workflow.
+            scenario=self.scen.get_scenario_all_desc(eda_output=None),
+            task_desc=target_task.get_task_information(),
+            spec=(
+                implementation.file_dict["spec/workflow.md"]
+                if DS_RD_SETTING.spec_enabled
+                else T("scenarios.data_science.share:component_spec.Workflow").r()
+            ),
+        )
+        user_prompt = T(".prompts:workflow_eval.user").r(
+            stdout=stdout.strip(),
+            code=implementation.file_dict["main.py"],
+        )
+        wfb = build_cls_from_json_with_retry(
+            WorkflowSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=WorkflowSingleFeedback.val_and_update_init_dict,
+        )
+        if score_ret_code != 0:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + score_check_text
+        if submission_ret_code != 0:
+            wfb.final_decision = False
+            wfb.return_checking += "\nSubmission file check failed."
+        return wfb

rdagent/components/coder/data_science/workflow/eval_tests/submission_format_test.txt

@@ -0,0 +1,77 @@
+from pathlib import Path
+import pandas as pd
+import hashlib
+
+def calculate_md5(file_path):
+    with open(file_path, "rb") as f:
+        file_hash = hashlib.md5(f.read()).hexdigest()
+    return file_hash
+
+file_md5 = calculate_md5("scores.csv")
+
+"""
+find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
+./denoising-dirty-documents/sampleSubmission.csv
+./the-icml-2013-whale-challenge-right-whale-redux/sampleSubmission.csv
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv.zip
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv
+./random-acts-of-pizza/sampleSubmission.csv
+./text-normalization-challenge-english-language/en_sample_submission_2.csv.zip
+./text-normalization-challenge-english-language/en_sample_submission_2.csv
+./detecting-insults-in-social-commentary/sample_submission_null.csv
+"""
+
+# Find sample submission file dynamically
+input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}")
+# Look for common variations of sample submission filenames
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
+                         list(input_dir.glob("*sampleSubmission*.csv"))
+
+assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}"
+
+# Use first matching file
+sample_submission_name = sample_submission_files[0].name
+SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+print(f"Using sample submission file: {sample_submission_name}")
+
+# Check if the sample submission file exists
+assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
+
+# Check if our submission file exists
+assert Path('submission.csv').exists(), "Error: submission.csv not found"
+
+sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+our_submission = pd.read_csv('submission.csv')
+
+success = True
+# Print the columns of the sample submission file
+print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+print("Columns in our_submission.csv:", our_submission.columns)
+
+for col in sample_submission.columns:
+    if col not in our_submission.columns:
+        success = False
+        print(f'Column {col} not found in submission.csv')
+
+if success:
+    print(f'submission.csv\'s columns aligns with {sample_submission_name} .')
+
+
+# Print the first 5 rows of the two submission files, with columns separated by commas.
+def print_first_rows(file_path, file_name, num_rows=5):
+    print(f"\nFirst {num_rows} rows of {file_name}:")
+    try:
+        with open(file_path, 'r') as file:
+            for i, line in enumerate(file):
+                if i < num_rows:
+                    print(line.strip())
+                else:
+                    break
+    except FileNotFoundError:
+        print(f"Error: {file_name} not found.")
+
+print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+print_first_rows('submission.csv', 'submission.csv')
+
+assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
+print(f"\nPlease Checked the content of the submission file(submission.csv should align with {sample_submission_name}). ")

rdagent/components/coder/data_science/workflow/exp.py

@@ -0,0 +1,14 @@
+import pickle
+import site
+import traceback
+from pathlib import Path
+from typing import Dict, Optional
+
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.core.utils import cache_with_pickle
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class WorkflowTask(CoSTEERTask):
+    def __init__(self, name: str = "Workflow", *args, **kwargs) -> None:
+        super().__init__(name=name, *args, **kwargs)

rdagent/components/coder/data_science/workflow/prompts.yaml

@@ -0,0 +1,137 @@
+workflow_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+
+    Here is the competition information for this task:
+    {{ competition_info }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.file_dict["main.py"] }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.file_dict["main.py"] }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+    ## Guidelines
+    1. Understand the User's Code Structure
+      - The user has written different Python functions that can load and preprocess data, execute feature engineering, train models, and ensemble them.
+      - Each functionality is in a separate Python file.
+    2. Your task is only to integrate the existing processes of load_data, feature, model, and ensemble into a complete workflow. Do not edit or modify the existing Python files. The final step should output the predictions in the required format.
+    3. The user may provide specific code organization rules and instructions. Ensure that the integration follows the given framework and structure.
+    4. After predicting the output, print the shape and other information of the output to stdout to help the evaluator assess the code.
+    5. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+  
+  user: |-
+    --------- Code Specification ---------
+    {{ code_spec }}
+
+    --------- load data code ---------
+    file: load_data.py
+    {{ load_data_code }}
+
+    --------- feature engineering code ---------
+    file: feature.py
+    {{ feature_code }}
+
+    --------- model training code ---------
+    Attention: The input and output of the model function is flexible. Training dataset is necessary, but validation and test dateset might be optional. The hyperparameters can either be passed as arguments or be set as default values in the function. You need to use the function correctly.
+    All model files share the same function name. Please import the model files with their name like: from {file_name} import {function_name}
+    {{ model_codes }}
+
+    --------- ensemble code ---------
+    Note, we will check the index of the score.csv, so please use the model name as the index to feed into ensemble function.
+    file: ensemble.py
+    {{ ensemble_code }}
+
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    {% endif %}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% endif %}
+
+workflow_eval:
+  system: |-
+    You are a data scientist responsible for evaluating workflow code generation.
+
+    ## Task Description
+    The user is trying to build a workflow in the following scenario:
+    {{ scenario }}
+
+    The main code generation task is as follows:
+    {{ task_desc }}
+
+    The user provides workflow information and its components.
+    The details on how to structure the workflow are given in the specification file:
+    ```markdown
+    {{ spec }}
+    ```
+
+    This workflow integrates multiple stages, including:
+    - Data loading
+    - Feature engineering
+    - Model training
+    - Ensembling
+
+    ## Evaluation Scope
+    Your focus is to check whether the workflow code:
+    1. Executes successfully, correctly organizing components and generating a final submission.
+    2. Generates predictions in the correct format, ensuring they align with the **sample submission** structure!
+
+    [Note] 
+    1. The individual components (data loading, feature engineering, model tuning, etc.) have already been evaluated by the user. You should only evaluate and improve the workflow code, unless there are critical issues in the components.
+    2. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
+    3. As long as the execution does not exceed the time limit, ensure that the code uses cross-validation to split the training data and train the model. If cross-validation is not used, mention it in the execution section and set `final_decision` to `false`.
+
+    ## Evaluation Criteria
+    You will be given the workflow execution output (`stdout`) to determine correctness.  
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the main workflow executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission, checking the index, column names, and CSV content.",
+        "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+  
+  user: |-
+    --------- Workflow test stdout ---------
+    {{ stdout }}
+    --------- Workflow code generated by user ---------
+    {{ code }}

rdagent/components/coder/data_science/workflow/test.py

@@ -0,0 +1,59 @@
+"""
+Generate dataset to test the workflow output
+"""
+
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
+from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
+from rdagent.components.coder.data_science.workflow.eval import (
+    WorkflowGeneralCaseSpecEvaluator,
+)
+from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
+from rdagent.core.experiment import FBWorkspace
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.scen import KaggleScen
+
+
+def develop_one_competition(competition: str):
+    scen = KaggleScen(competition=competition)
+    workflow_coder = WorkflowCoSTEER(scen)
+
+    wt = WorkflowTask(
+        name="WorkflowTask",
+        description="Integrate the existing processes of load_data, feature, model, and ensemble into a complete workflow.",
+        base_code="",
+    )
+
+    tpl_ex_path = Path(__file__).resolve() / Path("rdagent/scenarios/kaggle/tpl_ex").resolve() / competition
+    injected_file_names = ["spec/workflow.md", "load_data.py", "feature.py", "model01.py", "ensemble.py", "main.py"]
+
+    workflowexp = FBWorkspace()
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        workflowexp.inject_files(**{file_name: file_path.read_text()})
+
+    wt.base_code += workflowexp.file_dict["main.py"]
+    exp = DSExperiment(
+        sub_tasks=[wt],
+    )
+
+    """es = WorkflowMultiProcessEvolvingStrategy(scen=scen, settings=CoSTEER_SETTINGS)
+    new_code = es.implement_one_task(target_task=wt, queried_knowledge=None, workspace = workflowexp)
+    print(new_code)"""
+
+    """eva = WorkflowGeneralCaseSpecEvaluator(scen=scen)
+    exp.feedback = eva.evaluate(target_task=wt, queried_knowledge=None, implementation=workflowexp, gt_implementation=None)
+    print(exp.feedback)"""
+
+    # Run the experiment
+    for file_name in injected_file_names:
+        file_path = tpl_ex_path / file_name
+        exp.experiment_workspace.inject_files(**{file_name: file_path.read_text()})
+
+    exp = workflow_coder.develop(exp)
+
+
+if __name__ == "__main__":
+    develop_one_competition("aerial-cactus-identification")
+    # dotenv run -- python rdagent/components/coder/data_science/workflow/test.py