chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,32 @@
|
||||
from rdagent.components.coder.CoSTEER import CoSTEER
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder
|
||||
from rdagent.components.coder.factor_coder.evolving_strategy import (
|
||||
FactorMultiProcessEvolvingStrategy,
|
||||
)
|
||||
from rdagent.core.experiment import Experiment
|
||||
from rdagent.core.scenario import Scenario
|
||||
|
||||
|
||||
class FactorCoSTEER(CoSTEER):
|
||||
def __init__(
|
||||
self,
|
||||
scen: Scenario,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
setting = FACTOR_COSTEER_SETTINGS
|
||||
eva = CoSTEERMultiEvaluator(FactorEvaluatorForCoder(scen=scen), scen=scen)
|
||||
es = FactorMultiProcessEvolvingStrategy(scen=scen, settings=FACTOR_COSTEER_SETTINGS)
|
||||
|
||||
super().__init__(*args, settings=setting, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs)
|
||||
|
||||
def develop(self, exp: Experiment) -> Experiment:
|
||||
try:
|
||||
exp = super().develop(exp)
|
||||
finally:
|
||||
if hasattr(self, "evolve_agent") and self.evolve_agent.evolving_trace:
|
||||
es = self.evolve_agent.evolving_trace[-1]
|
||||
exp.prop_dev_feedback = es.feedback
|
||||
return exp
|
||||
@@ -0,0 +1,49 @@
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from pydantic_settings import SettingsConfigDict
|
||||
|
||||
from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
|
||||
from rdagent.utils.env import CondaConf, Env, LocalEnv
|
||||
|
||||
|
||||
class FactorCoSTEERSettings(CoSTEERSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="FACTOR_CoSTEER_")
|
||||
|
||||
data_folder: str = "git_ignore_folder/factor_implementation_source_data"
|
||||
"""Path to the folder containing financial data (default is fundamental data in Qlib)"""
|
||||
|
||||
data_folder_debug: str = "git_ignore_folder/factor_implementation_source_data_debug"
|
||||
"""Path to the folder containing partial financial data (for debugging)"""
|
||||
|
||||
simple_background: bool = False
|
||||
"""Whether to use simple background information for code feedback"""
|
||||
|
||||
file_based_execution_timeout: int = 3600
|
||||
"""Timeout in seconds for each factor implementation execution"""
|
||||
|
||||
select_method: str = "random"
|
||||
"""Method for the selection of factors implementation"""
|
||||
|
||||
python_bin: str = "python"
|
||||
"""Path to the Python binary"""
|
||||
|
||||
|
||||
def get_factor_env(
|
||||
conf_type: Optional[str] = None,
|
||||
extra_volumes: dict = {},
|
||||
running_timeout_period: int = 600,
|
||||
enable_cache: Optional[bool] = None,
|
||||
) -> Env:
|
||||
conf = FactorCoSTEERSettings()
|
||||
if hasattr(conf, "python_bin"):
|
||||
env = LocalEnv(conf=(CondaConf(conda_env_name=os.environ.get("CONDA_DEFAULT_ENV"))))
|
||||
env.conf.extra_volumes = extra_volumes.copy()
|
||||
env.conf.running_timeout_period = running_timeout_period
|
||||
if enable_cache is not None:
|
||||
env.conf.enable_cache = enable_cache
|
||||
env.prepare()
|
||||
return env
|
||||
|
||||
|
||||
FACTOR_COSTEER_SETTINGS = FactorCoSTEERSettings()
|
||||
@@ -0,0 +1,550 @@
|
||||
import io
|
||||
import json
|
||||
from abc import abstractmethod
|
||||
from typing import Dict, Tuple
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
from rdagent.core.experiment import Task, Workspace
|
||||
from rdagent.oai.llm_conf import LLM_SETTINGS
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
class FactorEvaluator:
|
||||
"""Although the init method is same to Evaluator, but we want to emphasize they are different"""
|
||||
|
||||
def __init__(self, scen=None) -> None:
|
||||
self.scen = scen
|
||||
|
||||
@abstractmethod
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: Task,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
**kwargs,
|
||||
) -> Tuple[str, object]:
|
||||
"""You can get the dataframe by
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
_, gen_df = implementation.execute()
|
||||
_, gt_df = gt_implementation.execute()
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tuple[str, object]
|
||||
- str: the text-based description of the evaluation result
|
||||
- object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result
|
||||
|
||||
"""
|
||||
raise NotImplementedError("Please implement the `evaluator` method")
|
||||
|
||||
def _get_df(self, gt_implementation: Workspace, implementation: Workspace):
|
||||
if gt_implementation is not None:
|
||||
_, gt_df = gt_implementation.execute()
|
||||
if isinstance(gt_df, pd.Series):
|
||||
gt_df = gt_df.to_frame("gt_factor")
|
||||
if isinstance(gt_df, pd.DataFrame):
|
||||
gt_df = gt_df.sort_index()
|
||||
else:
|
||||
gt_df = None
|
||||
|
||||
_, gen_df = implementation.execute()
|
||||
if isinstance(gen_df, pd.Series):
|
||||
gen_df = gen_df.to_frame("source_factor")
|
||||
if isinstance(gen_df, pd.DataFrame):
|
||||
gen_df = gen_df.sort_index()
|
||||
return gt_df, gen_df
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorCodeEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
execution_feedback: str,
|
||||
value_feedback: str = "",
|
||||
gt_implementation: Workspace = None,
|
||||
**kwargs,
|
||||
):
|
||||
factor_information = target_task.get_task_information()
|
||||
code = implementation.all_codes
|
||||
|
||||
system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r(
|
||||
scenario=(
|
||||
self.scen.get_scenario_all_desc(
|
||||
target_task,
|
||||
filtered_tag="feature",
|
||||
simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
|
||||
)
|
||||
if self.scen is not None
|
||||
else "No scenario description."
|
||||
)
|
||||
)
|
||||
|
||||
execution_feedback_to_render = execution_feedback
|
||||
for _ in range(10): # 10 times to split the content is enough
|
||||
user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r(
|
||||
factor_information=factor_information,
|
||||
code=code,
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
value_feedback=value_feedback,
|
||||
gt_code=gt_implementation.code if gt_implementation else None,
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
)
|
||||
> APIBackend().chat_token_limit
|
||||
):
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
|
||||
else:
|
||||
break
|
||||
critic_response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
|
||||
return critic_response, None
|
||||
|
||||
|
||||
class FactorInfEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
_, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
INF_count = gen_df.isin([float("inf"), -float("inf")]).sum().sum()
|
||||
if INF_count == 0:
|
||||
return "The source dataframe does not have any infinite values.", True
|
||||
else:
|
||||
return (
|
||||
f"The source dataframe has {INF_count} infinite values. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
class FactorSingleColumnEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
_, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
if len(gen_df.columns) == 1:
|
||||
return "The source dataframe has only one column which is correct.", True
|
||||
else:
|
||||
return (
|
||||
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
class FactorOutputFormatEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Skip the evaluation of the output format.",
|
||||
False,
|
||||
)
|
||||
buffer = io.StringIO()
|
||||
gen_df.info(buf=buffer)
|
||||
gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
|
||||
system_prompt = T(".prompts:evaluator_output_format_system").r(
|
||||
scenario=(
|
||||
self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
|
||||
if self.scen is not None
|
||||
else "No scenario description."
|
||||
)
|
||||
)
|
||||
|
||||
# TODO: with retry_context(retry_n=3, except_list=[KeyError]):
|
||||
max_attempts = 3
|
||||
attempts = 0
|
||||
final_evaluation_dict = None
|
||||
|
||||
while attempts < max_attempts:
|
||||
try:
|
||||
api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
|
||||
resp = api.build_messages_and_create_chat_completion(
|
||||
user_prompt=gen_df_info_str,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str | bool | int],
|
||||
)
|
||||
resp_dict = json.loads(resp)
|
||||
resp_dict["output_format_decision"] = str(resp_dict["output_format_decision"]).lower() in ["true", "1"]
|
||||
|
||||
return (
|
||||
str(resp_dict["output_format_feedback"]),
|
||||
resp_dict["output_format_decision"],
|
||||
)
|
||||
except (KeyError, json.JSONDecodeError) as e:
|
||||
attempts += 1
|
||||
if attempts >= max_attempts:
|
||||
raise KeyError(
|
||||
"Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
|
||||
) from e
|
||||
|
||||
return "Failed to evaluate output format after multiple attempts.", False
|
||||
|
||||
|
||||
class FactorDatetimeDailyEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str | object]:
|
||||
_, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return "The source dataframe is None. Skip the evaluation of the datetime format.", False
|
||||
|
||||
if "datetime" not in gen_df.index.names:
|
||||
return "The source dataframe does not have a datetime index. Please check the implementation.", False
|
||||
|
||||
try:
|
||||
pd.to_datetime(gen_df.index.get_level_values("datetime"))
|
||||
except Exception:
|
||||
return (
|
||||
f"The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.\n The head of the output dataframe is: \n{gen_df.head()}",
|
||||
False,
|
||||
)
|
||||
|
||||
time_diff = pd.to_datetime(gen_df.index.get_level_values("datetime")).to_series().diff().dropna().unique()
|
||||
if pd.Timedelta(minutes=1) in time_diff:
|
||||
return (
|
||||
"The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
return "The generated dataframe is daily.", True
|
||||
|
||||
|
||||
class FactorRowCountEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
|
||||
return (
|
||||
(
|
||||
f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
|
||||
+ "Please verify the implementation. "
|
||||
if ratio <= 0.99
|
||||
else ""
|
||||
),
|
||||
ratio,
|
||||
)
|
||||
|
||||
|
||||
class FactorIndexEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index)
|
||||
similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set))
|
||||
return (
|
||||
(
|
||||
f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. "
|
||||
+ "Please check the implementation."
|
||||
if similarity <= 0.99
|
||||
else ""
|
||||
),
|
||||
similarity,
|
||||
)
|
||||
|
||||
|
||||
class FactorMissingValuesEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
|
||||
return "Both dataframes have the same missing values.", True
|
||||
else:
|
||||
return (
|
||||
f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
class FactorEqualValueRatioEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
-1,
|
||||
)
|
||||
try:
|
||||
close_values = gen_df.sub(gt_df).abs().lt(1e-6)
|
||||
result_int = close_values.astype(int)
|
||||
pos_num = result_int.sum().sum()
|
||||
acc_rate = pos_num / close_values.size
|
||||
except:
|
||||
close_values = gen_df
|
||||
if close_values.all().iloc[0]:
|
||||
return (
|
||||
"All values in the dataframes are equal within the tolerance of 1e-6.",
|
||||
acc_rate,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
|
||||
acc_rate,
|
||||
)
|
||||
|
||||
|
||||
class FactorCorrelationEvaluator(FactorEvaluator):
|
||||
def __init__(self, hard_check: bool, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.hard_check = hard_check
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
concat_df = pd.concat([gen_df, gt_df], axis=1)
|
||||
concat_df.columns = ["source", "gt"]
|
||||
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
|
||||
ric = (
|
||||
concat_df.groupby("datetime")
|
||||
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
|
||||
.dropna()
|
||||
.mean()
|
||||
)
|
||||
|
||||
if self.hard_check:
|
||||
if ic > 0.99 and ric > 0.99:
|
||||
return (
|
||||
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
|
||||
True,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
|
||||
False,
|
||||
)
|
||||
else:
|
||||
return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
|
||||
|
||||
|
||||
class FactorValueEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
version: int = 1, # 1 for qlib factors and 2 for kaggle factors
|
||||
**kwargs,
|
||||
) -> Tuple:
|
||||
conclusions = []
|
||||
|
||||
# Initialize result variables
|
||||
row_result = 0
|
||||
index_result = 0
|
||||
output_format_result = None
|
||||
equal_value_ratio_result = 0
|
||||
high_correlation_result = False
|
||||
row_result = None
|
||||
|
||||
# Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
|
||||
if version == 1:
|
||||
feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
elif version == 2:
|
||||
input_shape = self.scen.input_shape
|
||||
_, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df.shape[-1] > input_shape[-1]:
|
||||
conclusions.append(
|
||||
"Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure."
|
||||
)
|
||||
|
||||
feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
# Check if the index of the dataframe is ("datetime", "instrument")
|
||||
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
if version == 1:
|
||||
feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
|
||||
implementation, gt_implementation
|
||||
)
|
||||
conclusions.append(feedback_str)
|
||||
else:
|
||||
daily_check_result = None
|
||||
|
||||
# Check dataframe format
|
||||
if gt_implementation is not None:
|
||||
feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
|
||||
implementation, gt_implementation
|
||||
)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
|
||||
implementation, gt_implementation
|
||||
)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
if index_result > 0.99:
|
||||
feedback_str, high_correlation_result = FactorCorrelationEvaluator(
|
||||
hard_check=True, scen=self.scen
|
||||
).evaluate(implementation, gt_implementation)
|
||||
else:
|
||||
high_correlation_result = False
|
||||
feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless"
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
# Combine all conclusions into a single string
|
||||
conclusion_str = "\n".join(conclusions)
|
||||
|
||||
if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
|
||||
decision_from_value_check = True
|
||||
elif (
|
||||
row_result is not None
|
||||
and row_result <= 0.99
|
||||
or output_format_result is False
|
||||
or daily_check_result is False
|
||||
or inf_evaluate_res is False
|
||||
):
|
||||
decision_from_value_check = False
|
||||
else:
|
||||
decision_from_value_check = None
|
||||
return conclusion_str, decision_from_value_check
|
||||
|
||||
|
||||
class FactorFinalDecisionEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
execution_feedback: str,
|
||||
value_feedback: str,
|
||||
code_feedback: str,
|
||||
**kwargs,
|
||||
) -> Tuple:
|
||||
system_prompt = T(".prompts:evaluator_final_decision_v1_system").r(
|
||||
scenario=(
|
||||
self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
|
||||
if self.scen is not None
|
||||
else "No scenario description."
|
||||
)
|
||||
)
|
||||
execution_feedback_to_render = execution_feedback
|
||||
|
||||
for _ in range(10): # 10 times to split the content is enough
|
||||
user_prompt = T(".prompts:evaluator_final_decision_v1_user").r(
|
||||
factor_information=target_task.get_task_information(),
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
code_feedback=code_feedback,
|
||||
value_feedback=(
|
||||
value_feedback
|
||||
if value_feedback is not None
|
||||
else "No Ground Truth Value provided, so no evaluation on value is performed."
|
||||
),
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
)
|
||||
> APIBackend().chat_token_limit
|
||||
):
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
|
||||
else:
|
||||
break
|
||||
|
||||
# TODO: with retry_context(retry_n=3, except_list=[KeyError]):
|
||||
final_evaluation_dict = None
|
||||
attempts = 0
|
||||
max_attempts = 3
|
||||
|
||||
while attempts < max_attempts:
|
||||
try:
|
||||
api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
|
||||
final_evaluation_dict = json.loads(
|
||||
api.build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
seed=attempts, # in case of useless retrying when cache enabled.
|
||||
json_target_type=Dict[str, str | bool | int],
|
||||
),
|
||||
)
|
||||
final_decision = final_evaluation_dict["final_decision"]
|
||||
final_feedback = final_evaluation_dict["final_feedback"]
|
||||
|
||||
final_decision = str(final_decision).lower() in ["true", "1"]
|
||||
return final_decision, final_feedback
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("Failed to decode JSON response from API.") from e
|
||||
except KeyError as e:
|
||||
attempts += 1
|
||||
if attempts >= max_attempts:
|
||||
raise KeyError(
|
||||
"Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts."
|
||||
) from e
|
||||
|
||||
return None, None
|
||||
@@ -0,0 +1,130 @@
|
||||
import re
|
||||
|
||||
from rdagent.components.coder.CoSTEER.evaluators import (
|
||||
CoSTEEREvaluator,
|
||||
CoSTEERMultiFeedback,
|
||||
CoSTEERSingleFeedbackDeprecated,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.eva_utils import (
|
||||
FactorCodeEvaluator,
|
||||
FactorFinalDecisionEvaluator,
|
||||
FactorValueEvaluator,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
from rdagent.core.evolving_framework import QueriedKnowledge
|
||||
from rdagent.core.experiment import Workspace
|
||||
|
||||
FactorSingleFeedback = CoSTEERSingleFeedbackDeprecated
|
||||
|
||||
|
||||
class FactorEvaluatorForCoder(CoSTEEREvaluator):
|
||||
"""This class is the v1 version of evaluator for a single factor implementation.
|
||||
It calls several evaluators in share modules to evaluate the factor implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.value_evaluator = FactorValueEvaluator(self.scen)
|
||||
self.code_evaluator = FactorCodeEvaluator(self.scen)
|
||||
self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace = None,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> FactorSingleFeedback:
|
||||
if implementation is None:
|
||||
return None
|
||||
|
||||
target_task_information = target_task.get_task_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
||||
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
||||
return FactorSingleFeedback(
|
||||
execution_feedback="This task has failed too many times, skip implementation.",
|
||||
value_generated_flag=False,
|
||||
code_feedback="This task has failed too many times, skip code evaluation.",
|
||||
value_feedback="This task has failed too many times, skip value evaluation.",
|
||||
final_decision=False,
|
||||
final_feedback="This task has failed too many times, skip final decision evaluation.",
|
||||
final_decision_based_on_gt=False,
|
||||
)
|
||||
else:
|
||||
factor_feedback = FactorSingleFeedback()
|
||||
|
||||
# 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
|
||||
(
|
||||
execution_feedback,
|
||||
gen_df,
|
||||
) = implementation.execute()
|
||||
|
||||
execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
|
||||
factor_feedback.execution_feedback = "\n".join(
|
||||
[line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
|
||||
)
|
||||
|
||||
# 2. Get factor value feedback
|
||||
if gen_df is None:
|
||||
factor_feedback.value_feedback = "No factor value generated, skip value evaluation."
|
||||
factor_feedback.value_generated_flag = False
|
||||
decision_from_value_check = None
|
||||
else:
|
||||
factor_feedback.value_generated_flag = True
|
||||
(
|
||||
factor_feedback.value_feedback,
|
||||
decision_from_value_check,
|
||||
) = self.value_evaluator.evaluate(
|
||||
implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
|
||||
)
|
||||
|
||||
factor_feedback.final_decision_based_on_gt = gt_implementation is not None
|
||||
|
||||
if decision_from_value_check is not None and decision_from_value_check is True:
|
||||
# To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
|
||||
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
|
||||
elif decision_from_value_check is not None and decision_from_value_check is False:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
|
||||
else:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
(
|
||||
factor_feedback.final_decision,
|
||||
factor_feedback.final_feedback,
|
||||
) = self.final_decision_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
code_feedback=factor_feedback.code_feedback,
|
||||
)
|
||||
return factor_feedback
|
||||
|
||||
|
||||
# TODO:
|
||||
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
|
||||
"""When the prompt is too long. We have to shorten it.
|
||||
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
|
||||
"""
|
||||
# TODO: this should replace most of code in
|
||||
# - FactorFinalDecisionEvaluator.evaluate
|
||||
# - FactorCodeEvaluator.evaluate
|
||||
@@ -0,0 +1,178 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Dict
|
||||
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
|
||||
from rdagent.components.coder.CoSTEER.evolving_strategy import (
|
||||
MultiProcessEvolvingStrategy,
|
||||
)
|
||||
from rdagent.components.coder.CoSTEER.knowledge_management import (
|
||||
CoSTEERQueriedKnowledge,
|
||||
CoSTEERQueriedKnowledgeV2,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.oai.llm_conf import LLM_SETTINGS
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.num_loop = 0
|
||||
self.haveSelected = False
|
||||
|
||||
def error_summary(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_former_failed_knowledge_to_render: list,
|
||||
queried_similar_error_knowledge_to_render: list,
|
||||
) -> str:
|
||||
error_summary_system_prompt = T(".prompts:evolving_strategy_error_summary_v2_system").r(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task),
|
||||
factor_information_str=target_task.get_task_information(),
|
||||
code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
|
||||
)
|
||||
for _ in range(10): # max attempt to reduce the length of error_summary_user_prompt
|
||||
error_summary_user_prompt = T(".prompts:evolving_strategy_error_summary_v2_user").r(
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt
|
||||
)
|
||||
< APIBackend().chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
error_summary_critics = APIBackend(
|
||||
use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
|
||||
).build_messages_and_create_chat_completion(
|
||||
user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt, json_mode=False
|
||||
)
|
||||
return error_summary_critics
|
||||
|
||||
def implement_one_task(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_knowledge: CoSTEERQueriedKnowledge,
|
||||
workspace: FBWorkspace | None = None,
|
||||
prev_task_feedback: CoSTEERSingleFeedback | None = None,
|
||||
) -> str:
|
||||
target_factor_task_information = target_task.get_task_information()
|
||||
|
||||
queried_similar_successful_knowledge = (
|
||||
queried_knowledge.task_to_similar_task_successful_knowledge[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
) # A list, [success task implement knowledge]
|
||||
|
||||
if isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2):
|
||||
queried_similar_error_knowledge = (
|
||||
queried_knowledge.task_to_similar_error_successful_knowledge[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else {}
|
||||
) # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
|
||||
else:
|
||||
queried_similar_error_knowledge = {}
|
||||
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.task_to_former_failed_traces[target_factor_task_information][0]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[
|
||||
target_factor_task_information
|
||||
][1]
|
||||
system_prompt = T(".prompts:evolving_strategy_factor_implementation_v1_system").r(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
|
||||
# 动态地防止prompt超长
|
||||
for _ in range(10): # max attempt to reduce the length of user_prompt
|
||||
# 总结error（可选）
|
||||
if (
|
||||
isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2)
|
||||
and FACTOR_COSTEER_SETTINGS.v2_error_summary
|
||||
and len(queried_similar_error_knowledge_to_render) != 0
|
||||
and len(queried_former_failed_knowledge_to_render) != 0
|
||||
):
|
||||
error_summary_critics = self.error_summary(
|
||||
target_task,
|
||||
queried_former_failed_knowledge_to_render,
|
||||
queried_similar_error_knowledge_to_render,
|
||||
)
|
||||
else:
|
||||
error_summary_critics = None
|
||||
# 构建user_prompt。开始写代码
|
||||
user_prompt = T(".prompts:evolving_strategy_factor_implementation_v2_user").r(
|
||||
factor_information_str=target_factor_task_information,
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
error_summary_critics=error_summary_critics,
|
||||
latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt)
|
||||
< APIBackend().chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_former_failed_knowledge_to_render) > 1:
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
|
||||
elif len(queried_similar_successful_knowledge_to_render) > len(
|
||||
queried_similar_error_knowledge_to_render,
|
||||
):
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[:-1]
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
for _ in range(10):
|
||||
try:
|
||||
response = APIBackend(
|
||||
use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
|
||||
).build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
|
||||
try:
|
||||
code = json.loads(response)["code"]
|
||||
except json.decoder.JSONDecodeError:
|
||||
# extract python code block
|
||||
match = re.search(r"```python(.*?)```", response, re.DOTALL)
|
||||
if match:
|
||||
code = match.group(1).strip()
|
||||
else:
|
||||
raise # continue to retry
|
||||
|
||||
return code
|
||||
|
||||
except (json.decoder.JSONDecodeError, KeyError):
|
||||
pass
|
||||
else:
|
||||
return "" # return empty code if failed to get code after 10 attempts
|
||||
|
||||
def assign_code_list_to_evo(self, code_list, evo):
|
||||
for index in range(len(evo.sub_tasks)):
|
||||
if code_list[index] is None:
|
||||
continue
|
||||
if evo.sub_workspace_list[index] is None:
|
||||
evo.sub_workspace_list[index] = FactorFBWorkspace(target_task=evo.sub_tasks[index])
|
||||
# Since the `implement_one_task` method is not standardized and the `code_list` has both `str` and `dict` data types,
|
||||
# we ended up getting an `TypeError` here, so we chose to fix the problem temporarily with this dirty method.
|
||||
if isinstance(code_list[index], dict):
|
||||
evo.sub_workspace_list[index].inject_files(**code_list[index])
|
||||
else:
|
||||
evo.sub_workspace_list[index].inject_files(**{"factor.py": code_list[index]})
|
||||
return evo
|
||||
@@ -0,0 +1,231 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Tuple, Union
|
||||
|
||||
import pandas as pd
|
||||
from filelock import FileLock
|
||||
|
||||
from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
|
||||
from rdagent.components.coder.CoSTEER.task import CoSTEERTask
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
|
||||
from rdagent.core.experiment import Experiment, FBWorkspace
|
||||
from rdagent.core.utils import cache_with_pickle
|
||||
from rdagent.oai.llm_utils import md5_hash
|
||||
|
||||
|
||||
class FactorTask(CoSTEERTask):
|
||||
# TODO: generalized the attributes into the Task
|
||||
# - factor_* -> *
|
||||
def __init__(
|
||||
self,
|
||||
factor_name,
|
||||
factor_description,
|
||||
factor_formulation,
|
||||
*args,
|
||||
variables: dict = {},
|
||||
resource: str = None,
|
||||
factor_implementation: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.factor_name = (
|
||||
factor_name # TODO: remove it in the later version. Keep it only for pickle version compatibility
|
||||
)
|
||||
self.factor_formulation = factor_formulation
|
||||
self.variables = variables
|
||||
self.factor_resources = resource
|
||||
self.factor_implementation = factor_implementation
|
||||
super().__init__(name=factor_name, description=factor_description, *args, **kwargs)
|
||||
|
||||
@property
|
||||
def factor_description(self):
|
||||
"""for compatibility"""
|
||||
return self.description
|
||||
|
||||
def get_task_information(self):
|
||||
return f"""factor_name: {self.factor_name}
|
||||
factor_description: {self.factor_description}
|
||||
factor_formulation: {self.factor_formulation}
|
||||
variables: {str(self.variables)}"""
|
||||
|
||||
def get_task_brief_information(self):
|
||||
return f"""factor_name: {self.factor_name}
|
||||
factor_description: {self.factor_description}
|
||||
factor_formulation: {self.factor_formulation}
|
||||
variables: {str(self.variables)}"""
|
||||
|
||||
def get_task_information_and_implementation_result(self):
|
||||
return {
|
||||
"factor_name": self.factor_name,
|
||||
"factor_description": self.factor_description,
|
||||
"factor_formulation": self.factor_formulation,
|
||||
"variables": str(self.variables),
|
||||
"factor_implementation": str(self.factor_implementation),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_dict(dict):
|
||||
return FactorTask(**dict)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<{self.__class__.__name__}[{self.factor_name}]>"
|
||||
|
||||
|
||||
class FactorFBWorkspace(FBWorkspace):
|
||||
"""
|
||||
This class is used to implement a factor by writing the code to a file.
|
||||
Input data and output factor value are also written to files.
|
||||
"""
|
||||
|
||||
# TODO: (Xiao) think raising errors may get better information for processing
|
||||
FB_EXEC_SUCCESS = "Execution succeeded without error."
|
||||
FB_CODE_NOT_SET = "code is not set."
|
||||
FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
|
||||
FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
|
||||
FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
raise_exception: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.raise_exception = raise_exception
|
||||
|
||||
def hash_func(self, data_type: str = "Debug") -> str:
|
||||
return (
|
||||
md5_hash(data_type + self.file_dict["factor.py"])
|
||||
if ("factor.py" in self.file_dict and not self.raise_exception)
|
||||
else None
|
||||
)
|
||||
|
||||
@cache_with_pickle(hash_func)
|
||||
def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
|
||||
"""
|
||||
execute the implementation and get the factor value by the following steps:
|
||||
1. make the directory in workspace path
|
||||
2. write the code to the file in the workspace path
|
||||
3. link all the source data to the workspace path folder
|
||||
if call_factor_py is True:
|
||||
4. execute the code
|
||||
else:
|
||||
4. generate a script from template to import the factor.py dump get the factor value to result.h5
|
||||
5. read the factor value from the output file in the workspace path folder
|
||||
returns the execution feedback as a string and the factor value as a pandas dataframe
|
||||
|
||||
|
||||
Regarding the cache mechanism:
|
||||
1. We will store the function's return value to ensure it behaves as expected.
|
||||
- The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception])
|
||||
|
||||
"""
|
||||
self.before_execute()
|
||||
if self.file_dict is None or "factor.py" not in self.file_dict:
|
||||
if self.raise_exception:
|
||||
raise CodeFormatError(self.FB_CODE_NOT_SET)
|
||||
else:
|
||||
return self.FB_CODE_NOT_SET, None
|
||||
with FileLock(self.workspace_path / "execution.lock"):
|
||||
if self.target_task.version == 1:
|
||||
source_data_path = (
|
||||
Path(
|
||||
FACTOR_COSTEER_SETTINGS.data_folder_debug,
|
||||
)
|
||||
if data_type == "Debug" # FIXME: (yx) don't think we should use a debug tag for this.
|
||||
else Path(
|
||||
FACTOR_COSTEER_SETTINGS.data_folder,
|
||||
)
|
||||
)
|
||||
elif self.target_task.version == 2:
|
||||
# TODO you can change the name of the data folder for a better understanding
|
||||
source_data_path = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition
|
||||
|
||||
source_data_path.mkdir(exist_ok=True, parents=True)
|
||||
code_path = self.workspace_path / f"factor.py"
|
||||
|
||||
self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path)
|
||||
|
||||
execution_feedback = self.FB_EXECUTION_SUCCEEDED
|
||||
execution_success = False
|
||||
execution_error = None
|
||||
|
||||
if self.target_task.version == 1:
|
||||
execution_code_path = code_path
|
||||
elif self.target_task.version == 2:
|
||||
execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
|
||||
execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
|
||||
|
||||
try:
|
||||
subprocess.check_output(
|
||||
f"{FACTOR_COSTEER_SETTINGS.python_bin} {execution_code_path}",
|
||||
shell=True,
|
||||
cwd=self.workspace_path,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout,
|
||||
)
|
||||
execution_success = True
|
||||
except subprocess.CalledProcessError as e:
|
||||
import site
|
||||
|
||||
execution_feedback = (
|
||||
e.output.decode()
|
||||
.replace(str(execution_code_path.parent.absolute()), r"/path/to")
|
||||
.replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
|
||||
)
|
||||
if len(execution_feedback) > 2000:
|
||||
execution_feedback = (
|
||||
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
|
||||
)
|
||||
if self.raise_exception:
|
||||
raise CustomRuntimeError(execution_feedback)
|
||||
else:
|
||||
execution_error = CustomRuntimeError(execution_feedback)
|
||||
except subprocess.TimeoutExpired:
|
||||
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds."
|
||||
if self.raise_exception:
|
||||
raise CustomRuntimeError(execution_feedback)
|
||||
else:
|
||||
execution_error = CustomRuntimeError(execution_feedback)
|
||||
|
||||
workspace_output_file_path = self.workspace_path / "result.h5"
|
||||
if workspace_output_file_path.exists() and execution_success:
|
||||
try:
|
||||
executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
|
||||
execution_feedback += self.FB_OUTPUT_FILE_FOUND
|
||||
except Exception as e:
|
||||
execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
|
||||
executed_factor_value_dataframe = None
|
||||
else:
|
||||
execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
|
||||
executed_factor_value_dataframe = None
|
||||
if self.raise_exception:
|
||||
raise NoOutputError(execution_feedback)
|
||||
else:
|
||||
execution_error = NoOutputError(execution_feedback)
|
||||
|
||||
return execution_feedback, executed_factor_value_dataframe
|
||||
|
||||
def __str__(self) -> str:
|
||||
# NOTE:
|
||||
# If the code cache works, the workspace will be None.
|
||||
return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
|
||||
@staticmethod
|
||||
def from_folder(task: FactorTask, path: Union[str, Path], **kwargs):
|
||||
path = Path(path)
|
||||
code_dict = {}
|
||||
for file_path in path.iterdir():
|
||||
if file_path.suffix == ".py":
|
||||
code_dict[file_path.name] = file_path.read_text()
|
||||
return FactorFBWorkspace(target_task=task, code_dict=code_dict, **kwargs)
|
||||
|
||||
|
||||
FactorExperiment = Experiment
|
||||
FeatureExperiment = Experiment
|
||||
@@ -0,0 +1,15 @@
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from factor import feature_engineering_cls
|
||||
|
||||
if os.path.exists("X_valid.pkl"):
|
||||
valid_df = pd.read_pickle("X_valid.pkl").head(1000)
|
||||
else:
|
||||
raise FileNotFoundError("No valid data found.")
|
||||
|
||||
cls = feature_engineering_cls()
|
||||
cls.fit(valid_df)
|
||||
new_feat = cls.transform(valid_df)
|
||||
new_feat.to_hdf("result.h5", key="data", mode="w")
|
||||
@@ -0,0 +1,209 @@
|
||||
|
||||
evaluator_code_feedback_v1_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
User will provide you the information of the factor.
|
||||
|
||||
Your job is to check whether user's code is align with the factor and the scenario.
|
||||
The user will provide the source python code and the execution error message if execution failed.
|
||||
The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic.
|
||||
|
||||
User has also compared the factor values calculated by the user's code and the ground truth code. The user will provide you some analyze result comparing two output. You may find some error in the code which caused the difference between the two output.
|
||||
|
||||
If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
|
||||
If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct.
|
||||
|
||||
Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX".
|
||||
|
||||
You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
|
||||
|
||||
You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output:
|
||||
critic 1: The critic message to critic 1
|
||||
critic 2: The critic message to critic 2
|
||||
|
||||
evaluator_code_feedback_v1_user: |-
|
||||
--------------Factor information:---------------
|
||||
{{ factor_information }}
|
||||
--------------Python code:---------------
|
||||
{{ code }}
|
||||
--------------Execution feedback:---------------
|
||||
{{ execution_feedback }}
|
||||
{% if value_feedback is not none %}
|
||||
--------------Factor value feedback:---------------
|
||||
{{ value_feedback }}
|
||||
{% endif %}
|
||||
{% if gt_code is not none %}
|
||||
--------------Ground truth Python code:---------------
|
||||
{{ gt_code }}
|
||||
{% endif %}
|
||||
|
||||
evolving_strategy_factor_implementation_v1_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
Your code is expected to align the scenario in any form which means The user needs to get the exact factor values with your code as expected.
|
||||
|
||||
To help you write the correct code, the user might provide multiple information that helps you write the correct code:
|
||||
1. The user might provide you the correct code to similar factors. Your should learn from these code to write the correct code.
|
||||
2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code.
|
||||
3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code.
|
||||
|
||||
Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
|
||||
|
||||
Notice that you should not add any other text before or after the json format.
|
||||
|
||||
{% if queried_former_failed_knowledge|length != 0 %}
|
||||
--------------Your former latest attempt:---------------
|
||||
=====Code to the former implementation=====
|
||||
{{ queried_former_failed_knowledge[-1].implementation.all_codes }}
|
||||
=====Feedback to the former implementation=====
|
||||
{{ queried_former_failed_knowledge[-1].feedback }}
|
||||
{% endif %}
|
||||
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
|
||||
evolving_strategy_factor_implementation_v2_user: |-
|
||||
--------------Target factor information:---------------
|
||||
{{ factor_information_str }}
|
||||
|
||||
{% if queried_similar_error_knowledge|length != 0 %}
|
||||
{% if error_summary_critics is none %}
|
||||
Recall your last failure, your implementation met some errors.
|
||||
When doing other tasks, you met some similar errors but you finally solve them. Here are some examples:
|
||||
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
|
||||
--------------Factor information to similar error ({{error_content}}):---------------
|
||||
{{ similar_error_knowledge[0].target_task.get_task_information() }}
|
||||
=====Code with similar error ({{error_content}}):=====
|
||||
{{ similar_error_knowledge[0].implementation.all_codes }}
|
||||
=====Success code to former code with similar error ({{error_content}}):=====
|
||||
{{ similar_error_knowledge[1].implementation.all_codes }}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
Recall your last failure, your implementation met some errors.
|
||||
After reviewing some similar errors and their solutions, here are some suggestions for you to correct your code:
|
||||
{{error_summary_critics}}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if queried_similar_successful_knowledge|length != 0 %}
|
||||
Here are some success implements of similar component tasks, take them as references:
|
||||
--------------Correct code to similar factors:---------------
|
||||
{% for similar_successful_knowledge in queried_similar_successful_knowledge %}
|
||||
=====Factor {{loop.index}}:=====
|
||||
{{ similar_successful_knowledge.target_task.get_task_information() }}
|
||||
=====Code:=====
|
||||
{{ similar_successful_knowledge.implementation.all_codes }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if latest_attempt_to_latest_successful_execution is not none %}
|
||||
You have tried to correct your former failed code but still met some errors. Here is the latest attempt to the latest successful execution, try not to get the same error to your new code:
|
||||
=====Your latest attempt=====
|
||||
{{ latest_attempt_to_latest_successful_execution.implementation.all_codes }}
|
||||
=====Feedback to your latest attempt=====
|
||||
{{ latest_attempt_to_latest_successful_execution.feedback }}
|
||||
{% endif %}
|
||||
|
||||
evolving_strategy_error_summary_v2_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
User is doing the following task:
|
||||
{{factor_information_str}}
|
||||
|
||||
You have written some code but it meets errors like the following:
|
||||
{{code_and_feedback}}
|
||||
|
||||
The user has found some tasks that met similar errors, and their final correct solutions.
|
||||
Please refer to these similar errors and their solutions, provide some clear, short and accurate critics that might help you solve the issues in your code.
|
||||
|
||||
You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
|
||||
|
||||
[NOTE]
|
||||
1. When processing data, avoid time leakage.
|
||||
|
||||
Please response the critic in the following format. Here is an example structure for the output:
|
||||
critic 1: The critic message to critic 1
|
||||
critic 2: The critic message to critic 2
|
||||
|
||||
evolving_strategy_error_summary_v2_user: |-
|
||||
{% if queried_similar_error_knowledge|length != 0 %}
|
||||
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
|
||||
--------------Factor information to similar error ({{error_content}}):---------------
|
||||
{{ similar_error_knowledge[0].target_task.get_task_information() }}
|
||||
=====Code with similar error ({{error_content}}):=====
|
||||
{{ similar_error_knowledge[0].implementation.all_codes }}
|
||||
=====Success code to former code with similar error ({{error_content}}):=====
|
||||
{{ similar_error_knowledge[1].implementation.all_codes }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
select_implementable_factor_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
Your job is to help the user select the easiest-to-implement factors. Some factors may be difficult to implement due to a lack of information or excessive complexity. The user will provide the number of factors you should pick and information about the factors, including their descriptions, formulas, and variable explanations.
|
||||
User will provide you the former attempt to implement the factor and the feedback to the implementation. You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors.
|
||||
Please analyze the difficulties of the each factors and provide the reason and response the indices of selected implementable factor in the json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"Analysis": "Analyze the difficulties of the each factors and provide the reason why the factor can be implemented or not."
|
||||
"selected_factor": "The indices of selected factor index in the list, like [0, 2, 3].The length should be the number of factor left after filtering.",
|
||||
}
|
||||
|
||||
select_implementable_factor_user: |-
|
||||
Number of factor you should pick: {{ factor_num }}
|
||||
{% for factor_info in sub_tasks %}
|
||||
=============Factor index:{{factor_info[0]}}:=============
|
||||
=====Factor name:=====
|
||||
{{ factor_info[1].factor_name }}
|
||||
=====Factor description:=====
|
||||
{{ factor_info[1].factor_description }}
|
||||
=====Factor formulation:=====
|
||||
{{ factor_info[1].factor_formulation }}
|
||||
{% if factor_info[2]|length != 0 %}
|
||||
--------------Your former attempt:---------------
|
||||
{% for former_attempt in factor_info[2] %}
|
||||
=====Code to attempt {{ loop.index }}=====
|
||||
{{ former_attempt.implementation.all_codes }}
|
||||
=====Feedback to attempt {{ loop.index }}=====
|
||||
{{ former_attempt.feedback }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
evaluator_output_format_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
User will provide you the format of the output. Please help to check whether the output is align with the format.
|
||||
Please respond in the JSON format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"output_format_decision": True,
|
||||
"output_format_feedback": "The output format is correct."
|
||||
}
|
||||
|
||||
|
||||
evaluator_final_decision_v1_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
User has finished evaluation and got some feedback from the evaluator.
|
||||
The evaluator run the code and get the factor value dataframe and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the scenario and factor description to give a final decision about the evaluation result. The final decision concludes whether the factor is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False.
|
||||
|
||||
The implementation final decision is considered in the following logic:
|
||||
1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
|
||||
2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
|
||||
3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description. The implementation cannot be considered correct if the code execution failed, no matter what the reason is.
|
||||
|
||||
Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
|
||||
{
|
||||
"final_decision": True,
|
||||
"final_feedback": "The final feedback message",
|
||||
}
|
||||
|
||||
evaluator_final_decision_v1_user: |-
|
||||
--------------Factor information:---------------
|
||||
{{ factor_information }}
|
||||
--------------Execution feedback:---------------
|
||||
{{ execution_feedback }}
|
||||
--------------Code feedback:---------------
|
||||
{{ code_feedback }}
|
||||
--------------Factor value feedback:---------------
|
||||
{{ value_feedback }}
|
||||
Reference in New Issue
Block a user