chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,49 @@
|
||||
import argparse
|
||||
import json
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Get agent's gaia score")
|
||||
parser.add_argument('--file', default='/Users/tangjiabin/Documents/reasoning/metachain/evaluation_results/gaia/system_triage_agent/claude-3-5-sonnet-20241022_maxiter/output.jsonl', type=str, help="Path to the agent's output.jsonl")
|
||||
args = parser.parse_args()
|
||||
this_log = args.file
|
||||
outs = []
|
||||
with open(this_log, 'r') as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
outs.append(json.loads(line))
|
||||
print(f'Reading {this_log}')
|
||||
print(f'Metadata:\n {outs[0]["metadata"]}')
|
||||
|
||||
total = 0
|
||||
success = 0
|
||||
l1_total = 0
|
||||
l1_success = 0
|
||||
l2_total = 0
|
||||
l2_success = 0
|
||||
l3_total = 0
|
||||
l3_success = 0
|
||||
for out in outs:
|
||||
total += 1
|
||||
if out['test_result']['score']:
|
||||
success += 1
|
||||
if out['instance']['Level'] == "1":
|
||||
l1_total += 1
|
||||
if out['test_result']['score']:
|
||||
l1_success += 1
|
||||
elif out['instance']['Level'] == "2":
|
||||
l2_total += 1
|
||||
if out['test_result']['score']:
|
||||
l2_success += 1
|
||||
elif out['instance']['Level'] == "3":
|
||||
l3_total += 1
|
||||
if out['test_result']['score']:
|
||||
l3_success += 1
|
||||
print(f'Success rate: {success}/{total} = {success/total * 100:.4f}%')
|
||||
print(f'L1 success rate: {l1_success}/{l1_total} = {l1_success/l1_total * 100:.4f}%')
|
||||
print(f'L2 success rate: {l2_success}/{l2_total} = {l2_success/l2_total * 100:.4f}%')
|
||||
print(f'L3 success rate: {l3_success}/{l3_total} = {l3_success/l3_total * 100:.4f}%')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,298 @@
|
||||
import argparse
|
||||
from constant import DOCKER_WORKPLACE_NAME
|
||||
from datasets import load_dataset
|
||||
import huggingface_hub
|
||||
from autoagent import MetaChain
|
||||
from autoagent.logger import MetaChainLogger, LoggerManager
|
||||
from evaluation.utils import make_metadata, prepare_dataset, update_progress, check_port_available, run_evaluation, clean_msg
|
||||
from evaluation.types import EvalMetadata, EvalOutput
|
||||
import autoagent.agents as agenthub
|
||||
import os.path as osp
|
||||
import pandas as pd
|
||||
import asyncio
|
||||
import re
|
||||
import os
|
||||
import shutil
|
||||
from autoagent.registry import registry
|
||||
from evaluation.gaia.scorer import question_scorer
|
||||
import json
|
||||
# from autoagent.util import run_command_in_container
|
||||
from autoagent.environment.docker_env import DockerEnv, DockerConfig, check_container_ports, check_container_exist, check_container_running
|
||||
from autoagent.environment.browser_env import BrowserEnv
|
||||
from autoagent.environment.markdown_browser import RequestsMarkdownBrowser
|
||||
from autoagent.types import Response
|
||||
from autoagent.util import function_to_json
|
||||
from autoagent.main import run_in_client, run_in_client_non_async
|
||||
from autoagent.agents.meta_agent.tool_editor import get_tool_editor_agent
|
||||
from autoagent.environment.utils import setup_metachain
|
||||
import subprocess
|
||||
DATASET_CACHE_DIR = osp.join(osp.dirname(__file__), 'data')
|
||||
# Note: You should run this script in the root directory of the project autoagent
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--container_name', type=str, default='gaia_test')
|
||||
parser.add_argument('--model', type=str, default='claude-3-5-sonnet-20241022')
|
||||
parser.add_argument('--git_clone', action='store_true', default=False)
|
||||
parser.add_argument('--setup_package', type=str, default=None)
|
||||
parser.add_argument('--test_pull_name', type=str, default='main')
|
||||
parser.add_argument('--debug', action='store_true', default=False)
|
||||
# metadata
|
||||
parser.add_argument('--agent_func', type=str, default='get_system_triage_agent')
|
||||
parser.add_argument('--eval_note', type=str, default=None)
|
||||
parser.add_argument('--eval_output_dir', type=str, default='./evaluation_results')
|
||||
parser.add_argument('--data_split', type=str, default=None)
|
||||
# gaia level
|
||||
parser.add_argument('--level', type=str, default='1')
|
||||
parser.add_argument('--eval_n_limit', type=int, default=None)
|
||||
parser.add_argument('--port', type=int, default=12345)
|
||||
parser.add_argument('--eval_num_workers', type=int, default=1)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
def get_config(metadata: EvalMetadata, instance_id: str):
|
||||
container_name = metadata.container_name+f'_{instance_id}'
|
||||
|
||||
port_info = check_container_ports(container_name)
|
||||
port = metadata.port
|
||||
if port_info:
|
||||
port = port_info[0]
|
||||
else:
|
||||
# while not check_port_available(port):
|
||||
# port += 1
|
||||
# 使用文件锁来确保端口分配的原子性
|
||||
import filelock
|
||||
lock_file = os.path.join(os.getcwd(), ".port_lock")
|
||||
lock = filelock.FileLock(lock_file)
|
||||
|
||||
with lock:
|
||||
port = metadata.port
|
||||
while not check_port_available(port):
|
||||
port += 1
|
||||
print(f'{port} is not available, trying {port+1}')
|
||||
# 立即标记该端口为已使用
|
||||
with open(os.path.join(os.getcwd(), f".port_{port}"), 'w') as f:
|
||||
f.write(container_name)
|
||||
local_root = os.path.join(os.getcwd(), f"workspace_gaia_whole", f"gaia_eval_{instance_id}")
|
||||
os.makedirs(local_root, exist_ok=True)
|
||||
docker_config = DockerConfig(
|
||||
workplace_name=DOCKER_WORKPLACE_NAME,
|
||||
container_name=container_name,
|
||||
communication_port=port,
|
||||
conda_path='/root/miniconda3',
|
||||
local_root=local_root,
|
||||
git_clone=metadata.git_clone,
|
||||
test_pull_name=metadata.test_pull_name,
|
||||
)
|
||||
return docker_config
|
||||
|
||||
def process_instance(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
logger: MetaChainLogger,
|
||||
) -> EvalOutput:
|
||||
docker_config = get_config(metadata, instance_id=instance['instance_id'])
|
||||
code_env = None
|
||||
try:
|
||||
|
||||
|
||||
code_env, web_env, file_env = create_environment(docker_config)
|
||||
local_workplace = code_env.local_workplace
|
||||
docker_workplace = code_env.docker_workplace
|
||||
|
||||
|
||||
# Setup the logger properly, so you can run multi-processing to parallelize the evaluation
|
||||
logger.info(f'Starting evaluation for instance {instance["instance_id"]}.')
|
||||
|
||||
if instance['file_name'] != '':
|
||||
assert metadata.data_split is not None
|
||||
src_file = os.path.join(
|
||||
DATASET_CACHE_DIR, '2023', metadata.data_split, instance['file_name']
|
||||
)
|
||||
assert os.path.exists(src_file)
|
||||
extension_name = instance['file_name'].split('.')[-1]
|
||||
dest_file = os.path.join(local_workplace, f'file.{extension_name}')
|
||||
shutil.copy(src_file, dest_file)
|
||||
file_name = dest_file.split('/')[-1]
|
||||
else:
|
||||
dest_file = None
|
||||
|
||||
|
||||
# Prepare instruction
|
||||
instruction = f"{instance['Question']}\n"
|
||||
logger.info(f'Instruction: {instruction}')
|
||||
if dest_file:
|
||||
instruction += f"\n\nThe mentioned file is provided in the workspace at: {osp.join(docker_workplace, file_name)}"
|
||||
|
||||
instruction += 'IMPORTANT: Any agent cannot stop using tools until the task is done. Don\'t tell me how to do bot do it using tools!\n'
|
||||
instruction += 'IMPORTANT: System Triage Agent must hand off the task to the suitable agents, and finally answer the question util there is no more sub-task to do.\n'
|
||||
instruction += 'IMPORTANT: When you meet something you are not sure about, you should use the `Web Surfer Agent` to search the web. And when you are required to compute something, you should use the `Programming Agent` to compute. Take Advantage of agents as much as possible.\n'
|
||||
instruction += 'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
|
||||
instruction += 'Please encapsulate your final answer (answer ONLY) within <solution> and </solution>.\n'
|
||||
instruction += (
|
||||
'For example: The answer to the question is <solution> 42 </solution>.\n'
|
||||
)
|
||||
|
||||
logger.info(f'Instruction:\n{instruction}')
|
||||
|
||||
system_triage_agent = registry.agents[metadata.agent_func](model=metadata.model)
|
||||
messages = [
|
||||
{
|
||||
'role': 'user',
|
||||
'content': instruction
|
||||
}
|
||||
]
|
||||
|
||||
context_variables = {"code_env": code_env, "web_env": web_env, "file_env": file_env}
|
||||
# Here's how you can run the agent (similar to the `main` function) and get the final task state
|
||||
tool_editor_agent = get_tool_editor_agent(model=metadata.model)
|
||||
response: Response | None = asyncio.run(
|
||||
run_in_client(
|
||||
agent=system_triage_agent,
|
||||
messages=messages,
|
||||
context_variables = context_variables,
|
||||
logger=logger,
|
||||
meta_agent=tool_editor_agent,
|
||||
docker_config=docker_config,
|
||||
code_env=code_env,
|
||||
)
|
||||
)
|
||||
# response: Response | None = run_in_client_non_async(
|
||||
# agent=system_triage_agent,
|
||||
# messages=messages,
|
||||
# context_variables = context_variables,
|
||||
# logger=logger
|
||||
# )
|
||||
messages.extend(response.messages)
|
||||
# save messages to a file
|
||||
messages_file = osp.join(metadata.eval_output_dir, f"gaia_{instance['instance_id']}", f'messages_{metadata.agent_func.replace("get_", "")}.json')
|
||||
os.makedirs(osp.dirname(messages_file), exist_ok=True)
|
||||
messages = clean_msg(messages)
|
||||
with open(messages_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(messages, f, ensure_ascii=False, indent=4)
|
||||
# ======= Attempt to evaluate the agent's edits =======
|
||||
# If you are working on simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
|
||||
# You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
|
||||
|
||||
if response is None:
|
||||
raise ValueError('Response should not be None.')
|
||||
|
||||
model_answer_raw = response.messages[-1]['content']
|
||||
|
||||
# attempt to parse model_answer
|
||||
model_answer = re.findall(r'<solution>(.*?)</solution>', model_answer_raw)
|
||||
if len(model_answer) == 0:
|
||||
logger.info(f'Failed to parse model answer: {model_answer_raw}', title='WARNING', color='yellow')
|
||||
model_answer = model_answer_raw
|
||||
else:
|
||||
model_answer = model_answer[0]
|
||||
|
||||
logger.info(
|
||||
f'Final message: {model_answer} | Ground truth: {instance["Final answer"]}',
|
||||
title='INFO', color='green'
|
||||
)
|
||||
score = question_scorer(
|
||||
model_answer=model_answer, ground_truth=instance['Final answer']
|
||||
)
|
||||
test_result = {
|
||||
'score': score,
|
||||
'model_answer_raw': model_answer_raw,
|
||||
'model_answer': model_answer,
|
||||
'ground_truth': instance['Final answer'],
|
||||
}
|
||||
|
||||
|
||||
# Save the output
|
||||
output = EvalOutput(
|
||||
instance_id=instance['instance_id'],
|
||||
instance=instance.to_dict(),
|
||||
instruction=instance['Question'],
|
||||
metadata=metadata,
|
||||
messages=messages,
|
||||
test_result=test_result,
|
||||
)
|
||||
finally:
|
||||
# 清理资源
|
||||
if code_env is not None:
|
||||
try:
|
||||
# 停止容器
|
||||
code_env.stop_container()
|
||||
logger.info(f"Container {docker_config.container_name} stopped successfully")
|
||||
# 可选：删除容器
|
||||
# subprocess.run(["docker", "rm", docker_config.container_name],
|
||||
# capture_output=True, text=True)
|
||||
# logger.info(f"Container {docker_config.container_name} removed successfully")
|
||||
|
||||
# 可选：删除工作目录
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cleanup: {str(e)}")
|
||||
|
||||
# 清理端口标记文件
|
||||
port_file = os.path.join(os.getcwd(), f".port_{docker_config.communication_port}")
|
||||
if os.path.exists(port_file):
|
||||
os.remove(port_file)
|
||||
logger.info(f"Port {docker_config.communication_port} released")
|
||||
return output
|
||||
|
||||
def map_instance_to_port(dataset: pd.DataFrame, metadata: EvalMetadata):
|
||||
port_dict = {}
|
||||
for idx, row in dataset.iterrows():
|
||||
port_dict[row['instance_id']] = metadata.port + idx
|
||||
|
||||
|
||||
def create_environment(docker_config: DockerConfig):
|
||||
"""
|
||||
1. create the code environment
|
||||
2. create the web environment
|
||||
3. create the file environment
|
||||
"""
|
||||
code_env = DockerEnv(docker_config)
|
||||
code_env.init_container()
|
||||
|
||||
web_env = BrowserEnv(browsergym_eval_env = None, local_root=docker_config.local_root, workplace_name=docker_config.workplace_name)
|
||||
file_env = RequestsMarkdownBrowser(viewport_size=1024 * 5, local_root=docker_config.local_root, workplace_name=docker_config.workplace_name, downloads_folder=os.path.join(docker_config.local_root, docker_config.workplace_name, "downloads"))
|
||||
|
||||
return code_env, web_env, file_env
|
||||
def main(args):
|
||||
metadata: EvalMetadata = make_metadata(
|
||||
model=args.model,
|
||||
dataset_name="gaia",
|
||||
agent_func=args.agent_func,
|
||||
eval_note=args.eval_note,
|
||||
eval_output_dir=args.eval_output_dir,
|
||||
data_split=args.data_split,
|
||||
details={'gaia-level': args.level},
|
||||
port=args.port,
|
||||
container_name=args.container_name,
|
||||
git_clone=args.git_clone,
|
||||
test_pull_name=args.test_pull_name,
|
||||
)
|
||||
log_path = osp.join(metadata.eval_output_dir, 'logs', f'agent_{metadata.model}.log')
|
||||
LoggerManager.set_logger(MetaChainLogger(log_path))
|
||||
|
||||
dataset = load_dataset('gaia-benchmark/GAIA', args.level)
|
||||
huggingface_hub.snapshot_download(
|
||||
'gaia-benchmark/GAIA',
|
||||
repo_type='dataset',
|
||||
local_dir=DATASET_CACHE_DIR,
|
||||
)
|
||||
gaia_tests = dataset[metadata.data_split].to_pandas()
|
||||
gaia_tests.rename(columns={'task_id': 'instance_id'}, inplace=True)
|
||||
|
||||
output_file = osp.join(metadata.eval_output_dir, 'output.jsonl')
|
||||
prepared_dataset = prepare_dataset(gaia_tests, output_file, args.eval_n_limit)
|
||||
|
||||
run_evaluation(
|
||||
dataset=prepared_dataset,
|
||||
metadata=metadata,
|
||||
output_file=output_file,
|
||||
num_workers=args.eval_num_workers,
|
||||
process_instance_func=process_instance,
|
||||
)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = get_args()
|
||||
main(args)
|
||||
# print(check_container_exist('gaia_lite_eval_c61d22de-5f6c-4958-a7f6-5e9707bd3466'))
|
||||
@@ -0,0 +1,102 @@
|
||||
import re
|
||||
import string
|
||||
import warnings
|
||||
|
||||
|
||||
def normalize_number_str(number_str: str) -> float:
|
||||
# we replace these common units and commas to allow
|
||||
# conversion to float
|
||||
for char in ['$', '%', ',']:
|
||||
number_str = number_str.replace(char, '')
|
||||
try:
|
||||
return float(number_str)
|
||||
except ValueError:
|
||||
print(f'String {number_str} cannot be normalized to number str.')
|
||||
return float('inf')
|
||||
|
||||
|
||||
def split_string(
|
||||
s: str,
|
||||
char_list: list[str] = None,
|
||||
) -> list[str]:
|
||||
if char_list is None:
|
||||
char_list = [',', ';']
|
||||
pattern = f"[{''.join(char_list)}]"
|
||||
return re.split(pattern, s)
|
||||
|
||||
|
||||
def question_scorer(
|
||||
model_answer: str,
|
||||
ground_truth: str,
|
||||
) -> bool:
|
||||
def is_float(element: any) -> bool:
|
||||
try:
|
||||
float(element)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
# if gt is a number
|
||||
if is_float(ground_truth):
|
||||
print(f'Evaluating {model_answer} as a number.')
|
||||
normalized_answer = normalize_number_str(model_answer)
|
||||
return normalized_answer == float(ground_truth)
|
||||
|
||||
# if gt is a list
|
||||
elif any(char in ground_truth for char in [',', ';']):
|
||||
print(f'Evaluating {model_answer} as a comma separated list.')
|
||||
# question with the fish: normalization removes punct
|
||||
|
||||
gt_elems = split_string(ground_truth)
|
||||
ma_elems = split_string(model_answer)
|
||||
|
||||
# check length is the same
|
||||
if len(gt_elems) != len(ma_elems):
|
||||
warnings.warn(
|
||||
'Answer lists have different lengths, returning False.',
|
||||
UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return False
|
||||
|
||||
# compare each element as float or str
|
||||
comparisons = []
|
||||
for ma_elem, gt_elem in zip(ma_elems, gt_elems):
|
||||
if is_float(gt_elem):
|
||||
normalized_ma_elem = normalize_number_str(ma_elem)
|
||||
comparisons.append(normalized_ma_elem == float(gt_elem))
|
||||
else:
|
||||
# we do not remove punct since comparisons can include punct
|
||||
comparisons.append(
|
||||
normalize_str(ma_elem, remove_punct=False)
|
||||
== normalize_str(gt_elem, remove_punct=False)
|
||||
)
|
||||
return all(comparisons)
|
||||
|
||||
# if gt is a str
|
||||
else:
|
||||
print(f'Evaluating {model_answer} as a string.')
|
||||
return normalize_str(model_answer) == normalize_str(ground_truth)
|
||||
|
||||
|
||||
def normalize_str(input_str, remove_punct=True) -> str:
|
||||
"""Normalize a string by:
|
||||
- Removing all white spaces
|
||||
- Optionally removing punctuation (if remove_punct is True)
|
||||
- Converting to lowercase
|
||||
Parameters:
|
||||
- input_str: str, the string to normalize
|
||||
- remove_punct: bool, whether to remove punctuation (default: True)
|
||||
|
||||
Returns:
|
||||
- str, the normalized string
|
||||
"""
|
||||
# Remove all white spaces. Required e.g for seagull vs. sea gull
|
||||
no_spaces = re.sub(r'\s', '', input_str)
|
||||
|
||||
# Remove punctuation, if specified.
|
||||
if remove_punct:
|
||||
translator = str.maketrans('', '', string.punctuation)
|
||||
return no_spaces.lower().translate(translator)
|
||||
else:
|
||||
return no_spaces.lower()
|
||||
@@ -0,0 +1,12 @@
|
||||
current_dir=$(dirname "$(readlink -f "$0")")
|
||||
|
||||
cd $current_dir
|
||||
cd ../../../
|
||||
export DOCKER_WORKPLACE_NAME=workplace
|
||||
export EVAL_MODE=True
|
||||
export DEBUG=True
|
||||
export BASE_IMAGES=tjbtech1/gaia-bookworm:v2
|
||||
export COMPLETION_MODEL=claude-3-5-sonnet-20241022
|
||||
|
||||
python evaluation/gaia/run_infer.py --container_name gaia_lite_eval --model ${COMPLETION_MODEL} --test_pull_name test_pull_1225 --debug --eval_num_workers 1 --port 12345 --data_split validation --level 2023_all --agent_func get_system_triage_agent --git_clone
|
||||
# python /Users/tangjiabin/Documents/reasoning/metachain/test_gaia_tool.py
|
||||
@@ -0,0 +1,17 @@
|
||||
|
||||
|
||||
from autoagent.environment.docker_env import DockerEnv, DockerConfig, check_container_ports, check_container_exist, check_container_running
|
||||
from autoagent.tools.files import create_file
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
os.environ["BASE_IMAGES"] = "tjbtech1/gaia-bookworm:amd64"
|
||||
config = DockerConfig(container_name = "gaia_amd64_test",
|
||||
workplace_name = "workplace_gaia_amd64_test",
|
||||
communication_port = 12345,
|
||||
conda_path = "/root/miniconda3"
|
||||
)
|
||||
env = DockerEnv(config)
|
||||
env.init_container()
|
||||
res = create_file(path = 'test.py', content = 'print("hello world")', env = env)
|
||||
print(res)
|
||||
@@ -0,0 +1,125 @@
|
||||
from pathlib import Path
|
||||
from tqdm import tqdm
|
||||
import multiprocessing
|
||||
from copy import deepcopy
|
||||
import re
|
||||
from lm_eval.tasks.minerva_math.utils import (
|
||||
last_boxed_only_string,
|
||||
normalize_final_answer,
|
||||
get_unnormalized_answer,
|
||||
remove_boxed,
|
||||
is_equiv,
|
||||
)
|
||||
|
||||
import yaml
|
||||
import argparse
|
||||
|
||||
def load_yaml(path: Path):
|
||||
with open(path, "r") as f:
|
||||
data = yaml.load(f, Loader=yaml.CLoader)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def save_yaml(path: Path, data, sort_keys=True):
|
||||
with open(path, "w") as f:
|
||||
yaml.dump(data, f, sort_keys=sort_keys)
|
||||
|
||||
|
||||
ANS_RE_GSM8k = re.compile(r"#### (\-?[\$0-9\.\,]+)")
|
||||
INVALID_ANS_GSM8k = "[invalid]"
|
||||
GSM8K_IGNORE_REGEXES = [",", "\\$", "\\.$"]
|
||||
|
||||
|
||||
def filter_ignores(st, regexes_to_ignore):
|
||||
if regexes_to_ignore is not None:
|
||||
for s in regexes_to_ignore:
|
||||
st = re.sub(s, "", st)
|
||||
return st
|
||||
|
||||
|
||||
def extract_answer_gsm8k(completion):
|
||||
match = ANS_RE_GSM8k.search(completion)
|
||||
if match:
|
||||
match_str = match.group(1).strip()
|
||||
match_str = filter_ignores(
|
||||
match_str,
|
||||
GSM8K_IGNORE_REGEXES,
|
||||
)
|
||||
return match_str
|
||||
else:
|
||||
return INVALID_ANS_GSM8k
|
||||
|
||||
|
||||
def is_correct_gsm8k(model_completion, gt_example):
|
||||
gt_answer = extract_answer_gsm8k(gt_example)
|
||||
assert gt_answer != INVALID_ANS_GSM8k
|
||||
model_answer = extract_answer_gsm8k(model_completion)
|
||||
return model_answer == gt_answer or is_equiv(model_answer, gt_answer)
|
||||
|
||||
def my_get_unnormalized_answer(og_pred):
|
||||
og_pred = get_unnormalized_answer(og_pred)
|
||||
print(og_pred)
|
||||
og_pred = re.sub(r"\\+[\(\[](.+?)\\+[\)\]]", "\\1", og_pred)
|
||||
return og_pred
|
||||
|
||||
|
||||
def is_correct_minerva(og_pred, gt):
|
||||
pred = normalize_final_answer(my_get_unnormalized_answer(og_pred))
|
||||
print(pred)
|
||||
print(gt)
|
||||
# gt = normalize_final_answer(remove_boxed(last_boxed_only_string(gt)))
|
||||
# string equality check needed because of https://github.com/EleutherAI/lm-evaluation-harness/issues/2212
|
||||
return pred == gt or is_equiv(pred, gt)
|
||||
|
||||
|
||||
|
||||
|
||||
def is_correct(sample: str, gt_answer: str, dset: str):
|
||||
if dset == "gsm8k":
|
||||
return is_correct_gsm8k(sample, gt_answer)
|
||||
elif dset == "math":
|
||||
return is_correct_minerva(sample, gt_answer)
|
||||
else:
|
||||
raise ValueError(f"Dataset {dset} not supported")
|
||||
|
||||
|
||||
|
||||
|
||||
def get_tasks(config):
|
||||
sample_paths = Path(config.samples_dir).glob("*.yaml")
|
||||
|
||||
tasks = []
|
||||
for sample_path in tqdm(sample_paths, desc="Loading generations"):
|
||||
save_path = config.save_dir / sample_path.name
|
||||
|
||||
task_config = deepcopy(config)
|
||||
task_config.sample_path = sample_path
|
||||
task_config.save_path = save_path
|
||||
|
||||
tasks.append(task_config)
|
||||
|
||||
return tasks
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
tasks = Path(args.save_dir).glob("*.yaml")
|
||||
|
||||
|
||||
corrects = []
|
||||
|
||||
for task in tqdm(tasks, desc="Evaluating"):
|
||||
result = load_yaml(task)
|
||||
|
||||
correct = is_correct(result["answer"], result["gt_answer"], "math")
|
||||
corrects.append(correct)
|
||||
|
||||
print(f"Accuracy: {sum(corrects) / len(corrects)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--save_dir", type=str, default="evaluation_results/math500/math_solver")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -0,0 +1,23 @@
|
||||
MATH_COT_PROMPT = """Problem:
|
||||
Find the domain of the expression $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}
|
||||
|
||||
Solution:
|
||||
The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.
|
||||
|
||||
Problem:
|
||||
If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$
|
||||
|
||||
Solution:
|
||||
We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.
|
||||
|
||||
Problem:
|
||||
Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?
|
||||
|
||||
Solution:
|
||||
If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.
|
||||
|
||||
Problem:
|
||||
If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.
|
||||
|
||||
Solution:
|
||||
If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct."""
|
||||
@@ -0,0 +1,121 @@
|
||||
import torch
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
import multiprocessing
|
||||
import random
|
||||
import requests
|
||||
from functools import partial
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
from metachain.agents.math.math_solver_agent import get_math_solver_agent
|
||||
from metachain import MetaChain
|
||||
from metachain.workflows.math_solver_workflow_flow import majority_voting
|
||||
import importlib
|
||||
import os
|
||||
import asyncio
|
||||
from evaluation.math500.prompts import MATH_COT_PROMPT
|
||||
|
||||
|
||||
def save_yaml(path: Path, data, sort_keys=True):
|
||||
with open(path, "w") as f:
|
||||
yaml.dump(data, f, sort_keys=sort_keys)
|
||||
|
||||
async def run_inference(item, save_dir, workflow):
|
||||
|
||||
outpath = save_dir / f"{item['id']}.yaml"
|
||||
if outpath.exists():
|
||||
return
|
||||
|
||||
prompt = MATH_COT_PROMPT + f"\n\nProblem:\n{item['problem']}\n\nYour task is to solve this problem."
|
||||
prompt += "Please given your final answer (answer ONLY) within the format of `Final Answer: The final answer is <answer>. I hope it is correct.` after your reasoning \n"
|
||||
prompt += "For example: According to ...\nFinal Answer: The final answer is $24$. I hope it is correct.\n"
|
||||
|
||||
if workflow == "majority_voting":
|
||||
answer = await majority_voting(prompt)
|
||||
elif workflow == None:
|
||||
agent = get_math_solver_agent(model="deepseek/deepseek-chat")
|
||||
client = MetaChain()
|
||||
messages = [
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
context_variables = {
|
||||
}
|
||||
|
||||
response = await client.run_async(agent, messages, context_variables)
|
||||
answer = response.messages[-1]['content']
|
||||
else: raise ValueError(f"Unknown workflow: {workflow}")
|
||||
|
||||
out = {
|
||||
"prompt": prompt,
|
||||
"question": item["problem"],
|
||||
"answer": answer,
|
||||
"gt_answer": item["answer"],
|
||||
}
|
||||
|
||||
save_yaml(outpath, out)
|
||||
|
||||
|
||||
async def main(args):
|
||||
|
||||
test_dataset = list(
|
||||
load_dataset(
|
||||
"HuggingFaceH4/MATH-500", "default", split="test", trust_remote_code=True
|
||||
)
|
||||
)
|
||||
|
||||
print(f"Number of test items: {len(test_dataset)}")
|
||||
|
||||
random.seed(12345)
|
||||
|
||||
|
||||
for i, data in enumerate(test_dataset):
|
||||
data["id"] = i
|
||||
|
||||
random.shuffle(test_dataset)
|
||||
|
||||
if args.limit is not None:
|
||||
limit = args.limit
|
||||
else:
|
||||
limit = len(test_dataset)
|
||||
|
||||
if args.stride is not None:
|
||||
stride = args.stride
|
||||
else:
|
||||
stride = 1
|
||||
|
||||
if args.offset is not None:
|
||||
offset = args.offset
|
||||
else:
|
||||
offset = 0
|
||||
|
||||
test_dataset = test_dataset[offset:limit:stride]
|
||||
|
||||
print(f"Total number of items to process: {len(test_dataset)}")
|
||||
|
||||
if args.workflow == None:
|
||||
save_dir = os.path.join(args.save_dir, "math_solver")
|
||||
save_dir = Path(save_dir)
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
else:
|
||||
save_dir = os.path.join(args.save_dir, args.workflow)
|
||||
save_dir = Path(save_dir)
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
|
||||
predictions = []
|
||||
for item in tqdm(test_dataset):
|
||||
predictions.append(await run_inference(item, save_dir, args.workflow))
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num_few_shot", type=int, default=2)
|
||||
parser.add_argument("--limit", type=int, default=3)
|
||||
parser.add_argument("--stride", type=int, default=1)
|
||||
parser.add_argument("--offset", type=int, default=0)
|
||||
parser.add_argument("--save_dir", type=str, default="evaluation_results/math500")
|
||||
parser.add_argument("--workflow", type=str, default=None)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args))
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1,92 @@
|
||||
from constant import DOCKER_WORKPLACE_NAME
|
||||
from autoagent.environment.docker_container import init_container
|
||||
from autoagent.io_utils import read_yaml_file, get_md5_hash_bytext
|
||||
from autoagent.agents import get_rag_agent
|
||||
from autoagent.core import AutoAgent
|
||||
from autoagent.environment.docker_env import DockerEnv, DockerConfig, with_env
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
from tqdm import trange
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser(description="working@tjb-tech")
|
||||
parser.add_argument('--container_name', type=str, default='gaia_test')
|
||||
parser.add_argument('--model', type=str, default='gpt-4o-mini-2024-07-18')
|
||||
parser.add_argument('--git_clone', action='store_true', default=False)
|
||||
parser.add_argument('--setup_package', type=str, default='lite_pkgs')
|
||||
parser.add_argument('--debug', action='store_true', default=False)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
def get_env(container_name: str = 'gaia_test', model: str = 'gpt-4o-mini-2024-07-18', git_clone: bool = False, setup_package: str = 'lite_pkgs', test_pull_name: str = 'test_pull_1010', debug: bool = True):
|
||||
workplace_name = DOCKER_WORKPLACE_NAME
|
||||
docker_config = DockerConfig(container_name=container_name, workplace_name=workplace_name, communication_port=12345, conda_path='/home/user/micromamba')
|
||||
docker_env = DockerEnv(docker_config)
|
||||
return docker_env
|
||||
|
||||
|
||||
def append_to_json(file_path, entry):
|
||||
if os.path.exists(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as json_file:
|
||||
data = json.load(json_file)
|
||||
else:
|
||||
data = []
|
||||
|
||||
data.append(entry)
|
||||
|
||||
with open(file_path, 'w', encoding='utf-8') as json_file:
|
||||
json.dump(data, json_file, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
|
||||
async def main(container_name: str = 'gaia_test', model: str = 'gpt-4o-mini-2024-07-18', git_clone: bool = False, setup_package: str = 'lite_pkgs', test_pull_name: str = 'test_pull_1010', debug: bool = True, task_instructions: str = None):
|
||||
workplace_name = DOCKER_WORKPLACE_NAME
|
||||
# docker_env = get_env(container_name, model, git_clone, setup_package, test_pull_name, debug)
|
||||
# docker_env.init_container()
|
||||
|
||||
csv_file_path = './MultiHopRAG.csv'
|
||||
json_path = './result.json'
|
||||
|
||||
question_list=[]
|
||||
GA_LIST=[]
|
||||
with open(csv_file_path, mode='r', encoding='utf-8') as question_file:
|
||||
reader = csv.DictReader(question_file)
|
||||
for row in reader:
|
||||
question_list.append(row['query'])
|
||||
GA_LIST.append(row['answer'])
|
||||
|
||||
row_count = 0
|
||||
|
||||
with open(json_path, 'r', encoding='utf-8') as json_file:
|
||||
data = json.load(json_file)
|
||||
row_count = len(data)
|
||||
|
||||
|
||||
for QUESTIONid in trange(row_count,len(question_list)):#
|
||||
|
||||
task_instructions = question_list[QUESTIONid]
|
||||
#answer: row[1]
|
||||
|
||||
codeact_agent = get_rag_agent(model)#, rag_env=docker_env)
|
||||
mc = MetaChain()
|
||||
# try:
|
||||
context_variables = {"working_dir": DOCKER_WORKPLACE_NAME,"user_query": task_instructions}
|
||||
messages = [{"role": "user", "content": task_instructions}]
|
||||
response = await mc.run_async(agent=codeact_agent, messages=messages,max_turns=10, context_variables=context_variables, debug=debug)
|
||||
|
||||
|
||||
data_new = {
|
||||
"query": task_instructions,
|
||||
"gold_answer": GA_LIST[QUESTIONid],
|
||||
"answer": response.messages[-1]['content']
|
||||
}
|
||||
|
||||
append_to_json(json_path, data_new)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = get_args()
|
||||
asyncio.run(main(args.container_name, args.model, args.git_clone, args.setup_package, args.test_pull_name, args.debug))
|
||||
@@ -0,0 +1,25 @@
|
||||
current_dir=$(dirname "$(readlink -f "$0")")
|
||||
|
||||
cd $current_dir
|
||||
cd ../
|
||||
export DOCKER_WORKPLACE_NAME=workplace_rag
|
||||
export EVAL_MODE=True
|
||||
export DEBUG=True
|
||||
export BASE_IMAGES=tjbtech1/gaia-bookworm:v2
|
||||
export COMPLETION_MODEL=claude-3-5-sonnet-20241022
|
||||
|
||||
|
||||
# export OPENAI_API_KEY=
|
||||
# model=gpt-4o-2024-08-06
|
||||
model=gpt-4o-mini-2024-07-18
|
||||
|
||||
|
||||
# export MISTRAL_API_KEY=\
|
||||
# model=mistral/mistral-large-2407
|
||||
|
||||
# export DEEPSEEK_API_KEY=
|
||||
# model=deepseek/deepseek-chat
|
||||
# model=deepseek/deepseek-coder
|
||||
|
||||
python run_rag.py --model $model
|
||||
# python AAAtestRAG_exp.py
|
||||
@@ -0,0 +1,65 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, List
|
||||
import json
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EvalMetadata(BaseModel):
|
||||
agent_func: str
|
||||
model: str
|
||||
eval_output_dir: str
|
||||
start_time: str
|
||||
dataset: str | None = None
|
||||
data_split: str | None = None
|
||||
details: dict[str, Any] | None = None
|
||||
container_name: str | None = None
|
||||
port: int | None = None
|
||||
git_clone: bool | None = None
|
||||
test_pull_name: str | None = None
|
||||
|
||||
def model_dump(self, *args, **kwargs):
|
||||
dumped_dict = super().model_dump(*args, **kwargs)
|
||||
# avoid leaking sensitive information
|
||||
return dumped_dict
|
||||
|
||||
def model_dump_json(self, *args, **kwargs):
|
||||
dumped = super().model_dump_json(*args, **kwargs)
|
||||
dumped_dict = json.loads(dumped)
|
||||
logger.debug(f'Dumped metadata: {dumped_dict}')
|
||||
return json.dumps(dumped_dict)
|
||||
|
||||
class EvalOutput(BaseModel):
|
||||
# NOTE: User-specified
|
||||
instance_id: str
|
||||
# output of the evaluation
|
||||
# store anything that is needed for the score calculation
|
||||
test_result: dict[str, Any]
|
||||
|
||||
instruction: str | None = None
|
||||
|
||||
# Interaction info
|
||||
metadata: EvalMetadata | None = None
|
||||
# list[tuple[dict[str, Any], dict[str, Any]]] - for compatibility with the old format
|
||||
messages: List | None = None
|
||||
error: str | None = None
|
||||
|
||||
# Optionally save the input test instance
|
||||
instance: dict[str, Any] | None = None
|
||||
|
||||
def model_dump(self, *args, **kwargs):
|
||||
dumped_dict = super().model_dump(*args, **kwargs)
|
||||
# Remove None values
|
||||
dumped_dict = {k: v for k, v in dumped_dict.items() if v is not None}
|
||||
# Apply custom serialization for metadata (to avoid leaking sensitive information)
|
||||
if self.metadata is not None:
|
||||
dumped_dict['metadata'] = self.metadata.model_dump()
|
||||
return dumped_dict
|
||||
|
||||
def model_dump_json(self, *args, **kwargs):
|
||||
dumped = super().model_dump_json(*args, **kwargs)
|
||||
dumped_dict = json.loads(dumped)
|
||||
# Apply custom serialization for metadata (to avoid leaking sensitive information)
|
||||
if 'metadata' in dumped_dict:
|
||||
dumped_dict['metadata'] = json.loads(self.metadata.model_dump_json())
|
||||
return json.dumps(dumped_dict)
|
||||
@@ -0,0 +1,379 @@
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any, TextIO, List, Dict
|
||||
from .types import EvalMetadata, EvalOutput
|
||||
import pandas as pd
|
||||
import json
|
||||
from typing import Callable, Awaitable
|
||||
from tqdm import tqdm
|
||||
from autoagent.logger import MetaChainLogger, LoggerManager
|
||||
import multiprocessing as mp
|
||||
import psutil
|
||||
import traceback
|
||||
import socket
|
||||
import queue # 添加这行导入
|
||||
|
||||
def make_metadata(
|
||||
model: str,
|
||||
dataset_name: str,
|
||||
agent_func: str,
|
||||
eval_note: str | None,
|
||||
eval_output_dir: str,
|
||||
data_split: str | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
port: int | None = None,
|
||||
container_name: str | None = None,
|
||||
git_clone: bool = False,
|
||||
test_pull_name: str | None = None,
|
||||
) -> EvalMetadata:
|
||||
eval_note = f'_N_{eval_note}' if eval_note else ''
|
||||
|
||||
eval_output_path = os.path.join(
|
||||
eval_output_dir,
|
||||
dataset_name,
|
||||
agent_func.replace('get_', ''),
|
||||
f'{model}_maxiter{eval_note}',
|
||||
)
|
||||
|
||||
pathlib.Path(eval_output_path).mkdir(parents=True, exist_ok=True)
|
||||
pathlib.Path(os.path.join(eval_output_path, 'logs')).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
|
||||
metadata = EvalMetadata(
|
||||
agent_func=agent_func,
|
||||
model=model,
|
||||
eval_output_dir=eval_output_path,
|
||||
start_time=time.strftime('%Y-%m-%d %H:%M:%S'),
|
||||
dataset=dataset_name,
|
||||
data_split=data_split,
|
||||
details=details,
|
||||
port=port,
|
||||
container_name=container_name,
|
||||
git_clone=git_clone,
|
||||
test_pull_name=test_pull_name,
|
||||
)
|
||||
metadata_json = metadata.model_dump_json()
|
||||
with open(os.path.join(eval_output_path, 'metadata.json'), 'w') as f:
|
||||
f.write(metadata_json)
|
||||
|
||||
return metadata
|
||||
|
||||
def prepare_dataset(
|
||||
dataset: pd.DataFrame,
|
||||
output_file: str,
|
||||
eval_n_limit: int,
|
||||
eval_ids: list[str] | None = None,
|
||||
skip_num: int | None = None,
|
||||
):
|
||||
assert (
|
||||
'instance_id' in dataset.columns
|
||||
), "Expected 'instance_id' column in the dataset. You should define your own unique identifier for each instance and use it as the 'instance_id' column."
|
||||
logger = LoggerManager.get_logger()
|
||||
id_column = 'instance_id'
|
||||
logger.info(f'Writing evaluation output to {output_file}')
|
||||
finished_ids: set[str] = set()
|
||||
if os.path.exists(output_file):
|
||||
with open(output_file, 'r') as f:
|
||||
for line in f:
|
||||
data = json.loads(line)
|
||||
finished_ids.add(str(data[id_column]))
|
||||
logger.info(
|
||||
f'\nOutput file {output_file} already exists. Loaded {len(finished_ids)} finished instances.', title='Warning', color='red'
|
||||
)
|
||||
|
||||
if eval_ids:
|
||||
eval_ids_converted = [dataset[id_column].dtype.type(id) for id in eval_ids]
|
||||
dataset = dataset[dataset[id_column].isin(eval_ids_converted)]
|
||||
logger.info(f'Limiting evaluation to {len(eval_ids)} specific instances.')
|
||||
elif skip_num and skip_num >= 0:
|
||||
skip_num = min(skip_num, len(dataset))
|
||||
dataset = dataset.iloc[skip_num:]
|
||||
logger.info(
|
||||
f'Starting evaluation with skipping first {skip_num} instances ({len(dataset)} instances to run).'
|
||||
)
|
||||
if eval_n_limit and eval_n_limit > 0:
|
||||
dataset = dataset.head(eval_n_limit)
|
||||
logger.info(f'Limiting evaluation to {eval_n_limit} instances.')
|
||||
elif eval_n_limit and eval_n_limit > 0:
|
||||
dataset = dataset.head(eval_n_limit)
|
||||
logger.info(f'Limiting evaluation to first {eval_n_limit} instances.')
|
||||
|
||||
new_dataset = [
|
||||
instance
|
||||
for _, instance in dataset.iterrows()
|
||||
if str(instance[id_column]) not in finished_ids
|
||||
]
|
||||
logger.info(
|
||||
f'Finished instances: {len(finished_ids)}, Remaining instances: {len(new_dataset)}'
|
||||
)
|
||||
|
||||
return pd.DataFrame(new_dataset)
|
||||
def _process_and_queue(process_instance_func, instance, metadata, use_mp, max_retries, queue):
|
||||
"""包装函数，将结果放入队列"""
|
||||
try:
|
||||
result = _process_instance_wrapper(
|
||||
process_instance_func, instance, metadata, use_mp, max_retries
|
||||
)
|
||||
queue.put(result)
|
||||
except Exception as e:
|
||||
print(f"Error processing instance {instance.get('instance_id', 'unknown')}: {str(e)}")
|
||||
traceback.print_exc()
|
||||
# 在发生错误时也要把错误结果放入队列，避免主进程等待
|
||||
queue.put(None) # 或者放入一个表示错误的特殊值
|
||||
# finally:
|
||||
# # 确保子进程中的资源被释放
|
||||
# queue.close()
|
||||
|
||||
def run_evaluation(
|
||||
dataset: pd.DataFrame,
|
||||
metadata: EvalMetadata | None,
|
||||
output_file: str,
|
||||
num_workers: int,
|
||||
process_instance_func: Callable[
|
||||
[pd.Series, EvalMetadata, bool], Awaitable[EvalOutput]
|
||||
],
|
||||
max_retries: int = 3, # number of retries for each instance
|
||||
):
|
||||
logger = LoggerManager.get_logger()
|
||||
use_multiprocessing = num_workers > 1
|
||||
|
||||
if metadata is not None:
|
||||
logger.info(
|
||||
f'Evaluation started with Agent {metadata.agent_func}\n'
|
||||
)
|
||||
else:
|
||||
logger.info('Running evaluation without metadata.', title='Warning', color='red')
|
||||
logger.info(f'Evaluation started with {num_workers} workers.')
|
||||
|
||||
total_instances = len(dataset)
|
||||
pbar = tqdm(total=total_instances, desc='Instances processed')
|
||||
output_fp = open(output_file, 'a')
|
||||
|
||||
try:
|
||||
if use_multiprocessing:
|
||||
# 使用队列来收集结果
|
||||
results_queue = mp.Queue()
|
||||
active_processes = []
|
||||
instances_iter = dataset.iterrows()
|
||||
instances_completed = 0
|
||||
|
||||
while instances_completed < total_instances:
|
||||
# 启动新进程，直到达到worker数量限制
|
||||
while len(active_processes) < num_workers and instances_completed < total_instances:
|
||||
try:
|
||||
_, instance = next(instances_iter)
|
||||
# 创建非守护进程
|
||||
p = mp.Process(
|
||||
target=_process_and_queue,
|
||||
args=(process_instance_func, instance, metadata, True, max_retries, results_queue),
|
||||
daemon=False # 关键：设置为非守护进程
|
||||
)
|
||||
p.start()
|
||||
time.sleep(3)
|
||||
active_processes.append((p, time.time())) # 记录进程启动时间
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
# 检查完成的进程
|
||||
for p, start_time in active_processes[:]:
|
||||
if not p.is_alive():
|
||||
try:
|
||||
# 给进程1分钟时间来清理资源
|
||||
p.join(timeout=60)
|
||||
if p.is_alive():
|
||||
logger.warning(f"Process {p.pid} cleanup timeout, force terminating...")
|
||||
p.terminate()
|
||||
p.join(timeout=5)
|
||||
if p.is_alive():
|
||||
p.kill()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error cleaning up process {p.pid}: {str(e)}")
|
||||
p.kill()
|
||||
finally:
|
||||
active_processes.remove((p, start_time))
|
||||
|
||||
# 处理队列中的结果
|
||||
try:
|
||||
while not results_queue.empty():
|
||||
result = results_queue.get_nowait()
|
||||
update_progress(result, pbar, output_fp)
|
||||
instances_completed += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing results: {str(e)}")
|
||||
|
||||
time.sleep(0.1) # 避免过度占用CPU
|
||||
|
||||
# 清理剩余进程
|
||||
logger.info("Cleaning up remaining processes...")
|
||||
for p, _ in active_processes:
|
||||
try:
|
||||
# 给进程一个较短的超时时间
|
||||
p.join(timeout=5)
|
||||
if p.is_alive():
|
||||
p.terminate()
|
||||
p.join(timeout=1)
|
||||
if p.is_alive():
|
||||
p.kill()
|
||||
except Exception as e:
|
||||
logger.info(f"Error cleaning up process {p.pid}: {str(e)}", title='warning', color='red')
|
||||
try:
|
||||
p.kill()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 快速清空队列
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
result = results_queue.get_nowait()
|
||||
update_progress(result, pbar, output_fp)
|
||||
instances_completed += 1
|
||||
except queue.Empty:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.info(f"Error processing final results: {str(e)}", title='Warning', color='red')
|
||||
else:
|
||||
for _, instance in dataset.iterrows():
|
||||
result = _process_instance_wrapper(
|
||||
process_instance_func=process_instance_func,
|
||||
instance=instance,
|
||||
metadata=metadata,
|
||||
use_mp=False,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
update_progress(result, pbar, output_fp)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print('\nKeyboardInterrupt received. Cleaning up...\n')
|
||||
if use_multiprocessing:
|
||||
for p, _ in active_processes:
|
||||
try:
|
||||
p.terminate()
|
||||
p.join(timeout=1)
|
||||
except Exception:
|
||||
p.kill()
|
||||
cleanup()
|
||||
finally:
|
||||
# 确保资源被释放
|
||||
output_fp.close()
|
||||
if use_multiprocessing:
|
||||
results_queue.close()
|
||||
results_queue.join_thread()
|
||||
|
||||
output_fp.close()
|
||||
logger.info('\nEvaluation finished.\n')
|
||||
def _process_instance_wrapper_mp(args):
|
||||
"""Wrapper for multiprocessing, especially for imap_unordered."""
|
||||
return _process_instance_wrapper(*args)
|
||||
|
||||
def _process_instance_wrapper(
|
||||
process_instance_func: Callable[[pd.Series, EvalMetadata, bool], EvalOutput],
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
use_mp: bool,
|
||||
max_retries: int = 5,
|
||||
) -> EvalOutput:
|
||||
"""Wrap the process_instance_func to handle retries and errors.
|
||||
|
||||
Retry an instance up to max_retries times if it fails (e.g., due to transient network/runtime issues).
|
||||
"""
|
||||
if use_mp:
|
||||
log_path = os.path.join(metadata.eval_output_dir, 'logs', f'agent_{metadata.model}_did_{instance["instance_id"]}.log')
|
||||
logger = MetaChainLogger(log_path)
|
||||
else:
|
||||
logger = LoggerManager.get_logger()
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
result = process_instance_func(instance, metadata, logger)
|
||||
return result
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
stacktrace = traceback.format_exc()
|
||||
if attempt == max_retries:
|
||||
logger.info(error, title='Error', color='red')
|
||||
msg = (
|
||||
'-' * 10
|
||||
+ '\n'
|
||||
+ f'Error in instance [{instance.instance_id}]: {error}. Stacktrace:\n{stacktrace}'
|
||||
+ '\n'
|
||||
+ f'[Encountered after {max_retries} retries. Please check the logs and report the issue.]'
|
||||
+ '-' * 10
|
||||
)
|
||||
# Raise an error after all retries & stop the evaluation
|
||||
logger.info(error, title='Error', color='red')
|
||||
raise RuntimeError(
|
||||
f'Maximum error retries reached for instance {instance.instance_id}'
|
||||
) from e
|
||||
msg = (
|
||||
'-' * 10
|
||||
+ '\n'
|
||||
+ f'Error in instance [{instance.instance_id}]: {error}. Stacktrace:\n{stacktrace}'
|
||||
+ '\n'
|
||||
+ '-' * 10
|
||||
+ f'[The above error occurred. Retrying... (attempt {attempt + 1} of {max_retries})]'
|
||||
+ '-' * 10
|
||||
+ '\n'
|
||||
)
|
||||
logger.info(msg, title='Error', color='red')
|
||||
if use_mp:
|
||||
print(msg) # use print to directly print to console
|
||||
time.sleep(5)
|
||||
|
||||
def update_progress(
|
||||
result: EvalOutput,
|
||||
pbar: tqdm,
|
||||
output_fp: TextIO,
|
||||
):
|
||||
"""Update the progress bar and write the result to the output file."""
|
||||
logger = LoggerManager.get_logger()
|
||||
pbar.update(1)
|
||||
pbar.set_description(f'Instance {result.instance_id}')
|
||||
pbar.set_postfix_str(f'Test Result: {str(result.test_result)[:300]}...')
|
||||
logger.info(
|
||||
f'Finished evaluation for instance {result.instance_id}: {str(result.test_result)[:300]}...\n'
|
||||
)
|
||||
output_fp.write(json.dumps(result.model_dump()) + '\n')
|
||||
output_fp.flush()
|
||||
|
||||
def cleanup():
|
||||
print('Cleaning up child processes...')
|
||||
for process in mp.active_children():
|
||||
print(f'Terminating child process: {process.name}')
|
||||
process.terminate()
|
||||
process.join()
|
||||
|
||||
|
||||
def check_port_available(port):
|
||||
"""check if the port is available"""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
try:
|
||||
# set the port reuse option
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
# try to bind the port
|
||||
s.bind(('0.0.0.0', port))
|
||||
# immediately close the connection
|
||||
s.close()
|
||||
return True # the port is available
|
||||
except socket.error:
|
||||
return False # the port is not available
|
||||
|
||||
def clean_msg(msg: List[Dict[str, Any]]):
|
||||
new_msg = []
|
||||
for m in msg:
|
||||
msg_content = m['content']
|
||||
if isinstance(msg_content, str):
|
||||
m['content'] = msg_content
|
||||
new_msg.append(m.copy())
|
||||
elif isinstance(msg_content, List):
|
||||
new_content = []
|
||||
for c in msg_content:
|
||||
if c['type'] == 'text':
|
||||
new_content.append(c.copy())
|
||||
elif c['type'] == 'image_url':
|
||||
new_content.append({'type': 'image_url', 'image_url': 'placeholder'})
|
||||
m['content'] = new_content
|
||||
new_msg.append(m.copy())
|
||||
return new_msg
|
||||
Reference in New Issue
Block a user