chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,193 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from datasets import load_dataset
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
class APPsReader:
|
||||
def __init__(self, split: str = "train", train_sub_split: str = ""):
|
||||
self.train_sub_val_ids = set(json.load(open("apps_train_sub_val_ids.json")))
|
||||
self.split = split
|
||||
self.train_sub_split = train_sub_split
|
||||
|
||||
def __call__(self, file_path):
|
||||
data = load_dataset(file_path, split=self.split, trust_remote_code=True).to_list()
|
||||
logger.info(len(data))
|
||||
|
||||
if self.split == "train" and self.train_sub_split:
|
||||
if self.train_sub_split == "train":
|
||||
data = [item for item in data if item["problem_id"] not in self.train_sub_val_ids]
|
||||
elif self.train_sub_split == "val":
|
||||
data = [item for item in data if item["problem_id"] in self.train_sub_val_ids]
|
||||
else:
|
||||
raise ValueError(f"Invalid train_sub_split: {self.train_sub_split} [train, val]")
|
||||
logger.info(f"Using {self.train_sub_split} split for training data")
|
||||
logger.info(len(data))
|
||||
|
||||
missing_solutions = 0
|
||||
missing_test_cases = 0
|
||||
for item in data:
|
||||
if item["solutions"]:
|
||||
item["solutions"] = json.loads(item["solutions"])
|
||||
else:
|
||||
missing_solutions += 1
|
||||
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
else:
|
||||
missing_test_cases += 1
|
||||
|
||||
print(f"Missing solutions: {missing_solutions}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
class APPsWithFunctionName:
|
||||
def __init__(self, split: str = "train", train_sub_split: str = "", use_starter_code: bool = False):
|
||||
self.train_sub_val_ids = set(json.load(open("apps_train_sub_val_ids.json")))
|
||||
self.split = split
|
||||
self.train_sub_split = train_sub_split
|
||||
self.use_starter_code = use_starter_code
|
||||
|
||||
def __call__(self, file_path):
|
||||
data = load_dataset(file_path, split=self.split, trust_remote_code=True).to_list()
|
||||
logger.info(len(data))
|
||||
|
||||
if self.split == "train" and self.train_sub_split:
|
||||
if self.train_sub_split == "train":
|
||||
data = [item for item in data if item["problem_id"] not in self.train_sub_val_ids]
|
||||
elif self.train_sub_split == "val":
|
||||
data = [item for item in data if item["problem_id"] in self.train_sub_val_ids]
|
||||
else:
|
||||
raise ValueError(f"Invalid train_sub_split: {self.train_sub_split} [train, val]")
|
||||
logger.info(f"Using {self.train_sub_split} split for training data")
|
||||
logger.info(len(data))
|
||||
|
||||
missing_solutions = 0
|
||||
missing_test_cases = 0
|
||||
for item in data:
|
||||
if item["solutions"]:
|
||||
item["solutions"] = json.loads(item["solutions"])
|
||||
else:
|
||||
missing_solutions += 1
|
||||
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
if "fn_name" in item["input_output"]:
|
||||
item["fn_name"] = item["input_output"]["fn_name"]
|
||||
if self.use_starter_code:
|
||||
assert item["starter_code"]
|
||||
item["question"] += f"\n\n{item['starter_code']}"
|
||||
else:
|
||||
item["question"] += f"\n\nYou should name the function as `{item['fn_name']}`."
|
||||
else:
|
||||
missing_test_cases += 1
|
||||
|
||||
print(f"Missing solutions: {missing_solutions}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
class APPsFlatTestCasesReader(APPsWithFunctionName):
|
||||
def __call__(self, file_path):
|
||||
data = super().__call__(file_path)
|
||||
outputs = []
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
inputs = item["input_output"]["inputs"]
|
||||
inputs = [str(_input) for _input in inputs]
|
||||
item["test_inputs"] = inputs
|
||||
outputs.append(item)
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
class PseudoInputsWithFunctionName:
|
||||
def __init__(self, use_starter_code: bool = False, train_sub_split: str = "train"):
|
||||
self.use_starter_code = use_starter_code
|
||||
self.train_sub_val_ids = set([f"apps-train-{x}" for x in json.load(open("apps_train_sub_val_ids.json"))])
|
||||
self.train_sub_split = train_sub_split
|
||||
|
||||
def __call__(self, file_path):
|
||||
data = json.load(open(file_path, encoding="utf-8"))
|
||||
logger.info(len(data))
|
||||
|
||||
if self.train_sub_split == "train":
|
||||
data = [item for item in data if item["problem_id"] not in self.train_sub_val_ids]
|
||||
elif self.train_sub_split == "val":
|
||||
data = [item for item in data if item["problem_id"] in self.train_sub_val_ids]
|
||||
else:
|
||||
raise ValueError(f"Invalid train_sub_split: {self.train_sub_split} [train, val]")
|
||||
logger.info(f"Using {self.train_sub_split} split for training data")
|
||||
logger.info(len(data))
|
||||
|
||||
missing_solutions = 0
|
||||
missing_test_cases = 0
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
if not isinstance(item["input_output"], dict):
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
if "fn_name" in item["input_output"]:
|
||||
item["fn_name"] = item["input_output"]["fn_name"]
|
||||
if self.use_starter_code:
|
||||
# FIXME: This is a bug for synthesized data, since we didn't generate starter code.
|
||||
# We should use `fn_name` to compose simple starter_code instead.
|
||||
# See PseudoInputsWithFunctionNameFixStarterCode for the fix.
|
||||
if "starter_code" in item and item["starter_code"]:
|
||||
item["question"] += f"\n\n{item['starter_code']}"
|
||||
else:
|
||||
item["question"] += f"\n\nYou should name the function as `{item['fn_name']}`."
|
||||
else:
|
||||
missing_test_cases += 1
|
||||
|
||||
print(f"Missing solutions: {missing_solutions}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
class PseudoInputsWithFunctionNameFixStarterCode:
|
||||
def __init__(self, use_starter_code: bool = False, train_sub_split: str = "train"):
|
||||
self.use_starter_code = use_starter_code
|
||||
self.train_sub_val_ids = set([f"apps-train-{x}" for x in json.load(open("apps_train_sub_val_ids.json"))])
|
||||
self.train_sub_split = train_sub_split
|
||||
|
||||
def __call__(self, file_path):
|
||||
data = json.load(open(file_path, encoding="utf-8"))
|
||||
logger.info(len(data))
|
||||
|
||||
if self.train_sub_split == "train":
|
||||
data = [item for item in data if item["problem_id"] not in self.train_sub_val_ids]
|
||||
elif self.train_sub_split == "val":
|
||||
data = [item for item in data if item["problem_id"] in self.train_sub_val_ids]
|
||||
else:
|
||||
raise ValueError(f"Invalid train_sub_split: {self.train_sub_split} [train, val]")
|
||||
logger.info(f"Using {self.train_sub_split} split for training data")
|
||||
logger.info(len(data))
|
||||
|
||||
missing_solutions = 0
|
||||
missing_test_cases = 0
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
if not isinstance(item["input_output"], dict):
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
if "fn_name" in item["input_output"]:
|
||||
item["fn_name"] = item["input_output"]["fn_name"]
|
||||
if self.use_starter_code and "starter_code" in item and item["starter_code"]:
|
||||
item["question"] += f"\n\n{item['starter_code']}"
|
||||
else:
|
||||
item["question"] += f"\n\nYou should name the function as `{item['fn_name']}`."
|
||||
else:
|
||||
missing_test_cases += 1
|
||||
|
||||
print(f"Missing solutions: {missing_solutions}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,87 @@
|
||||
import os.path
|
||||
|
||||
from datasets import load_dataset
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class CodeContestReader:
|
||||
def __call__(self, file_path):
|
||||
if os.path.exists(file_path):
|
||||
data = load_dataset("parquet", data_files=file_path)["train"]
|
||||
else:
|
||||
data = load_dataset("parquet", data_files=list(glob(file_path)))["train"]
|
||||
print(len(data))
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
description = item["description"]
|
||||
title = item["name"]
|
||||
if item["time_limit"]:
|
||||
time_limits = item["time_limit"]["seconds"]
|
||||
else:
|
||||
time_limits = "N/A"
|
||||
if item["memory_limit_bytes"]:
|
||||
mem_limits = item["memory_limit_bytes"]
|
||||
else:
|
||||
mem_limits = "N/A"
|
||||
pos_solution = item["solutions"]["solution"]
|
||||
neg_solution = item["incorrect_solutions"]["solution"]
|
||||
|
||||
outputs.append({
|
||||
"name": title,
|
||||
"description": description,
|
||||
"time_limit": time_limits,
|
||||
"memory_limit": mem_limits,
|
||||
"pos_solution": pos_solution,
|
||||
"neg_solution": neg_solution,
|
||||
"id": f"{title}",
|
||||
})
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
class CodeContestFlatReader:
|
||||
def __call__(self, file_path):
|
||||
if os.path.exists(file_path):
|
||||
data = load_dataset("parquet", data_files=file_path)["train"]
|
||||
else:
|
||||
data = load_dataset("parquet", data_files=list(glob(file_path)))["train"]
|
||||
print(len(data))
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
description = item["description"]
|
||||
title = item["name"]
|
||||
if item["time_limit"]:
|
||||
time_limits = item["time_limit"]["seconds"]
|
||||
else:
|
||||
time_limits = "N/A"
|
||||
if item["memory_limit_bytes"]:
|
||||
mem_limits = item["memory_limit_bytes"]
|
||||
else:
|
||||
mem_limits = "N/A"
|
||||
pos_solution = item["solutions"]["solution"]
|
||||
neg_solution = item["incorrect_solutions"]["solution"]
|
||||
|
||||
for i, solution in enumerate(pos_solution):
|
||||
outputs.append({
|
||||
"name": title,
|
||||
"description": description,
|
||||
"time_limit": time_limits,
|
||||
"memory_limit": mem_limits,
|
||||
"solution": solution,
|
||||
"id": f"{title}_{i}",
|
||||
"value": 1,
|
||||
})
|
||||
|
||||
for i, solution in enumerate(neg_solution):
|
||||
outputs.append({
|
||||
"name": title,
|
||||
"description": description,
|
||||
"time_limit": time_limits,
|
||||
"memory_limit": mem_limits,
|
||||
"solution": solution,
|
||||
"id": f"{title}_{i}",
|
||||
"value": 0,
|
||||
})
|
||||
|
||||
return outputs
|
||||
@@ -0,0 +1,392 @@
|
||||
import collections
|
||||
import json
|
||||
import os.path
|
||||
import random
|
||||
from typing import List, Dict, Callable
|
||||
|
||||
import hydra
|
||||
from omegaconf import DictConfig
|
||||
from torch.utils.data import Dataset
|
||||
from transformers import PreTrainedTokenizer
|
||||
|
||||
from data.input_aligner import empty_aligner
|
||||
from data.input_utils import json_read_fn
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
|
||||
class ResponseAlignDataset(Dataset):
|
||||
def __init__(self,
|
||||
file_path: str,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
template: str,
|
||||
aligner: Callable = empty_aligner,
|
||||
instruction: str = "",
|
||||
few_shot_prompt: str = "",
|
||||
api_based: bool = False, message_compose_fn: Callable = None,
|
||||
service_based: bool = False, service_processor: Callable = None,
|
||||
flush_file: str = None,
|
||||
split_size: int = -1,
|
||||
split_id: int = 0,
|
||||
index_field: str = "id",
|
||||
max_data_num: int = -1,
|
||||
read_fn: Callable = json_read_fn,
|
||||
replacement: Dict[str, str] = None,
|
||||
):
|
||||
self.tokenizer = tokenizer
|
||||
self.template = template
|
||||
self.instruction = instruction
|
||||
self.few_shot_prompt = few_shot_prompt
|
||||
self.api_based = api_based
|
||||
self.message_compose_fn = message_compose_fn
|
||||
self.service_based = service_based
|
||||
self.service_processor = service_processor
|
||||
self.flush_file = flush_file
|
||||
self.split_size = split_size
|
||||
self.split_id = split_id
|
||||
self.index_field = index_field
|
||||
self.max_data_num = max_data_num
|
||||
self.replacement = replacement
|
||||
|
||||
data = read_fn(file_path)
|
||||
self.data: List[Dict] = aligner(data)
|
||||
|
||||
for item in self.data:
|
||||
if self.instruction:
|
||||
item["instruction"] = self.instruction
|
||||
if self.few_shot_prompt:
|
||||
item["few_shot_prompt"] = self.few_shot_prompt
|
||||
|
||||
flushed_data = set()
|
||||
flushed_full_data = []
|
||||
if flush_file is not None and os.path.exists(flush_file):
|
||||
tmp = open(flush_file, "r", encoding="utf-8").readlines()
|
||||
has_reading_error = False
|
||||
for line in tmp:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Error reading line: {line}")
|
||||
has_reading_error = True
|
||||
continue
|
||||
if "response" in item and item["response"]:
|
||||
if item["id"] in flushed_data:
|
||||
continue
|
||||
flushed_data.add(item["id"])
|
||||
flushed_full_data.append(item)
|
||||
logger.info(f"Loaded flushed data: {len(flushed_data)} from {flush_file}")
|
||||
if has_reading_error:
|
||||
logger.warning("There are reading errors in the flush file")
|
||||
with open(flush_file, "w", encoding="utf-8") as f:
|
||||
for item in flushed_full_data:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
if split_size > 0:
|
||||
batch_size = (len(self.data) + split_size - 1) // split_size
|
||||
self.data = self.data[split_id * batch_size: (split_id + 1) * batch_size]
|
||||
|
||||
self.data = [item for item in self.data if item[self.index_field] not in flushed_data and str(item[self.index_field]) not in flushed_data]
|
||||
|
||||
def __len__(self):
|
||||
if self.max_data_num > 0:
|
||||
return min(self.max_data_num, len(self.data))
|
||||
return len(self.data)
|
||||
|
||||
def api_getitem(self, index):
|
||||
item = self.data[index]
|
||||
if self.replacement:
|
||||
tmp = self.template
|
||||
for k, v in self.replacement.items():
|
||||
tmp = tmp.replace(k, item[v])
|
||||
text = tmp
|
||||
else:
|
||||
text = self.template.format(**item)
|
||||
if self.message_compose_fn is not None:
|
||||
text = self.message_compose_fn(text)
|
||||
item["text"] = text
|
||||
return {
|
||||
"text": text,
|
||||
"meta_data": item,
|
||||
}
|
||||
|
||||
def service_getitem(self, index):
|
||||
inputs = self.api_getitem(index)
|
||||
response = self.service_processor(inputs["text"])
|
||||
inputs["response"] = response
|
||||
return inputs
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if self.api_based:
|
||||
return self.api_getitem(idx)
|
||||
if self.service_based:
|
||||
return self.service_getitem(idx)
|
||||
item = self.data[idx]
|
||||
if self.replacement:
|
||||
tmp = self.template
|
||||
for k, v in self.replacement.items():
|
||||
tmp = tmp.replace(k, item[v])
|
||||
text = tmp
|
||||
else:
|
||||
text = self.template.format(**item)
|
||||
item["text"] = text
|
||||
return {
|
||||
"text": text,
|
||||
"meta_data": item,
|
||||
}
|
||||
|
||||
|
||||
class PromptResponseDataset(Dataset):
|
||||
def __init__(self,
|
||||
file_path: str,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
prompt_template: str,
|
||||
response_template: str,
|
||||
aligner: Callable = empty_aligner,
|
||||
instruction: str = "",
|
||||
few_shot_prompt: str = "",
|
||||
api_based: bool = False,
|
||||
service_based: bool = False, service_processor: Callable = None,
|
||||
flush_file: str = None,
|
||||
split_size: int = -1,
|
||||
split_id: int = 0,
|
||||
index_field: str = "id",
|
||||
max_data_num: int = -1,
|
||||
read_fn: Callable = json_read_fn,
|
||||
kv_mapping: Dict[str, str] = None,
|
||||
):
|
||||
self.tokenizer = tokenizer
|
||||
self.prompt_template = prompt_template
|
||||
self.response_template = response_template
|
||||
self.instruction = instruction
|
||||
self.few_shot_prompt = few_shot_prompt
|
||||
self.api_based = api_based
|
||||
self.service_based = service_based
|
||||
self.service_processor = service_processor
|
||||
self.flush_file = flush_file
|
||||
self.split_size = split_size
|
||||
self.split_id = split_id
|
||||
self.index_field = index_field
|
||||
self.max_data_num = max_data_num
|
||||
self.kv_mapping = kv_mapping
|
||||
|
||||
data = read_fn(file_path)
|
||||
self.data: List[Dict] = aligner(data)
|
||||
|
||||
for item in self.data:
|
||||
if self.instruction:
|
||||
item["instruction"] = self.instruction
|
||||
if self.few_shot_prompt:
|
||||
item["few_shot_prompt"] = self.few_shot_prompt
|
||||
|
||||
flushed_data = set()
|
||||
if flush_file is not None and os.path.exists(flush_file):
|
||||
tmp = open(flush_file, "r", encoding="utf-8").readlines()
|
||||
for line in tmp:
|
||||
item = json.loads(line)
|
||||
if "response" in item and item["response"].strip() != "":
|
||||
flushed_data.add(item["id"])
|
||||
logger.info(f"Loaded flushed data: {len(flushed_data)} from {flush_file}")
|
||||
|
||||
self.data = [item for item in self.data if item[self.index_field] not in flushed_data]
|
||||
|
||||
if split_size > 0:
|
||||
batch_size = (len(self.data) + split_size - 1) // split_size
|
||||
self.data = self.data[split_id * batch_size: (split_id + 1) * batch_size]
|
||||
|
||||
def __len__(self):
|
||||
if self.max_data_num > 0:
|
||||
return min(self.max_data_num, len(self.data))
|
||||
return len(self.data)
|
||||
|
||||
def api_getitem(self, index):
|
||||
raise NotImplementedError
|
||||
|
||||
def service_getitem(self, index):
|
||||
raise NotImplementedError
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if self.api_based:
|
||||
return self.api_getitem(idx)
|
||||
if self.service_based:
|
||||
return self.service_getitem(idx)
|
||||
item = self.data[idx]
|
||||
prompt = self.prompt_template.format(**item)
|
||||
response = self.response_template.format(**item)
|
||||
text = prompt + response
|
||||
item["text"] = text
|
||||
item["prompt"] = prompt
|
||||
|
||||
if not self.kv_mapping:
|
||||
return {
|
||||
"text": text,
|
||||
"meta_data": item,
|
||||
}
|
||||
|
||||
res = {v: item[k] for k, v in self.kv_mapping.items()}
|
||||
res["meta_data"] = item
|
||||
return res
|
||||
|
||||
|
||||
class MultiMappingDataset(Dataset):
|
||||
def __init__(self,
|
||||
file_path: str,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
template: Dict[str, str],
|
||||
aligner: Callable = empty_aligner,
|
||||
instruction: str = "",
|
||||
few_shot_prompt: str = "",
|
||||
api_based: bool = False,
|
||||
service_based: bool = False, service_processor: Callable = None,
|
||||
flush_file: str = None,
|
||||
split_size: int = -1,
|
||||
split_id: int = 0,
|
||||
index_field: str = "id",
|
||||
max_data_num: int = -1,
|
||||
read_fn: Callable = json_read_fn,
|
||||
kv_mapping: Dict[str, str] = None,
|
||||
replacement: Dict[str, str] = None,
|
||||
):
|
||||
self.tokenizer = tokenizer
|
||||
self.template = template
|
||||
self.instruction = instruction
|
||||
self.few_shot_prompt = few_shot_prompt
|
||||
self.api_based = api_based
|
||||
self.service_based = service_based
|
||||
self.service_processor = service_processor
|
||||
self.flush_file = flush_file
|
||||
self.split_size = split_size
|
||||
self.split_id = split_id
|
||||
self.index_field = index_field
|
||||
self.max_data_num = max_data_num
|
||||
self.kv_mapping = kv_mapping
|
||||
self.replacement = replacement
|
||||
|
||||
data = read_fn(file_path)
|
||||
self.data: List[Dict] = aligner(data)
|
||||
|
||||
for item in self.data:
|
||||
if self.instruction:
|
||||
item["instruction"] = self.instruction
|
||||
if self.few_shot_prompt:
|
||||
item["few_shot_prompt"] = self.few_shot_prompt
|
||||
|
||||
flushed_data = set()
|
||||
if flush_file is not None and os.path.exists(flush_file):
|
||||
tmp = open(flush_file, "r", encoding="utf-8").readlines()
|
||||
for line in tmp:
|
||||
item = json.loads(line)
|
||||
if "response" in item and item["response"]:
|
||||
flushed_data.add(item["id"])
|
||||
logger.info(f"Loaded flushed data: {len(flushed_data)} from {flush_file}")
|
||||
|
||||
self.data = [item for item in self.data if item[self.index_field] not in flushed_data]
|
||||
|
||||
if split_size > 0:
|
||||
batch_size = (len(self.data) + split_size - 1) // split_size
|
||||
self.data = self.data[split_id * batch_size: (split_id + 1) * batch_size]
|
||||
|
||||
def __len__(self):
|
||||
if self.max_data_num > 0:
|
||||
return min(self.max_data_num, len(self.data))
|
||||
return len(self.data)
|
||||
|
||||
def api_getitem(self, index):
|
||||
raise NotImplementedError
|
||||
|
||||
def service_getitem(self, index):
|
||||
raise NotImplementedError
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if self.api_based:
|
||||
return self.api_getitem(idx)
|
||||
if self.service_based:
|
||||
return self.service_getitem(idx)
|
||||
item = self.data[idx]
|
||||
|
||||
inputs = {}
|
||||
if self.replacement:
|
||||
for name, tem in self.template.items():
|
||||
tmp = tem
|
||||
for k, v in self.replacement.items():
|
||||
tmp = tmp.replace(k, item[v])
|
||||
inputs[name] = tmp
|
||||
item[name] = tmp
|
||||
else:
|
||||
for k, v in self.template.items():
|
||||
item[k] = v.format(**item)
|
||||
inputs[k] = item[k]
|
||||
inputs["meta_data"] = item
|
||||
|
||||
if not self.kv_mapping:
|
||||
return inputs
|
||||
|
||||
res = {v: item[k] for k, v in self.kv_mapping.items()}
|
||||
res["meta_data"] = item
|
||||
return res
|
||||
|
||||
|
||||
class MultiMappingDatasetGrouping(MultiMappingDataset):
|
||||
def __init__(self,
|
||||
file_path: str,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
template: Dict[str, str],
|
||||
aligner: Callable = empty_aligner,
|
||||
instruction: str = "",
|
||||
few_shot_prompt: str = "",
|
||||
api_based: bool = False,
|
||||
service_based: bool = False, service_processor: Callable = None,
|
||||
flush_file: str = None,
|
||||
split_size: int = -1,
|
||||
split_id: int = 0,
|
||||
index_field: str = "id",
|
||||
max_data_num: int = -1,
|
||||
read_fn: Callable = json_read_fn,
|
||||
kv_mapping: Dict[str, str] = None,
|
||||
group_field: str = "id",
|
||||
):
|
||||
super().__init__(file_path,
|
||||
tokenizer,
|
||||
template,
|
||||
aligner,
|
||||
instruction,
|
||||
few_shot_prompt,
|
||||
api_based,
|
||||
service_based,
|
||||
service_processor,
|
||||
flush_file,
|
||||
split_size,
|
||||
split_id,
|
||||
index_field,
|
||||
max_data_num,
|
||||
read_fn,
|
||||
kv_mapping)
|
||||
|
||||
random.shuffle(self.data)
|
||||
groups = collections.defaultdict(list)
|
||||
for item in self.data:
|
||||
groups[item[group_field]].append(item)
|
||||
new_data = []
|
||||
for group in groups.values():
|
||||
new_data.extend(group)
|
||||
self.data = new_data
|
||||
|
||||
|
||||
class ReplayDataset(Dataset):
|
||||
def __init__(self, file_path: str, tokenizer: PreTrainedTokenizer, new_dataset_cfg: DictConfig, old_dataset_cfg: DictConfig, replay_ratio: float = 0.1):
|
||||
logger.info(f"Loading new dataset from {file_path}")
|
||||
self.new_dataset = hydra.utils.instantiate(new_dataset_cfg, file_path=file_path, tokenizer=tokenizer)
|
||||
logger.info(f"Loading old dataset from {old_dataset_cfg}")
|
||||
old_dataset = hydra.utils.instantiate(old_dataset_cfg, tokenizer=tokenizer)
|
||||
logger.info(f"Replay ratio: {replay_ratio}")
|
||||
self.replay_ratio = replay_ratio
|
||||
self.old_data = random.sample([item for item in old_dataset], int(len(old_dataset) * replay_ratio))
|
||||
|
||||
def __len__(self):
|
||||
return len(self.new_dataset) + len(self.old_data)
|
||||
|
||||
def __getitem__(self, index):
|
||||
if index < len(self.new_dataset):
|
||||
return self.new_dataset[index]
|
||||
else:
|
||||
return self.old_data[index - len(self.new_dataset)]
|
||||
@@ -0,0 +1,354 @@
|
||||
# Copied from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/data_processing/answer_extraction.py
|
||||
import re
|
||||
import regex
|
||||
|
||||
|
||||
def _fix_fracs(string):
|
||||
substrs = string.split("\\frac")
|
||||
new_str = substrs[0]
|
||||
if len(substrs) > 1:
|
||||
substrs = substrs[1:]
|
||||
for substr in substrs:
|
||||
new_str += "\\frac"
|
||||
if len(substr) > 0 and substr[0] == "{":
|
||||
new_str += substr
|
||||
else:
|
||||
try:
|
||||
assert len(substr) >= 2
|
||||
except:
|
||||
return string
|
||||
a = substr[0]
|
||||
b = substr[1]
|
||||
if b != "{":
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}{" + b + "}" + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}{" + b + "}"
|
||||
else:
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}" + b + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}" + b
|
||||
string = new_str
|
||||
return string
|
||||
|
||||
|
||||
def _fix_a_slash_b(string):
|
||||
if len(string.split("/")) != 2:
|
||||
return string
|
||||
a = string.split("/")[0]
|
||||
b = string.split("/")[1]
|
||||
try:
|
||||
if "sqrt" not in a:
|
||||
a = int(a)
|
||||
if "sqrt" not in b:
|
||||
b = int(b)
|
||||
assert string == "{}/{}".format(a, b)
|
||||
new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
|
||||
return new_string
|
||||
except:
|
||||
return string
|
||||
|
||||
|
||||
def _fix_sqrt(string):
|
||||
_string = re.sub(r"\\sqrt(-?[0-9.a-zA-Z]+)", r"\\sqrt{\1}", string)
|
||||
_string = re.sub(r"\\sqrt\s+(\w+)$", r"\\sqrt{\1}", _string)
|
||||
return _string
|
||||
|
||||
|
||||
def _fix_tan(string):
|
||||
_string = re.sub(r"\\tan(-?[0-9.a-zA-Z]+)", r"\\tan{\1}", string)
|
||||
_string = re.sub(r"\\tan\s+(\w+)$", r"\\tan{\1}", _string)
|
||||
return _string
|
||||
|
||||
|
||||
def strip_string(string):
|
||||
string = str(string).strip()
|
||||
# linebreaks
|
||||
string = string.replace("\n", "")
|
||||
|
||||
# right "."
|
||||
string = string.rstrip(".")
|
||||
|
||||
# remove inverse spaces
|
||||
string = string.replace("\\!", "")
|
||||
# string = string.replace("\\ ", "")
|
||||
|
||||
# replace \\ with \
|
||||
# string = string.replace("\\\\", "\\")
|
||||
# string = string.replace("\\\\", "\\")
|
||||
|
||||
if string.startswith("\\text{") and string.endswith("}"):
|
||||
string = string.split("{", 1)[1][:-1]
|
||||
|
||||
# replace tfrac and dfrac with frac
|
||||
string = string.replace("tfrac", "frac")
|
||||
string = string.replace("dfrac", "frac")
|
||||
string = string.replace("cfrac", "frac")
|
||||
|
||||
# remove \left and \right
|
||||
string = string.replace("\\left", "")
|
||||
string = string.replace("\\right", "")
|
||||
|
||||
# Remove unit: miles, dollars if after is not none
|
||||
_string = re.sub(r"\\text{.*?}$", "", string).strip()
|
||||
if _string != "" and _string != string:
|
||||
# print("Warning: unit not removed: '{}' -> '{}'".format(string, _string))
|
||||
string = _string
|
||||
|
||||
# Remove circ (degrees)
|
||||
string = string.replace("^{\\circ}", "").strip()
|
||||
string = string.replace("^\\circ", "").strip()
|
||||
|
||||
string = regex.sub(r"\{(c|m)?m\}(\^(2|3))?", "", string).strip()
|
||||
string = regex.sub(r"p\.m\.$", "", string).strip()
|
||||
string = regex.sub(r"(\d)\s*t$", r"\1", string).strip()
|
||||
|
||||
# remove dollar signs
|
||||
string = string.replace("\\$", "")
|
||||
string = string.replace("$", "")
|
||||
|
||||
# string = string.replace("\\text", "")
|
||||
string = string.replace("x\\in", "")
|
||||
|
||||
# remove percentage
|
||||
string = string.replace("\\%", "%")
|
||||
string = string.replace("\%", "%")
|
||||
# string = string.replace("%", "")
|
||||
|
||||
# " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
|
||||
string = string.replace(" .", " 0.")
|
||||
string = string.replace("{.", "{0.")
|
||||
|
||||
# cdot
|
||||
string = string.replace("\\cdot", "")
|
||||
|
||||
# inf
|
||||
string = string.replace("infinity", "\\infty")
|
||||
if "\\infty" not in string:
|
||||
string = string.replace("inf", "\\infty")
|
||||
string = string.replace("+\\inity", "\\infty")
|
||||
|
||||
# and
|
||||
# string = string.replace("and", "")
|
||||
string = string.replace("\\mathbf", "")
|
||||
string = string.replace("\\mathrm", "")
|
||||
|
||||
# use regex to remove \mbox{...}
|
||||
string = re.sub(r"\\mbox{.*?}", "", string)
|
||||
|
||||
# quote
|
||||
string.replace("'", "")
|
||||
string.replace("\"", "")
|
||||
|
||||
# i, j
|
||||
if "j" in string and "i" not in string:
|
||||
string = string.replace("j", "i")
|
||||
|
||||
# replace a.000b where b is not number or b is end, with ab, use regex
|
||||
string = re.sub(r"(\d+)\.0+([^\d])", r"\1\2", string)
|
||||
string = re.sub(r"(\d+)\.0+$", r"\1", string)
|
||||
|
||||
# if empty, return empty string
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if string[0] == ".":
|
||||
string = "0" + string
|
||||
|
||||
# to consider: get rid of e.g. "k = " or "q = " at beginning
|
||||
# if len(string.split("=")) == 2:
|
||||
# if len(string.split("=")[0]) <= 2:
|
||||
# string = string.split("=")[1]
|
||||
|
||||
string = _fix_sqrt(string)
|
||||
string = _fix_tan(string)
|
||||
string = string.replace(" ", "")
|
||||
|
||||
# \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
|
||||
string = _fix_fracs(string)
|
||||
|
||||
# NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
|
||||
string = _fix_a_slash_b(string)
|
||||
|
||||
string = regex.sub(r"(\\|,|\.)+$", "", string)
|
||||
|
||||
return string
|
||||
|
||||
|
||||
def extract_boxed_answers(text):
|
||||
answers = []
|
||||
for piece in text.split('boxed{')[1:]:
|
||||
n = 0
|
||||
for i in range(len(piece)):
|
||||
if piece[i] == '{':
|
||||
n += 1
|
||||
elif piece[i] == '}':
|
||||
n -= 1
|
||||
if n < 0:
|
||||
if i + 1 < len(piece) and piece[i + 1] == '%':
|
||||
answers.append(piece[: i + 1])
|
||||
else:
|
||||
answers.append(piece[:i])
|
||||
break
|
||||
return answers
|
||||
|
||||
|
||||
def extract_program_output(pred_str):
|
||||
"""
|
||||
extract output between the last ```output\n...\n```
|
||||
"""
|
||||
if "```output" not in pred_str:
|
||||
return ""
|
||||
if '```output' in pred_str:
|
||||
pred_str = pred_str.split('```output')[-1]
|
||||
if '```' in pred_str:
|
||||
pred_str = pred_str.split('```')[0]
|
||||
output = pred_str.strip()
|
||||
return output
|
||||
|
||||
|
||||
def extract_answer(pred_str, exhaust=False):
|
||||
pred = []
|
||||
if 'final answer is $' in pred_str and '$. I hope' in pred_str:
|
||||
tmp = pred_str.split('final answer is $', 1)[1]
|
||||
pred = [tmp.split('$. I hope', 1)[0].strip()]
|
||||
elif 'boxed' in pred_str:
|
||||
pred = extract_boxed_answers(pred_str)
|
||||
elif ('he answer is' in pred_str):
|
||||
pred = [pred_str.split('he answer is')[-1].strip()]
|
||||
else:
|
||||
program_output = extract_program_output(pred_str)
|
||||
if program_output != "":
|
||||
# fall back to program
|
||||
pred.append(program_output)
|
||||
else: # use the last number
|
||||
pattern = '-?\d*\.?\d+'
|
||||
ans = re.findall(pattern, pred_str.replace(",", ""))
|
||||
if (len(ans) >= 1):
|
||||
ans = ans[-1]
|
||||
else:
|
||||
ans = ''
|
||||
if ans:
|
||||
pred.append(ans)
|
||||
|
||||
# multiple line
|
||||
_pred = []
|
||||
for ans in pred:
|
||||
ans = ans.strip().split("\n")[0]
|
||||
ans = ans.lstrip(":")
|
||||
ans = ans.rstrip(".")
|
||||
ans = ans.rstrip("/")
|
||||
ans = strip_string(ans)
|
||||
_pred.append(ans)
|
||||
if exhaust:
|
||||
return _pred
|
||||
else:
|
||||
return _pred[-1] if _pred else ""
|
||||
|
||||
|
||||
def extract_math_answer(question, reasoning, task):
|
||||
answer = []
|
||||
for ans in extract_answer(reasoning, exhaust=True):
|
||||
if 'separated by commas' in question and all(ch not in ans for ch in '()[]'):
|
||||
answer.extend([a.strip() for a in ans.split(",")])
|
||||
elif regex.search(r"\\text\{\s*and\s*\}", ans):
|
||||
answer.extend([a.strip() for a in regex.sub(r"\\text\{\s*and\s*\}", "[SEP]", ans).split("[SEP]")])
|
||||
else:
|
||||
answer.append(ans.strip())
|
||||
return answer
|
||||
|
||||
|
||||
def extract_math_few_shot_cot_answer(question, reasoning, task):
|
||||
if 'Problem:' in reasoning:
|
||||
reasoning = reasoning.split("Problem:", 1)[0]
|
||||
return extract_math_answer(question, reasoning, task)
|
||||
|
||||
|
||||
def extract_last_single_answer(question, reasoning, task):
|
||||
return extract_answer(reasoning, exhaust=False)
|
||||
|
||||
|
||||
def extract_gsm_few_shot_cot_answer(question, reasoning, task):
|
||||
if 'Q: ' in reasoning:
|
||||
reasoning = reasoning.split("Q: ", 1)[0]
|
||||
pred = [s for s in regex.findall(r'-?\d+\.?\d*', reasoning)]
|
||||
if pred:
|
||||
return pred[-1]
|
||||
else:
|
||||
return "[invalid]"
|
||||
|
||||
|
||||
def extract_agieval_gaokao_mathcloze_few_shot_cot_test(question, reasoning, task):
|
||||
if '问题 ' in reasoning:
|
||||
reasoning = reasoning.split("问题 ", 1)[0]
|
||||
if '答案是' in reasoning:
|
||||
ans = reasoning.split('答案是', 1)[1].strip()
|
||||
ans = ans.split("\n")[0].strip()
|
||||
ans = [ans.strip("$")]
|
||||
else:
|
||||
ans = ['placeholder']
|
||||
return ans
|
||||
|
||||
|
||||
def extract_agieval_gaokao_mathqa_few_shot_cot_test(question, reasoning, task):
|
||||
if '问题 ' in reasoning:
|
||||
reasoning = reasoning.split("问题 ", 1)[0]
|
||||
if '答案是' in reasoning:
|
||||
ans = reasoning.split('答案是', 1)[1].strip()
|
||||
ans = ans.split("\n")[0].strip()
|
||||
else:
|
||||
ans = 'placeholder'
|
||||
return ans
|
||||
|
||||
|
||||
def extract_sat_few_shot_answer(question, reasoning, task):
|
||||
if 'Problem:' in reasoning:
|
||||
reasoning = reasoning.split("Problem:", 1)[0]
|
||||
patt = regex.search(r"the final answer is \(?(?P<ans>[abcd])\)?", reasoning.lower())
|
||||
if patt is not None:
|
||||
return patt.group('ans').upper()
|
||||
return 'placeholder'
|
||||
|
||||
|
||||
def extract_ocwcourses_few_shot_answer(question, reasoning, task):
|
||||
if 'Problem:' in reasoning:
|
||||
reasoning = reasoning.split("Problem:", 1)[0]
|
||||
patt = regex.search(r"final answer is (?P<ans>.*)\. I hope it is correct.", reasoning)
|
||||
if patt is None:
|
||||
pred = "[invalid]"
|
||||
print(f"DEBUG >>>\n{reasoning}", flush=True)
|
||||
else:
|
||||
pred = patt.group('ans')
|
||||
return pred
|
||||
|
||||
|
||||
def extract_mmlu_stem(question, reasoning, task):
|
||||
if 'Problem:' in reasoning:
|
||||
reasoning = reasoning.split("Problem:", 1)[0]
|
||||
return extract_sat_few_shot_answer(question, reasoning, task)
|
||||
|
||||
|
||||
def extract_minif2f_isabelle(question, reasoning, task):
|
||||
if 'Informal:' in reasoning:
|
||||
reasoning = reasoning.split("Informal:", 1)[0]
|
||||
return reasoning.strip()
|
||||
|
||||
|
||||
def extract_cmath_few_shot_test(question, reasoning, task):
|
||||
if '问题：' in reasoning:
|
||||
reasoning = reasoning.split("问题：", 1)[0]
|
||||
if '答案是' in reasoning:
|
||||
ans = reasoning.split('答案是', 1)[1].strip()
|
||||
ans = ans.split("\n")[0]
|
||||
ans = ans.strip("：")
|
||||
ans = ans.strip("。")
|
||||
try:
|
||||
ans = [s for s in regex.findall(r'-?\d+\.?\d*', ans)][-1]
|
||||
except:
|
||||
print(f"DEBUG CMATH: {reasoning}", flush=True)
|
||||
ans = "[invalid]"
|
||||
else:
|
||||
ans = extract_last_single_answer(question, reasoning, task)
|
||||
return ans
|
||||
@@ -0,0 +1,181 @@
|
||||
import regex
|
||||
from copy import deepcopy
|
||||
from data.deepseek_math_utils.eval_utils import math_equal
|
||||
from data.deepseek_math_utils.ocwcourses_eval_utils import normalize_numeric, numeric_equality, normalize_symbolic_equation, SymbolicMathMixin
|
||||
|
||||
|
||||
def is_correct(item, pred_key='prediction', prec=1e-3):
|
||||
pred = item[pred_key]
|
||||
ans = item['answer']
|
||||
if isinstance(pred, list) and isinstance(ans, list):
|
||||
pred_matched = set()
|
||||
ans_matched = set()
|
||||
for i in range(len(pred)):
|
||||
for j in range(len(ans)):
|
||||
item_cpy = deepcopy(item)
|
||||
item_cpy.update({
|
||||
pred_key: pred[i],
|
||||
'answer': ans[j]
|
||||
})
|
||||
if is_correct(item_cpy, pred_key=pred_key, prec=prec):
|
||||
pred_matched.add(i)
|
||||
ans_matched.add(j)
|
||||
if item_cpy[pred_key] == '2,3,4':
|
||||
print(item, flush=True)
|
||||
print("wtf", flush=True)
|
||||
return len(pred_matched) == len(pred) and len(ans_matched) == len(ans)
|
||||
elif isinstance(pred, str) and isinstance(ans, str):
|
||||
if '\\cup' in pred and '\\cup' in ans:
|
||||
item = deepcopy(item)
|
||||
item.update({
|
||||
pred_key: pred.split('\\cup'),
|
||||
'answer': ans.split('\\cup'),
|
||||
})
|
||||
return is_correct(item, pred_key=pred_key, prec=prec)
|
||||
else:
|
||||
label = False
|
||||
try:
|
||||
label = abs(float(regex.sub(r',', '', str(pred))) - float(regex.sub(r',', '', str(ans)))) < prec
|
||||
except:
|
||||
pass
|
||||
label = label or (ans and pred == ans) or math_equal(pred, ans)
|
||||
return label
|
||||
else:
|
||||
print(item, flush=True)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def eval_math(item, pred_key='prediction', prec=1e-3):
|
||||
pred = item[pred_key]
|
||||
if pred_key == 'program_output' and isinstance(pred, str):
|
||||
pred = [pred]
|
||||
ans = item['answer']
|
||||
if isinstance(pred, list) and isinstance(ans, list):
|
||||
# for some questions in MATH, `reference` repeats answers
|
||||
_ans = []
|
||||
for a in ans:
|
||||
if a not in _ans:
|
||||
_ans.append(a)
|
||||
ans = _ans
|
||||
# some predictions for MATH questions also repeats answers
|
||||
_pred = []
|
||||
for a in pred:
|
||||
if a not in _pred:
|
||||
_pred.append(a)
|
||||
# some predictions mistakenly box non-answer strings
|
||||
pred = _pred[-len(ans):]
|
||||
|
||||
item.update({
|
||||
pred_key: pred,
|
||||
'answer': ans
|
||||
})
|
||||
return is_correct(item, pred_key=pred_key, prec=prec)
|
||||
|
||||
|
||||
def eval_last_single_answer(item, pred_key='prediction', prec=1e-3):
|
||||
for key in [pred_key, 'answer']:
|
||||
assert isinstance(item[key], str), f"{key} = `{item[key]}` is not a str"
|
||||
return is_correct(item, pred_key=pred_key, prec=prec)
|
||||
|
||||
|
||||
def eval_agieval_gaokao_math_cloze(item, pred_key='prediction', prec=1e-3):
|
||||
if pred_key == 'program_output' and isinstance(item[pred_key], str):
|
||||
item[pred_key] = [item[pred_key]]
|
||||
for key in [pred_key, 'answer']:
|
||||
assert isinstance(item[key], list), f"{key} = `{item[key]}` is not a list"
|
||||
pred = item[pred_key]
|
||||
ans = item['answer']
|
||||
_pred = []
|
||||
for p in pred:
|
||||
p = p + ";"
|
||||
while p:
|
||||
left_brackets = 0
|
||||
for i in range(len(p)):
|
||||
if p[i] == ';' or (p[i] == ',' and left_brackets == 0):
|
||||
_p, p = p[:i].strip(), p[i + 1:].strip()
|
||||
if _p not in _pred:
|
||||
_pred.append(_p)
|
||||
break
|
||||
elif p[i] in '([{':
|
||||
left_brackets += 1
|
||||
elif p[i] in ')]}':
|
||||
left_brackets -= 1
|
||||
pred = _pred[-len(ans):]
|
||||
if len(pred) == len(ans):
|
||||
for p, a in zip(pred, ans):
|
||||
item.update({
|
||||
pred_key: p,
|
||||
'answer': a,
|
||||
})
|
||||
if not is_correct(item, pred_key=pred_key, prec=prec):
|
||||
return False
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def eval_agieval_gaokao_mathqa(item, pred_key='prediction', prec=1e-3):
|
||||
if pred_key == 'program_output' and isinstance(item[pred_key], str):
|
||||
item[pred_key] = [item[pred_key]]
|
||||
pred_str = " ".join(item[pred_key])
|
||||
ans = item['answer']
|
||||
tag = None
|
||||
idx = -1
|
||||
for t in 'ABCD':
|
||||
if t in pred_str and pred_str.index(t) > idx:
|
||||
tag = t
|
||||
idx = pred_str.index(t)
|
||||
return tag == ans
|
||||
|
||||
|
||||
def eval_math_sat(item, pred_key='prediction', prec=1e-3):
|
||||
for key in [pred_key, 'answer']:
|
||||
assert isinstance(item[key], str), f"{key} = `{item[key]}` is not a str"
|
||||
return item[pred_key].lower() == item['answer'].lower()
|
||||
|
||||
|
||||
def eval_mmlu_stem(item, pred_key='prediction', prec=1e-3):
|
||||
return eval_math_sat(item, pred_key=pred_key, prec=prec)
|
||||
|
||||
|
||||
def eval_ocwcourses(item, pred_key='prediction', prec=1e-3):
|
||||
INVALID_ANSWER = "[invalidanswer]"
|
||||
for key in [pred_key, 'answer']:
|
||||
assert isinstance(item[key], str), f"{key} = `{item[key]}` is not a str"
|
||||
pred = item[pred_key]
|
||||
ans = item['answer']
|
||||
|
||||
try:
|
||||
float(ans)
|
||||
normalize_fn = normalize_numeric
|
||||
is_equiv = numeric_equality
|
||||
answer_type = "numeric"
|
||||
except ValueError:
|
||||
if "=" in ans:
|
||||
normalize_fn = normalize_symbolic_equation
|
||||
is_equiv = lambda x, y: x == y
|
||||
answer_type = "equation"
|
||||
else:
|
||||
normalize_fn = SymbolicMathMixin().normalize_tex
|
||||
is_equiv = SymbolicMathMixin().is_tex_equiv
|
||||
answer_type = "expression"
|
||||
|
||||
correct_answer = normalize_fn(ans)
|
||||
|
||||
unnormalized_answer = pred if pred else INVALID_ANSWER
|
||||
model_answer = normalize_fn(unnormalized_answer)
|
||||
|
||||
if unnormalized_answer == INVALID_ANSWER:
|
||||
acc = 0
|
||||
elif model_answer == INVALID_ANSWER:
|
||||
acc = 0
|
||||
elif is_equiv(model_answer, correct_answer):
|
||||
acc = 1
|
||||
else:
|
||||
acc = 0
|
||||
|
||||
return acc
|
||||
|
||||
|
||||
def eval_minif2f_isabelle(item, pred_key='prediction', prec=1e-3):
|
||||
return True
|
||||
@@ -0,0 +1,330 @@
|
||||
import multiprocessing
|
||||
from math import isclose
|
||||
import numpy as np
|
||||
from typing import Union, Any, Dict
|
||||
|
||||
from sympy import simplify, N
|
||||
from sympy.parsing.sympy_parser import parse_expr
|
||||
from sympy.parsing.latex import parse_latex
|
||||
import re
|
||||
import regex
|
||||
|
||||
from data.deepseek_math_utils.answer_extraction import extract_answer, extract_program_output, strip_string
|
||||
|
||||
|
||||
def extract_program(result: str, last_only=True):
|
||||
"""
|
||||
extract the program after "```python", and before "```"
|
||||
"""
|
||||
program = ""
|
||||
start = False
|
||||
for line in result.split("\n"):
|
||||
if line.startswith("```python"):
|
||||
if last_only:
|
||||
program = "" # only extract the last program
|
||||
else:
|
||||
program += "\n# ========\n"
|
||||
start = True
|
||||
elif line.startswith("```"):
|
||||
start = False
|
||||
elif start:
|
||||
program += line + "\n"
|
||||
return program
|
||||
|
||||
|
||||
def parse_ground_truth(example: Dict[str, Any], data_name):
|
||||
if 'gt_cot' in example:
|
||||
return example['gt_cot'], strip_string(example['gt'])
|
||||
|
||||
# parse ground truth
|
||||
if data_name in ["math", 'ocw']:
|
||||
gt_cot = example['solution']
|
||||
gt_ans = extract_answer(gt_cot)
|
||||
elif data_name == "gsm8k":
|
||||
gt_cot, gt_ans = example['answer'].split("####")
|
||||
elif data_name == "gsm-hard":
|
||||
gt_cot, gt_ans = example['code'], example['target']
|
||||
elif data_name == "svamp":
|
||||
gt_cot, gt_ans = example['Equation'], example['Answer']
|
||||
elif data_name == "asdiv":
|
||||
gt_cot = example['formula']
|
||||
gt_ans = re.sub(r"\(.*?\)", "", example['answer'])
|
||||
elif data_name == "mawps":
|
||||
gt_cot, gt_ans = None, example['target']
|
||||
elif data_name == "tabmwp":
|
||||
gt_cot = example['solution']
|
||||
gt_ans = example['answer']
|
||||
if example['ans_type'] in ['integer_number', 'decimal_number']:
|
||||
if '/' in gt_ans:
|
||||
gt_ans = int(gt_ans.split('/')[0]) / int(gt_ans.split('/')[1])
|
||||
elif ',' in gt_ans:
|
||||
gt_ans = float(gt_ans.replace(',', ''))
|
||||
elif '%' in gt_ans:
|
||||
gt_ans = float(gt_ans.split('%')[0]) / 100
|
||||
else:
|
||||
gt_ans = float(gt_ans)
|
||||
elif data_name == "bbh":
|
||||
gt_cot, gt_ans = None, example['target']
|
||||
else:
|
||||
raise NotImplementedError(data_name)
|
||||
# post process
|
||||
gt_cot = str(gt_cot).strip()
|
||||
gt_ans = strip_string(gt_ans)
|
||||
return gt_cot, gt_ans
|
||||
|
||||
|
||||
def parse_question(example, data_name):
|
||||
question = ""
|
||||
if data_name == "asdiv":
|
||||
question = f"{example['body'].strip()} {example['question'].strip()}"
|
||||
elif data_name == "svamp":
|
||||
body = example["Body"].strip()
|
||||
if not body.endswith("."):
|
||||
body = body + "."
|
||||
question = f'{body} {example["Question"].strip()}'
|
||||
elif data_name == "tabmwp":
|
||||
title_str = f'regarding "{example["table_title"]}" ' if example['table_title'] else ""
|
||||
question = f'Read the following table {title_str}and answer a question:\n'
|
||||
question += f'{example["table"]}\n{example["question"]}'
|
||||
if example['choices']:
|
||||
question += f' Please select from the following options: {example["choices"]}'
|
||||
else:
|
||||
for key in ['question', 'problem', 'Question', 'input']:
|
||||
if key in example:
|
||||
question = example[key]
|
||||
break
|
||||
assert question != ""
|
||||
return question.strip()
|
||||
|
||||
|
||||
def run_execute(executor, result, prompt_type, execute=False):
|
||||
if not result or result == 'error':
|
||||
return None, None
|
||||
report = None
|
||||
|
||||
if "program_only" in prompt_type:
|
||||
prediction = extract_program_output(result)
|
||||
elif prompt_type in ["pot", "pal"] and execute:
|
||||
code = extract_program(result)
|
||||
prediction, report = executor.apply(code)
|
||||
else:
|
||||
prediction = extract_answer(result)
|
||||
|
||||
prediction = strip_string(prediction)
|
||||
return prediction, report
|
||||
|
||||
|
||||
def parse_digits(num):
|
||||
# format: 234.23 || 23%
|
||||
num = regex.sub(',', '', str(num))
|
||||
try:
|
||||
return float(num)
|
||||
except:
|
||||
if num.endswith('%'):
|
||||
num = num[:-1]
|
||||
if num.endswith('\\'):
|
||||
num = num[:-1]
|
||||
try:
|
||||
return float(num) / 100
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def is_digit(num):
|
||||
# paired with parse_digits
|
||||
return parse_digits(num) is not None
|
||||
|
||||
|
||||
def normalize_prediction(prediction):
|
||||
try: # 1. numerical equal
|
||||
if is_digit(prediction):
|
||||
prediction = np.round(float(str(prediction).replace(",", "")), 6)
|
||||
return str(prediction)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 2. symbolic equal
|
||||
prediction = str(prediction).strip()
|
||||
|
||||
## deal with [], (), {}
|
||||
brackets = []
|
||||
while prediction.startswith("[") and prediction.endswith("]") or (prediction.startswith("(") and prediction.endswith(")")):
|
||||
bracket = prediction[0]
|
||||
prediction = prediction[1:-1]
|
||||
if brackets and ',' in prediction:
|
||||
pred_parts = [normalize_prediction(part) for part in prediction.split(",")]
|
||||
prediction = ",".join(pred_parts)
|
||||
|
||||
if brackets:
|
||||
for b in reversed(brackets):
|
||||
if b == '[':
|
||||
prediction = '[' + prediction + ']'
|
||||
else:
|
||||
assert b == '('
|
||||
prediction = '(' + prediction + ')'
|
||||
|
||||
def _parse(s):
|
||||
for f in [parse_latex, parse_expr]:
|
||||
try:
|
||||
return f(s)
|
||||
except:
|
||||
pass
|
||||
return s
|
||||
|
||||
prediction = _parse(prediction)
|
||||
|
||||
for s in ['{', "}", "(", ")"]:
|
||||
prediction = prediction.replace(s, "")
|
||||
|
||||
return prediction
|
||||
|
||||
|
||||
def math_equal(prediction: Union[bool, float, str],
|
||||
reference: Union[float, str],
|
||||
include_percentage: bool = True,
|
||||
is_close: bool = True,
|
||||
timeout: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Exact match of math if and only if:
|
||||
1. numerical equal: both can convert to float and are equal
|
||||
2. symbolic equal: both can convert to sympy expression and are equal
|
||||
"""
|
||||
if str(prediction) == str(reference):
|
||||
return True
|
||||
|
||||
try: # 1. numerical equal
|
||||
if is_digit(prediction) and is_digit(reference):
|
||||
prediction = parse_digits(prediction)
|
||||
reference = parse_digits(reference)
|
||||
# number questions
|
||||
if include_percentage:
|
||||
gt_result = [reference / 100, reference, reference * 100]
|
||||
else:
|
||||
gt_result = [reference]
|
||||
for item in gt_result:
|
||||
try:
|
||||
if is_close:
|
||||
if isclose(item, prediction, abs_tol=1e-3):
|
||||
return True
|
||||
else:
|
||||
if item == prediction:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
except:
|
||||
pass
|
||||
|
||||
if not prediction and prediction not in [0, False]:
|
||||
return False
|
||||
|
||||
# 2. symbolic equal
|
||||
reference = str(reference).strip()
|
||||
prediction = str(prediction).strip()
|
||||
|
||||
if regex.match(r'(\(|\[).+(\)|\])', prediction) is not None and regex.match(r'(\(|\[).+(\)|\])', reference) is not None:
|
||||
pred_parts = prediction[1:-1].split(",")
|
||||
ref_parts = reference[1:-1].split(",")
|
||||
if len(pred_parts) == len(ref_parts):
|
||||
if all([math_equal(pred_parts[i], ref_parts[i], include_percentage, is_close) for i in range(len(pred_parts))]):
|
||||
return True
|
||||
|
||||
if (prediction.startswith("\\begin{pmatrix}") or prediction.startswith("\\begin{bmatrix}")) and (
|
||||
prediction.endswith("\\end{pmatrix}") or prediction.endswith("\\end{bmatrix}")) and \
|
||||
(reference.startswith("\\begin{pmatrix}") or reference.startswith("\\begin{bmatrix}")) and (
|
||||
reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}")):
|
||||
pred_lines = [line.strip() for line in prediction[len("\\begin{pmatrix}"): -len("\\end{pmatrix}")].split("\\\\") if line.strip()]
|
||||
ref_lines = [line.strip() for line in reference[len("\\begin{pmatrix}"): -len("\\end{pmatrix}")].split("\\\\") if line.strip()]
|
||||
matched = True
|
||||
if len(pred_lines) == len(ref_lines):
|
||||
for pred_line, ref_line in zip(pred_lines, ref_lines):
|
||||
pred_parts = pred_line.split("&")
|
||||
ref_parts = ref_line.split("&")
|
||||
if len(pred_parts) == len(ref_parts):
|
||||
if not all([math_equal(pred_parts[i], ref_parts[i], include_percentage, is_close) for i in range(len(pred_parts))]):
|
||||
matched = False
|
||||
break
|
||||
else:
|
||||
matched = False
|
||||
if not matched:
|
||||
break
|
||||
else:
|
||||
matched = False
|
||||
if matched:
|
||||
return True
|
||||
|
||||
if prediction.count('=') == 1 and reference.count('=') == 1:
|
||||
pred = prediction.split('=')
|
||||
pred = f"{pred[0].strip()} - ({pred[1].strip()})"
|
||||
ref = reference.split('=')
|
||||
ref = f"{ref[0].strip()} - ({ref[1].strip()})"
|
||||
if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref):
|
||||
return True
|
||||
elif prediction.count('=') == 1 and len(prediction.split('=')[0].strip()) <= 2 and '=' not in reference:
|
||||
if math_equal(prediction.split('=')[1], reference, include_percentage, is_close):
|
||||
return True
|
||||
elif reference.count('=') == 1 and len(reference.split('=')[0].strip()) <= 2 and '=' not in prediction:
|
||||
if math_equal(prediction, reference.split('=')[1], include_percentage, is_close):
|
||||
return True
|
||||
|
||||
# symbolic equal with sympy
|
||||
if timeout:
|
||||
if call_with_timeout(symbolic_equal_process, prediction, reference):
|
||||
return True
|
||||
else:
|
||||
if symbolic_equal(prediction, reference):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def math_equal_process(param):
|
||||
return math_equal(param[-2], param[-1])
|
||||
|
||||
|
||||
def symbolic_equal(a, b):
|
||||
def _parse(s):
|
||||
for f in [parse_latex, parse_expr]:
|
||||
try:
|
||||
return f(s)
|
||||
except:
|
||||
pass
|
||||
return s
|
||||
|
||||
a = _parse(a)
|
||||
b = _parse(b)
|
||||
|
||||
try:
|
||||
if simplify(str(a - b)) == 0:
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
if isclose(N(a), N(b), abs_tol=1e-3):
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def symbolic_equal_process(a, b, output_queue):
|
||||
result = symbolic_equal(a, b)
|
||||
output_queue.put(result)
|
||||
|
||||
|
||||
def call_with_timeout(func, *args, timeout=1, **kwargs):
|
||||
output_queue = multiprocessing.Queue()
|
||||
process_args = args + (output_queue,)
|
||||
process = multiprocessing.Process(target=func, args=process_args, kwargs=kwargs)
|
||||
process.start()
|
||||
process.join(timeout)
|
||||
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join()
|
||||
return False
|
||||
|
||||
return output_queue.get()
|
||||
@@ -0,0 +1,268 @@
|
||||
import re
|
||||
import numpy as np
|
||||
import sympy
|
||||
from sympy.core.sympify import SympifyError
|
||||
from sympy.parsing.latex import parse_latex
|
||||
|
||||
import signal
|
||||
|
||||
INVALID_ANSWER = "[invalidanswer]"
|
||||
|
||||
|
||||
class timeout:
|
||||
def __init__(self, seconds=1, error_message="Timeout"):
|
||||
self.seconds = seconds
|
||||
self.error_message = error_message
|
||||
|
||||
def handle_timeout(self, signum, frame):
|
||||
raise TimeoutError(self.error_message)
|
||||
|
||||
def __enter__(self):
|
||||
signal.signal(signal.SIGALRM, self.handle_timeout)
|
||||
signal.alarm(self.seconds)
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
signal.alarm(0)
|
||||
|
||||
|
||||
def normalize_numeric(s):
|
||||
if s is None:
|
||||
return None
|
||||
for unit in [
|
||||
"eV",
|
||||
" \\mathrm{~kg} \\cdot \\mathrm{m} / \\mathrm{s}",
|
||||
" kg m/s",
|
||||
"kg*m/s",
|
||||
"kg",
|
||||
"m/s",
|
||||
"m / s",
|
||||
"m s^{-1}",
|
||||
"\\text{ m/s}",
|
||||
" \\mathrm{m/s}",
|
||||
" \\text{ m/s}",
|
||||
"g/mole",
|
||||
"g/mol",
|
||||
"\\mathrm{~g}",
|
||||
"\\mathrm{~g} / \\mathrm{mol}",
|
||||
"W",
|
||||
"erg/s",
|
||||
"years",
|
||||
"year",
|
||||
"cm",
|
||||
]:
|
||||
s = s.replace(unit, "")
|
||||
s = s.strip()
|
||||
for maybe_unit in ["m", "s", "cm"]:
|
||||
s = s.replace("\\mathrm{" + maybe_unit + "}", "")
|
||||
s = s.replace("\\mathrm{~" + maybe_unit + "}", "")
|
||||
s = s.strip()
|
||||
s = s.strip("$")
|
||||
try:
|
||||
return float(eval(s))
|
||||
except:
|
||||
try:
|
||||
expr = parse_latex(s)
|
||||
if expr.is_number:
|
||||
return float(expr)
|
||||
return INVALID_ANSWER
|
||||
except:
|
||||
return INVALID_ANSWER
|
||||
|
||||
|
||||
def numeric_equality(n1, n2, threshold=0.01):
|
||||
if n1 is None or n2 is None:
|
||||
return False
|
||||
if np.isclose(n1, 0) or np.isclose(n2, 0) or np.isclose(n1 - n2, 0):
|
||||
return np.abs(n1 - n2) < threshold * (n1 + n2) / 2
|
||||
else:
|
||||
return np.isclose(n1, n2)
|
||||
|
||||
|
||||
def normalize_symbolic_equation(s):
|
||||
if not isinstance(s, str):
|
||||
return INVALID_ANSWER
|
||||
if s.startswith("\\["):
|
||||
s = s[2:]
|
||||
if s.endswith("\\]"):
|
||||
s = s[:-2]
|
||||
s = s.replace("\\left(", "(")
|
||||
s = s.replace("\\right)", ")")
|
||||
s = s.replace("\\\\", "\\")
|
||||
if s.startswith("$") or s.endswith("$"):
|
||||
s = s.strip("$")
|
||||
try:
|
||||
maybe_expression = parse_latex(s)
|
||||
if not isinstance(maybe_expression, sympy.core.relational.Equality):
|
||||
# we have equation, not expression
|
||||
return INVALID_ANSWER
|
||||
else:
|
||||
return maybe_expression
|
||||
except:
|
||||
return INVALID_ANSWER
|
||||
|
||||
|
||||
class SymbolicMathMixin:
|
||||
"""
|
||||
Methods useful for parsing mathematical expressions from text and determining equivalence of expressions.
|
||||
"""
|
||||
|
||||
SUBSTITUTIONS = [ # used for text normalize
|
||||
("an ", ""),
|
||||
("a ", ""),
|
||||
(".$", "$"),
|
||||
("\\$", ""),
|
||||
(r"\ ", ""),
|
||||
(" ", ""),
|
||||
("mbox", "text"),
|
||||
(",\\text{and}", ","),
|
||||
("\\text{and}", ","),
|
||||
("\\text{m}", "\\text{}"),
|
||||
]
|
||||
REMOVED_EXPRESSIONS = [ # used for text normalizer
|
||||
"square",
|
||||
"ways",
|
||||
"integers",
|
||||
"dollars",
|
||||
"mph",
|
||||
"inches",
|
||||
"ft",
|
||||
"hours",
|
||||
"km",
|
||||
"units",
|
||||
"\\ldots",
|
||||
"sue",
|
||||
"points",
|
||||
"feet",
|
||||
"minutes",
|
||||
"digits",
|
||||
"cents",
|
||||
"degrees",
|
||||
"cm",
|
||||
"gm",
|
||||
"pounds",
|
||||
"meters",
|
||||
"meals",
|
||||
"edges",
|
||||
"students",
|
||||
"childrentickets",
|
||||
"multiples",
|
||||
"\\text{s}",
|
||||
"\\text{.}",
|
||||
"\\text{\ns}",
|
||||
"\\text{}^2",
|
||||
"\\text{}^3",
|
||||
"\\text{\n}",
|
||||
"\\text{}",
|
||||
r"\mathrm{th}",
|
||||
r"^\circ",
|
||||
r"^{\circ}",
|
||||
r"\;",
|
||||
r",\!",
|
||||
"{,}",
|
||||
'"',
|
||||
"\\dots",
|
||||
]
|
||||
|
||||
def normalize_tex(self, final_answer: str) -> str:
|
||||
"""
|
||||
Normalizes a string representing a mathematical expression.
|
||||
Used as a preprocessing step before parsing methods.
|
||||
|
||||
Copied character for character from appendix D of Lewkowycz et al. (2022)
|
||||
"""
|
||||
final_answer = final_answer.split("=")[-1]
|
||||
|
||||
for before, after in self.SUBSTITUTIONS:
|
||||
final_answer = final_answer.replace(before, after)
|
||||
for expr in self.REMOVED_EXPRESSIONS:
|
||||
final_answer = final_answer.replace(expr, "")
|
||||
|
||||
# Extract answer that is in LaTeX math, is bold,
|
||||
# is surrounded by a box, etc.
|
||||
final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer)
|
||||
final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer)
|
||||
final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer)
|
||||
final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer)
|
||||
final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer)
|
||||
|
||||
# Normalize shorthand TeX:
|
||||
# \fracab -> \frac{a}{b}
|
||||
# \frac{abc}{bef} -> \frac{abc}{bef}
|
||||
# \fracabc -> \frac{a}{b}c
|
||||
# \sqrta -> \sqrt{a}
|
||||
# \sqrtab -> sqrt{a}b
|
||||
final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer)
|
||||
final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer)
|
||||
final_answer = final_answer.replace("$", "")
|
||||
|
||||
# Normalize 100,000 -> 100000
|
||||
if final_answer.replace(",", "").isdigit():
|
||||
final_answer = final_answer.replace(",", "")
|
||||
|
||||
return final_answer
|
||||
|
||||
def parse_tex(self, text: str, time_limit: int = 5) -> sympy.Basic:
|
||||
"""
|
||||
Wrapper around `sympy.parse_text` that outputs a SymPy expression.
|
||||
Typically, you want to apply `normalize_text` as a preprocessing step.
|
||||
"""
|
||||
try:
|
||||
with timeout(seconds=time_limit):
|
||||
parsed = parse_latex(text)
|
||||
except (
|
||||
# general error handling: there is a long tail of possible sympy/other
|
||||
# errors we would like to catch
|
||||
Exception
|
||||
) as e:
|
||||
print(f"failed to parse {text} with exception {e}")
|
||||
return None
|
||||
|
||||
return parsed
|
||||
|
||||
def is_exp_equiv(self, x1: sympy.Basic, x2: sympy.Basic, time_limit=5) -> bool:
|
||||
"""
|
||||
Determines whether two sympy expressions are equal.
|
||||
"""
|
||||
try:
|
||||
with timeout(seconds=time_limit):
|
||||
try:
|
||||
diff = x1 - x2
|
||||
except (SympifyError, ValueError, TypeError) as e:
|
||||
print(
|
||||
f"Couldn't subtract {x1} and {x2} with exception {e}"
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
if sympy.simplify(diff) == 0:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except (SympifyError, ValueError, TypeError) as e:
|
||||
print(f"Failed to simplify {x1}-{x2} with {e}")
|
||||
return False
|
||||
except TimeoutError as e:
|
||||
print(f"Timed out comparing {x1} and {x2}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"failed on unrecognized exception {e}")
|
||||
return False
|
||||
|
||||
def is_tex_equiv(self, x1: str, x2: str, time_limit=5) -> bool:
|
||||
"""
|
||||
Determines whether two (ideally normalized using `normalize_text`) TeX expressions are equal.
|
||||
|
||||
Does so by first checking for string exact-match, then falls back on sympy-equivalence,
|
||||
following the (Lewkowycz et al. 2022) methodology.
|
||||
"""
|
||||
if x1 == x2:
|
||||
# don't resort to sympy if we have full string match, post-normalization
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
parsed_x2 = self.parse_tex(x2)
|
||||
if not parsed_x2:
|
||||
# if our reference fails to parse into a Sympy object,
|
||||
# we forgo parsing + checking our generated answer.
|
||||
return False
|
||||
return self.is_exp_equiv(self.parse_tex(x1), parsed_x2, time_limit=time_limit)
|
||||
@@ -0,0 +1,383 @@
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import os.path
|
||||
import random
|
||||
from glob import glob
|
||||
from typing import List, Dict, Tuple, Union, Any, Callable, Optional
|
||||
|
||||
import torch
|
||||
from omegaconf import DictConfig
|
||||
from omegaconf.listconfig import ListConfig
|
||||
from torch.utils.data import Dataset
|
||||
from transformers import PreTrainedTokenizer
|
||||
from data.math import decompose_deepseek_math_cot_v2
|
||||
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
|
||||
class DPOCollator:
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int, padding: str = "longest"):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
self.padding = padding
|
||||
|
||||
def __call__(self, batch):
|
||||
chosen = [item["chosen"] for item in batch]
|
||||
reject = [item["reject"] for item in batch]
|
||||
indices = [item["index"] for item in batch]
|
||||
text_inputs = chosen + reject
|
||||
|
||||
text_prompts = []
|
||||
for item in batch:
|
||||
if "chosen_prompt" in item:
|
||||
text_prompts.append(item["chosen_prompt"])
|
||||
else:
|
||||
text_prompts.append(item["prompt"])
|
||||
for item in batch:
|
||||
if "reject_prompt" in item:
|
||||
text_prompts.append(item["reject_prompt"])
|
||||
else:
|
||||
text_prompts.append(item["prompt"])
|
||||
|
||||
encoded_prompts = self.tokenizer(text_prompts, padding=self.padding, truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
input_lens = torch.sum(encoded_prompts["attention_mask"], dim=-1)
|
||||
|
||||
encoded_inputs = self.tokenizer(text_inputs, padding=self.padding, truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
if self.tokenizer.padding_side == "left":
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
input_lens = input_lens + padding_len
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
prompt_mask = torch.arange(encoded_inputs["input_ids"].size(1))[None, :] < input_lens[:, None]
|
||||
# TODO: @2024/09/13
|
||||
# There is another case that the chosen prompt is sth. like <prompt> + <space> + <eos>
|
||||
# Since usually I also set pad_token as eos_token, then the labels here could be all pad_token.
|
||||
# This could cause NAN loss when computing SFT loss.
|
||||
if prompt_mask.sum() == labels.numel(): # FIXME: This could also induce NAN loss during DPO with SFT loss. @2024/08/09
|
||||
logger.warning(f"Prompt mask is all True. Indices: {indices}")
|
||||
prompt_mask[0, -1] = False
|
||||
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
"prompt": text_prompts,
|
||||
"chosen": chosen,
|
||||
"reject": reject,
|
||||
}
|
||||
return encoded_inputs
|
||||
|
||||
|
||||
class DPODataSFTCollator:
|
||||
"""
|
||||
Note that when you are using the DPO pair dataset, you may overlook the oversampling of chosen samples.
|
||||
"""
|
||||
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
|
||||
def __call__(self, batch):
|
||||
prompt = [item["prompt"] for item in batch]
|
||||
chosen = [item["chosen"] for item in batch]
|
||||
indices = [item["index"] for item in batch]
|
||||
|
||||
text_prompts = prompt
|
||||
text_inputs = chosen
|
||||
|
||||
encoded_prompts = self.tokenizer(text_prompts, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
input_lens = torch.sum(encoded_prompts["attention_mask"], dim=-1)
|
||||
|
||||
encoded_inputs = self.tokenizer(text_inputs, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
if self.tokenizer.padding_side == "left":
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
input_lens = input_lens + padding_len
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
prompt_mask = torch.arange(encoded_inputs["input_ids"].size(1))[None, :] < input_lens[:, None]
|
||||
if prompt_mask.sum() == labels.numel():
|
||||
logger.warning(f"Prompt mask is all True. Indices: {indices}")
|
||||
prompt_mask[0, -1] = False
|
||||
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
"prompt": prompt,
|
||||
"chosen": chosen,
|
||||
"response": chosen,
|
||||
}
|
||||
if "label" in batch[0]:
|
||||
encoded_inputs["meta_data"]["label"] = [item["label"] for item in batch]
|
||||
return encoded_inputs
|
||||
|
||||
|
||||
class DPOCollatorWithExtraInputs:
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int, padding: str = "longest", extra_keys: List[str] = None):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
self.padding = padding
|
||||
self.extra_keys = extra_keys
|
||||
|
||||
def __call__(self, batch):
|
||||
chosen = [item["chosen"] for item in batch]
|
||||
reject = [item["reject"] for item in batch]
|
||||
indices = [item["index"] for item in batch]
|
||||
text_inputs = chosen + reject
|
||||
|
||||
text_prompts = []
|
||||
for item in batch:
|
||||
if "chosen_prompt" in item:
|
||||
text_prompts.append(item["chosen_prompt"])
|
||||
else:
|
||||
text_prompts.append(item["prompt"])
|
||||
for item in batch:
|
||||
if "reject_prompt" in item:
|
||||
text_prompts.append(item["reject_prompt"])
|
||||
else:
|
||||
text_prompts.append(item["prompt"])
|
||||
|
||||
encoded_prompts = self.tokenizer(text_prompts, padding=self.padding, truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
input_lens = torch.sum(encoded_prompts["attention_mask"], dim=-1)
|
||||
|
||||
encoded_inputs = self.tokenizer(text_inputs, padding=self.padding, truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
if self.tokenizer.padding_side == "left":
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
input_lens = input_lens + padding_len
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
prompt_mask = torch.arange(encoded_inputs["input_ids"].size(1))[None, :] < input_lens[:, None]
|
||||
# TODO: @2024/09/13
|
||||
# There is another case that the chosen prompt is sth. like <prompt> + <space> + <eos>
|
||||
# Since usually I also set pad_token as eos_token, then the labels here could be all pad_token.
|
||||
# This could cause NAN loss when computing SFT loss.
|
||||
if prompt_mask.sum() == labels.numel(): # FIXME: This could also induce NAN loss during DPO with SFT loss. @2024/08/09
|
||||
logger.warning(f"Prompt mask is all True. Indices: {indices}")
|
||||
prompt_mask[0, -1] = False
|
||||
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
|
||||
for k in self.extra_keys:
|
||||
_ex_inputs = [item[k] for item in batch]
|
||||
_ex_inputs = torch.tensor(_ex_inputs, dtype=torch.float)
|
||||
encoded_inputs[k] = _ex_inputs
|
||||
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
"prompt": text_prompts,
|
||||
"chosen": chosen,
|
||||
"reject": reject,
|
||||
}
|
||||
return encoded_inputs
|
||||
|
||||
|
||||
class Trajectory2ValueCollator:
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
|
||||
def __call__(self, batch):
|
||||
prompt = [item["prompt"] for item in batch]
|
||||
inputs = [item["input"] for item in batch]
|
||||
indices = [item["index"] for item in batch]
|
||||
values = [item["value"] for item in batch]
|
||||
|
||||
text_prompts = prompt
|
||||
text_inputs = inputs
|
||||
|
||||
encoded_prompts = self.tokenizer(text_prompts, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
input_lens = torch.sum(encoded_prompts["attention_mask"], dim=-1)
|
||||
|
||||
encoded_inputs = self.tokenizer(text_inputs, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
if self.tokenizer.padding_side == "left":
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
input_lens = input_lens + padding_len
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
prompt_mask = torch.arange(encoded_inputs["input_ids"].size(1))[None, :] < input_lens[:, None]
|
||||
if prompt_mask.sum() == labels.numel():
|
||||
logger.warning(f"Prompt mask is all True. Indices: {indices}")
|
||||
prompt_mask[0, -1] = False
|
||||
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
encoded_inputs["values"] = torch.tensor(values, dtype=torch.long)
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
"prompt": prompt,
|
||||
"input": inputs,
|
||||
"response": inputs,
|
||||
"label": values,
|
||||
}
|
||||
return encoded_inputs
|
||||
|
||||
|
||||
class StepEndingsCollator:
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
|
||||
def __call__(self, batch):
|
||||
prompt = [item["prompt"] for item in batch]
|
||||
chosen = [item["chosen"] for item in batch]
|
||||
indices = [item["index"] for item in batch]
|
||||
|
||||
text_prompts = prompt
|
||||
text_inputs = chosen
|
||||
|
||||
encoded_prompts = self.tokenizer(text_prompts, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
input_lens = torch.sum(encoded_prompts["attention_mask"], dim=-1)
|
||||
|
||||
encoded_inputs = self.tokenizer(text_inputs, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
if self.tokenizer.padding_side == "left":
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
input_lens = input_lens + padding_len
|
||||
else:
|
||||
padding_len = torch.zeros(len(batch), dtype=torch.long)
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
prompt_mask = torch.arange(encoded_inputs["input_ids"].size(1))[None, :] < input_lens[:, None]
|
||||
if prompt_mask.sum() == labels.numel():
|
||||
logger.warning(f"Prompt mask is all True. Indices: {indices}")
|
||||
prompt_mask[0, -1] = False
|
||||
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
endings = []
|
||||
for b, item in enumerate(batch):
|
||||
ending = decompose_deepseek_math_cot_v2(item["prompt"], item["response"], self.max_seq_length, self.tokenizer)
|
||||
ending = [e + padding_len[b].item() for e in ending]
|
||||
endings.append(ending)
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
"prompt": prompt,
|
||||
"chosen": chosen,
|
||||
"response": [item["response"] for item in batch],
|
||||
"ending": endings,
|
||||
"type": [None] * len(endings),
|
||||
}
|
||||
if "label" in batch[0]:
|
||||
encoded_inputs["meta_data"]["label"] = [item["label"] for item in batch]
|
||||
return encoded_inputs
|
||||
|
||||
|
||||
def iterative_mask(text_segment_list: List[List[str]], masks: List[int], tokenizer: PreTrainedTokenizer, **tokenize_kwargs):
|
||||
if len(text_segment_list) == 0:
|
||||
raise ValueError("Input groups should be greater than 0.")
|
||||
if len(text_segment_list) == 1:
|
||||
return tokenizer(text_segment_list[0], **tokenize_kwargs), None
|
||||
assert len(masks) == 1 or masks[0] == 0, "The prefix should always be masked if there are multiple groups of inputs"
|
||||
|
||||
all_input_lens = []
|
||||
all_inputs = []
|
||||
for group in text_segment_list:
|
||||
group_inputs = tokenizer(group, **tokenize_kwargs)
|
||||
all_inputs.append(group_inputs)
|
||||
all_input_lens.append(torch.sum(group_inputs["attention_mask"], dim=-1, keepdim=True))
|
||||
|
||||
if tokenizer.padding_side == "left":
|
||||
# If left padding, we should first compute the padding length at last
|
||||
padding_len = torch.sum(1 - all_inputs[-1]["attention_mask"], dim=-1, keepdim=True)
|
||||
else:
|
||||
padding_len = 0
|
||||
|
||||
last_len = torch.zeros(all_inputs[-1]["input_ids"].size(0), 1, dtype=torch.long)
|
||||
prompt_mask = torch.zeros(all_inputs[-1]["input_ids"].shape, dtype=torch.long)
|
||||
seq_range = torch.arange(all_inputs[-1]["input_ids"].size(1))
|
||||
for _acc_lens, mask in zip(all_input_lens, masks):
|
||||
_condition_lens = padding_len + _acc_lens
|
||||
if mask == 0:
|
||||
group_mask = (seq_range[None, :] < _condition_lens) & (last_len <= seq_range[None, :])
|
||||
prompt_mask += group_mask
|
||||
last_len = _condition_lens
|
||||
return all_inputs[-1], prompt_mask
|
||||
|
||||
|
||||
class SFTFoldAttnMaskCollator:
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int, text_keys: List[str], text_masks: List[int]):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
self.text_keys = text_keys
|
||||
self.text_masks = text_masks
|
||||
|
||||
def __call__(self, batch):
|
||||
text_segment_list = []
|
||||
|
||||
for b in batch:
|
||||
assert self.text_keys[0] in b, f"At least the first group of inputs is contained in the batch, got {list(b.keys())}"
|
||||
last_input = b[self.text_keys[0]]
|
||||
for k in self.text_keys[1:]:
|
||||
if k not in b:
|
||||
b[k] = last_input
|
||||
last_input = b[k]
|
||||
|
||||
for i, key in enumerate(self.text_keys):
|
||||
batch_item = [b[key] for b in batch]
|
||||
text_segment_list.append(batch_item)
|
||||
|
||||
indices = [item["index"] for item in batch]
|
||||
|
||||
encoded_inputs, prompt_mask = iterative_mask(text_segment_list, self.text_masks, self.tokenizer,
|
||||
padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
if prompt_mask is not None:
|
||||
if prompt_mask.sum() == labels.numel():
|
||||
logger.warning(f"Prompt mask is all True. Indices: {indices}")
|
||||
prompt_mask[0, -1] = False
|
||||
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
}
|
||||
encoded_inputs["meta_data"].update({
|
||||
k: text_segment_list[i] for i, k in enumerate(self.text_keys)
|
||||
})
|
||||
if "label" in batch[0]:
|
||||
encoded_inputs["meta_data"]["label"] = [item["label"] for item in batch]
|
||||
return encoded_inputs
|
||||
|
||||
|
||||
class TextPromptCollator:
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int, padding: str = "longest", extra_text_inputs: DictConfig[str, bool] = None,
|
||||
**kwargs):
|
||||
self.tokenizer: PreTrainedTokenizer = tokenizer
|
||||
|
||||
self.max_seq_length = max_seq_length
|
||||
self.padding = padding
|
||||
|
||||
self.extra_text_inputs = extra_text_inputs
|
||||
|
||||
def __call__(self, batch):
|
||||
inputs = [b["input"] for b in batch]
|
||||
index = [b["index"] for b in batch]
|
||||
|
||||
model_inputs = self.tokenizer(inputs, padding=self.padding, truncation=True, max_length=self.max_seq_length,
|
||||
return_tensors="pt")
|
||||
|
||||
if self.extra_text_inputs is not None:
|
||||
for k, v in self.extra_text_inputs.items():
|
||||
_ex_inputs = [b[k] for b in batch]
|
||||
if v:
|
||||
model_inputs[k] = self.tokenizer(_ex_inputs, padding=self.padding, truncation=True, max_length=self.max_seq_length,
|
||||
return_tensors="pt")
|
||||
else:
|
||||
model_inputs[k] = _ex_inputs
|
||||
|
||||
model_inputs["meta_data"] = {
|
||||
"inputs": inputs,
|
||||
"index": index,
|
||||
}
|
||||
return model_inputs
|
||||
@@ -0,0 +1,49 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from datasets import load_dataset
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
class HumanEvalReader:
|
||||
def __call__(self, file_path: str = "openai_humaneval"):
|
||||
data = load_dataset(file_path, split="test").to_list()
|
||||
logger.info(len(data))
|
||||
|
||||
return data
|
||||
|
||||
|
||||
class MBPPReader:
|
||||
def __init__(self, sanitized: bool = True):
|
||||
self.sanitized = sanitized
|
||||
|
||||
def __call__(self, file_path: str = "mbpp"):
|
||||
if self.sanitized:
|
||||
dataset = load_dataset("mbpp", "sanitized", split="test").to_list()
|
||||
else:
|
||||
dataset = load_dataset("mbpp", split="test").to_list()
|
||||
|
||||
for item in dataset:
|
||||
item["test_list"] = "\n".join(item["test_list"])
|
||||
|
||||
return dataset
|
||||
|
||||
|
||||
class MBPPReaderFixed:
|
||||
def __init__(self, sanitized: bool = True):
|
||||
self.sanitized = sanitized
|
||||
|
||||
def __call__(self, file_path: str = "mbpp"):
|
||||
if self.sanitized:
|
||||
dataset = load_dataset("mbpp", "sanitized", split="test").to_list()
|
||||
else:
|
||||
dataset = load_dataset("mbpp", split="test").to_list()
|
||||
|
||||
for item in dataset:
|
||||
item["test_list"] = "\n".join(item["test_list"])
|
||||
|
||||
return dataset
|
||||
@@ -0,0 +1,592 @@
|
||||
import copy
|
||||
import json
|
||||
import os.path
|
||||
import random
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from multiprocessing.pool import Pool
|
||||
from typing import Dict, List, Callable, Union
|
||||
|
||||
from omegaconf.listconfig import ListConfig
|
||||
from tqdm import tqdm
|
||||
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
|
||||
def _format_option_list(option_list: List[str], _rank2option: List[str]) -> str:
|
||||
res = []
|
||||
for op_id, op in enumerate(option_list):
|
||||
res.append(f"{_rank2option[op_id]}. {op}")
|
||||
return "\n".join(res)
|
||||
|
||||
|
||||
def option_id2str_aligner():
|
||||
option_id2str = ["A", "B", "C", "D", "E"]
|
||||
|
||||
def func(data: List[Dict]):
|
||||
for sample in data:
|
||||
sample["str_label"] = option_id2str[sample["label"]]
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def key_based_filter_aligner(key, value):
|
||||
if isinstance(value, ListConfig):
|
||||
value = list(value)
|
||||
if not isinstance(value, list):
|
||||
value = [value]
|
||||
|
||||
def func(data: List[Dict]):
|
||||
return [item for item in data if item[key] in value]
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_confidence_ratio_filter(lower_bound: float, upper_bound: float, pos_field: str, response_field: str):
|
||||
def func(item):
|
||||
pos_num = len(item[pos_field])
|
||||
total_num = len(item[response_field])
|
||||
ratio = pos_num / total_num
|
||||
if lower_bound <= ratio <= upper_bound:
|
||||
return True
|
||||
return False
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def filter_aligner(filter_func: Callable):
|
||||
def func(data: List[Dict]):
|
||||
return [item for item in data if filter_func(item)]
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def json_field2str(key, val: str = None, indent: int = 4):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
if val:
|
||||
item[val] = json.dumps(item[key], indent=indent, ensure_ascii=False)
|
||||
else:
|
||||
item[key] = json.dumps(item[key], indent=indent, ensure_ascii=False)
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def starts_with_filter(key, value):
|
||||
def func(data: List[Dict]):
|
||||
return [item for item in data if item[key].startswith(value)]
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def not_none_filter(key):
|
||||
def func(data: List[Dict]):
|
||||
return [item for item in data if key in item and item[key] not in ["", None, []]]
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def rename_field_aligner(kv_pair: Dict):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
for k, v in kv_pair.items():
|
||||
tmp = item.pop(k)
|
||||
item[v] = tmp
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def field_extract_aligner(input_index_field: str, extract_index_field: str, extract_fields: List[str], extra_file: str, renamed_fields: Dict[str, str] = None):
|
||||
if os.path.exists(extra_file):
|
||||
extra_data = json.load(open(extra_file, encoding="utf-8"))
|
||||
else:
|
||||
extra_data = []
|
||||
for file in glob(extra_file):
|
||||
extra_data += json.load(open(file))
|
||||
if len(extra_data) == 0:
|
||||
raise ValueError(f"No data found in {extra_file}")
|
||||
id2extra_data = {str(item[extract_index_field]): item for item in extra_data}
|
||||
renaming = {}
|
||||
for _field in extract_fields:
|
||||
if renamed_fields and _field in renamed_fields:
|
||||
renaming[_field] = renamed_fields[_field]
|
||||
else:
|
||||
renaming[_field] = _field
|
||||
|
||||
def func(data: List[Dict]):
|
||||
missing = 0
|
||||
missing_field = 0
|
||||
outputs = []
|
||||
for item in data:
|
||||
item_id = str(item[input_index_field])
|
||||
if item_id not in id2extra_data:
|
||||
missing += 1
|
||||
continue
|
||||
extra_item = id2extra_data[item_id]
|
||||
if any(x not in extra_item for x in extract_fields):
|
||||
missing_field += 1
|
||||
continue
|
||||
for field in extract_fields:
|
||||
item[renaming[field]] = extra_item[field]
|
||||
outputs.append(item)
|
||||
|
||||
logger.info(f"Extracted {len(outputs)} items from {extra_file}")
|
||||
logger.info(f"Missing {missing} items in {extra_file}")
|
||||
logger.info(f"Missing {missing_field} fields in {extra_file}")
|
||||
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def flat_aligner(input_index_field: str, extract_field: Union[str, List[str]], mode: str = "single"):
|
||||
if isinstance(extract_field, str):
|
||||
extract_field = [extract_field]
|
||||
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in data:
|
||||
item_id = item[input_index_field]
|
||||
# if not all(item[_field] for _field in extract_field):
|
||||
# continue
|
||||
if any(item[_field] in [None, "", []] for _field in extract_field):
|
||||
continue
|
||||
|
||||
num = len(item[extract_field[0]])
|
||||
for _field in extract_field[1:]:
|
||||
assert len(item[_field]) == num, f"Length not match: {item[_field]}"
|
||||
|
||||
tmp_list = []
|
||||
for i in range(num):
|
||||
new_item = copy.deepcopy(item)
|
||||
if any(item[_field][i] in [None, "", []] for _field in extract_field):
|
||||
continue
|
||||
for _field in extract_field:
|
||||
new_item[_field] = item[_field][i]
|
||||
new_item[input_index_field] = f"{item_id}_{i}"
|
||||
tmp_list.append(new_item)
|
||||
if mode == "single":
|
||||
break
|
||||
|
||||
if len(tmp_list) == 0:
|
||||
continue
|
||||
|
||||
if mode == "single":
|
||||
outputs.append(tmp_list[0])
|
||||
elif mode == "random":
|
||||
outputs.append(random.choice(tmp_list))
|
||||
else:
|
||||
outputs += tmp_list
|
||||
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def option_flatten_aligner():
|
||||
def func(data: List[Dict]):
|
||||
for sample in data:
|
||||
sample["option_list"] = _format_option_list(sample["options"], ["A", "B", "C", "D"])
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def empty_aligner(data: List[Dict]):
|
||||
return data
|
||||
|
||||
|
||||
def add_id_aligner(id_field: str = "id"):
|
||||
def func(data: List[Dict]):
|
||||
for i, item in enumerate(data):
|
||||
item[id_field] = i
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def concat_aligner(aligners: List[Callable]):
|
||||
def func(data: List[Dict]):
|
||||
for aligner in aligners:
|
||||
data = aligner(data)
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_pair_aligner_cleaned(response_field: str = "response",
|
||||
id_field: str = "id",
|
||||
do_sample: bool = False, ):
|
||||
"""
|
||||
This aligner only accepts the cleaned file, which has removing all empty responses and combined with original data.
|
||||
:return: Callable
|
||||
"""
|
||||
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in data:
|
||||
pos_resp = []
|
||||
neg_resp = []
|
||||
for i, (resp, pred) in enumerate(zip(item[response_field], item["pred"])):
|
||||
assert resp
|
||||
# assert pred
|
||||
if isinstance(resp, list):
|
||||
assert isinstance(resp[0], str)
|
||||
# assert "The answer is" in resp[-1], resp
|
||||
resp = "".join(resp)
|
||||
|
||||
if isinstance(item["label"], str):
|
||||
if pred == item["label"]:
|
||||
pos_resp.append((i, resp))
|
||||
else:
|
||||
neg_resp.append((i, resp))
|
||||
elif isinstance(item["label"], int):
|
||||
if pred and ord(pred) - ord("A") == item["label"]:
|
||||
pos_resp.append((i, resp))
|
||||
else:
|
||||
neg_resp.append((i, resp))
|
||||
else:
|
||||
raise ValueError(f"Unknown type of label: {type(item['label'])}")
|
||||
|
||||
if not (len(pos_resp) and len(neg_resp)):
|
||||
continue
|
||||
|
||||
if do_sample:
|
||||
pos = random.choice(pos_resp)
|
||||
neg = random.choice(neg_resp)
|
||||
pos_resp = [pos]
|
||||
neg_resp = [neg]
|
||||
|
||||
for pos in pos_resp:
|
||||
for neg in neg_resp:
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["pos"] = pos[1]
|
||||
new_item["neg"] = neg[1]
|
||||
new_item["pos_id"] = f"{item[id_field]}_{pos[0]}"
|
||||
new_item["neg_id"] = f"{item[id_field]}_{neg[0]}"
|
||||
outputs.append(new_item)
|
||||
|
||||
logger.info(f"Counted {len(outputs)} DPO contrastive pairs.")
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_pair_aligner(pos_field: Union[str, ListConfig], neg_field: Union[str, ListConfig]):
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
if isinstance(pos_field, str):
|
||||
_pos_fields = [pos_field]
|
||||
else:
|
||||
_pos_fields = list(pos_field)
|
||||
if isinstance(neg_field, str):
|
||||
_neg_fields = [neg_field]
|
||||
else:
|
||||
_neg_fields = list(neg_field)
|
||||
|
||||
for item in tqdm(data, desc="DPO pair aligner", total=len(data)):
|
||||
pos_resp = []
|
||||
neg_resp = []
|
||||
for _field in _pos_fields:
|
||||
pos_resp += item[_field]
|
||||
for _field in _neg_fields:
|
||||
neg_resp += item[_field]
|
||||
|
||||
for pos in pos_resp:
|
||||
for neg in neg_resp:
|
||||
new_item = copy.deepcopy(item)
|
||||
|
||||
# To save memory
|
||||
for _field in _pos_fields:
|
||||
new_item.pop(_field)
|
||||
for _field in _neg_fields:
|
||||
new_item.pop(_field)
|
||||
|
||||
new_item["pos"] = pos
|
||||
new_item["neg"] = neg
|
||||
outputs.append(new_item)
|
||||
|
||||
logger.info(f"Counted {len(outputs)} DPO contrastive pairs.")
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def eval_multiple_choice(item):
|
||||
if isinstance(item, dict):
|
||||
pred = item["prediction"]
|
||||
label = item["answer"]
|
||||
elif isinstance(item, tuple):
|
||||
pred, label = item
|
||||
else:
|
||||
raise ValueError(f"Unknown type of item: {type(item)}")
|
||||
|
||||
if isinstance(label, str):
|
||||
if pred == label:
|
||||
return True
|
||||
return False
|
||||
if isinstance(label, int):
|
||||
if pred and ord(pred) - ord("A") == label:
|
||||
return True
|
||||
return False
|
||||
|
||||
raise ValueError(f"Unknown type of label: {type(item['label'])}")
|
||||
|
||||
|
||||
def prompt_fill_aligner(prompt_file: str, mapping: Dict[str, str], prompt_field: str = "prompt"):
|
||||
full_prompt = open(prompt_file).read()
|
||||
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
prompt = copy.deepcopy(full_prompt)
|
||||
for k, v in mapping.items():
|
||||
prompt = prompt.replace(k, item[v])
|
||||
item[prompt_field] = prompt
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def value2pair_aligner(field: str, pos_field: str, neg_field: str, value_field: str):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
pair_data = item.pop(field)
|
||||
values = item.pop(value_field)
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
for x, v in zip(pair_data, values):
|
||||
if v:
|
||||
pos.append(x)
|
||||
else:
|
||||
neg.append(x)
|
||||
item[pos_field] = pos
|
||||
item[neg_field] = neg
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def return_threshold_mapping(value_threshold: float):
|
||||
def func(v):
|
||||
if v >= value_threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def return_binary_mapping():
|
||||
def func(v):
|
||||
if v is True:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def value_mapping_aligner(value_field: str, value_mapping_func: Callable, return_int: bool = False):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
if return_int:
|
||||
item[value_field] = int(value_mapping_func(item[value_field]))
|
||||
else:
|
||||
item[value_field] = value_mapping_func(item[value_field])
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_pair2value_aligner(pos_field: str, neg_field: str, seq_field: str, value_field: str, flatten: bool = True):
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in data:
|
||||
pos = item.pop(pos_field)
|
||||
neg = item.pop(neg_field)
|
||||
if flatten:
|
||||
item_a = copy.deepcopy(item)
|
||||
item_a[seq_field] = pos
|
||||
item_a[value_field] = 1
|
||||
|
||||
item_b = copy.deepcopy(item)
|
||||
item_b[seq_field] = neg
|
||||
item_b[value_field] = 0
|
||||
|
||||
outputs.append(item_a)
|
||||
outputs.append(item_b)
|
||||
else:
|
||||
item[seq_field] = [pos, neg]
|
||||
item[value_field] = [1, 0]
|
||||
outputs.append(item)
|
||||
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def value2pair_mapping_aligner(field: str, pos_field: str, neg_field: str, value_field: str, value_mapping_func: Callable):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
pair_data = item.pop(field)
|
||||
values = item.pop(value_field)
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
for x, v in zip(pair_data, values):
|
||||
if value_mapping_func(v):
|
||||
pos.append(x)
|
||||
else:
|
||||
neg.append(x)
|
||||
item[pos_field] = pos
|
||||
item[neg_field] = neg
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_random_choice_aligner(anchor_field: str, paired_field: str):
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in tqdm(data, desc="DPO random choice aligner", total=len(data)):
|
||||
if len(item[anchor_field]) == 0:
|
||||
continue
|
||||
if len(item[paired_field]) == 0:
|
||||
continue
|
||||
for anchor in item[anchor_field]:
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item[anchor_field] = anchor
|
||||
new_item[paired_field] = random.choice(item[paired_field])
|
||||
outputs.append(new_item)
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_flat_random_choice_aligner(paired_field: str):
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in tqdm(data, desc="DPO random choice aligner", total=len(data)):
|
||||
if len(item[paired_field]) == 0:
|
||||
continue
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item[paired_field] = random.choice(item[paired_field])
|
||||
outputs.append(new_item)
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def dpo_paired_random_choice_aligner(anchor_field: str, paired_field, sort_accord_to_len: bool = False, top_k: int = 5, num_workers: int = 16):
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in tqdm(data, desc="processing dpo pairs", total=len(data)):
|
||||
if len(item[anchor_field]) == 0:
|
||||
continue
|
||||
if len(item[paired_field]) == 0:
|
||||
continue
|
||||
assert len(item[anchor_field]) == len(item[paired_field]), (item[anchor_field], item[paired_field])
|
||||
for anchor, targets in zip(item[anchor_field], item[paired_field]):
|
||||
if len(targets) == 0:
|
||||
continue
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item[anchor_field] = anchor
|
||||
new_item[paired_field] = random.choice(targets)
|
||||
outputs.append(new_item)
|
||||
|
||||
return outputs
|
||||
|
||||
def func_sorted(data: List[Dict]):
|
||||
outputs_before_sort = []
|
||||
for item in tqdm(data, desc="DPO paired random choice aligner", total=len(data)):
|
||||
if len(item[anchor_field]) == 0:
|
||||
continue
|
||||
if len(item[paired_field]) == 0:
|
||||
continue
|
||||
assert len(item[anchor_field]) == len(item[paired_field]), (item[anchor_field], item[paired_field])
|
||||
for anchor, targets in zip(item[anchor_field], item[paired_field]):
|
||||
if len(targets) == 0:
|
||||
continue
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item[anchor_field] = anchor
|
||||
new_item[paired_field] = targets
|
||||
outputs_before_sort.append(new_item)
|
||||
|
||||
_annotate = partial(_sort_worker, _pos_field=anchor_field, _neg_field=paired_field, _top_k=top_k)
|
||||
with Pool(num_workers) as p:
|
||||
outputs_after_sort = list(tqdm(p.imap(_annotate, outputs_before_sort), total=len(outputs_before_sort)))
|
||||
|
||||
for item in outputs_after_sort:
|
||||
item[paired_field] = random.choice(item[paired_field])
|
||||
|
||||
return outputs_after_sort
|
||||
|
||||
if sort_accord_to_len:
|
||||
return func_sorted
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def sample_steps(response: str):
|
||||
lines = response.split("\n")
|
||||
lines = [line for line in lines if line.strip()]
|
||||
return len(lines)
|
||||
|
||||
|
||||
def _sort_worker(item, _pos_field: str, _neg_field: str, _top_k: int = 5):
|
||||
anchor = item[_pos_field]
|
||||
targets = item[_neg_field]
|
||||
anchor_steps = sample_steps(anchor)
|
||||
|
||||
sorted_targets = sorted(targets, key=lambda x: abs(anchor_steps - sample_steps(x)))
|
||||
item[_neg_field] = sorted_targets[:_top_k]
|
||||
return item
|
||||
|
||||
|
||||
def dpo_bi_random_choice_aligner(pos_field: str, neg_field: str, sort_accord_to_len: bool = False, top_k: int = 5, num_workers: int = 16):
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
for item in tqdm(data, desc="DPO random choice aligner", total=len(data)):
|
||||
if len(item[pos_field]) == 0:
|
||||
continue
|
||||
if len(item[neg_field]) == 0:
|
||||
continue
|
||||
pos = random.choice(item[pos_field])
|
||||
neg = random.choice(item[neg_field])
|
||||
item[pos_field] = pos
|
||||
item[neg_field] = neg
|
||||
outputs.append(item)
|
||||
return outputs
|
||||
|
||||
def func_sorted(data: List[Dict]):
|
||||
outputs_before_sort = []
|
||||
for item in tqdm(data, desc="DPO random choice aligner", total=len(data)):
|
||||
if len(item[pos_field]) == 0:
|
||||
continue
|
||||
if len(item[neg_field]) == 0:
|
||||
continue
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["pos"] = random.choice(item[pos_field])
|
||||
new_item["neg"] = item[neg_field]
|
||||
outputs_before_sort.append(new_item)
|
||||
|
||||
_annotate = partial(_sort_worker, _pos_field="pos", _neg_field="neg", _top_k=top_k)
|
||||
with Pool(num_workers) as p:
|
||||
outputs_after_sort = list(tqdm(p.imap(_annotate, outputs_before_sort), total=len(outputs_before_sort)))
|
||||
|
||||
for item in outputs_after_sort:
|
||||
item["neg"] = random.choice(item["neg"])
|
||||
|
||||
return outputs_after_sort
|
||||
|
||||
if sort_accord_to_len:
|
||||
return func_sorted
|
||||
|
||||
return func
|
||||
@@ -0,0 +1,49 @@
|
||||
import json
|
||||
|
||||
from datasets import load_dataset
|
||||
from typing import Dict
|
||||
|
||||
|
||||
def read_text(file_path: str):
|
||||
return open(file_path, encoding="utf-8").read()
|
||||
|
||||
|
||||
def json_read_fn(file_path: str):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
|
||||
|
||||
def hf_datasets_load_fn(**kwargs):
|
||||
def func(file_path):
|
||||
return load_dataset(file_path, **kwargs)
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def jsonl_read_fn():
|
||||
def func(file_path):
|
||||
return [json.loads(line) for line in open(file_path, encoding="utf-8").readlines()]
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def compose_message(system_prompt: str = ""):
|
||||
def compose_fn(text: str):
|
||||
messages = []
|
||||
if system_prompt:
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
messages.append({"role": "user", "content": text})
|
||||
|
||||
return messages
|
||||
|
||||
return compose_fn
|
||||
|
||||
|
||||
def recompose_template(units: Dict[str, str], compositions: Dict[str, str]) -> Dict[str, str]:
|
||||
templates = {}
|
||||
for k, v in compositions.items():
|
||||
templates[k] = v.format(**units)
|
||||
return templates
|
||||
|
||||
|
||||
def compose_template(units: Dict[str, str], composition: str) -> str:
|
||||
return composition.format(**units)
|
||||
@@ -0,0 +1,361 @@
|
||||
# Copied from https://github.com/meta-math/MetaMath/blob/main/eval_math.py
|
||||
# and https://github.com/meta-math/MetaMath/blob/main/eval_gsm8k.py
|
||||
import re
|
||||
import sys
|
||||
from typing import List, Dict
|
||||
|
||||
from fraction import Fraction
|
||||
from data.math_util import is_equiv, last_boxed_only_string
|
||||
from transformers import PreTrainedTokenizer
|
||||
from general_util.logger import get_child_logger
|
||||
from data.deepseek_math_utils.answer_extraction import extract_math_answer, extract_last_single_answer
|
||||
import torch
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
MAX_INT = sys.maxsize
|
||||
|
||||
|
||||
def is_number(s):
|
||||
try:
|
||||
float(s)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
import unicodedata
|
||||
unicodedata.numeric(s)
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def extract_answer_number(completion, separator: str = "The answer is: "):
|
||||
text = completion.split(separator)
|
||||
if len(text) > 1:
|
||||
extract_ans = text[-1].strip()
|
||||
match = re.search(r'[\-+]?\d*[\.,/]?\d+', extract_ans)
|
||||
if match:
|
||||
if '/' in match.group():
|
||||
denominator = match.group().split('/')[1]
|
||||
numerator = match.group().split('/')[0]
|
||||
if is_number(denominator) == True and is_number(numerator) == True:
|
||||
if denominator == '0':
|
||||
return round(float(numerator.replace(',', '')))
|
||||
else:
|
||||
frac = Fraction(match.group().replace(',', ''))
|
||||
num_numerator = frac.numerator
|
||||
num_denominator = frac.denominator
|
||||
return round(float(num_numerator / num_denominator))
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
if float(match.group().replace(',', '')) == float('inf'):
|
||||
return None
|
||||
return round(float(match.group().replace(',', '')))
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def gsk8k_answer_cleaner(separator: str = "The answer is: "):
|
||||
def func(completion: str):
|
||||
res = extract_answer_number(completion, separator=separator)
|
||||
if res is None:
|
||||
return ""
|
||||
return str(res) # To be compatible with `OpenAICallBack`
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def number_answer_extractor(separator: str = "The answer is: ", completion_field: str = "response"):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
res = extract_answer_number(item[completion_field], separator=separator)
|
||||
if res is None:
|
||||
item["label"] = ""
|
||||
else:
|
||||
item["label"] = str(res)
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def gsm8k_gold_answer_extractor(response_field: str = "response"):
|
||||
def func(data: List[Dict]):
|
||||
for i, item in enumerate(data):
|
||||
temp_ans = item[response_field].split('#### ')[1]
|
||||
temp_ans = int(temp_ans.replace(',', ''))
|
||||
item["label"] = str(temp_ans)
|
||||
item["index"] = i
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def remove_boxed(s):
|
||||
left = "\\boxed{"
|
||||
try:
|
||||
assert s[:len(left)] == left
|
||||
assert s[-1] == "}"
|
||||
return s[len(left):-1]
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def math_gold_answer_extractor(response_field: str = "output", kv_mapping: Dict = None):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
item["label"] = remove_boxed(last_boxed_only_string(item[response_field]))
|
||||
|
||||
if kv_mapping is not None: # "instruction" is maintained for composition. So the `instruction` key in MATH dataset should be changed.
|
||||
for k, v in kv_mapping.items():
|
||||
item[v] = item.pop(k)
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def math_boxed_answer_cleaner():
|
||||
def func(s):
|
||||
return remove_boxed(last_boxed_only_string(s))
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def math_boxed_answer_cleaner_proxy():
|
||||
def func(question, reasoning, task):
|
||||
return remove_boxed(last_boxed_only_string(reasoning))
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def math_gold_answer_extractor_deepseek(query_field: str = "instruction", response_field: str = "output", kv_mapping: Dict = None):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
item["label"] = extract_math_answer(item[query_field], item[response_field], "cot")
|
||||
|
||||
if kv_mapping is not None: # "instruction" is maintained for composition. So the `instruction` key in MATH dataset should be changed.
|
||||
for k, v in kv_mapping.items():
|
||||
item[v] = item.pop(k)
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def gsm8k_gold_answer_extractor_deepseek(query_field: str = "instruction", response_field: str = "output", kv_mapping: Dict = None):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
item["label"] = extract_last_single_answer(item[query_field], item[response_field], "cot")
|
||||
|
||||
if kv_mapping is not None:
|
||||
for k, v in kv_mapping.items():
|
||||
item[v] = item.pop(k)
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
# This is the original one from MetaMath repository.
|
||||
def process_results(doc, completion, answer):
|
||||
split_ans = completion.split('The answer is: ')
|
||||
if len(split_ans) > 1:
|
||||
ans = split_ans[-1]
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
if is_equiv(extract_ans, answer):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
temp = {'question': doc, 'output': completion, 'answer': answer}
|
||||
# invalid_outputs.append(temp)
|
||||
return False
|
||||
|
||||
|
||||
def math_answer_cleaner(separator: str = "The answer is: "):
|
||||
def func(completion: str):
|
||||
split_ans = completion.split(separator)
|
||||
if len(split_ans) > 1:
|
||||
ans = split_ans[-1]
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
|
||||
if "$" in extract_ans: # Added new filtering
|
||||
extract_ans = extract_ans.replace("$", "")
|
||||
|
||||
if "=" in extract_ans:
|
||||
extract_ans = extract_ans.split('=')[-1].strip()
|
||||
|
||||
return extract_ans
|
||||
else:
|
||||
return ""
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def meta_math_gold_answer_extractor(response_field: str = "response"):
|
||||
cleaner = math_answer_cleaner(separator="The answer is: ")
|
||||
|
||||
def func(data: List[Dict]):
|
||||
outputs = []
|
||||
cnt = 0
|
||||
for item in data:
|
||||
label = cleaner(item[response_field])
|
||||
if label:
|
||||
item["label"] = label
|
||||
outputs.append(item)
|
||||
else:
|
||||
cnt += 1
|
||||
logger.info(f"Counted {len(outputs)} items, {cnt} items are invalid")
|
||||
|
||||
return outputs
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def decompose_rap(prompt: str, response: str, max_seq_length: int, tokenizer: PreTrainedTokenizer):
|
||||
# raw_steps = response.strip().split("\n")
|
||||
raw_steps = response.split("\n")
|
||||
steps = [raw_steps[0]]
|
||||
for line in raw_steps[1:]:
|
||||
if line.replace("#", "").strip() == "":
|
||||
continue
|
||||
if not (line.startswith("SubQuestion ") or line.startswith("Answer ")):
|
||||
steps[-1] += "\n" + line
|
||||
else:
|
||||
steps.append(line)
|
||||
|
||||
endings = []
|
||||
acc_step = prompt
|
||||
for i, step in enumerate(steps):
|
||||
if i == 0:
|
||||
acc_step = acc_step + step
|
||||
else:
|
||||
acc_step = acc_step + "\n" + step
|
||||
input_ids = tokenizer(acc_step, truncation=True, max_length=max_seq_length)["input_ids"]
|
||||
endings.append(len(input_ids) - 1)
|
||||
|
||||
assert len(endings) > 0, (prompt, response)
|
||||
return endings
|
||||
|
||||
|
||||
def decompose_cot(prompt: str, response: str, max_seq_length: int, tokenizer: PreTrainedTokenizer):
|
||||
steps = response.split("\n")
|
||||
|
||||
endings = []
|
||||
acc_step = prompt
|
||||
for i, step in enumerate(steps):
|
||||
if i == 0:
|
||||
acc_step = acc_step + step
|
||||
else:
|
||||
acc_step = acc_step + "\n" + step
|
||||
input_ids = tokenizer(acc_step, truncation=True, max_length=max_seq_length)["input_ids"]
|
||||
endings.append(len(input_ids) - 1)
|
||||
|
||||
assert len(endings) > 0, (prompt, response)
|
||||
return endings
|
||||
|
||||
|
||||
def decompose_deepseek_math_cot_v2(prompt: str, response: str, max_seq_length: int, tokenizer: PreTrainedTokenizer):
|
||||
assert isinstance(prompt, str), prompt
|
||||
assert isinstance(response, str), response
|
||||
steps = response.split("\n")
|
||||
|
||||
endings = []
|
||||
acc_step = prompt
|
||||
for i, step in enumerate(steps):
|
||||
if i == 0:
|
||||
acc_step = acc_step + step
|
||||
else:
|
||||
acc_step = acc_step + "\n" + step
|
||||
if step.strip():
|
||||
input_ids = tokenizer(acc_step, truncation=False)["input_ids"]
|
||||
endings.append(len(input_ids) - 1)
|
||||
|
||||
full_text = prompt + response
|
||||
true_input_ids = tokenizer(full_text, truncation=True, max_length=max_seq_length)["input_ids"]
|
||||
endings = [e for e in endings if e < len(true_input_ids)]
|
||||
|
||||
# assert len(endings) > 0, (prompt, response)
|
||||
if len(endings) == 0:
|
||||
logger.warning(f"Warning: Bad response:\n\n=========================Prompt====================\n{prompt}\n\n"
|
||||
f"=======================Response=====================\n{response}\n\n")
|
||||
return endings
|
||||
|
||||
|
||||
# def decompose_deepseek_math_cot_v2_aligner(tokenizer: PreTrainedTokenizer, max_seq_length: int, response_field: str, prompt_field: str):
|
||||
# def func(data: List[Dict]):
|
||||
# for item in data:
|
||||
# item["ending"] = decompose_deepseek_math_cot_v2(item[prompt_field], item[response_field], max_seq_length, tokenizer)
|
||||
# return data
|
||||
#
|
||||
# return func
|
||||
|
||||
|
||||
class RAPResponseStepRewardCollator:
|
||||
_decompose_fns = {
|
||||
"rap": decompose_rap,
|
||||
"cot": decompose_cot,
|
||||
}
|
||||
|
||||
def __init__(self, tokenizer: PreTrainedTokenizer, max_seq_length: int, cot_type: str = "rap"):
|
||||
self.tokenizer = tokenizer
|
||||
self.max_seq_length = max_seq_length
|
||||
self.decompose_fn = self._decompose_fns[cot_type]
|
||||
|
||||
def __call__(self, batch):
|
||||
prompt = [item["prompt"] for item in batch]
|
||||
inputs = [item["input"] for item in batch]
|
||||
indices = [item["index"] for item in batch]
|
||||
|
||||
text_prompts = prompt
|
||||
text_inputs = inputs
|
||||
|
||||
encoded_prompts = self.tokenizer(text_prompts, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
input_lens = torch.sum(encoded_prompts["attention_mask"], dim=-1)
|
||||
|
||||
encoded_inputs = self.tokenizer(text_inputs, padding="longest", truncation=True, max_length=self.max_seq_length, return_tensors="pt")
|
||||
if self.tokenizer.padding_side == "left":
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
input_lens = input_lens + padding_len
|
||||
|
||||
labels = encoded_inputs["input_ids"].clone()
|
||||
prompt_mask = torch.arange(encoded_inputs["input_ids"].size(1))[None, :] < input_lens[:, None]
|
||||
labels[prompt_mask] = self.tokenizer.pad_token_id
|
||||
|
||||
endings = []
|
||||
padding_len = torch.sum(1 - encoded_inputs["attention_mask"], dim=-1)
|
||||
for b, item in enumerate(batch):
|
||||
# ending = decompose_rap(item["prompt"], item["response"], self.max_seq_length, self.tokenizer)
|
||||
ending = self.decompose_fn(item["prompt"], item["response"], self.max_seq_length, self.tokenizer)
|
||||
if self.tokenizer.padding_side == "left":
|
||||
ending = [e + padding_len[b].item() for e in ending]
|
||||
endings.append(ending)
|
||||
|
||||
encoded_inputs["labels"] = labels
|
||||
encoded_inputs["meta_data"] = {
|
||||
"index": indices,
|
||||
"prompt": prompt,
|
||||
"input": inputs,
|
||||
"response": [item["response"] for item in batch],
|
||||
"ending": endings,
|
||||
"type": [None] * len(endings),
|
||||
}
|
||||
return encoded_inputs
|
||||
@@ -0,0 +1,40 @@
|
||||
import json
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
|
||||
class MetaMathReaderHF:
|
||||
def __init__(self, category: str):
|
||||
self.category = category
|
||||
|
||||
def __call__(self, file_path: str = "meta-math/MetaMathQA"):
|
||||
data = load_dataset(file_path, split="train").to_list()
|
||||
for i, data in enumerate(data):
|
||||
data["id"] = i
|
||||
|
||||
if not self.category or (self.category and self.category == "all"):
|
||||
return data
|
||||
|
||||
if self.category == "math":
|
||||
return [item for item in data if "MATH" in item["type"]]
|
||||
if self.category == "gsm":
|
||||
return [item for item in data if "GSM" in item["type"]]
|
||||
raise ValueError(f"Invalid category: {self.category}")
|
||||
|
||||
|
||||
class MetaMathReader:
|
||||
def __init__(self, category: str = "math"):
|
||||
self.category = category
|
||||
|
||||
def __call__(self, file_path: str = "meta-math/MetaMathQA"):
|
||||
data = json.load(open(file_path))
|
||||
|
||||
if not self.category or (self.category and self.category == "all"):
|
||||
return data
|
||||
|
||||
if self.category == "math":
|
||||
return [item for item in data if "MATH" in item["type"]]
|
||||
if self.category == "gsm":
|
||||
return [item for item in data if "GSM" in item["type"]]
|
||||
raise ValueError(f"Invalid category: {self.category}")
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
# Copied from https://github.com/meta-math/MetaMath/blob/main/util.py
|
||||
import pprint
|
||||
import re
|
||||
|
||||
|
||||
def last_boxed_only(sample):
|
||||
q, a = sample
|
||||
a = last_boxed_only_string(a)
|
||||
if a is None:
|
||||
return None
|
||||
return q, a
|
||||
|
||||
|
||||
def last_boxed_only_string(string):
|
||||
idx = string.rfind("\\boxed")
|
||||
if idx < 0:
|
||||
idx = string.rfind("\\fbox")
|
||||
if idx < 0:
|
||||
return None
|
||||
|
||||
i = idx
|
||||
right_brace_idx = None
|
||||
num_left_braces_open = 0
|
||||
while i < len(string):
|
||||
if string[i] == "{":
|
||||
num_left_braces_open += 1
|
||||
if string[i] == "}":
|
||||
num_left_braces_open -= 1
|
||||
if num_left_braces_open == 0:
|
||||
right_brace_idx = i
|
||||
break
|
||||
i += 1
|
||||
|
||||
if right_brace_idx is None:
|
||||
retval = None
|
||||
else:
|
||||
retval = string[idx:right_brace_idx + 1]
|
||||
|
||||
return retval
|
||||
|
||||
|
||||
def only_until_first_boxed_from_tokens(string, tokens):
|
||||
idx = string.find("\\boxed")
|
||||
if idx < 0:
|
||||
idx = string.find("\\fbox")
|
||||
if idx < 0:
|
||||
return None
|
||||
|
||||
cum_length = 0
|
||||
for i, t in enumerate(tokens):
|
||||
cum_length += len(t)
|
||||
if cum_length >= idx:
|
||||
break
|
||||
|
||||
return tokens[:i]
|
||||
|
||||
|
||||
def clean_numbers(sample):
|
||||
if not sample:
|
||||
return None
|
||||
new_sample = list()
|
||||
for s in sample:
|
||||
new_sample.append(_clean_numbers(s))
|
||||
|
||||
return tuple(new_sample)
|
||||
|
||||
|
||||
def _clean_numbers(string):
|
||||
"""
|
||||
Clean Numbers in the given string
|
||||
|
||||
>>> _clean_numbers(None, "Hello 123")
|
||||
'Hello 123'
|
||||
>>> _clean_numbers(None, "Hello 1234")
|
||||
'Hello 1,234'
|
||||
>>> _clean_numbers(None, "Hello 1234324asdasd")
|
||||
'Hello 1,234,324asdasd'
|
||||
"""
|
||||
num_prev_digits = 0
|
||||
new_string = ""
|
||||
for i, c in enumerate(string):
|
||||
# isdigit() doesnt work here because of weird unicode chars.
|
||||
if c in {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0'}:
|
||||
num_prev_digits += 1
|
||||
else:
|
||||
if num_prev_digits > 3:
|
||||
# Some fixing
|
||||
string_number = new_string[-num_prev_digits:]
|
||||
new_string = new_string[:-num_prev_digits] + "{0:,}".format(int(string_number))
|
||||
num_prev_digits = 0
|
||||
new_string += c
|
||||
|
||||
if num_prev_digits > 3:
|
||||
# Some fixing
|
||||
string_number = new_string[-num_prev_digits:]
|
||||
new_string = new_string[:-num_prev_digits] + "{0:,}".format(int(string_number))
|
||||
|
||||
return new_string
|
||||
|
||||
|
||||
def fix_fracs(string):
|
||||
substrs = string.split("\\frac")
|
||||
new_str = substrs[0]
|
||||
if len(substrs) > 1:
|
||||
substrs = substrs[1:]
|
||||
for substr in substrs:
|
||||
new_str += "\\frac"
|
||||
if substr[0] == "{":
|
||||
new_str += substr
|
||||
else:
|
||||
try:
|
||||
assert len(substr) >= 2
|
||||
except AssertionError:
|
||||
return string
|
||||
a = substr[0]
|
||||
b = substr[1]
|
||||
if b != "{":
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}{" + b + "}" + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}{" + b + "}"
|
||||
else:
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}" + b + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}" + b
|
||||
string = new_str
|
||||
return string
|
||||
|
||||
|
||||
def fix_a_slash_b(string):
|
||||
if len(string.split("/")) != 2:
|
||||
return string
|
||||
a = string.split("/")[0]
|
||||
b = string.split("/")[1]
|
||||
try:
|
||||
a = int(a)
|
||||
b = int(b)
|
||||
assert string == "{}/{}".format(a, b)
|
||||
new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
|
||||
return new_string
|
||||
except AssertionError:
|
||||
return string
|
||||
|
||||
|
||||
def remove_right_units(string):
|
||||
# "\\text{ " only ever occurs (at least in the val set) when describing units
|
||||
if "\\text{ " in string:
|
||||
splits = string.split("\\text{ ")
|
||||
assert len(splits) == 2
|
||||
return splits[0]
|
||||
else:
|
||||
return string
|
||||
|
||||
|
||||
def fix_sqrt(string):
|
||||
if "\\sqrt" not in string:
|
||||
return string
|
||||
splits = string.split("\\sqrt")
|
||||
new_string = splits[0]
|
||||
for split in splits[1:]:
|
||||
if split[0] != "{":
|
||||
a = split[0]
|
||||
new_substr = "\\sqrt{" + a + "}" + split[1:]
|
||||
else:
|
||||
new_substr = "\\sqrt" + split
|
||||
new_string += new_substr
|
||||
return new_string
|
||||
|
||||
|
||||
def strip_string(string):
|
||||
# linebreaks
|
||||
string = string.replace("\n", "")
|
||||
|
||||
# remove inverse spaces
|
||||
string = string.replace("\\!", "")
|
||||
|
||||
# replace \\ with \
|
||||
string = string.replace("\\\\", "\\")
|
||||
|
||||
# replace tfrac and dfrac with frac
|
||||
string = string.replace("tfrac", "frac")
|
||||
string = string.replace("dfrac", "frac")
|
||||
|
||||
# remove \left and \right
|
||||
string = string.replace("\\left", "")
|
||||
string = string.replace("\\right", "")
|
||||
|
||||
# Remove circ (degrees)
|
||||
string = string.replace("^{\\circ}", "")
|
||||
string = string.replace("^\\circ", "")
|
||||
|
||||
# remove dollar signs
|
||||
string = string.replace("\\$", "")
|
||||
|
||||
# remove units (on the right)
|
||||
string = remove_right_units(string)
|
||||
|
||||
# remove percentage
|
||||
string = string.replace("\\%", "")
|
||||
string = string.replace("\%", "") # noqa: W605
|
||||
|
||||
# " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
|
||||
string = string.replace(" .", " 0.")
|
||||
string = string.replace("{.", "{0.")
|
||||
# if empty, return empty string
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if string[0] == ".":
|
||||
string = "0" + string
|
||||
|
||||
# to consider: get rid of e.g. "k = " or "q = " at beginning
|
||||
if len(string.split("=")) == 2:
|
||||
if len(string.split("=")[0]) <= 2:
|
||||
string = string.split("=")[1]
|
||||
|
||||
# fix sqrt3 --> sqrt{3}
|
||||
string = fix_sqrt(string)
|
||||
|
||||
# remove spaces
|
||||
string = string.replace(" ", "")
|
||||
|
||||
# \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
|
||||
string = fix_fracs(string)
|
||||
|
||||
# manually change 0.5 --> \frac{1}{2}
|
||||
if string == "0.5":
|
||||
string = "\\frac{1}{2}"
|
||||
|
||||
# NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
|
||||
string = fix_a_slash_b(string)
|
||||
|
||||
return string
|
||||
|
||||
|
||||
def is_equiv(str1, str2, verbose=False):
|
||||
if str1 is None and str2 is None:
|
||||
print("WARNING: Both None")
|
||||
return True
|
||||
if str1 is None or str2 is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
ss1 = strip_string(str1)
|
||||
ss2 = strip_string(str2)
|
||||
# pdb.set_trace()
|
||||
if verbose:
|
||||
print(ss1, ss2)
|
||||
return ss1 == ss2
|
||||
except Exception:
|
||||
return str1 == str2
|
||||
|
||||
|
||||
class NotEqual:
|
||||
def __eq__(self, other):
|
||||
return False
|
||||
@@ -0,0 +1,546 @@
|
||||
# ======================== MATH Scale Utils ======================================
|
||||
# https://github.com/microsoft/unilm/blob/master/mathscale/MWPBench/eval_vllm/util.py#L282-L336
|
||||
|
||||
import re
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
def fix_fracs(string):
|
||||
substrs = string.split("\\frac")
|
||||
new_str = substrs[0]
|
||||
if len(substrs) > 1:
|
||||
substrs = substrs[1:]
|
||||
for substr in substrs:
|
||||
new_str += "\\frac"
|
||||
if substr[0] == "{":
|
||||
new_str += substr
|
||||
else:
|
||||
try:
|
||||
assert len(substr) >= 2
|
||||
except AssertionError:
|
||||
return string
|
||||
a = substr[0]
|
||||
b = substr[1]
|
||||
if b != "{":
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}{" + b + "}" + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}{" + b + "}"
|
||||
else:
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}" + b + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}" + b
|
||||
string = new_str
|
||||
return string
|
||||
|
||||
|
||||
def fix_a_slash_b(string):
|
||||
if len(string.split("/")) != 2:
|
||||
return string
|
||||
a = string.split("/")[0]
|
||||
b = string.split("/")[1]
|
||||
try:
|
||||
a = int(a)
|
||||
b = int(b)
|
||||
assert string == "{}/{}".format(a, b)
|
||||
new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
|
||||
return new_string
|
||||
except Exception as e:
|
||||
return string
|
||||
|
||||
|
||||
def remove_right_units(string):
|
||||
# "\\text{ " only ever occurs (at least in the val set) when describing units
|
||||
if "\\text{ " in string:
|
||||
splits = string.split("\\text{ ")
|
||||
assert len(splits) == 2
|
||||
return splits[0]
|
||||
else:
|
||||
return string
|
||||
|
||||
|
||||
def fix_sqrt(string):
|
||||
if "\\sqrt" not in string:
|
||||
return string
|
||||
splits = string.split("\\sqrt")
|
||||
new_string = splits[0]
|
||||
for split in splits[1:]:
|
||||
if split[0] != "{":
|
||||
a = split[0]
|
||||
new_substr = "\\sqrt{" + a + "}" + split[1:]
|
||||
else:
|
||||
new_substr = "\\sqrt" + split
|
||||
new_string += new_substr
|
||||
return new_string
|
||||
|
||||
|
||||
def is_number(s):
|
||||
s = s.strip("$")
|
||||
|
||||
try:
|
||||
# Try to convert the string to an integer
|
||||
int(s)
|
||||
return True
|
||||
except ValueError:
|
||||
try:
|
||||
# Try to convert the string to a float
|
||||
float(s)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def unbox_and_extract(text):
|
||||
start_indices = [m.start() for m in re.finditer(r'\\boxed{', text)]
|
||||
extracted_contents = []
|
||||
for start in start_indices:
|
||||
brace_count = 0
|
||||
for i, char in enumerate(text[start:]):
|
||||
if char == '{':
|
||||
brace_count += 1
|
||||
elif char == '}':
|
||||
brace_count -= 1
|
||||
if brace_count == 0:
|
||||
end = start + i + 1
|
||||
extracted_contents.append(text[start + 7:end - 1]) # +7 to skip '\\boxed{'
|
||||
break
|
||||
# Replace '\\boxed{...}' with the content inside it
|
||||
unboxed_text = re.sub(r'\\boxed{(.*?)}', r'\1', text)
|
||||
return unboxed_text, extracted_contents
|
||||
|
||||
|
||||
def convert_to_latex_fraction(text: str) -> str:
|
||||
# Use regex to find all occurrences of ((num)/(denom))
|
||||
pattern = re.compile(r"\(\(([\d]+)\)/\(([\d]+)\)\)")
|
||||
|
||||
matches = pattern.findall(text)
|
||||
|
||||
for match in matches:
|
||||
num, denom = match
|
||||
latex_frac = f"\\\\frac{{{num}}}{{{denom}}}"
|
||||
|
||||
# Replace the old expression with the LaTeX fraction
|
||||
text = text.replace(f"(({num})/({denom}))", latex_frac)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def strip_string(string):
|
||||
# convert ((3)/(4)) -> \\frac{3}{4}
|
||||
string = convert_to_latex_fraction(string)
|
||||
|
||||
# remove ,
|
||||
string = string.replace(",", "")
|
||||
|
||||
# linebreaks
|
||||
string = string.replace("\n", "")
|
||||
|
||||
# remove inverse spaces
|
||||
string = string.replace("\\!", "")
|
||||
|
||||
# replace \\ with \
|
||||
string = string.replace("\\\\", "\\")
|
||||
|
||||
# replace tfrac and dfrac with frac
|
||||
string = string.replace("tfrac", "frac")
|
||||
string = string.replace("dfrac", "frac")
|
||||
|
||||
# remove \left and \right
|
||||
string = string.replace("\\left", "")
|
||||
string = string.replace("\\right", "")
|
||||
|
||||
# Remove circ (degrees)
|
||||
string = string.replace("^{\\circ}", "")
|
||||
string = string.replace("^\\circ", "")
|
||||
|
||||
# remove dollar signs
|
||||
string = string.replace("\\$", "")
|
||||
|
||||
# remove units (on the right)
|
||||
string = remove_right_units(string)
|
||||
|
||||
# remove percentage
|
||||
string = string.replace("\\%", "")
|
||||
string = string.replace("\%", "") # noqa: W605
|
||||
|
||||
# " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
|
||||
string = string.replace(" .", " 0.")
|
||||
string = string.replace("{.", "{0.")
|
||||
# if empty, return empty string
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if string[0] == ".":
|
||||
string = "0" + string
|
||||
|
||||
# to consider: get rid of e.g. "k = " or "q = " at beginning
|
||||
if len(string.split("=")) == 2:
|
||||
if len(string.split("=")[0]) <= 2:
|
||||
string = string.split("=")[1]
|
||||
|
||||
# fix sqrt3 --> sqrt{3}
|
||||
string = fix_sqrt(string)
|
||||
|
||||
# My own
|
||||
string = string.replace("\\quad", " ")
|
||||
# remove spaces
|
||||
string = string.replace(" ", "")
|
||||
|
||||
# \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
|
||||
string = fix_fracs(string)
|
||||
|
||||
# manually change 0.5 --> \frac{1}{2}
|
||||
if string == "0.5":
|
||||
string = "\\frac{1}{2}"
|
||||
|
||||
# NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
|
||||
string = fix_a_slash_b(string)
|
||||
|
||||
return string
|
||||
|
||||
|
||||
def is_single_inline_math(expression: str) -> bool:
|
||||
# Use regex to check for a pattern that starts and ends with dollar signs,
|
||||
# and contains no other dollar signs in between.
|
||||
pattern = re.compile(r"^\$[^$]+\$$")
|
||||
|
||||
match = pattern.match(expression)
|
||||
|
||||
return bool(match)
|
||||
|
||||
|
||||
def mathscale_is_equiv(prediction_ans, reference_ans, verbose=False):
|
||||
if prediction_ans is None and reference_ans is None:
|
||||
print("WARNING: Both None")
|
||||
return True, prediction_ans, reference_ans
|
||||
if prediction_ans is None or reference_ans is None:
|
||||
return False, prediction_ans, reference_ans
|
||||
|
||||
try:
|
||||
clean_prediction_ans = strip_string(prediction_ans)
|
||||
clean_reference_ans = strip_string(reference_ans)
|
||||
|
||||
if is_number(clean_prediction_ans) and is_number(clean_reference_ans):
|
||||
judge = float(clean_prediction_ans.strip("$")) == float(clean_reference_ans.strip("$"))
|
||||
# print(f"1 judge: {judge}")
|
||||
elif is_single_inline_math(clean_reference_ans):
|
||||
judge = (clean_reference_ans.strip("$") in clean_prediction_ans.strip("$"))
|
||||
# print(f"2 judge: {judge}")
|
||||
elif (len(clean_prediction_ans) >= 3) and (not is_number(clean_prediction_ans)) and (not clean_prediction_ans.startswith("-")) and (
|
||||
not clean_reference_ans.startswith("-")) and (clean_prediction_ans in clean_reference_ans):
|
||||
judge = True
|
||||
# print(f"3 judge: {judge}")
|
||||
elif (len(clean_reference_ans) >= 3) and (not is_number(clean_reference_ans)) and (not clean_prediction_ans.startswith("-")) and (
|
||||
not clean_reference_ans.startswith("-")) and (clean_reference_ans in clean_prediction_ans):
|
||||
judge = True
|
||||
# print(f"4 judge: {judge}")
|
||||
else:
|
||||
judge = clean_prediction_ans == clean_reference_ans
|
||||
# print(f"5 judge: {judge}")
|
||||
if verbose:
|
||||
print(f"clean_prediction_ans: {clean_prediction_ans} | clean_reference_ans: {clean_reference_ans} | judge: {judge}")
|
||||
return judge, clean_prediction_ans, clean_reference_ans
|
||||
except Exception as e:
|
||||
# print(e)
|
||||
return prediction_ans == reference_ans, prediction_ans, reference_ans
|
||||
|
||||
|
||||
def mathscale_is_equiv_proxy(prediction_ans, reference_ans, verbose=False):
|
||||
return mathscale_is_equiv(prediction_ans, reference_ans.lower(), verbose=verbose)[0]
|
||||
|
||||
|
||||
def is_correct(completion, answer, verbose=False):
|
||||
completion = completion.lower()
|
||||
answer = answer.lower()
|
||||
|
||||
# Extract short answer from completion
|
||||
extract_ans = ""
|
||||
|
||||
try:
|
||||
clean_reference_ans = strip_string(answer)
|
||||
except Exception as e:
|
||||
clean_reference_ans = answer
|
||||
print(f"Failed to strip answer string: {answer} =============== {e}")
|
||||
is_reference_ans_number = is_number(clean_reference_ans)
|
||||
|
||||
# First extract boxed answer
|
||||
unbox_long_answer, box_short_answers = unbox_and_extract(completion)
|
||||
if box_short_answers != []:
|
||||
extract_ans = box_short_answers[-1].strip()
|
||||
# print(f"1 extract_ans: {extract_ans}")
|
||||
# extract the last number answer
|
||||
elif is_reference_ans_number:
|
||||
numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion)
|
||||
if numbers:
|
||||
extract_ans = numbers[-1]
|
||||
# print(f"2 extract_ans: {extract_ans}")
|
||||
# extract "the answer is ..." answer
|
||||
elif ("answer is" in completion) or ("solution is" in completion):
|
||||
if "answer is" in completion:
|
||||
split_ans = completion.split('answer is')
|
||||
else:
|
||||
split_ans = completion.split('solution is')
|
||||
ans = split_ans[-1].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"3 extract_ans: {extract_ans}")
|
||||
# extract "therefore xx is xxx" answer
|
||||
elif "is" in completion:
|
||||
pos = completion.rfind("is")
|
||||
ans = completion[pos + 2:].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"4 extract_ans: {extract_ans}")
|
||||
else:
|
||||
return False, f"failed extracting answer from completion", clean_reference_ans
|
||||
|
||||
judge, clean_prediction_ans, clean_reference_ans = mathscale_is_equiv(extract_ans, answer, verbose=verbose)
|
||||
return judge, clean_prediction_ans, clean_reference_ans
|
||||
|
||||
|
||||
def mathscale_extract_answer():
|
||||
def func(completion: str):
|
||||
completion = completion.lower()
|
||||
|
||||
# Extract short answer from completion
|
||||
extract_ans = ""
|
||||
|
||||
# First extract boxed answer
|
||||
unbox_long_answer, box_short_answers = unbox_and_extract(completion)
|
||||
if box_short_answers != []:
|
||||
extract_ans = box_short_answers[-1].strip()
|
||||
# print(f"1 extract_ans: {extract_ans}")
|
||||
# extract the last number answer
|
||||
# elif is_reference_ans_number:
|
||||
# numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion)
|
||||
# if numbers:
|
||||
# extract_ans = numbers[-1]
|
||||
# print(f"2 extract_ans: {extract_ans}")
|
||||
# extract "the answer is ..." answer
|
||||
elif ("answer is" in completion) or ("solution is" in completion):
|
||||
if "answer is" in completion:
|
||||
split_ans = completion.split('answer is')
|
||||
else:
|
||||
split_ans = completion.split('solution is')
|
||||
ans = split_ans[-1].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"3 extract_ans: {extract_ans}")
|
||||
# extract "therefore xx is xxx" answer
|
||||
elif "is" in completion:
|
||||
pos = completion.rfind("is")
|
||||
ans = completion[pos + 2:].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"4 extract_ans: {extract_ans}")
|
||||
else:
|
||||
return ""
|
||||
|
||||
return extract_ans
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def mathscale_extract_answer_fn_v2_list(separator: str = "The answer is: ", completion_field: str = "response"):
|
||||
from data.math import extract_answer_number
|
||||
|
||||
mathscale_extract_fn = mathscale_extract_answer()
|
||||
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
res = extract_answer_number(item[completion_field], separator=separator)
|
||||
if not res:
|
||||
item["label"] = str(mathscale_extract_fn(item[completion_field]))
|
||||
else:
|
||||
item["label"] = str(res)
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def mathscale_extract_answer_v2(completion: str):
|
||||
completion = completion.lower()
|
||||
|
||||
# Extract short answer from completion
|
||||
extract_ans = ""
|
||||
|
||||
# First extract boxed answer
|
||||
unbox_long_answer, box_short_answers = unbox_and_extract(completion)
|
||||
numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion)
|
||||
if box_short_answers != []:
|
||||
extract_ans = box_short_answers[-1].strip()
|
||||
# print(f"1 extract_ans: {extract_ans}")
|
||||
# extract the last number answer
|
||||
# elif is_reference_ans_number:
|
||||
elif numbers:
|
||||
extract_ans = numbers[-1]
|
||||
# print(f"2 extract_ans: {extract_ans}")
|
||||
# extract "the answer is ..." answer
|
||||
elif ("answer is" in completion) or ("solution is" in completion):
|
||||
if "answer is" in completion:
|
||||
split_ans = completion.split('answer is')
|
||||
else:
|
||||
split_ans = completion.split('solution is')
|
||||
ans = split_ans[-1].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"3 extract_ans: {extract_ans}")
|
||||
# extract "therefore xx is xxx" answer
|
||||
elif "is" in completion:
|
||||
pos = completion.rfind("is")
|
||||
ans = completion[pos + 2:].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"4 extract_ans: {extract_ans}")
|
||||
else:
|
||||
return ""
|
||||
|
||||
try:
|
||||
extract_ans = strip_string(extract_ans)
|
||||
except Exception as e:
|
||||
print(f"Failed to strip answer string: {extract_ans} =============== {e}")
|
||||
|
||||
return extract_ans
|
||||
|
||||
|
||||
def mathscale_extract_answer_v3(completion: str):
|
||||
completion = completion.lower()
|
||||
|
||||
# Extract short answer from completion
|
||||
extract_ans = ""
|
||||
|
||||
# First extract boxed answer
|
||||
unbox_long_answer, box_short_answers = unbox_and_extract(completion)
|
||||
if "answer is" in completion:
|
||||
numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion.split("answer is")[-1])
|
||||
elif "solution is" in completion:
|
||||
numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion.split("solution is")[-1])
|
||||
else:
|
||||
numbers = []
|
||||
if box_short_answers != []:
|
||||
extract_ans = box_short_answers[-1].strip()
|
||||
# print(f"1 extract_ans: {extract_ans}")
|
||||
# extract the last number answer
|
||||
# elif is_reference_ans_number:
|
||||
elif numbers:
|
||||
extract_ans = numbers[0]
|
||||
# print(f"2 extract_ans: {extract_ans}")
|
||||
# extract "the answer is ..." answer
|
||||
elif ("answer is" in completion) or ("solution is" in completion):
|
||||
if "answer is" in completion:
|
||||
split_ans = completion.split('answer is')
|
||||
else:
|
||||
split_ans = completion.split('solution is')
|
||||
ans = split_ans[-1].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"3 extract_ans: {extract_ans}")
|
||||
# extract "therefore xx is xxx" answer
|
||||
elif "is" in completion:
|
||||
pos = completion.rfind("is")
|
||||
ans = completion[pos + 2:].strip().lstrip(":").strip()
|
||||
extract_ans_temp = ans.split('.\n')[0]
|
||||
extract_ans_temp = extract_ans_temp.strip()
|
||||
extract_ans_temp = extract_ans_temp.strip('.')
|
||||
if len(extract_ans_temp) > 0 and extract_ans_temp[-1] == '.':
|
||||
extract_ans = extract_ans_temp[0:-1]
|
||||
else:
|
||||
extract_ans = extract_ans_temp
|
||||
extract_ans = extract_ans.strip()
|
||||
# print(f"4 extract_ans: {extract_ans}")
|
||||
else:
|
||||
return ""
|
||||
|
||||
try:
|
||||
extract_ans = strip_string(extract_ans)
|
||||
except Exception as e:
|
||||
print(f"Failed to strip answer string: {extract_ans}")
|
||||
|
||||
return extract_ans
|
||||
|
||||
|
||||
def mathscale_extract_answer_fn_v3(completion_field: str = "response"):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
res = mathscale_extract_answer_v2(item[completion_field])
|
||||
if res:
|
||||
item["label"] = str(res)
|
||||
else:
|
||||
item["label"] = ""
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def mathscale_extract_answer_fn_v4(completion_field: str = "response"):
|
||||
def func(data: List[Dict]):
|
||||
for item in data:
|
||||
res = mathscale_extract_answer_v3(item[completion_field])
|
||||
if res:
|
||||
item["label"] = str(res)
|
||||
else:
|
||||
item["label"] = ""
|
||||
return data
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def extract_pure_prompt_aligner():
|
||||
def func(data: List[Dict]):
|
||||
for i, item in enumerate(data):
|
||||
prompt = item["prompt"]
|
||||
|
||||
_s = prompt.index("### Instruction:") + len("### Instruction:")
|
||||
_e = prompt.index("### Response:")
|
||||
question = prompt[_s:_e].strip()
|
||||
|
||||
item["question"] = question
|
||||
item["id"] = i
|
||||
|
||||
return data
|
||||
|
||||
return func
|
||||
@@ -0,0 +1,22 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from datasets import load_dataset
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
|
||||
class NuminaMathReader:
|
||||
def __init__(self, id_field: str = "id", split: str = "train"):
|
||||
self.id_field = id_field
|
||||
self.split = split
|
||||
|
||||
def __call__(self, file_path):
|
||||
data = load_dataset(file_path, split=self.split).to_list()
|
||||
logger.info(len(data))
|
||||
|
||||
for i, item in enumerate(data):
|
||||
item[self.id_field] = i
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,201 @@
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from typing import Union, List
|
||||
|
||||
from openai import OpenAI, AzureOpenAI
|
||||
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger(__name__)
|
||||
|
||||
|
||||
class GPTAPIInterface:
|
||||
def __init__(self, model: str, max_tokens: int, api_time_interval: int = 2, api_key: str = None):
|
||||
self.model = model
|
||||
self.max_tokens = max_tokens
|
||||
self.api_time_interval = api_time_interval
|
||||
if api_key:
|
||||
self.client = OpenAI(
|
||||
# This is the default and can be omitted
|
||||
api_key=api_key,
|
||||
)
|
||||
else:
|
||||
self.client = OpenAI(
|
||||
# This is the default and can be omitted
|
||||
api_key=os.environ.get("OPENAI_API_KEY"),
|
||||
)
|
||||
|
||||
def __call__(self, text: str):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class GPTTurbo(GPTAPIInterface):
|
||||
def __init__(self,
|
||||
model: str = "gpt-3.5-turbo",
|
||||
max_tokens: int = 2048,
|
||||
temperature: float = 0.0,
|
||||
top_p: int = 1,
|
||||
frequency_penalty: float = 0.0,
|
||||
presence_penalty: float = 0.0,
|
||||
api_time_interval: int = 2,
|
||||
organization: str = "",
|
||||
n: int = 1,
|
||||
api_key: str = None):
|
||||
super().__init__(model, max_tokens, api_time_interval, api_key)
|
||||
self.temperature = temperature
|
||||
self.top_p = top_p
|
||||
self.frequency_penalty = frequency_penalty
|
||||
self.presence_penalty = presence_penalty
|
||||
self.organization = organization
|
||||
self.n = n
|
||||
|
||||
def __call__(self, text: Union[str, List[str]]):
|
||||
if isinstance(text, list):
|
||||
if isinstance(text[0], str):
|
||||
assert len(text) == 1, "Currently we only support one input in single batch."
|
||||
text = text[0]
|
||||
else:
|
||||
assert isinstance(text[0], dict)
|
||||
assert "role" in text[0] and "content" in text[0]
|
||||
|
||||
flag = False
|
||||
error_time = 0
|
||||
response = None
|
||||
max_tokens = self.max_tokens
|
||||
while not flag:
|
||||
try:
|
||||
if isinstance(text, str):
|
||||
messages = [{"role": "user", "content": text}]
|
||||
else:
|
||||
messages = text
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=self.temperature,
|
||||
top_p=self.top_p,
|
||||
frequency_penalty=self.frequency_penalty,
|
||||
presence_penalty=self.presence_penalty,
|
||||
n=self.n,
|
||||
)
|
||||
error_time = 0
|
||||
flag = True
|
||||
except Exception as exc:
|
||||
logger.warning(exc)
|
||||
logger.warning(">>>>>>>>>>>>>>>>>>>>>>>>>>>>")
|
||||
err_msg = traceback.format_exc()
|
||||
# logger.warning(traceback.print_exc())
|
||||
logger.warning(err_msg)
|
||||
logger.warning(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
|
||||
if "maximum context" in err_msg:
|
||||
max_tokens -= 100
|
||||
logger.warning("max_tokens: {}".format(max_tokens))
|
||||
error_time += 1
|
||||
if error_time > 20:
|
||||
logger.warning("Too many errors. Sleep 60s.")
|
||||
time.sleep(60)
|
||||
return {"response": ""}
|
||||
|
||||
if self.api_time_interval:
|
||||
time.sleep(self.api_time_interval)
|
||||
|
||||
response = [choice.message.content for choice in response.choices]
|
||||
if len(response) == 1:
|
||||
response = response[0]
|
||||
return {"response": response}
|
||||
|
||||
|
||||
class AzureGPTEndpoint:
|
||||
def __init__(self,
|
||||
model_name: str = "gpt-4-32k",
|
||||
max_tokens: int = 2048,
|
||||
temperature: float = 0.0,
|
||||
top_p: int = 1,
|
||||
# frequency_penalty: float = 0.0,
|
||||
# presence_penalty: float = 0.0,
|
||||
api_time_interval: int = 0,
|
||||
# organization: str = "",
|
||||
n: int = 1,
|
||||
return_text: bool = True,
|
||||
):
|
||||
if model_name == "gpt-4o":
|
||||
self.client = AzureOpenAI(
|
||||
azure_endpoint="https://gcrgpt4aoai5.openai.azure.com/",
|
||||
api_key=os.getenv("OPENAI_API_KEY"),
|
||||
api_version="2024-02-01"
|
||||
)
|
||||
else:
|
||||
self.client = AzureOpenAI(
|
||||
azure_endpoint="https://gcraoai5sw1.openai.azure.com//",
|
||||
api_key=os.getenv("OPENAI_API_KEY"),
|
||||
api_version="2023-03-15-preview"
|
||||
)
|
||||
|
||||
self.model = model_name
|
||||
self.max_tokens = max_tokens
|
||||
self.temperature = temperature
|
||||
self.top_p = top_p
|
||||
# self.frequency_penalty = frequency_penalty
|
||||
# self.presence_penalty = presence_penalty
|
||||
# self.organization = organization
|
||||
self.n = n
|
||||
self.api_time_interval = api_time_interval
|
||||
self.return_text = return_text
|
||||
|
||||
def __call__(self, text: Union[str, List[str]]):
|
||||
if isinstance(text, list):
|
||||
if isinstance(text[0], str):
|
||||
assert len(text) == 1, "Currently we only support one input in single batch."
|
||||
text = text[0]
|
||||
else:
|
||||
assert isinstance(text[0], dict)
|
||||
assert "role" in text[0] and "content" in text[0]
|
||||
|
||||
flag = False
|
||||
error_time = 0
|
||||
response = None
|
||||
max_tokens = self.max_tokens
|
||||
while not flag:
|
||||
try:
|
||||
if isinstance(text, str):
|
||||
messages = [{"role": "user", "content": text}]
|
||||
else:
|
||||
messages = text
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=self.temperature,
|
||||
top_p=self.top_p,
|
||||
n=self.n,
|
||||
)
|
||||
error_time = 0
|
||||
flag = True
|
||||
except Exception as exc:
|
||||
logger.warning(exc)
|
||||
logger.warning(">>>>>>>>>>>>>>>>>>>>>>>>>>>>")
|
||||
err_msg = traceback.format_exc()
|
||||
logger.warning(err_msg)
|
||||
logger.warning(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
|
||||
if "maximum context" in err_msg:
|
||||
max_tokens -= 100
|
||||
logger.warning("max_tokens: {}".format(max_tokens))
|
||||
error_time += 1
|
||||
if error_time > 20:
|
||||
logger.warning("Too many errors. Sleep 60s.")
|
||||
time.sleep(60)
|
||||
return {"response": ""}
|
||||
|
||||
if self.api_time_interval:
|
||||
time.sleep(self.api_time_interval)
|
||||
|
||||
response = [choice.message.content for choice in response.choices]
|
||||
if len(response) == 1:
|
||||
response = response[0]
|
||||
|
||||
if self.return_text:
|
||||
return response
|
||||
return {"response": response}
|
||||
|
||||
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Zhibin Gou
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,52 @@
|
||||
### Requirements
|
||||
You can install the required packages with the following command:
|
||||
```bash
|
||||
cd latex2sympy
|
||||
pip install -e .
|
||||
cd ..
|
||||
pip install -r requirements.txt
|
||||
pip install vllm==0.5.1 --no-build-isolation
|
||||
pip install transformers==4.42.3
|
||||
```
|
||||
|
||||
### Evaluation
|
||||
You can evaluate Qwen2.5/Qwen2-Math-Instruct series model with the following command:
|
||||
```bash
|
||||
# Qwen2.5-Math-Instruct Series
|
||||
PROMPT_TYPE="qwen25-math-cot"
|
||||
# Qwen2.5-Math-1.5B-Instruct
|
||||
export CUDA_VISIBLE_DEVICES="0"
|
||||
MODEL_NAME_OR_PATH="Qwen/Qwen2.5-Math-1.5B-Instruct"
|
||||
bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
|
||||
|
||||
# Qwen2.5-Math-7B-Instruct
|
||||
export CUDA_VISIBLE_DEVICES="0"
|
||||
MODEL_NAME_OR_PATH="Qwen/Qwen2.5-Math-7B-Instruct"
|
||||
bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
|
||||
|
||||
# Qwen2.5-Math-72B-Instruct
|
||||
export CUDA_VISIBLE_DEVICES="0,1,2,3"
|
||||
MODEL_NAME_OR_PATH="Qwen/Qwen2.5-Math-72B-Instruct"
|
||||
bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
|
||||
|
||||
|
||||
# Qwen2-Math-Instruct Series
|
||||
PROMPT_TYPE="qwen-boxed"
|
||||
# Qwen2-Math-1.5B-Instruct
|
||||
export CUDA_VISIBLE_DEVICES="0"
|
||||
MODEL_NAME_OR_PATH="Qwen/Qwen2-Math-1.5B-Instruct"
|
||||
bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
|
||||
|
||||
# Qwen2-Math-7B-Instruct
|
||||
export CUDA_VISIBLE_DEVICES="0"
|
||||
MODEL_NAME_OR_PATH="Qwen/Qwen2-Math-7B-Instruct"
|
||||
bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
|
||||
|
||||
# Qwen2-Math-72B-Instruct
|
||||
export CUDA_VISIBLE_DEVICES="0,1,2,3"
|
||||
MODEL_NAME_OR_PATH="Qwen/Qwen2-Math-72B-Instruct"
|
||||
bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
|
||||
```
|
||||
|
||||
## Acknowledgement
|
||||
The codebase is adapted from [math-evaluation-harness](https://github.com/ZubinGou/math-evaluation-harness).
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import datasets
|
||||
from datasets import load_dataset, Dataset, concatenate_datasets
|
||||
from utils import load_jsonl, lower_keys
|
||||
|
||||
|
||||
def load_data(data_name, split, data_dir="./data"):
|
||||
data_file = f"{data_dir}/{data_name}/{split}.jsonl"
|
||||
if os.path.exists(data_file):
|
||||
examples = list(load_jsonl(data_file))
|
||||
else:
|
||||
if data_name == "math":
|
||||
dataset = load_dataset(
|
||||
"competition_math",
|
||||
split=split,
|
||||
name="main",
|
||||
cache_dir=f"{data_dir}/temp",
|
||||
)
|
||||
elif data_name == "gsm8k":
|
||||
dataset = load_dataset(data_name, split=split)
|
||||
elif data_name == "svamp":
|
||||
# evaluate on training set + test set
|
||||
dataset = load_dataset("ChilleD/SVAMP", split="train")
|
||||
dataset = concatenate_datasets(
|
||||
[dataset, load_dataset("ChilleD/SVAMP", split="test")]
|
||||
)
|
||||
elif data_name == "asdiv":
|
||||
dataset = load_dataset("EleutherAI/asdiv", split="validation")
|
||||
dataset = dataset.filter(
|
||||
lambda x: ";" not in x["answer"]
|
||||
) # remove multi-answer examples
|
||||
elif data_name == "mawps":
|
||||
examples = []
|
||||
# four sub-tasks
|
||||
for data_name in ["singleeq", "singleop", "addsub", "multiarith"]:
|
||||
sub_examples = list(load_jsonl(f"{data_dir}/mawps/{data_name}.jsonl"))
|
||||
for example in sub_examples:
|
||||
example["type"] = data_name
|
||||
examples.extend(sub_examples)
|
||||
dataset = Dataset.from_list(examples)
|
||||
elif data_name == "mmlu_stem":
|
||||
dataset = load_dataset("hails/mmlu_no_train", "all", split="test")
|
||||
# only keep stem subjects
|
||||
stem_subjects = [
|
||||
"abstract_algebra",
|
||||
"astronomy",
|
||||
"college_biology",
|
||||
"college_chemistry",
|
||||
"college_computer_science",
|
||||
"college_mathematics",
|
||||
"college_physics",
|
||||
"computer_security",
|
||||
"conceptual_physics",
|
||||
"electrical_engineering",
|
||||
"elementary_mathematics",
|
||||
"high_school_biology",
|
||||
"high_school_chemistry",
|
||||
"high_school_computer_science",
|
||||
"high_school_mathematics",
|
||||
"high_school_physics",
|
||||
"high_school_statistics",
|
||||
"machine_learning",
|
||||
]
|
||||
dataset = dataset.rename_column("subject", "type")
|
||||
dataset = dataset.filter(lambda x: x["type"] in stem_subjects)
|
||||
elif data_name == "carp_en":
|
||||
dataset = load_jsonl(f"{data_dir}/carp_en/test.jsonl")
|
||||
else:
|
||||
raise NotImplementedError(data_name)
|
||||
|
||||
examples = list(dataset)
|
||||
examples = [lower_keys(example) for example in examples]
|
||||
dataset = Dataset.from_list(examples)
|
||||
os.makedirs(f"{data_dir}/{data_name}", exist_ok=True)
|
||||
dataset.to_json(data_file)
|
||||
|
||||
# add 'idx' in the first column
|
||||
if "idx" not in examples[0]:
|
||||
examples = [{"idx": i, **example} for i, example in enumerate(examples)]
|
||||
|
||||
# dedepulicate & sort
|
||||
examples = sorted(examples, key=lambda x: x["idx"])
|
||||
return examples
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
import argparse
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
from pebble import ProcessPool
|
||||
from concurrent.futures import TimeoutError
|
||||
|
||||
from grader import *
|
||||
|
||||
from parser import *
|
||||
from utils import load_jsonl
|
||||
from python_executor import PythonExecutor
|
||||
|
||||
|
||||
def evaluate(data_name, prompt_type, samples: list=None, file_path: str=None, max_num_samples=None, execute=False):
|
||||
assert samples or file_path, "samples or file_path must be provided"
|
||||
if not samples:
|
||||
samples = list(load_jsonl(file_path))
|
||||
if 'idx' in samples[0]:
|
||||
samples = {sample['idx']: sample for sample in samples}.values()
|
||||
samples = sorted(samples, key=lambda x: x['idx'])
|
||||
else:
|
||||
samples = [dict(idx=idx, **sample) for idx, sample in enumerate(samples)]
|
||||
|
||||
if max_num_samples:
|
||||
print(f"max_num_samples: {max_num_samples} / {len(samples)}")
|
||||
samples = samples[:max_num_samples]
|
||||
|
||||
# parse gt
|
||||
for sample in samples:
|
||||
sample['gt_cot'], sample['gt'] = parse_ground_truth(sample, data_name)
|
||||
params = [(idx, pred, sample['gt']) for idx, sample in enumerate(samples) for pred in sample['pred']]
|
||||
|
||||
scores = []
|
||||
timeout_cnt = 0
|
||||
|
||||
with ProcessPool(max_workers=1) as pool:
|
||||
future = pool.map(math_equal_process, params, timeout=3)
|
||||
iterator = future.result()
|
||||
with tqdm(total=len(samples), desc="Evaluate") as progress_bar:
|
||||
while True:
|
||||
try:
|
||||
result = next(iterator)
|
||||
scores.append(result)
|
||||
except StopIteration:
|
||||
break
|
||||
except TimeoutError as error:
|
||||
print(error)
|
||||
scores.append(False)
|
||||
timeout_cnt += 1
|
||||
except Exception as error:
|
||||
print(error.traceback)
|
||||
exit()
|
||||
progress_bar.update(1)
|
||||
|
||||
idx = 0
|
||||
score_mat = []
|
||||
for sample in samples:
|
||||
sample['score'] = scores[idx: idx+len(sample['pred'])]
|
||||
assert len(sample['score']) == len(sample['pred'])
|
||||
score_mat.append(sample['score'])
|
||||
idx += len(sample['pred'])
|
||||
|
||||
max_len = max([len(s) for s in score_mat])
|
||||
|
||||
for i, s in enumerate(score_mat):
|
||||
if len(s) < max_len:
|
||||
score_mat[i] = s + [s[-1]] * (max_len - len(s)) # pad
|
||||
|
||||
# output mean of each column of scores
|
||||
col_means= np.array(score_mat).mean(axis=0)
|
||||
mean_score = list(np.round(col_means * 100, decimals=1))
|
||||
|
||||
result_json = {
|
||||
"num_samples": len(samples),
|
||||
"num_scores": len(scores),
|
||||
"timeout_samples": timeout_cnt,
|
||||
"empty_samples": len([s for s in samples if not s['pred'][-1]]),
|
||||
"acc": mean_score[0]
|
||||
}
|
||||
|
||||
# each type score
|
||||
if "type" in samples[0]:
|
||||
type_scores = {}
|
||||
for sample in samples:
|
||||
if sample['type'] not in type_scores:
|
||||
type_scores[sample['type']] = []
|
||||
type_scores[sample['type']].append(sample['score'][-1])
|
||||
type_scores = {k: np.round(np.array(v).mean() * 100, decimals=1) for k, v in type_scores.items()}
|
||||
type_scores = {k: v for k, v in sorted(type_scores.items(), key=lambda item: item[0])}
|
||||
result_json['type_acc'] = type_scores
|
||||
|
||||
print(result_json)
|
||||
return samples, result_json
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_name", type=str, default="math")
|
||||
parser.add_argument("--prompt_type", type=str, default="tool-integrated")
|
||||
parser.add_argument("--file_path", type=str, default=None, required=True)
|
||||
parser.add_argument("--max_num_samples", type=int, default=None)
|
||||
parser.add_argument("--execute", action="store_true")
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
evaluate(data_name=args.data_name, prompt_type=args.prompt_type, file_path=args.file_path,
|
||||
max_num_samples=args.max_num_samples, execute=args.execute)
|
||||
Executable
+378
@@ -0,0 +1,378 @@
|
||||
import json
|
||||
|
||||
|
||||
def get_examples():
|
||||
examples = {}
|
||||
examples["gsm8k"] = [
|
||||
(
|
||||
"There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?",
|
||||
"There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have been 21 - 15 = 6. The answer is 6.",
|
||||
),
|
||||
(
|
||||
"If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?",
|
||||
"There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.",
|
||||
),
|
||||
(
|
||||
"Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?",
|
||||
"Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The answer is 39.",
|
||||
),
|
||||
(
|
||||
"Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?",
|
||||
"Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8. The answer is 8.",
|
||||
),
|
||||
(
|
||||
"Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?",
|
||||
"Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9. The answer is 9.",
|
||||
),
|
||||
(
|
||||
"There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?",
|
||||
"There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The answer is 29.",
|
||||
),
|
||||
(
|
||||
"Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?",
|
||||
"Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The answer is 33.",
|
||||
),
|
||||
(
|
||||
"Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
|
||||
"Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The answer is 8.",
|
||||
),
|
||||
]
|
||||
examples["gsm8k-pal"] = [
|
||||
(
|
||||
"Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
|
||||
'```python\ndef solution():\n """Olivia has $23. She bought five bagels for $3 each. How much money does she have left?"""\n money_initial = 23\n bagels = 5\n bagel_cost = 3\n money_spent = bagels * bagel_cost\n money_left = money_initial - money_spent\n result = money_left\n return result\n```',
|
||||
),
|
||||
(
|
||||
"Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?",
|
||||
'```python\ndef solution():\n """Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?"""\n golf_balls_initial = 58\n golf_balls_lost_tuesday = 23\n golf_balls_lost_wednesday = 2\n golf_balls_left = golf_balls_initial - golf_balls_lost_tuesday - golf_balls_lost_wednesday\n result = golf_balls_left\n return result\n```',
|
||||
),
|
||||
(
|
||||
"There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?",
|
||||
'```python\ndef solution():\n """There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?"""\n computers_initial = 9\n computers_per_day = 5\n num_days = 4 # 4 days between monday and thursday\n computers_added = computers_per_day * num_days\n computers_total = computers_initial + computers_added\n result = computers_total\n return result\n```',
|
||||
),
|
||||
]
|
||||
examples["gsm8k-tora"] = [
|
||||
(
|
||||
"Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
|
||||
"```python\ndef money_left():\n money_initial = 23\n bagels = 5\n bagel_cost = 3\n money_spent = bagels * bagel_cost\n remaining_money = money_initial - money_spent\n return remaining_money\n \nremaining_money = money_left()\nprint(remaining_money)\n```\n```output\n8\n```\nOlivia has $\\boxed{8}$ dollars left.",
|
||||
),
|
||||
(
|
||||
"Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?",
|
||||
"```python\ndef remaining_golf_balls():\n golf_balls_initial = 58\n golf_balls_lost_tuesday = 23\n golf_balls_lost_wednesday = 2\n golf_balls_left = golf_balls_initial - golf_balls_lost_tuesday - golf_balls_lost_wednesday\n remaining_golf_balls = golf_balls_left\n return remaining_golf_balls\n\nanswer = remaining_golf_balls() \nprint(answer)\n```\n```output\n33\n```\nMichael had $\\boxed{33}$ golf balls at the end of Wednesday.",
|
||||
),
|
||||
(
|
||||
"There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?",
|
||||
"```python\ndef total_computers():\n computers_initial = 9\n computers_per_day = 5\n num_days = 4 # 4 days between monday and thursday\n computers_added = computers_per_day * num_days\n computers_total = computers_initial + computers_added\n return computers_total\n\ntotal_computers = total_computers()\nprint(total_computers)\n```\n```output\n29\n```\nThere're $\\boxed{29}$ computers in the server room.",
|
||||
),
|
||||
]
|
||||
|
||||
examples["math"] = [
|
||||
(
|
||||
"Kevin Kangaroo begins hopping on a number line at 0. He wants to get to 1, but he can hop only $\\frac{1}{3}$ of the distance. Each hop tires him out so that he continues to hop $\\frac{1}{3}$ of the remaining distance. How far has he hopped after five hops? Express your answer as a common fraction.",
|
||||
"Let's think step by step\nKevin hops $1/3$ of the remaining distance with every hop.\nHis first hop takes $1/3$ closer.\nFor his second hop, he has $2/3$ left to travel, so he hops forward $(2/3)(1/3)$.\nFor his third hop, he has $(2/3)^2$ left to travel, so he hops forward $(2/3)^2(1/3)$.\nIn general, Kevin hops forward $(2/3)^{k-1}(1/3)$ on his $k$th hop.\nWe want to find how far he has hopped after five hops.\nThis is a finite geometric series with first term $1/3$, common ratio $2/3$, and five terms.\nThus, Kevin has hopped $\\frac{\\frac{1}{3}\\left(1-\\left(\\frac{2}{3}\\right)^5\\right)}{1-\\frac{2}{3}} = \\boxed{\\frac{211}{243}}$.\nThe answer is \\frac{211}{243}}",
|
||||
),
|
||||
(
|
||||
"What is the area of the region defined by the equation $x^2+y^2 - 7 = 4y-14x+3$?",
|
||||
"Let's think step by step\nWe rewrite the equation as $x^2 + 14x + y^2 - 4y = 10$ and then complete the square,\nresulting in $(x+7)^2-49 + (y-2)^2-4=10$,\nor $(x+7)^2+(y-2)^2=63$.\nThis is the equation of a circle with center $(-7, 2)$ and radius $\\sqrt{63},$\nso the area of this region is $\\pi r^2 = \\boxed{63\\pi}$.\nThe answer is 63\\pi",
|
||||
),
|
||||
(
|
||||
"If $x^2+y^2=1$, what is the largest possible value of $|x|+|y|$?",
|
||||
"Let's think step by step\nIf $(x,y)$ lies on the circle,\nso does $(x,-y),$ $(-x,-y),$ and $(-x,-y),$ (which all give the same value of $|x| + |y|$),\nso we can assume that $x \\ge 0$ and $y \\ge 0.$\nThen $|x| + |y| = x + y.$ Squaring, we get\n\\[(x + y)^2 = x^2 + 2xy + y^2 = 1 + 2xy.\\]\nNote that $(x - y)^2 \\ge 0.$\nExpanding, we get $x^2 - 2xy + y^2 \\ge 0,$ so $2xy \\le x^2 + y^2 = 1.$\nHence,\\[1 + 2xy \\le 2,\\]which means $x + y \\le \\sqrt{2}.$\nEquality occurs when $x = y = \\frac{1}{\\sqrt{2}},$\nso the maximum value of $|x| + |y|$ is $\\boxed{\\sqrt{2}}.$\nThe answer is \\sqrt{2}",
|
||||
),
|
||||
(
|
||||
"If $f(x)=\\frac{ax+b}{cx+d}, abcd\\not=0$ and $f(f(x))=x$ for all $x$ in the domain of $f$, what is the value of $a+d$?",
|
||||
"Let's think step by step\nThe condition $f(f(x))$ means that $f$ is the inverse of itself,\nso its graph is symmetrical about the line $y = x$.\nWith a rational function of this form, we will have two asymptotes:\na vertical one at $x=-d/c$ if $cx+d$ does not divide $ax+b$,\nand a horizontal one at $y=a/c$,\nif we take the limit of $f(x)$ as $x$ goes to $\\pm\\infty$.\nIn order for $f$ to be its own inverse, the intersection of the asymptotes must lie on the line $y=x$\nso that it and its asymptotes reflect onto themselves.\nThis means that $-d/c=a/c$,\nand therefore $-d=a$ and $a+d=\\boxed{0}$.\nThe answer is 0",
|
||||
),
|
||||
(
|
||||
"Expand $(2z^2 + 5z - 6)(3z^3 - 2z + 1)$.",
|
||||
"Let's think step by step\n$$\\begin{array}{crrrrrrr}\n& & & 3z^3 & & -2z & + 1 & \\\\\n\\times & & & & 2z^2 & +5z & -6 \\\\\n\\cline{1-7}\\rule{0pt}{0.17in}\n& & & -18z^3 & & +12z & -6 & \\\\\n& & +15z^4 & & -10z^2 & +5z & & \\\\\n+ & 6z^5 & & -4z^3 & +2z^2 & & & \\\\\n\\cline{1-7}\\rule{0pt}{0.17in}\n& 6z^5 & +15z^4 & -22z^3 & - 8z^2 &+17z & -6 &\n\\end{array}$$\nThe answer is 6z^5+15z^4-22z^3-8z^2+17z-6",
|
||||
),
|
||||
]
|
||||
|
||||
examples["math_pal"] = [
|
||||
(
|
||||
"Display the final result in LaTeX.\n\n Find the coefficient of $x^3$ when $3(x^2 - x^3+x) +3(x +2x^3- 3x^2 + 3x^5+x^3) -5(1+x-4x^3 - x^2)$ is simplifie.",
|
||||
"```python\nfrom sympy import symbols, simplify\n\ndef solution():\n x = symbols('x')\n expr = 3*(x**2 - x**3 + x) + 3*(x + 2*x**3 - 3*x**2 + 3*x**5 + x**3) - 5*(1 + x - 4*x**3 - x**2)\n simplified_expr = simplify(expr)\n\n x3_coefficient = simplified_expr.as_coefficients_dict()[x**3]\n result = x3_coefficient\n return result\n```",
|
||||
),
|
||||
(
|
||||
"The surface area of a sphere with radius $r$ is $4\\pi r^2$. Including the area of its circular base, what is the total surface area of a hemisphere with radius 6 cm? Express your answer in terms of $\\pi$.",
|
||||
"```python\nimport math\n\ndef solution():\n radius = 6\n\n # Surface area of the hemisphere\n hemisphere_area = 2 * math.pi * radius**2\n\n # Area of the circular base\n base_area = math.pi * radius**2\n\n # Total surface area\n total_surface_area = hemisphere_area + base_area\n\n # Formatting the result in LaTeX\n result = r'{}\\\\pi'.format(total_surface_area / math.pi)\n return result\n```",
|
||||
),
|
||||
(
|
||||
"Monica tosses a fair 6-sided die. If the roll is a prime number, then she wins that amount of dollars (so that, for example, if she rolls 3, then she wins 3 dollars). If the roll is composite, she wins nothing. Otherwise, she loses 3 dollars. What is the expected value of her winnings on one die toss? Express your answer as a dollar value to the nearest cent.",
|
||||
'```python\ndef solution():\n # Probabilities of each outcome\n prime_prob = 1 / 6\n composite_prob = 1 / 3\n otherwise_prob = 1 / 6\n\n # Expected value of each outcome\n prime_expected_value = (2 * prime_prob) + (3 * prime_prob) + (5 * prime_prob)\n composite_expected_value = 0 * composite_prob\n otherwise_expected_value = -3 * otherwise_prob\n\n # Total expected value\n total_expected_value = prime_expected_value + composite_expected_value + otherwise_expected_value\n\n # Dollar value to the nearest cent\n result = "{:.2f}".format(total_expected_value)\n return result\n```',
|
||||
),
|
||||
(
|
||||
"Given $\\mathbf{a} = \\begin{pmatrix} -7 \\\\ 0 \\\\ 1 \\end{pmatrix}$ and $\\mathbf{b} = \\begin{pmatrix} 4 \\\\ 2 \\\\ -1 \\end{pmatrix},$ find $\\mathbf{a} - 3 \\mathbf{b}.$",
|
||||
"```python\nimport numpy as np\n\ndef solution()\n a = np.array([-7, 0, 1])\n b = np.array([4, 2, -1])\n\n result = a - 3 * b\n\n result = r'\\begin{{pmatrix}} {} \\\\ {} \\\\ {} \\end{{pmatrix}}'.format(result[0], result[1], result[2])\n return result\n```",
|
||||
),
|
||||
]
|
||||
examples["math-tora"] = [
|
||||
(
|
||||
"Convert the point $(0, -3 \\sqrt{3}, 3)$ in rectangular coordinates to spherical coordinates. Enter your answer in the form $(\\rho,\\theta,\\phi),$ where $\\rho > 0,$ $0 \\le \\theta < 2 \\pi,$ and $0 \\le \\phi \\le \\pi.$",
|
||||
"To convert the point $(x, y, z)$ in rectangular coordinates to spherical coordinates $(\\rho, \\theta, \\phi)$, we use the following formulas:\n1. $\\rho = \\sqrt{x^2 + y^2 + z^2}$\n2. $\\theta = \\arctan{\\frac{y}{x}}$\n3. $\\phi = \\arccos{\\frac{z}{\\rho}}$\n\nLet's write a function:\n\n```python\nfrom sympy import sqrt, atan2, acos, pi\n\ndef rectangular_to_spherical():\n x, y, z = 0, -3*sqrt(3), 3\n rho = sqrt(x**2 + y**2 + z**2)\n theta = atan2(y, x)\n phi = acos(z/rho)\n return rho, theta, phi\n\nspherical_coordinates = rectangular_to_spherical()\nprint(spherical_coordinates)\n```\n```output\n(6, -pi/2, pi/3)\n```\n\nThe required form of the answer is $(\\rho, \\theta, \\phi)$, where $\\rho > 0$, $0 \\le \\theta < 2\\pi$, and $0 \\le \\phi \\le \\pi$. Since our initial output has $\\theta = -\\pi/2$, which doesn't meet the condition for $\\theta$, we add $2\\pi$ to adjust its value.\n\nHence, the spherical coordinates for the point $(0, -3\\sqrt{3}, 3)$ are $\\boxed{(6, \\frac{3\\pi}{2}, \\frac{\\pi}{3})}$.",
|
||||
),
|
||||
(
|
||||
"Compute $1011_2 + 101_2 - 1100_2 + 1101_2$. Express your answer in base 2.",
|
||||
'Let\'s write a single python function for binary computation:\n\n```python\ndef binary_sum_diff():\n # Convert binary to decimal\n num1 = int("1011", 2)\n num2 = int("101", 2)\n num3 = int("1100", 2)\n num4 = int("1101", 2)\n\n # Perform the operations\n result = num1 + num2 - num3 + num4\n\n # Convert the result back to binary\n result_binary = format(result, "b")\n\n return result_binary\n\nresult = binary_sum_diff()\nprint(result)\n```\n```output\n10001\n```\nThe answer expressed in base 2 is $\\boxed{10001_2}$.',
|
||||
),
|
||||
(
|
||||
"Find the matrix that corresponds to projecting onto the vector $\\begin{pmatrix} 2 \\\\ -3 \\end{pmatrix}.$",
|
||||
"```python\nfrom sympy import Matrix, simplify\n\ndef projection_matrix():\n v = Matrix([2, -3])\n\n # Calculate the projection matrix\n P = (v * v.T) / (v.T * v)[0]\n\n # Simplify the elements\n P = P.applyfunc(simplify)\n\n return P\n\nresult = projection_matrix()\nprint(result)\n```\n```output\nMatrix([[4/13, -6/13], [-6/13, 9/13]])\n```\n\nThe matrix is $\\boxed{\\begin{pmatrix} \\frac{4}{13} & -\\frac{6}{13} \\ -\\frac{6}{13} & \\frac{9}{13} \\end{pmatrix}}.$",
|
||||
),
|
||||
(
|
||||
"Solve the inequality\n\\[\\frac{x^2 - 25}{x + 5} < 0.\\]",
|
||||
"Let's use `sympy` to solve the inequality:\n\n```python\nfrom sympy import symbols, simplify\nfrom sympy.solvers.inequalities import solve_univariate_inequality\nfrom sympy.core.relational import StrictLessThan\n\ndef solve_inequality():\n x = symbols('x')\n expression = (x**2 - 25) / (x + 5)\n inequality = StrictLessThan(expression, 0)\n solution = solve_univariate_inequality(inequality, x, relational=False)\n simplified_solution = simplify(solution)\n \n return simplified_solution\n\nresult = solve_inequality()\nprint(result)\n```\n```output\nUnion(Interval.open(-oo, -5), Interval.Lopen(-5, 5))\n```\nThe answer is $\\boxed{(-\\infty,-5)\\cup(-5,5)}$",
|
||||
),
|
||||
(
|
||||
'In the figure, triangles $ABC$ and $BCD$ are equilateral triangles. What is the value of $AD \\div BC$ when expressed in simplest radical form?\n\n[asy]\ndraw((0,0)--(5,8.7)--(10,0)--cycle);\ndraw((10,0)--(15,8.7)--(5,8.7));\nlabel("$A$",(0,0),SW);\nlabel("$B$",(5,8.7),N);\nlabel("$C$",(10,0),SE);\nlabel("$D$",(15,8.7),NE);\n[/asy]',
|
||||
"```python\nfrom sympy import Rational, sqrt, simplify\n\ndef ad_divided_by_bc():\n\n x = Rational(1, 1) # Side length of equilateral triangles\n\n ad_squared = 2 * x**2 * (1 + Rational(1, 2)) # Using the law of cosines with cos(2*pi/3) = -1/2\n ad = sqrt(ad_squared)\n\n bc = x # BC is the side length of the equilateral triangles\n\n simplified_ratio = simplify(ad / bc)\n\n return simplified_ratio\n\nresult = ad_divided_by_bc()\nprint(result)\n```\n```output\nsqrt(3)\n```\nThe value of $AD \\div BC$ is $\\boxed{\\sqrt{3}}$.",
|
||||
),
|
||||
]
|
||||
examples["carp_en"] = [
|
||||
(
|
||||
"If $3 a ^ { m + 2 } b$ and $\\frac { 1 } { 2 } ab ^ { n - 1 }$ are similar terms, then $m + n$ is equal to.",
|
||||
"Let's think step by step\n$3 a ^ { m + 2 } b$ and $\\frac { 1 } { 2 } ab ^ { n - 1 }$ are like terms. We can obtain $m + 2 = 1$ and $n - 1 = 1$. Solving for $m$ and $n$, we get $m = - 1$ and $n = 2$. Therefore, $m + n = - 1 + 2 = 1$.\nThe answer is: 1",
|
||||
),
|
||||
(
|
||||
"The solution to the equation $y - \\frac { y - 1 } { 2 } = - \\frac { y + 2 } { 5 }$ is ____ ?",
|
||||
"Let's think step by step\nTo eliminate the denominator, we have $10 y - 5 ( y - 1 ) = - 2 ( y + 2 )$. Expanding the brackets gives $10 y - 5 y + 5 = - 2 y - 4$. Rearranging terms gives $10 y - 5 y + 2 y = - 4 - 5$, which simplifies to $7 y = - 9$. Dividing both sides by 7 gives $y = - \\frac { 9 } { 7 }$.\nThe answer is: y = - \\frac { 9 } { 7 }",
|
||||
),
|
||||
(
|
||||
"If $( m + 4 ) ^ 2 + | n - 3 | = 0$, then $\\frac { 1 } { 2 } m - n$ = ____?",
|
||||
"Let's think step by step\n$\\because ( m + 4 ) ^ 2 + | n - 3 | = 0$, $\\therefore m + 4 = 0$, $n - 3 = 0$, which means $m = - 4$, $n = 3$. Then the original expression equals $- 2 - 3 = - 5$.\nThe answer is: - 5",
|
||||
),
|
||||
(
|
||||
"Given a quadratic equation in one variable $x$, $x ^ 2 + x + m = 0$, with one root being $x = 1$, what is the other root of this equation?",
|
||||
"Let's think step by step\nSuppose the quadratic equation in one variable about $x$ is $x ^ 2 + x + m = 0$, and $\\alpha$ is another real root of the equation. Since one real root of the quadratic equation in one variable about $x$ is $1$, we have $\\alpha + 1 = - 1$. Therefore, $\\alpha = - 2$.\nThe answer is: - 2",
|
||||
),
|
||||
(
|
||||
"The parabola $y = - 5 { x } ^ 2 + 1$ is translated $2$ units upward and $1$ unit to the left, resulting in the parabola _____.",
|
||||
"Let's think step by step\nThe parabola $y = - 5 { x } ^ 2 + 1$ is first shifted upward by 2 units, resulting in $y = - 5 { x } ^ 2 + 3$. Then it is shifted left by 1 unit, resulting in $y = - 5 {( x + 1 )} ^ 2 + 3$.\nThe answer is: y = - 5 ( x + 1 ) ^ { 2 } + 3",
|
||||
),
|
||||
(
|
||||
"If the radical $\\sqrt { x - 8 }$ is defined, then the range of real numbers for $x$ is ____ ?",
|
||||
"Let's think step by step\nSince the radical $\\sqrt { x - 8 }$ is defined, therefore $x - 8 \\ge 0$, which implies $x \\ge 8$.\nThe answer is: x \\ge 8",
|
||||
),
|
||||
(
|
||||
"If $a ^ { m } \\times a ^ { 2 } = a ^ { 7 }$, then the value of $m$ is ____?",
|
||||
"Let's think step by step\nAccording to the multiplication rule of powers with the same base: when multiplying powers with the same base, keep the base the same and add the exponents. We have $m + 2 = 7$, so solving for $m$ gives $m = 5$.\nThe answer is: 5",
|
||||
),
|
||||
(
|
||||
"If line segment $a$ and $b$ satisfy $\\frac { a } { b } = \\frac { 5 } { 2 }$, then the value of $\\frac { a - b } { b }$ is ____?",
|
||||
"Let's think step by step\n$\\because \\frac { a } { b } = \\frac { 5 } { 2 }$, $\\therefore$ we can assume $a = 5 k$, then $b = 2 k$, $\\therefore \\frac { a - b } { b } = \\frac { 5 k - 2 k } { 2 k } = \\frac { 3 } { 2 }$.\nThe answer is: \\frac { 3 } { 2 }",
|
||||
),
|
||||
]
|
||||
|
||||
examples["minerva_math"] = [
|
||||
(
|
||||
"Find the domain of the expression $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}",
|
||||
"The expressions inside each square root must be non-negative.\nTherefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$.\nAlso, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$.\nTherefore, the domain of the expression is $\\boxed{[2,5)}$.",
|
||||
),
|
||||
(
|
||||
"If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$",
|
||||
"We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$",
|
||||
),
|
||||
(
|
||||
"Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?",
|
||||
"If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{align*}\n30n&=480\\\\\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}",
|
||||
),
|
||||
(
|
||||
"If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\\\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{a}{b},$ assuming $b$ is nonzero.",
|
||||
"If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$",
|
||||
),
|
||||
]
|
||||
|
||||
examples["aqua"] = [
|
||||
(
|
||||
"John found that the average of 15 numbers is 40. If 10 is added to each number then the mean of the numbers is?\nAnswer Choices: (A) 50 (B) 45 (C) 65 (D) 78 (E) 64",
|
||||
"If 10 is added to each number, then the mean of the numbers also increases by 10. So the new mean would be 50. The answer is (A).",
|
||||
),
|
||||
(
|
||||
"If a / b = 3/4 and 8a + 5b = 22,then find the value of a.\nAnswer Choices: (A) 1/2 (B) 3/2 (C) 5/2 (D) 4/2 (E) 7/2",
|
||||
"a / b = 3/4, then b = 4a / 3. So 8a + 5(4a / 3) = 22. This simplifies to 8a + 20a / 3 = 22, which means 44a / 3 = 22. So a is equal to 3/2. The answer is (B).",
|
||||
),
|
||||
(
|
||||
"A person is traveling at 20 km/hr and reached his destiny in 2.5 hr then find the distance?\nAnswer Choices: (A) 53 km (B) 55 km (C) 52 km (D) 60 km (E) 50 km",
|
||||
"The distance that the person traveled would have been 20 km/hr * 2.5 hrs = 50 km. The answer is (E).",
|
||||
),
|
||||
(
|
||||
"How many keystrokes are needed to type the numbers from 1 to 500?\nAnswer Choices: (A) 1156 (B) 1392 (C) 1480 (D) 1562 (E) 1788",
|
||||
"There are 9 one-digit numbers from 1 to 9. There are 90 two-digit numbers from 10 to 99. There are 401 three-digit numbers from 100 to 500. 9 + 90(2) + 401(3) = 1392. The answer is (B).",
|
||||
),
|
||||
]
|
||||
examples["sat_math"] = [
|
||||
(
|
||||
"If $\frac{x-1}{3}=k$ and $k=3$, what is the value of $x$ ? \nAnswer Choices: (A) 2 (B) 4 (C) 9 (D) 10",
|
||||
"If k = 3, then x - 1 = 3 * 3, therfore, x - 1 = 9 and x = 10. The answer is D",
|
||||
),
|
||||
(
|
||||
"For $i=\\sqrt{-1}$, what is the sum $(7+3 i)+(-8+9 i)$ ? \nAnswer Choices: (A) $-1+12 i$ (B) $-1-6 i$ (C) $15+12 i$ (D) $15-6 i$ 3",
|
||||
"For (7+3 i)+(-8+9 i), the real part is 7 + (-8) = -1, the imageinary part is 3 i + 9 i = 12 i. The answer is A",
|
||||
),
|
||||
(
|
||||
"On Saturday afternoon, Armand sent $m$ text messages each hour for 5 hours, and Tyrone sent $p$ text messages each hour for 4 hours. Which of the following represents the total number of messages sent by Armand and Tyrone on Saturday afternoon?\nAnswer Choices: (A) $9 m p$ (B) $20 m p$ (C) $5 m+4 p$ (D) $4 m+5 p$",
|
||||
"Armand texts m messages each hour for 5 hours, which leads to 5m messages. Tyrone texts p messages each hour for 4 hours, which leds to 4p messages. The total is 5m + 4p. The answer is C.",
|
||||
),
|
||||
(
|
||||
"$$\begin{array}{r}3 x+4 y=-23 \\2 y-x=-19\\end{array}$$What is the solution $(x, y)$ to the system of equations above?\nAnswer Choices: (A) $(-5,-2)$ (B) $(3,-8)$ (C) $(4,-6)$ (D) $(9,-6)$",
|
||||
"By solving this equation, we found that x = 3 and y = -8. The answer is B.",
|
||||
),
|
||||
]
|
||||
examples["mmlu_mathematics"] = [
|
||||
(
|
||||
"Simplify and write the result with a rational denominator: $$\\sqrt{\\sqrt[3]{\\sqrt{\frac{1}{729}}}}$$\nAnswer Choices: (A) \\frac{3\\sqrt{3}}{3} (B) \\frac{1}{3} (C) \\sqrt{3} (D) \\frac{\\sqrt{3}}{3}",
|
||||
"Factoring $729=3^6$ and combining the roots $\frac{1}{2}\frac{1}{3}\frac{1}{2}=\frac{1}{12}$, we get that $\\sqrt{\\sqrt[3]{\\sqrt{\frac{1}{729}}}}=\\left(\frac{1}{3^6}\right)^{\frac{1}{12}}=\frac{1}{3^{\frac{1}{2}}}=\frac{3}{\\sqrt{3}}$. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"Five thousand dollars compounded annually at an $x\\%$ interest rate takes six years to double. At the same interest rate, how many years will it take $\\$300$ to grow to $\\$9600$?\nAnswer Choices:(A) 12 (B) 1 (C) 30 (D) 5",
|
||||
"To go from $\\$300$ to $\\$9600$, the value must go up by a factor of $9600/300=32=2^5$. Since at this interest rate it takes six years for it to double, it will take $5*6=30$ years to grow to $\\$9600$. The answer is (C).",
|
||||
),
|
||||
(
|
||||
"Ten students take a biology test and receive the following scores: 45, 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students’ test scores?\nAnswer Choices: (A) 55 (B) 60 (C) 62 (D) 65",
|
||||
"There are 10 students and the sum of their scores is $45 + 55 + 50 + 70 + 65 + 80 + 40 + 90 + 70 + 85 = 650$, the mean is $650/10=65$. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"The variable $x$ varies directly as the square of $y$, and $y$ varies directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is the value of $x$ when $z$ equals $\frac{1}{2}$?\nAnswer Choices: (A) -1 (B) 16 (C) -\frac{1}{256} (D) \\frac{1}{16}",
|
||||
"We know that $x \\propto y^2$ and $y \\propto z^3$, so $x = k z^6$ for some constant $k$. Plugging in for $x=-16$ and $z=2$, the constant value is $k=\frac{x}{z^6}=\frac{-16}{64}=-\frac{1}{4}$. So, when $z=\frac{1}{2}$, the value of $x$ is $x=kz^6=-\frac{1}{4}\frac{1}{2^6}=-\frac{1}{256}$. The answer is (C).",
|
||||
),
|
||||
(
|
||||
"Joe was in charge of lights for a dance. The red light blinks every two seconds, the yellow light every three seconds, and the blue light every five seconds. If we include the very beginning and very end of the dance, how many times during a seven minute dance will all the lights come on at the same time? (Assume that all three lights blink simultaneously at the very beginning of the dance.)\nAnswer Choices: (A) 3 (B) 15 (C) 6 (D) 5",
|
||||
"The least common multiple of 2, 3 and 5 is 30, so during a 7 minute dance, all the three lights will come on at the same time $2*7+1=15$ times. The answer is (B).",
|
||||
),
|
||||
]
|
||||
examples["mmlu_physics"] = [
|
||||
(
|
||||
"A microwave oven is connected to an outlet, 120 V, and draws a current of 2 amps. At what rate is energy being used by the microwave oven?\nAnswer Choices: (A) 10 W (B) 30 W (C) 60 W (D) 240 W",
|
||||
"Rate of energy usage is known as power; in an dissipative electrical circuit, power is given by voltage times current. So in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"A point charge, Q = +1 mC, is fixed at the origin. How much work is required to move a charge, Q = +8 µC, from the point (0, 4 meters) to the point (3 meters, 0)?\nAnswer Choices: (A) 3.5 J (B) 6.0 J (C) 22.5 J (D) 40 J",
|
||||
"To calculate the work required to move a charge from one location to another in a fixed electric field, it is enough to calculate the potential difference between the two locations. Here, the potential only depends on the distance between the charges; it’s $k q_1 q_2 / r$, where $k$ is Coulomb’s constant. Plugging in values $q_1 = $ 1 mC, $q_2 = 8 \\mu$ C, gives the answer as 5.992 J, which rounds to 6 J. The answer is (B).",
|
||||
),
|
||||
(
|
||||
"Which of the following conditions will ensure that angular momentum is conserved? I. Conservation of linear momentum II. Zero net external force III. Zero net external torque.\nAnswer Choices: (A) I and II only (B) I and III only (C) II and III only (D) III only",
|
||||
"Torque is defined as the change in angular momentum; if there is zero external torque, angular momentum is conserved. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"A photocell of work function ϕ = 2eV is connected to a resistor in series. Light of frequency f = 1 × 10^15 Hz hits a metal plate of the photocell. If the power of the light is P = 100 W, what is the current through the resistor?\nAnswer Choices: (A) 2:00 AM (B) 6:00 AM (C) 12:00 AM (D) 24 A",
|
||||
"The only answer above which has units of current is D, 24 A. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"A pipe full of air is closed at one end. A standing wave is produced in the pipe, causing the pipe to sound a note. Which of the following is a correct statement about the wave’s properties at the closed end of the pipe?\nAnswer Choices: (A) The pressure is at a node, but the particle displacement is at an antinode. (B) The pressure is at an antinode, but the particle displacement is at a node. (C) The pressure and the particle displacement are both at nodes. (D) The pressure and the particle displacement are both at antinodes.",
|
||||
"At the closed end of the pipe, the particles cannot have any net displacement because the pipe closure stops them. So the particle displacement is at a node. This closure also causes the pressure to be maximal, i.e. an antinode. The answer is (B).",
|
||||
),
|
||||
]
|
||||
examples["mmlu_chemistry"] = [
|
||||
(
|
||||
"Which of the following is considered an acid anhydride?\nAnswer Choices: (A) HCl (B) H2SO3 (C) SO2 (D) Al(NO3)3",
|
||||
"An acid anhydride is a compound that is derived by removing water from an acid. The chemical formula for water is H2O, which means that we need to determine which of these options, when combined with H2O, forms an acid. SO2, or Sulfur dioxide, when combined with H2O, makes H2SO4, or sulfuric acid. The answer is (C).",
|
||||
),
|
||||
(
|
||||
"Which of the following is expected to be a polar molecule?\nAnswer Choices: (A) PCl4F (B) BF3 (C) CO2 (D) Si(CH3)4",
|
||||
"A polar molecule is one that has a slightly positive charge on one end of the molecule and a slightly negative charge on the other end. Boron trifluoride (BF3) has Boron as the center atom and three fluorine atoms attached to it; it is trigonal planar and symmetric, so it is nonpolar. Carbon Dioxide (CO2) has Carbon as the central atom with double bonds to two Oxygen atoms - this is also symmetrical and therefore nonpolar. The same is the case for tetramethyl silane (SI(CH3)4), which is a Silicon atom surrounded by four methyl groups. The structure of PCL4F is that Phosphorus is the central atom, attached to four chlorines and one fluorine atom. This is asymmetrical, and therefore has a net dipole and is expected to be a polar molecule. The answer is (A).",
|
||||
),
|
||||
(
|
||||
"From the solubility rules, which of the following is true?\nAnswer Choices: (A) All chlorides, bromides, and iodides are soluble (B) All sulfates are soluble (C) All hydroxides are soluble (D) All ammonium-containing compounds are soluble",
|
||||
"The chlorides, bromides, and iodides of lead, silver, and mercury are not soluble in water. This rules out (A). The sulfates of lead, barium, and calcium are not soluble in water, which rules out (B). The hydroxides of any metal besides sodium, potassium, ammonium, calcium, and barium are insoluble. This rules out (C). Typically ammonium ions indicate a soluble ionic substance. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"A new compound is synthesized and found to be a monoprotic acid with a molar mass of 248 g/mol. When 0.0050 mol of this acid are dissolved in 0.500 L of water, the pH is measured as 3.89. What is the pKa of this acid?\nAnswer Choices: (A) 3.89 (B) 7.78 (C) 5.78 (D) 2.33",
|
||||
"Recall that $[A] = [H^{+}]$. Here, this is equal to $$10^{-3.89}$. Then we have $K_{a} = $frac{[H^{+}][A^{-}]}{[HA]} = \\frac{10^{-3.89} \\cdot 10^{-3.89}}{10^{-2}}. The resulting exponent is $-3.89 + (-3.89) - (-2) = 5.78$, therefore $K_a = 10^{-5.78}$. The $pK_a$ is the negative log of $K_a$, which is equal to $5.78$. The answer is (C).",
|
||||
),
|
||||
(
|
||||
"A solution contains 2.00 mole of acetic acid, CH3COOH, and 1.00 mole of calcium acetate, Ca(CH3COO)2. The solution is able to resist the addition of a small amount of strong acid or strong base with only minor changes in the pH of the solution. Larger quantities of strong acid or strong base can cause a significant change in pH. How many moles of nitric acid, HNO3, may be added before the pH begins to change significantly?\nAnswer Choices: (A) 0.500 mole (B) 1.00 mole (C) 2.00 mole (D) 3.00 mole",
|
||||
"We would like to compute the buffer capacity of this solution. First we write the equation for the ionization of the weak acid, in this case of acetic acid. $CH_{3}COOH (aq) + H_{2}O \rightarrow H_{3}O^{+} + CH3COO^{-}$. The conjugate base is therefore the acetate ion. The added strong acid, Nitric acid, will react with the conjugate base. Therefore the maximum amount of acid that can be added will be equal to the amount of acetate ion, or 2 moles. The answer is (C).",
|
||||
),
|
||||
]
|
||||
examples["mmlu_biology"] = [
|
||||
(
|
||||
"In animal cells, which of the following represents the most likely pathway that a secretory protein takes as it is synthesized in a cell?\nAnswer Choices: (A) Plasma membrane–Golgi apparatus–ribosome–secretory vesicle–rough ER (B) Ribosome–Golgi apparatus–rough ER–secretory vesicle–plasma membrane (C) Plasma membrane–Golgi apparatus–ribosome–secretory vesicle–rough ER (D) Ribosome–rough ER–Golgi apparatus–secretory vesicle–plasma membrane",
|
||||
"Protein synthesis starts at the ribosome, so we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum and moves from there to the Golgi apparatus, where it is modified and packaged into a vesicle. The vesicle then floats to the plasma membrane and is secreted. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"A mutation in a bacterial enzyme changed a previously polar amino acid into a nonpolar amino acid. This amino acid was located at a site distant from the enzyme’s active site. How might this mutation alter the enzyme’s substrate specificity?\nAnswer Choices: (A) By changing the enzyme’s pH optimum (B) By changing the enzyme’s location in the cell (C) By changing the shape of the protein (D) An amino acid change away from the active site cannot alter the enzyme’s substrate specificity.",
|
||||
"A change in an amino acid leads to a change in the primary structure of the protein. A change in the primary structure may lead to a change in the secondary and the tertiary structure of the protein. A change in the tertiary structure means a change in the shape of the protein, so (C) has to be correct. Since the change does not affect the active site of the enzyme, we do not expect the activity of the enzyme to be affected. The answer is (C).",
|
||||
),
|
||||
(
|
||||
"Which of the following is not a way to form recombinant DNA?\nAnswer Choices: (A) Translation (B) Conjugation (C) Specialized transduction (D) Transformation",
|
||||
"The introduction of foreign DNA or RNA into bacteria or eukaryotic cells is a common technique in molecular biology and scientific research. There are multiple ways foreign DNA can be introduced into cells including transformation, transduction, conjugation, and transfection. In contrast, (A) is not a way to form DNA: during translation the ribosomes synthesize proteins from RNA. The answer is (A).",
|
||||
),
|
||||
(
|
||||
"Homologous structures are often cited as evidence for the process of natural selection. All of the following are examples of homologous structures EXCEPT\nAnswer Choices: (A) the wings of a bird and the wings of a bat (B) the flippers of a whale and the arms of a man (C) the pectoral fins of a porpoise and the flippers of a seal (D) the forelegs of an insect and the forelimbs of a dog",
|
||||
"Homologous structures are similar physical features in organisms that share a common ancestor but different functions. Comparisons (B) and (C) are clearly homologous because they share a common ancestor and the structures serve different purposes. Bat wings and birg wings are also homologous, while they are both wings, the forelimbs serve different purposes. Insects and dogs are very far ancestors since one is vertebrate while the other is invertebrate and the forelimbs serve the same purpose, so they are not homologous. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"Which of the following is not known to be involved in the control of cell division?\nAnswer Choices: (A) Cyclins (B) Protein kinases (C) Checkpoints (D) Fibroblast cells",
|
||||
"Normal cells move through the cell cycle in a regulated way. At the checkpoint stage, they use information about their own internal state and cues from the environment around them to decide whether to proceed with cell division. Cues like these act by changing the activity of core cell cycle regulators inside the cell. The most common regulators are cyclins and cyclin-dependent kinases. Fibroblast cells do not play any role in cell division. The answer is (D).",
|
||||
),
|
||||
]
|
||||
examples["mmlu_computer"] = [
|
||||
(
|
||||
"Which of the following is an example of the use of a device on the Internet of Things (IoT) ?\nAnswer Choices: (A) A car alerts a driver that it is about to hit an object. (B) A hiker uses a G P S watch to keep track of her position. (C) A refrigerator orders milk from an online delivery service when the milk in the refrigerator is almost gone. (D) A runner uses a watch with optical sensors to monitor his heart rate.",
|
||||
"The term Internet of Things (IoT) refers to common devices which are connected to the internet, enabling new functionality. Choice A is incorrect because it does not describe an internet connected device. In choice B, the watch is only described as having GPS functionality but no internet connectivity. Choice C describes a common device (a refrigerator) which has internet connectivity enabling new functionality (online ordering). Choice D does not mention internet connectivity for the watch, only optical sensors. The answer is (C).",
|
||||
),
|
||||
(
|
||||
"Many Web browsers allow users to open anonymous windows. During a browsing session in an anonymous window, the browser does not record a browsing history or a list of downloaded files. When the anonymous window is exited, cookies created during the session are deleted. Which of the following statements about browsing sessions in an anonymous window is true?\nAnswer Choices: (A) The activities of a user browsing in an anonymous window will not be visible to people who monitor the user's network, such as the system administrator. (B) Items placed in a Web store's shopping cart for future purchase during the anonymous browsing session will not be saved on the user's computer. (C) A user will not be able to log in to e-mail or social media accounts during the anonymous browsing session. (D) A user browsing in an anonymous window will be protected from viruses launched from any web sites visited or files downloaded.",
|
||||
"Choice A is incorrect as it only describes network traffic, which an anonymous browser does not change. Choice B is correct as it correctly describes how an anonymous browser will prevent saving data on the user’s computer after the session is ended. Choice C is incorrect because an anonymous browser will not prevent logging in to email or social media accounts. Choice D is incorrect because an anonymous browser in itself performs no virus protection. The answer is (B).",
|
||||
),
|
||||
(
|
||||
'What is the output of "abc"[::-1] in Python 3? \nAnswer Choices: (A) Error (B) abc (C) cba (D) c',
|
||||
'We know that the slicing operator [::-1] takes all of the elements in the string in reverse order, so we reverse the order of the string "abc", resulting in "cba". The answer is (C).',
|
||||
),
|
||||
(
|
||||
'In the program below, the initial value of X is 5 and the initial value of Y is 10.\nIF (X < 0){\n DISPLAY ("Foxtrot")\n} ELSE {\n IF (X > Y){\n DISPLAY ("Hotel")\n } ELSE {\n IF (Y > 0){\n DISPLAY ("November")\n } ELSE {\n DISPLAY ("Yankee")\n }\n }\n}\nWhat is displayed as a result of running the program?\nAnswer Choices: (A) Foxtrot (B) Hotel (C) November (D) Yankee',
|
||||
'Because X has the value 5, the first conditional IF (X < 0) is false, so we move to the first ELSE clause. Because X is 5 and Y is 10, the second conditional IF (X > Y) is false, so we move to the following ELSE clause. Since Y is 10, the conditional IF (Y > 0) is true, so the command DISPLAY ("November") is executed. The answer is (C).',
|
||||
),
|
||||
(
|
||||
"A list of numbers has n elements, indexed from 1 to n. The following algorithm is intended to display the number of elements in the list that have a value greater than 100. The algorithm uses the variables count and position. Steps 3 and 4 are missing.\n Step 1: Set count to 0 and position to 1.\n Step 2: If the value of the element at index position is greater than 100, increase the value of count by 1.\n Step 3: (missing step)\n Step 4: (missing step)\n Step 5: Display the value of count.\nWhich of the following could be used to replace steps 3 and 4 so that the algorithm works as intended?\nAnswer Choices: (A) Step 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until the value of count is greater than 100.\n(B) Step 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until the value of position is greater than n.\n(C) Step 3: Repeat step 2 until the value of count is greater than 100.\n Step 4: Increase the value of position by 1.\n(D) Step 3: Repeat step 2 until the value of position is greater than n.\n Step 4: Increase the value of count by 1.",
|
||||
"Choice A is incorrect, because its Step 4 has an incorrect termination condition, stopping when count is greater than 100. We need to stop after inspecting all elements in the list. Choice B is correct because it correctly increments both count and position, and correctly repeats these steps and terminates when all elements in the list have been inspected. Choice C is incorrect because it incorrectly increments the variable count until its value is greater than 100, regardless of the elements in the list. Choice D is incorrect because its step 3 does not increment the value of position, so it will repeat forever. The answer is (B).",
|
||||
),
|
||||
]
|
||||
# mammoth
|
||||
examples["mmlu_stem"] = [
|
||||
(
|
||||
"Simplify and write the result with a rational denominator: $$\\sqrt{\\sqrt[3]{\\sqrt{\frac{1}{729}}}}$$\nAnswer Choices: (A) \\frac{3\\sqrt{3}}{3} (B) \\frac{1}{3} (C) \\sqrt{3} (D) \\frac{\\sqrt{3}}{3}",
|
||||
"Factoring $729=3^6$ and combining the roots $\\frac{1}{2}\\frac{1}{3}\\frac{1}{2}=\\frac{1}{12}$, we get that $\\sqrt{\\sqrt[3]{\\sqrt{\frac{1}{729}}}}=\\left(\frac{1}{3^6}\right)^{\frac{1}{12}}=\frac{1}{3^{\frac{1}{2}}}=\frac{3}{\\sqrt{3}}$. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"In animal cells, which of the following represents the most likely pathway that a secretory protein takes as it is synthesized in a cell?\nAnswer Choices: (A) Plasma membrane–Golgi apparatus–ribosome–secretory vesicle–rough ER (B) Ribosome–Golgi apparatus–rough ER–secretory vesicle–plasma membrane (C) Plasma membrane–Golgi apparatus–ribosome–secretory vesicle–rough ER (D) Ribosome–rough ER–Golgi apparatus–secretory vesicle–plasma membrane",
|
||||
"Protein synthesis starts at the ribosome, so we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum and moves from there to the Golgi apparatus, where it is modified and packaged into a vesicle. The vesicle then floats to the plasma membrane and is secreted. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"A microwave oven is connected to an outlet, 120 V, and draws a current of 2 amps. At what rate is energy being used by the microwave oven?\nAnswer Choices: (A) 10 W (B) 30 W (C) 60 W (D) 240 W",
|
||||
"Rate of energy usage is known as power; in an dissipative electrical circuit, power is given by voltage times current. So in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D).",
|
||||
),
|
||||
(
|
||||
"Which of the following is considered an acid anhydride?\nAnswer Choices: (A) HCl (B) H2SO3 (C) SO2 (D) Al(NO3)3",
|
||||
"An acid anhydride is a compound that is derived by removing water from an acid. The chemical formula for water is H2O, which means that we need to determine which of these options, when combined with H2O, forms an acid. SO2, or Sulfur dioxide, when combined with H2O, makes H2SO4, or sulfuric acid. The answer is (C).",
|
||||
),
|
||||
(
|
||||
'What is the output of "abc"[::-1] in Python 3? \nAnswer Choices: (A) Error (B) abc (C) cba (D) c',
|
||||
'We know that the slicing operator [::-1] takes all of the elements in the string in reverse order, so we reverse the order of the string "abc", resulting in "cba". The answer is (C).',
|
||||
),
|
||||
]
|
||||
examples["gaokao"] = [
|
||||
(
|
||||
"已知 $\\alpha, \\beta, \\gamma$ 是互不相同的锐角, 则在 $\\sin \\alpha \\cos \\beta, \\sin \\beta \\cos \\gamma, \\sin \\gamma \\cos \\alpha$ 三个值中, 大于 $\\frac{1}{2}$ 的个数的最大值是 ( )\n从以下选项中选择:\n(A) 0\n(B) 1\n(C) 2\n(D) 3",
|
||||
"1. 如果 $\\alpha, \\beta, \\gamma$ 均小于 $60^\\circ$，那么他们的正弦值都小于 $\\frac{1}{2}$，因此三个值中不可能有大于 $\\frac{1}{2}$ 的值。\n2. 如果有一个角大于 $60^\\circ$，假设为 $\\alpha$，那么对应的正弦值大于 $\\frac{1}{2}$。此时，由于三角形内角和为 $180^\\circ$，所以 $\\beta + \\gamma < 120^\\circ$。这意味着 $\\beta, \\gamma$ 的余弦值均大于 $\\frac{1}{2}$，所以此时 $\\sin \\alpha \\cos \\beta > \\frac{1}{2}, \\sin \\beta \\cos \\gamma > \\frac{1}{2}$。\n3. 如果有两个角大于 $60^\\circ$，例如 $\\alpha$ 和 $\\beta$，那么由于三角形内角和为 $180^\\circ$，我们可以得到 $\\gamma < 60^\\circ$，此时 $\\sin \\gamma < \\frac{1}{2}$。由于 $\\alpha$ 和 $\\beta$ 的余弦值都小于 $\\frac{1}{2}$，因此三个值中不可能有大于 $\\frac{1}{2}$ 的值。\n4. 如果三个角都大于 $60^\\circ$，显然不符合题意。\n综上所述，当有一个角大于 $60^\\circ$ 时，大于 $\\frac{1}{2}$ 的个数的最大值是 2。\n答案是 C",
|
||||
),
|
||||
(
|
||||
"正方体 $A B C D-A_{1} B_{1} C_{1} D_{1}$ 中, $B B_{1}$ 与平面 $A C D_{1}$ 所成角的余弦值为 ( )\n从以下选项中选择:\n(A) $\\frac{\\sqrt{2}}{3}$\n(B) $\\frac{\\sqrt{3}}{3}$\n(C) $\\frac{2}{3}$\n(D) $\\frac{\\sqrt{6}}{3}$",
|
||||
"设上下底面的中心分别为 $\\mathrm{O}_{1}, \\mathrm{O}$, 设正方体的棱长等于 1 , 则 $O_{1} O$ 与平面 $A C D_{1}$ 所成角就是 $B B_{1}$ 与平面 $A C D_{1}$ 所成角, 即 $\\angle O_{1} O D_{1}$, 直角三角形 $\\mathrm{OO}_{1} \\mathrm{D}_{1}$ 中, $\\cos \\angle \\mathrm{O}_{1} \\mathrm{OD}_{1}=\\frac{\\mathrm{O}_{1} \\mathrm{O}}{\\mathrm{OD}_{1}}=\\frac{\\frac{1}{\\sqrt{6}}}{2}=\\frac{\\sqrt{6}}{3}$. \n答案是 C",
|
||||
),
|
||||
(
|
||||
"设函数 $f(x)=\\left\\{\\begin{array}{ll}1+\\log _{2}(2-x), & x<1 \\ 2^{x-1}, & x \\geqslant 1,\\end{array}\\right.$ 则 $f(-2)+f\\left(\\log _{2} 12\\right)=$ ( )\n从以下选项中选择:\n(A) 3\n(B) 6\n(C) 9\n(D) 12",
|
||||
"首先，我们可以根据定义计算 $f(-2)$ 和 $f(\\log_2 12)$：\n$f(-2)=1+\\log_2(2-(-2))=1+\\log_2 4=3$\n$f(\\log_2 12)=2^{\\log_2 12-1}=6$\n因此，$f(-2)+f(\\log_2 12)=3+6=9$。\n答案是 C",
|
||||
),
|
||||
(
|
||||
"已知函数 $f(x)=a x^{3}-3 x^{2}+1$, 若 $f(x)$ 存在唯一的零点 $x_{0}$, 且 $x_{0}>$ 0 , 则实数 $\\mathrm{a}$ 的取值范围是 ( )\n从以下选项中选择:\n(A) $(1,+\\infty)$\n(B) $(2,+\\infty)$\n(C) $(-\\infty,-1)$\n(D) $(-\\infty,-2)$",
|
||||
"首先，我们可以通过求出函数的导函数 $f'(x)$ 来判断函数在 $x>0$ 区间内的单调性。在这里，我们求出导函数 $f'(x)$ 为 $f'(x)=3ax^2-6x$。\n然后，我们需要求出导函数 $f'(x)$ 的零点，以确定函数 $f(x)$ 在 $x>0$ 区间内的单调性。导函数 $f'(x)$ 的零点为 $x=0$ 和 $x=\\frac{2}{\\sqrt{a}}$。注意到 $x>0$，所以我们得到 $a<0$。此外，由于函数 $f(x)$ 在 $x=0$ 处的函数值为 $1$，因此不能有 $a=\\frac{4}{3}$。\n综上所述，当 $a$ 的取值范围为 $a<-\\frac{4}{3}$ 时，函数 $f(x)$ 在 $x>0$ 区间内是单调递减的，此时存在唯一的零点 $x_0$。因此，答案为 $(-\\infty,-2)$。\n答案是 D",
|
||||
),
|
||||
(
|
||||
"设 $\\left\\{a_{n}\\right\\}$ 是公差不为 0 的无穷等差数列, 则“ $\\left\\{a_{n}\\right\\}$ 为递增数列”是“存在正整数 $N_{0}$, 当 $n>N_{0}$ 时, $a_{n}>0$ ”的 ( )\n从以下选项中选择:\n(A) 充分而不必要条件\n(B) 必要而不充分条件\n(C) 充分必要条件\n(D) 既不充分也不必要条件",
|
||||
"首先，我们可以通过举例来判断该条件是充分还是必要条件。如果一个数列递增，那么它的公差一定大于 0，也就是存在正整数 $N_{0}$，当 $n>N_{0}$ 时，$a_{n}>0$。因此，“ $\\left\\{a_{n}\\right\\}$ 为递增数列”是“存在正整数 $N_{0}$, 当 $n>N_{0}$ 时, $a_{n}>0$ ”的必要条件。\n接下来，我们需要判断是否充分。也就是说，如果存在正整数 $N_{0}$，当 $n>N_{0}$ 时，$a_{n}>0$，那么能否得出“ $\\left\\{a_{n}\\right\\}$ 为递增数列”这一结论。答案是肯定的。因为如果 $a_{n}>0$，那么 $a_{n+1}-a_{n}>0$，即公差大于 0，因此该数列是递增的。因此，该条件是充分条件。\n综上所述，选项为 (C) 充分必要条件。\n答案是 C",
|
||||
),
|
||||
]
|
||||
|
||||
return examples
|
||||
Executable
+395
@@ -0,0 +1,395 @@
|
||||
"""
|
||||
This logic is largely copied from the Hendrycks' MATH release (math_equivalence), and borrowed from:
|
||||
- https://github.com/microsoft/ProphetNet/tree/master/CRITIC
|
||||
- https://github.com/openai/prm800k
|
||||
- https://github.com/microsoft/ToRA/blob/main/src/eval/grader.py
|
||||
- https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/eval/eval_utils.py
|
||||
"""
|
||||
|
||||
import re
|
||||
import regex
|
||||
import multiprocessing
|
||||
from math import isclose
|
||||
from typing import Union
|
||||
from collections import defaultdict
|
||||
|
||||
from sympy import simplify, N
|
||||
from sympy.parsing.sympy_parser import parse_expr
|
||||
from sympy.parsing.latex import parse_latex
|
||||
from latex2sympy2 import latex2sympy
|
||||
|
||||
# from .parser import choice_answer_clean, strip_string
|
||||
# from parser import choice_answer_clean
|
||||
|
||||
|
||||
def choice_answer_clean(pred: str):
|
||||
pred = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":")
|
||||
# Clean the answer based on the dataset
|
||||
tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper())
|
||||
if tmp:
|
||||
pred = tmp
|
||||
else:
|
||||
pred = [pred.strip().strip(".")]
|
||||
pred = pred[-1]
|
||||
# Remove the period at the end, again!
|
||||
pred = pred.rstrip(".").rstrip("/")
|
||||
return pred
|
||||
|
||||
|
||||
def parse_digits(num):
|
||||
num = regex.sub(",", "", str(num))
|
||||
try:
|
||||
return float(num)
|
||||
except:
|
||||
if num.endswith("%"):
|
||||
num = num[:-1]
|
||||
if num.endswith("\\"):
|
||||
num = num[:-1]
|
||||
try:
|
||||
return float(num) / 100
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def is_digit(num):
|
||||
# paired with parse_digits
|
||||
return parse_digits(num) is not None
|
||||
|
||||
|
||||
def str_to_pmatrix(input_str):
|
||||
input_str = input_str.strip()
|
||||
matrix_str = re.findall(r"\{.*,.*\}", input_str)
|
||||
pmatrix_list = []
|
||||
|
||||
for m in matrix_str:
|
||||
m = m.strip("{}")
|
||||
pmatrix = r"\begin{pmatrix}" + m.replace(",", "\\") + r"\end{pmatrix}"
|
||||
pmatrix_list.append(pmatrix)
|
||||
|
||||
return ", ".join(pmatrix_list)
|
||||
|
||||
|
||||
def math_equal(
|
||||
prediction: Union[bool, float, str],
|
||||
reference: Union[float, str],
|
||||
include_percentage: bool = True,
|
||||
is_close: bool = True,
|
||||
timeout: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Exact match of math if and only if:
|
||||
1. numerical equal: both can convert to float and are equal
|
||||
2. symbolic equal: both can convert to sympy expression and are equal
|
||||
"""
|
||||
# print("Judge:", prediction, reference)
|
||||
if prediction is None or reference is None:
|
||||
return False
|
||||
if str(prediction.strip().lower()) == str(reference.strip().lower()):
|
||||
return True
|
||||
if (
|
||||
reference in ["A", "B", "C", "D", "E"]
|
||||
and choice_answer_clean(prediction) == reference
|
||||
):
|
||||
return True
|
||||
|
||||
try: # 1. numerical equal
|
||||
if is_digit(prediction) and is_digit(reference):
|
||||
prediction = parse_digits(prediction)
|
||||
reference = parse_digits(reference)
|
||||
# number questions
|
||||
if include_percentage:
|
||||
gt_result = [reference / 100, reference, reference * 100]
|
||||
else:
|
||||
gt_result = [reference]
|
||||
for item in gt_result:
|
||||
try:
|
||||
if is_close:
|
||||
if numeric_equal(prediction, item):
|
||||
return True
|
||||
else:
|
||||
if item == prediction:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
except:
|
||||
pass
|
||||
|
||||
if not prediction and prediction not in [0, False]:
|
||||
return False
|
||||
|
||||
# 2. symbolic equal
|
||||
reference = str(reference).strip()
|
||||
prediction = str(prediction).strip()
|
||||
|
||||
## pmatrix (amps)
|
||||
if "pmatrix" in prediction and not "pmatrix" in reference:
|
||||
reference = str_to_pmatrix(reference)
|
||||
|
||||
## deal with [], (), {}
|
||||
pred_str, ref_str = prediction, reference
|
||||
if (
|
||||
prediction.startswith("[")
|
||||
and prediction.endswith("]")
|
||||
and not reference.startswith("(")
|
||||
) or (
|
||||
prediction.startswith("(")
|
||||
and prediction.endswith(")")
|
||||
and not reference.startswith("[")
|
||||
):
|
||||
pred_str = pred_str.strip("[]()")
|
||||
ref_str = ref_str.strip("[]()")
|
||||
for s in ["{", "}", "(", ")"]:
|
||||
ref_str = ref_str.replace(s, "")
|
||||
pred_str = pred_str.replace(s, "")
|
||||
if pred_str.lower() == ref_str.lower():
|
||||
return True
|
||||
|
||||
## [a, b] vs. [c, d], return a==c and b==d
|
||||
if (
|
||||
regex.match(r"(\(|\[).+(\)|\])", prediction) is not None
|
||||
and regex.match(r"(\(|\[).+(\)|\])", reference) is not None
|
||||
):
|
||||
pred_parts = prediction[1:-1].split(",")
|
||||
ref_parts = reference[1:-1].split(",")
|
||||
if len(pred_parts) == len(ref_parts):
|
||||
if all(
|
||||
[
|
||||
math_equal(
|
||||
pred_parts[i], ref_parts[i], include_percentage, is_close
|
||||
)
|
||||
for i in range(len(pred_parts))
|
||||
]
|
||||
):
|
||||
return True
|
||||
if (
|
||||
(
|
||||
prediction.startswith("\\begin{pmatrix}")
|
||||
or prediction.startswith("\\begin{bmatrix}")
|
||||
)
|
||||
and (
|
||||
prediction.endswith("\\end{pmatrix}")
|
||||
or prediction.endswith("\\end{bmatrix}")
|
||||
)
|
||||
and (
|
||||
reference.startswith("\\begin{pmatrix}")
|
||||
or reference.startswith("\\begin{bmatrix}")
|
||||
)
|
||||
and (
|
||||
reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}")
|
||||
)
|
||||
):
|
||||
pred_lines = [
|
||||
line.strip()
|
||||
for line in prediction[
|
||||
len("\\begin{pmatrix}") : -len("\\end{pmatrix}")
|
||||
].split("\\\\")
|
||||
if line.strip()
|
||||
]
|
||||
ref_lines = [
|
||||
line.strip()
|
||||
for line in reference[
|
||||
len("\\begin{pmatrix}") : -len("\\end{pmatrix}")
|
||||
].split("\\\\")
|
||||
if line.strip()
|
||||
]
|
||||
matched = True
|
||||
if len(pred_lines) == len(ref_lines):
|
||||
for pred_line, ref_line in zip(pred_lines, ref_lines):
|
||||
pred_parts = pred_line.split("&")
|
||||
ref_parts = ref_line.split("&")
|
||||
if len(pred_parts) == len(ref_parts):
|
||||
if not all(
|
||||
[
|
||||
math_equal(
|
||||
pred_parts[i],
|
||||
ref_parts[i],
|
||||
include_percentage,
|
||||
is_close,
|
||||
)
|
||||
for i in range(len(pred_parts))
|
||||
]
|
||||
):
|
||||
matched = False
|
||||
break
|
||||
else:
|
||||
matched = False
|
||||
if not matched:
|
||||
break
|
||||
else:
|
||||
matched = False
|
||||
if matched:
|
||||
return True
|
||||
|
||||
if prediction.count("=") == 1 and reference.count("=") == 1:
|
||||
pred = prediction.split("=")
|
||||
pred = f"{pred[0].strip()} - ({pred[1].strip()})"
|
||||
ref = reference.split("=")
|
||||
ref = f"{ref[0].strip()} - ({ref[1].strip()})"
|
||||
if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref):
|
||||
return True
|
||||
elif (
|
||||
prediction.count("=") == 1
|
||||
and len(prediction.split("=")[0].strip()) <= 2
|
||||
and "=" not in reference
|
||||
):
|
||||
if math_equal(
|
||||
prediction.split("=")[1], reference, include_percentage, is_close
|
||||
):
|
||||
return True
|
||||
elif (
|
||||
reference.count("=") == 1
|
||||
and len(reference.split("=")[0].strip()) <= 2
|
||||
and "=" not in prediction
|
||||
):
|
||||
if math_equal(
|
||||
prediction, reference.split("=")[1], include_percentage, is_close
|
||||
):
|
||||
return True
|
||||
|
||||
# symbolic equal with sympy
|
||||
if timeout:
|
||||
if call_with_timeout(symbolic_equal_process, prediction, reference):
|
||||
return True
|
||||
else:
|
||||
if symbolic_equal(prediction, reference):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def math_equal_process(param):
|
||||
return math_equal(param[-2], param[-1])
|
||||
|
||||
|
||||
def numeric_equal(prediction: float, reference: float):
|
||||
# Note that relative tolerance has significant impact
|
||||
# on the result of the synthesized GSM-Hard dataset
|
||||
# if reference.is_integer():
|
||||
# return isclose(reference, round(prediction), abs_tol=1e-4)
|
||||
# else:
|
||||
# prediction = round(prediction, len(str(reference).split(".")[-1]))
|
||||
return isclose(reference, prediction, rel_tol=1e-4)
|
||||
|
||||
|
||||
def symbolic_equal(a, b):
|
||||
def _parse(s):
|
||||
for f in [parse_latex, parse_expr, latex2sympy]:
|
||||
try:
|
||||
return f(s.replace("\\\\", "\\"))
|
||||
except:
|
||||
try:
|
||||
return f(s)
|
||||
except:
|
||||
pass
|
||||
return s
|
||||
|
||||
a = _parse(a)
|
||||
b = _parse(b)
|
||||
|
||||
# direct equal
|
||||
try:
|
||||
if str(a) == str(b) or a == b:
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
# simplify equal
|
||||
try:
|
||||
if a.equals(b) or simplify(a - b) == 0:
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
# equation equal
|
||||
try:
|
||||
if (abs(a.lhs - a.rhs)).equals(abs(b.lhs - b.rhs)):
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
if numeric_equal(float(N(a)), float(N(b))):
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
# matrix
|
||||
try:
|
||||
# if a and b are matrix
|
||||
if a.shape == b.shape:
|
||||
_a = a.applyfunc(lambda x: round(x, 3))
|
||||
_b = b.applyfunc(lambda x: round(x, 3))
|
||||
if _a.equals(_b):
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def symbolic_equal_process(a, b, output_queue):
|
||||
result = symbolic_equal(a, b)
|
||||
output_queue.put(result)
|
||||
|
||||
|
||||
def call_with_timeout(func, *args, timeout=1, **kwargs):
|
||||
output_queue = multiprocessing.Queue()
|
||||
process_args = args + (output_queue,)
|
||||
process = multiprocessing.Process(target=func, args=process_args, kwargs=kwargs)
|
||||
process.start()
|
||||
process.join(timeout)
|
||||
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join()
|
||||
return False
|
||||
|
||||
return output_queue.get()
|
||||
|
||||
def _test_math_equal():
|
||||
# print(math_equal("0.0833333333333333", "\\frac{1}{12}"))
|
||||
# print(math_equal("(1,4.5)", "(1,\\frac{9}{2})"))
|
||||
# print(math_equal("\\frac{x}{7}+\\frac{2}{7}", "\\frac{x+2}{7}", timeout=True))
|
||||
# print(math_equal("\\sec^2(y)", "\\tan^2(y)+1", timeout=True))
|
||||
# print(math_equal("\\begin{pmatrix}-\\frac{7}{4}&-2\\\\4&\\frac{1}{4}\\end{pmatrix}", "(\\begin{pmatrix}-\\frac{7}{4}&-2\\\\4&\\frac{1}{4}\\\\\\end{pmatrix})", timeout=True))
|
||||
|
||||
# pred = '\\begin{pmatrix}\\frac{1}{3x^{2/3}}&0&0\\\\0&1&0\\\\-\\sin(x)&0&0\\end{pmatrix}'
|
||||
# gt = '(\\begin{pmatrix}\\frac{1}{3\\sqrt[3]{x}^2}&0&0\\\\0&1&0\\\\-\\sin(x)&0&0\\\\\\end{pmatrix})'
|
||||
|
||||
# pred= '-\\frac{8x^2}{9(x^2-2)^{5/3}}+\\frac{2}{3(x^2-2)^{2/3}}'
|
||||
# gt= '-\\frac{2(x^2+6)}{9(x^2-2)\\sqrt[3]{x^2-2}^2}'
|
||||
|
||||
# pred = '-34x-45y+20z-100=0'
|
||||
# gt = '34x+45y-20z+100=0'
|
||||
|
||||
# pred = '\\frac{100}{3}'
|
||||
# gt = '33.3'
|
||||
|
||||
# pred = '\\begin{pmatrix}0.290243531202435\\\\0.196008371385084\\\\-0.186381278538813\\end{pmatrix}'
|
||||
# gt = '(\\begin{pmatrix}0.29\\\\0.196\\\\-0.186\\\\\\end{pmatrix})'
|
||||
|
||||
# pred = '\\frac{\\sqrt{\\sqrt{11}+\\sqrt{194}}}{2\\sqrt{33}+15}'
|
||||
# gt = '\\frac{\\sqrt{\\sqrt{11}+\\sqrt{194}}}{15+2\\sqrt{33}}'
|
||||
|
||||
# pred = '(+5)(b+2)'
|
||||
# gt = '(a+5)(b+2)'
|
||||
|
||||
# pred = '\\frac{1+\\sqrt{5}}{2}'
|
||||
# gt = '2'
|
||||
|
||||
# pred = '\\frac{34}{16}+\\frac{\\sqrt{1358}}{16}', gt = '4'
|
||||
# pred = '1', gt = '1\\\\sqrt{19}'
|
||||
|
||||
# pred = "(0.6,2.6667]"
|
||||
# gt = "(\\frac{3}{5},\\frac{8}{3}]"
|
||||
|
||||
gt = "x+2n+1"
|
||||
pred = "x+1"
|
||||
|
||||
print(math_equal(pred, gt, timeout=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test_math_equal()
|
||||
Executable
+405
@@ -0,0 +1,405 @@
|
||||
import random
|
||||
import os
|
||||
import argparse
|
||||
import time
|
||||
from vllm import LLM, SamplingParams
|
||||
from datetime import datetime
|
||||
from tqdm import tqdm
|
||||
|
||||
import torch
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
|
||||
from evaluate import evaluate
|
||||
from utils import set_seed, load_jsonl, save_jsonl, construct_prompt
|
||||
from parser import *
|
||||
from trajectory import *
|
||||
from data_loader import load_data
|
||||
from python_executor import PythonExecutor
|
||||
from model_utils import load_hf_lm_and_tokenizer, generate_completions
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_names", default="gsm8k,math", type=str)
|
||||
parser.add_argument("--data_dir", default="./data", type=str)
|
||||
parser.add_argument("--model_name_or_path", default="gpt-4", type=str)
|
||||
parser.add_argument("--output_dir", default="./output", type=str)
|
||||
parser.add_argument("--prompt_type", default="tool-integrated", type=str)
|
||||
parser.add_argument("--split", default="test", type=str)
|
||||
parser.add_argument("--num_test_sample", default=-1, type=int) # -1 for full data
|
||||
parser.add_argument("--seed", default=0, type=int)
|
||||
parser.add_argument("--start", default=0, type=int)
|
||||
parser.add_argument("--end", default=-1, type=int)
|
||||
parser.add_argument("--temperature", default=0, type=float)
|
||||
parser.add_argument("--n_sampling", default=1, type=int)
|
||||
parser.add_argument("--top_p", default=1, type=float)
|
||||
parser.add_argument("--max_tokens_per_call", default=2048, type=int)
|
||||
parser.add_argument("--shuffle", action="store_true")
|
||||
parser.add_argument("--use_vllm", action="store_true")
|
||||
parser.add_argument("--save_outputs", action="store_true")
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
parser.add_argument("--use_safetensors", action="store_true")
|
||||
parser.add_argument("--num_shots", type=int, default=0)
|
||||
parser.add_argument(
|
||||
"--apply_chat_template",
|
||||
action="store_true",
|
||||
help="Apply chat template to prompt.",
|
||||
)
|
||||
parser.add_argument("--pipeline_parallel_size", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"--adapt_few_shot",
|
||||
action="store_true",
|
||||
help="Few shot for multiple-choice questions, zero shot for others.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
args.top_p = (
|
||||
1 if args.temperature == 0 else args.top_p
|
||||
) # top_p must be 1 when using greedy sampling (vllm)
|
||||
return args
|
||||
|
||||
|
||||
def prepare_data(data_name, args):
|
||||
examples = load_data(data_name, args.split, args.data_dir)
|
||||
|
||||
# sample `num_test_sample` from dataset
|
||||
if args.num_test_sample > 0:
|
||||
# examples = random.sample(examples, min(args.num_test_sample, len(examples)))
|
||||
examples = examples[: args.num_test_sample]
|
||||
|
||||
# shuffle
|
||||
if args.shuffle:
|
||||
random.seed(datetime.now().timestamp())
|
||||
random.shuffle(examples)
|
||||
|
||||
# select start and end
|
||||
examples = examples[args.start : len(examples) if args.end == -1 else args.end]
|
||||
|
||||
# get out_file name
|
||||
dt_string = datetime.now().strftime("%m-%d_%H-%M")
|
||||
model_name = "/".join(args.model_name_or_path.split("/")[-2:])
|
||||
out_file_prefix = f"{args.split}_{args.prompt_type}_{args.num_test_sample}_seed{args.seed}_t{args.temperature}"
|
||||
output_dir = args.output_dir
|
||||
if not os.path.exists(output_dir):
|
||||
output_dir = f"outputs/{output_dir}"
|
||||
out_file = f"{output_dir}/{data_name}/{out_file_prefix}_s{args.start}_e{args.end}.jsonl"
|
||||
os.makedirs(f"{output_dir}/{data_name}", exist_ok=True)
|
||||
|
||||
# load all processed samples
|
||||
processed_samples = []
|
||||
if not args.overwrite:
|
||||
processed_files = [
|
||||
f
|
||||
for f in os.listdir(f"{output_dir}/{data_name}/")
|
||||
if f.endswith(".jsonl") and f.startswith(out_file_prefix)
|
||||
]
|
||||
for f in processed_files:
|
||||
processed_samples.extend(
|
||||
list(load_jsonl(f"{output_dir}/{data_name}/{f}"))
|
||||
)
|
||||
|
||||
# dedepulicate
|
||||
processed_samples = {sample["idx"]: sample for sample in processed_samples}
|
||||
processed_idxs = list(processed_samples.keys())
|
||||
processed_samples = list(processed_samples.values())
|
||||
examples = [example for example in examples if example["idx"] not in processed_idxs]
|
||||
return examples, processed_samples, out_file
|
||||
|
||||
|
||||
def setup(args):
|
||||
# load model
|
||||
available_gpus = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
|
||||
if args.use_vllm:
|
||||
llm = LLM(
|
||||
model=args.model_name_or_path,
|
||||
tensor_parallel_size=len(available_gpus) // args.pipeline_parallel_size,
|
||||
pipeline_parallel_size=args.pipeline_parallel_size,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
tokenizer = None
|
||||
if args.apply_chat_template:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
args.model_name_or_path, trust_remote_code=True
|
||||
)
|
||||
else:
|
||||
llm, tokenizer = load_hf_lm_and_tokenizer(
|
||||
model_name_or_path=args.model_name_or_path,
|
||||
load_in_half=True,
|
||||
use_fast_tokenizer=True,
|
||||
use_safetensors=args.use_safetensors,
|
||||
)
|
||||
|
||||
# infer & eval
|
||||
data_list = args.data_names.split(",")
|
||||
results = []
|
||||
for data_name in data_list:
|
||||
results.append(main(llm, tokenizer, data_name, args))
|
||||
|
||||
# add "avg" result to data_list and results
|
||||
data_list.append("avg")
|
||||
results.append(
|
||||
{
|
||||
"acc": sum([result["acc"] for result in results]) / len(results),
|
||||
}
|
||||
)
|
||||
|
||||
# print all results
|
||||
pad = max([len(data_name) for data_name in data_list])
|
||||
print("\t".join(data_name.ljust(pad, " ") for data_name in data_list))
|
||||
print("\t".join([f"{result['acc']:.1f}".ljust(pad, " ") for result in results]))
|
||||
|
||||
|
||||
def is_multi_choice(answer):
|
||||
for c in answer:
|
||||
if c not in ["A", "B", "C", "D", "E"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main(llm, tokenizer, data_name, args):
|
||||
examples, processed_samples, out_file = prepare_data(data_name, args)
|
||||
print("=" * 50)
|
||||
print("data:", data_name, " ,remain samples:", len(examples))
|
||||
if len(examples) > 0:
|
||||
print(examples[0])
|
||||
|
||||
# init python executor
|
||||
if "pal" in args.prompt_type:
|
||||
executor = PythonExecutor(get_answer_expr="solution()")
|
||||
else:
|
||||
executor = PythonExecutor(get_answer_from_stdout=True)
|
||||
|
||||
samples = []
|
||||
for example in tqdm(examples, total=len(examples)):
|
||||
idx = example["idx"]
|
||||
|
||||
# parse question and answer
|
||||
example["question"] = parse_question(example, data_name)
|
||||
if example["question"] == "":
|
||||
continue
|
||||
gt_cot, gt_ans = parse_ground_truth(example, data_name)
|
||||
example["gt_ans"] = gt_ans
|
||||
full_prompt = construct_prompt(example, data_name, args)
|
||||
|
||||
if idx == args.start:
|
||||
print(full_prompt)
|
||||
|
||||
sample = {
|
||||
"idx": idx,
|
||||
"question": example["question"],
|
||||
"gt_cot": gt_cot,
|
||||
"gt": gt_ans,
|
||||
"prompt": full_prompt,
|
||||
}
|
||||
|
||||
# add remain fields
|
||||
for key in [
|
||||
"level",
|
||||
"type",
|
||||
"unit",
|
||||
"solution_type",
|
||||
"choices",
|
||||
"solution",
|
||||
"ques_type",
|
||||
"ans_type",
|
||||
"answer_type",
|
||||
"dataset",
|
||||
"subfield",
|
||||
"filed",
|
||||
"theorem",
|
||||
"answer",
|
||||
]:
|
||||
if key in example:
|
||||
sample[key] = example[key]
|
||||
samples.append(sample)
|
||||
|
||||
# repeat n times
|
||||
input_prompts = [
|
||||
sample["prompt"] for sample in samples for _ in range(args.n_sampling)
|
||||
]
|
||||
if args.apply_chat_template:
|
||||
input_prompts = [
|
||||
tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt.strip()}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
for prompt in input_prompts
|
||||
]
|
||||
remain_prompts = input_prompts
|
||||
remain_prompts = [(i, prompt) for i, prompt in enumerate(remain_prompts)]
|
||||
end_prompts = []
|
||||
|
||||
max_func_call = 1 if args.prompt_type in ["cot", "pal"] else 4
|
||||
|
||||
stop_words = ["</s>", "<|im_end|>", "<|endoftext|>"]
|
||||
|
||||
if args.prompt_type in ["cot"]:
|
||||
stop_words.append("\n\nQuestion:")
|
||||
if args.prompt_type in ["pal", "tool-integrated", "jiuzhang_tora"]:
|
||||
stop_words.extend(["\n\n---", "```output"])
|
||||
elif args.prompt_type in ["wizard_zs", "platypus_fs"]:
|
||||
stop_words.extend(["Instruction", "Response"])
|
||||
elif "jiuzhang" in args.prompt_type:
|
||||
stop_words.append("\n\n## Question")
|
||||
elif "numina" in args.prompt_type:
|
||||
stop_words.append("\n### Problem")
|
||||
elif "pure" in args.prompt_type:
|
||||
stop_words.append("\n\n\n")
|
||||
|
||||
# start inference
|
||||
# measure time use
|
||||
start_time = time.time()
|
||||
for epoch in range(max_func_call):
|
||||
print("-" * 20, "Epoch", epoch)
|
||||
current_prompts = remain_prompts
|
||||
if len(current_prompts) == 0:
|
||||
break
|
||||
|
||||
# get all outputs
|
||||
prompts = [item[1] for item in current_prompts]
|
||||
if args.use_vllm:
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
SamplingParams(
|
||||
temperature=args.temperature,
|
||||
top_p=args.top_p,
|
||||
max_tokens=args.max_tokens_per_call,
|
||||
n=1,
|
||||
stop=stop_words,
|
||||
stop_token_ids=(
|
||||
[151645, 151643]
|
||||
if "qwen2" in args.model_name_or_path.lower()
|
||||
else None
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
outputs = sorted(
|
||||
outputs, key=lambda x: int(x.request_id)
|
||||
) # sort outputs by request_id
|
||||
outputs = [output.outputs[0].text for output in outputs]
|
||||
else:
|
||||
outputs = generate_completions(
|
||||
model=llm,
|
||||
tokenizer=tokenizer,
|
||||
prompts=prompts,
|
||||
max_new_tokens=args.max_tokens_per_call,
|
||||
batch_size=16,
|
||||
stop_id_sequences=stop_words,
|
||||
)
|
||||
|
||||
assert len(outputs) == len(current_prompts)
|
||||
|
||||
# process all outputs
|
||||
remain_prompts = []
|
||||
remain_codes = []
|
||||
for (i, query), output in zip(current_prompts, outputs):
|
||||
output = output.rstrip()
|
||||
query += output
|
||||
if args.prompt_type == "pal":
|
||||
remain_prompts.append((i, query))
|
||||
if "```python" in output:
|
||||
output = extract_program(query)
|
||||
remain_codes.append(output)
|
||||
elif args.prompt_type == "cot":
|
||||
end_prompts.append((i, query))
|
||||
elif "boxed" not in output and output.endswith("```"):
|
||||
program = extract_program(query)
|
||||
remain_prompts.append((i, query))
|
||||
remain_codes.append(program)
|
||||
else:
|
||||
end_prompts.append((i, query))
|
||||
|
||||
# execute the remain prompts
|
||||
remain_results = executor.batch_apply(remain_codes)
|
||||
for k in range(len(remain_prompts)):
|
||||
i, query = remain_prompts[k]
|
||||
res, report = remain_results[k]
|
||||
exec_result = res if res else report
|
||||
if "pal" in args.prompt_type:
|
||||
exec_result = "\\boxed{" + exec_result + "}"
|
||||
exec_result = f"\n```output\n{exec_result}\n```\n"
|
||||
query += exec_result
|
||||
# not end
|
||||
if epoch == max_func_call - 1:
|
||||
query += "\nReach max function call limit."
|
||||
remain_prompts[k] = (i, query)
|
||||
|
||||
# unsolved samples
|
||||
print("Unsolved samples:", len(remain_prompts))
|
||||
end_prompts.extend(remain_prompts)
|
||||
# sort by idx
|
||||
end_prompts = sorted(end_prompts, key=lambda x: x[0])
|
||||
|
||||
# remove input_prompt from end_prompt
|
||||
codes = []
|
||||
assert len(input_prompts) == len(end_prompts)
|
||||
for i in range(len(input_prompts)):
|
||||
_, end_prompt = end_prompts[i]
|
||||
code = end_prompt.split(input_prompts[i])[-1].strip()
|
||||
for stop_word in stop_words:
|
||||
if stop_word in code:
|
||||
code = code.split(stop_word)[0].strip()
|
||||
codes.append(code)
|
||||
|
||||
# extract preds
|
||||
results = [
|
||||
run_execute(executor, code, args.prompt_type, data_name) for code in codes
|
||||
]
|
||||
time_use = time.time() - start_time
|
||||
|
||||
# put results back to examples
|
||||
all_samples = []
|
||||
for i, sample in enumerate(samples):
|
||||
code = codes[i * args.n_sampling : (i + 1) * args.n_sampling]
|
||||
result = results[i * args.n_sampling : (i + 1) * args.n_sampling]
|
||||
preds = [item[0] for item in result]
|
||||
reports = [item[1] for item in result]
|
||||
for j in range(len(preds)):
|
||||
if sample["gt"] in ["A", "B", "C", "D", "E"] and preds[j] not in [
|
||||
"A",
|
||||
"B",
|
||||
"C",
|
||||
"D",
|
||||
"E",
|
||||
]:
|
||||
preds[j] = choice_answer_clean(code[j])
|
||||
elif is_multi_choice(sample["gt"]) and not is_multi_choice(preds[j]):
|
||||
# remove any non-choice char
|
||||
preds[j] = "".join(
|
||||
[c for c in preds[j] if c in ["A", "B", "C", "D", "E"]]
|
||||
)
|
||||
|
||||
sample.pop("prompt")
|
||||
sample.update({"code": code, "pred": preds, "report": reports})
|
||||
all_samples.append(sample)
|
||||
|
||||
# add processed samples
|
||||
all_samples.extend(processed_samples)
|
||||
all_samples, result_json = evaluate(
|
||||
samples=all_samples,
|
||||
data_name=data_name,
|
||||
prompt_type=args.prompt_type,
|
||||
execute=True,
|
||||
)
|
||||
|
||||
# save outputs
|
||||
if len(processed_samples) < len(all_samples) and args.save_outputs:
|
||||
save_jsonl(all_samples, out_file)
|
||||
|
||||
result_json["time_use_in_second"] = time_use
|
||||
result_json["time_use_in_minite"] = (
|
||||
f"{int(time_use // 60)}:{int(time_use % 60):02d}"
|
||||
)
|
||||
|
||||
with open(
|
||||
out_file.replace(".jsonl", f"_{args.prompt_type}_metrics.json"), "w"
|
||||
) as f:
|
||||
json.dump(result_json, f, indent=4)
|
||||
return result_json
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
set_seed(args.seed)
|
||||
setup(args)
|
||||
Executable
+267
@@ -0,0 +1,267 @@
|
||||
import re
|
||||
import time
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import string
|
||||
from enum import Enum, auto
|
||||
from tqdm import tqdm
|
||||
from collections import OrderedDict
|
||||
import dataclasses
|
||||
import pandas as pd
|
||||
import timeout_decorator
|
||||
import mpmath
|
||||
import sympy as sp
|
||||
from sympy.parsing.latex import parse_latex
|
||||
import sympy as sp
|
||||
from sympy import simplify
|
||||
from sympy.printing import latex
|
||||
from sympy.core.relational import Relational
|
||||
from sympy.solvers.solveset import solvify
|
||||
from sympy.solvers.inequalities import reduce_inequalities
|
||||
from sympy.parsing.sympy_parser import (
|
||||
parse_expr,
|
||||
standard_transformations,
|
||||
implicit_multiplication,
|
||||
)
|
||||
|
||||
|
||||
def compare_numerical_ans(ans_p, ans_l):
|
||||
if ans_p is None:
|
||||
return False
|
||||
ans_p = ans_p.replace(",", "").replace("$", "")
|
||||
ans_l = ans_l.replace(",", "").replace("$", "")
|
||||
try:
|
||||
if ans_p.endswith("%"):
|
||||
ans_p = float(ans_p.rstrip("%")) / 100
|
||||
if isinstance(ans_p, str):
|
||||
ans_p = float(ans_p)
|
||||
if isinstance(ans_l, str):
|
||||
ans_l = float(ans_l)
|
||||
except Exception as e:
|
||||
return False
|
||||
return abs(ans_p - float(ans_l)) < 1e-3
|
||||
|
||||
|
||||
def my_parse_latex(expr_str):
|
||||
expr_str = expr_str.replace("dfrac", "frac")
|
||||
expr = parse_latex(expr_str)
|
||||
if "\\pi" in expr_str:
|
||||
expr = expr.subs({sp.Symbol("pi"): sp.pi})
|
||||
expr = expr.subs({sp.Symbol("i"): sp.I})
|
||||
return expr
|
||||
|
||||
|
||||
def is_number(element: str) -> bool:
|
||||
try:
|
||||
float(element.replace(" ", ""))
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def percentage_to_fraction(text):
|
||||
pattern = r"(\d+(\.\d+)?%)"
|
||||
matches = re.findall(pattern, text)
|
||||
for match in matches:
|
||||
percentage_str = match[0]
|
||||
percentage = float(percentage_str.strip("%")) / 100
|
||||
fraction = str(percentage)
|
||||
text = text.replace(percentage_str, fraction)
|
||||
return text
|
||||
|
||||
|
||||
def clean_expr_str(expr_str):
|
||||
expr_str = (
|
||||
expr_str.replace(" . ", ".")
|
||||
.replace(". ", ".")
|
||||
.replace("**", "^")
|
||||
.replace("\\pm", "")
|
||||
.replace("*", "\\times ")
|
||||
.replace("\\\\", "\\")
|
||||
.replace("\\ne ", "\\neq ")
|
||||
.replace("!=", "\\neq")
|
||||
.replace(">=", "\\ge")
|
||||
.replace("<=", "\\le")
|
||||
.replace("≠", "\\neq")
|
||||
.replace("dfrac", "frac")
|
||||
.replace("tfrac", "frac")
|
||||
.replace("\\$", "")
|
||||
.replace("$", "")
|
||||
.replace("\\%", "")
|
||||
.replace("%", "")
|
||||
.replace("\\!", "")
|
||||
.replace("^\circ", "\\times \\pi / 180")
|
||||
.replace("//", "/")
|
||||
.replace('"', "")
|
||||
# .replace(",", "") # TODO
|
||||
)
|
||||
# expr_str = re.sub(r"\^\s(.*)", r"\^\s{\1}", expr_str)
|
||||
expr_str = re.sub(r"\\+", r"\\", expr_str)
|
||||
expr_str = re.sub(r"\^\s?\((.*?)\)", r"^{\1}", expr_str)
|
||||
expr_str = re.sub(r"\\frac\s?(\d)\s?(\d+)", r"\\frac{\1}{\2}", expr_str)
|
||||
expr_str = re.sub(r"\\log_\s?(\d)\s?(\d+)", r"\\log_{\1}{\2}", expr_str)
|
||||
expr_str = re.sub(r"\\frac\s?{(.*?)}\s?(\d)", r"\\frac{\1}{\2}", expr_str)
|
||||
expr_str = re.sub(r"\\frac\s?(\d)\s?{(.*?)}", r"\\frac{\1}{\2}", expr_str)
|
||||
expr_str = re.sub(r"\\sqrt\s?(\d)", r"\\sqrt{\1}", expr_str)
|
||||
expr_str = re.sub(r"sqrt\s?\((\d+)\)", r"\\sqrt{\1}", expr_str)
|
||||
expr_str = re.sub(r"sqrt\s?\((.*?)\)", r"\\sqrt{\1}", expr_str)
|
||||
expr_str = expr_str.replace(" sqrt", "\\sqrt")
|
||||
expr_str = (
|
||||
expr_str.replace("\\left", "").replace("\\right.", "").replace("\\right", "")
|
||||
)
|
||||
return expr_str
|
||||
|
||||
|
||||
def parse_latex_answer(sample):
|
||||
if isinstance(sample, int) or isinstance(sample, float):
|
||||
sample = str(sample)
|
||||
# return sample
|
||||
sample = clean_expr_str(sample)
|
||||
try:
|
||||
expr = my_parse_latex(sample)
|
||||
except:
|
||||
print("[parse failed]", sample)
|
||||
return None
|
||||
return expr
|
||||
|
||||
|
||||
def my_equals(ans_p, ans_l):
|
||||
return ans_p.equals(ans_l)
|
||||
|
||||
|
||||
def is_expr_equal(ans_p, ans_l, is_strict=False):
|
||||
def is_equ_num_equal(equation, number):
|
||||
if (
|
||||
isinstance(equation, sp.Eq)
|
||||
# and isinstance(equation.lhs, sp.Symbol)
|
||||
and equation.rhs.is_number
|
||||
and number.is_number
|
||||
):
|
||||
try:
|
||||
ret = my_equals(equation.rhs, number)
|
||||
return bool(ret)
|
||||
except:
|
||||
return equation.rhs == number
|
||||
|
||||
if ans_p is None or ans_l is None:
|
||||
return False
|
||||
if isinstance(ans_l, str):
|
||||
return ans_p == ans_l
|
||||
|
||||
if (
|
||||
not is_strict
|
||||
and is_equ_num_equal(ans_l, ans_p)
|
||||
or is_equ_num_equal(ans_p, ans_l)
|
||||
):
|
||||
return True
|
||||
|
||||
if ans_p.free_symbols != ans_l.free_symbols:
|
||||
return False
|
||||
|
||||
if ans_p == ans_l:
|
||||
return True
|
||||
|
||||
if isinstance(ans_l, sp.core.relational.Relational):
|
||||
try:
|
||||
if (
|
||||
type(ans_l) == type(ans_p)
|
||||
and my_equals(ans_p.lhs, ans_l.lhs)
|
||||
and my_equals(ans_p.rhs, ans_l.rhs)
|
||||
):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(ans_p, ans_l, e)
|
||||
try:
|
||||
ret = my_equals(ans_p, ans_l)
|
||||
return bool(ret)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
# @timeout_decorator.timeout(5)
|
||||
# def compare_ans(ans_p_str, ans_l_str, is_strict=False):
|
||||
# ans_p_str = clean_expr_str(ans_p_str)
|
||||
# ans_p_str = ans_p_str.replace(",", "").replace("$", "")
|
||||
# ans_l_str = clean_expr_str(ans_l_str)
|
||||
# ans_l_str = ans_l_str.replace(",", "").replace("$", "")
|
||||
# if ans_p_str is None:
|
||||
# return False
|
||||
# if ans_p_str.replace(" ", "") == ans_l_str.replace(" ", ""):
|
||||
# return True
|
||||
# ans_p = parse_latex_answer(ans_p_str)
|
||||
# if ans_p is None:
|
||||
# return False
|
||||
# ans_l = parse_latex_answer(ans_l_str)
|
||||
# if ans_l is None:
|
||||
# return False
|
||||
# return is_expr_equal(ans_p, ans_l, is_strict=is_strict)
|
||||
|
||||
|
||||
def extract_answer_number(sentence: str) -> float:
|
||||
sentence = sentence.replace(",", "")
|
||||
pred = [s for s in re.findall(r"-?\d+\.?\d*", sentence)]
|
||||
if not pred:
|
||||
return ""
|
||||
return pred[-1]
|
||||
|
||||
|
||||
@timeout_decorator.timeout(5)
|
||||
def compare_ans(ans_p_str, ans_l_str, is_strict=False):
|
||||
ans_p_str = clean_expr_str(ans_p_str)
|
||||
ans_p_str = ans_p_str.replace(",", "").replace("$", "")
|
||||
ans_l_str = clean_expr_str(ans_l_str)
|
||||
ans_l_str = ans_l_str.replace(",", "").replace("$", "")
|
||||
if ans_p_str is None:
|
||||
return False
|
||||
if ans_p_str.replace(" ", "") == ans_l_str.replace(" ", ""):
|
||||
return True
|
||||
ans_p = parse_latex_answer(ans_p_str)
|
||||
if ans_p is None:
|
||||
return False
|
||||
ans_l = parse_latex_answer(ans_l_str)
|
||||
if ans_l is None:
|
||||
return False
|
||||
if is_expr_equal(ans_p, ans_l, is_strict=is_strict):
|
||||
return True
|
||||
# TODO not suitable
|
||||
ans_p_str = extract_answer_number(ans_p_str)
|
||||
if is_number(ans_p_str):
|
||||
ans_p = parse_latex_answer(ans_p_str)
|
||||
if is_expr_equal(ans_p, ans_l, is_strict=is_strict):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def vote(answers):
|
||||
counter = Counter(answers)
|
||||
return counter.most_common(1)[0][0]
|
||||
|
||||
|
||||
def contains_number(s):
|
||||
return any(i.isdigit() for i in s)
|
||||
|
||||
|
||||
def rough_compare_ans(generation, answer):
|
||||
for line in generation.split("\n")[::-1]:
|
||||
if contains_number(line):
|
||||
break
|
||||
words = line.split()
|
||||
for i, w in enumerate(words):
|
||||
if i > 0 and words[i - 1] in ["+", "-", "*", "/", "^"]:
|
||||
continue
|
||||
if i < len(words) - 1 and words[i + 1] in ["+", "-", "*", "/", "^"]:
|
||||
continue
|
||||
if not contains_number(w):
|
||||
continue
|
||||
if compare_numerical_ans(w.replace("$", ""), answer) and "=" not in " ".join(
|
||||
w[i:]
|
||||
):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pred = "To find the minimum value of $f(x)$, we need to find the critical points where the derivative of $f(x)$ is either zero or undefined. Let's write a function to solve this problem:\n\n```python\nfrom sympy import symbols, diff, solve\n\ndef find_minimum_value():\n x, p = symbols('x p')\n f = abs(x - p) + abs(x - 15) + abs(x - p - 15)\n f_prime = diff(f, x)\n critical_points = solve(f_prime, x)\n \n # Evaluate f(x) at the critical points and endpoints\n values = [f.subs(x, p), f.subs(x, 15)]\n for point in critical_points:\n values.append(f.subs(x, point))\n \n # Find the minimum value\n minimum_value = min(values)\n \n return minimum_value\n\nresult = find_minimum_value()\nprint(result)\n```\n```output\nValueError(\"\\nCan't calculate derivative wrt re(x) + I*im(x).\")\n```\nIt seems that there is an error in the code. Let's modify the code to handle the case when the derivative is undefined.\n\n```python\nfrom sympy import symbols, diff, solve, oo\n\ndef find_minimum_value():\n x, p = symbols('x p')\n f = abs(x - p) + abs(x - 15) + abs(x - p - 15)\n f_prime = diff(f, x)\n critical_points = solve(f_prime, x)\n \n # Evaluate f(x) at the critical points and endpoints\n values = [f.subs(x, p), f.subs(x, 15)]\n for point in critical_points:\n if point.is_real:\n values.append(f.subs(x, point))\n \n # Find the minimum value\n minimum_value = min(values)\n \n return minimum_value\n\nresult = find_minimum_value()\nprint(result)\n```\n```output\nValueError(\"\\nCan't calculate derivative wrt re(x) + I*im(x).\")\n```\nApologies for the error in the code. It seems that the derivative calculation is causing an issue. Let's modify the code to handle the absolute value function and find the minimum value using a different approach.\n\n```python\nfrom sympy import symbols, Piecewise, Min, oo\n\ndef find_minimum_value():\n x, p = symbols('x p')\n f = abs(x - p) + abs(x - 15) + abs(x - p - 15)\n \n # Define the piecewise function for f(x)\n f_piecewise = Piecewise((f, x <= p), (f.subs(x, p), p < x <= 15), (f.subs(x, 15), x > 15))\n \n # Find the minimum value\n minimum_value = Min(f_piecewise, (x, p, 15))\n \n return minimum_value\n\nresult = find_minimum_value()\nprint(result)\n```\n```output\nTypeError('cannot determine truth value of Relational')\n```\nApologies for the error in the code. It seems that the piecewise function is causing an issue. Let's modify the code to handle the absolute value function and find the minimum value using a different approach.\n\n```python\nfrom sympy import symbols, Min, oo\n\ndef find_minimum_value():\n x, p = symbols('x p')\n f = abs(x - p) + abs(x - 15) + abs(x - p - 15)\n \n # Find the minimum value\n minimum_value = Min(f, (x, p, 15))\n \n return minimum_value\n\nresult = find_minimum_value()\nprint(result)\n```\n"
|
||||
label = "15"
|
||||
print(compare_ans(pred, label))
|
||||
Executable
+235
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
https://github.com/allenai/open-instruct
|
||||
"""
|
||||
import torch
|
||||
import tqdm
|
||||
from transformers import StoppingCriteria, StoppingCriteriaList
|
||||
|
||||
|
||||
class KeywordsStoppingCriteria(StoppingCriteria):
|
||||
def __init__(self, keywords_str, tokenizer):
|
||||
StoppingCriteria.__init__(self)
|
||||
self.current_context = []
|
||||
self.tokenizer = tokenizer
|
||||
self.keywords_str = keywords_str
|
||||
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
|
||||
if len(self.current_context) == 0:
|
||||
self.current_context = [[] for _ in range(input_ids.shape[0])]
|
||||
|
||||
# self.current_context.append(input_ids[0][-1].item())
|
||||
sequences_should_be_stopped = []
|
||||
for i in range(input_ids.shape[0]):
|
||||
_id = input_ids[i][-1].item()
|
||||
self.current_context[i].append(_id)
|
||||
current_context = self.tokenizer.decode(self.current_context[i])
|
||||
should_be_stopped = False
|
||||
for word in self.keywords_str:
|
||||
if word in current_context:
|
||||
should_be_stopped = True
|
||||
break
|
||||
sequences_should_be_stopped.append(should_be_stopped)
|
||||
return all(sequences_should_be_stopped)
|
||||
|
||||
|
||||
class KeyWordsCriteriaTrunc(StoppingCriteria):
|
||||
def __init__(self, stop_id_sequences, prompt_length):
|
||||
assert isinstance(stop_id_sequences[0], list), "stop_id_sequences should be a list of list of ids"
|
||||
self.stop_sequences = stop_id_sequences
|
||||
self.prompt_length = prompt_length
|
||||
|
||||
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
|
||||
sequences_should_be_stopped = []
|
||||
for i in range(input_ids.shape[0]):
|
||||
ids = input_ids[i][self.prompt_length:].tolist()
|
||||
should_be_stopped = False
|
||||
for stop_sequence in self.stop_sequences:
|
||||
if input_ids.shape[0] == 1:
|
||||
_ids = ids[-len(stop_sequence):]
|
||||
else:
|
||||
_ids = ids
|
||||
for j in range(len(_ids), 0, -len(stop_sequence)):
|
||||
if _ids[max(j - len(stop_sequence), 0): j] == stop_sequence:
|
||||
should_be_stopped = True
|
||||
break
|
||||
if should_be_stopped:
|
||||
break
|
||||
sequences_should_be_stopped.append(should_be_stopped)
|
||||
return all(sequences_should_be_stopped)
|
||||
|
||||
|
||||
class KeyWordsCriteria(StoppingCriteria):
|
||||
def __init__(self, stop_id_sequences):
|
||||
assert isinstance(stop_id_sequences[0], list), "stop_id_sequences should be a list of list of ids"
|
||||
self.stop_sequences = stop_id_sequences
|
||||
|
||||
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
|
||||
sequences_should_be_stopped = []
|
||||
for i in range(input_ids.shape[0]):
|
||||
sequence_should_be_stopped = False
|
||||
for stop_sequence in self.stop_sequences:
|
||||
if input_ids[i][-len(stop_sequence):].tolist() == stop_sequence:
|
||||
sequence_should_be_stopped = True
|
||||
break
|
||||
sequences_should_be_stopped.append(sequence_should_be_stopped)
|
||||
return all(sequences_should_be_stopped)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def generate_completions(model, tokenizer, prompts, batch_size=1, stop_id_sequences=None, add_special_tokens=True, disable_tqdm=False, **generation_kwargs):
|
||||
generations = []
|
||||
if not disable_tqdm:
|
||||
progress = tqdm.tqdm(total=len(prompts), desc="Generating Completions")
|
||||
|
||||
num_return_sequences = generation_kwargs.get("num_return_sequences", 1)
|
||||
for i in range(0, len(prompts), batch_size):
|
||||
batch_prompts = prompts[i:i+batch_size]
|
||||
tokenized_prompts = tokenizer(batch_prompts, padding="longest", return_tensors="pt", add_special_tokens=add_special_tokens)
|
||||
batch_input_ids = tokenized_prompts.input_ids
|
||||
attention_mask = tokenized_prompts.attention_mask
|
||||
|
||||
if model.device.type == "cuda":
|
||||
batch_input_ids = batch_input_ids.cuda()
|
||||
attention_mask = attention_mask.cuda()
|
||||
|
||||
# try:
|
||||
stop_criteria = KeywordsStoppingCriteria(stop_id_sequences, tokenizer)
|
||||
batch_outputs = model.generate(
|
||||
input_ids=batch_input_ids,
|
||||
attention_mask=attention_mask,
|
||||
stopping_criteria=StoppingCriteriaList([stop_criteria]),
|
||||
# stopping_criteria=[KeyWordsCriteria(stop_id_sequences)] if stop_id_sequences else None,
|
||||
# stopping_criteria=[KeyWordsCriteriaTrunc(stop_id_sequences, batch_input_ids.size(1))] if stop_id_sequences else None,
|
||||
**generation_kwargs
|
||||
)
|
||||
|
||||
# the stopping criteria is applied at batch level, so if other examples are not stopped, the entire batch will continue to generate.
|
||||
# so some outputs still have the stop sequence, which we need to remove.
|
||||
# if stop_id_sequences:
|
||||
# for output_idx in range(batch_outputs.shape[0]):
|
||||
# for token_idx in range(batch_input_ids.shape[1], batch_outputs.shape[1]):
|
||||
# if any(batch_outputs[output_idx, token_idx: token_idx+len(stop_sequence)].tolist() == stop_sequence for stop_sequence in stop_id_sequences):
|
||||
# batch_outputs[output_idx, token_idx:] = tokenizer.pad_token_id
|
||||
# break
|
||||
|
||||
# remove the prompt from the output
|
||||
# we need to re-encode the prompt because we need to make sure the special tokens are treated the same way as in the outputs.
|
||||
# we changed our previous way of truncating the output token ids dicrectly because some tokenizer (e.g., llama) won't add space token before the first token.
|
||||
# space is important for some tasks (e.g., code completion).
|
||||
batch_outputs = tokenizer.batch_decode(batch_outputs, skip_special_tokens=True)
|
||||
batch_prompts = tokenizer.batch_decode(batch_input_ids, skip_special_tokens=True)
|
||||
# duplicate the prompts to match the number of return sequences
|
||||
batch_prompts = [prompt for prompt in batch_prompts for _ in range(num_return_sequences)]
|
||||
batch_generations = [
|
||||
output[len(prompt):] for prompt, output in zip(batch_prompts, batch_outputs)
|
||||
]
|
||||
|
||||
# remove the remain stop sequence from the output.
|
||||
for idx, prediction in enumerate(batch_generations):
|
||||
for stop_sequence in stop_id_sequences:
|
||||
batch_generations[idx] = prediction.split(stop_sequence)[0]
|
||||
|
||||
generations += batch_generations
|
||||
|
||||
if not disable_tqdm:
|
||||
progress.update(len(batch_prompts)//num_return_sequences)
|
||||
|
||||
assert len(generations) == len(prompts) * num_return_sequences, "number of generations should be equal to number of prompts * num_return_sequences"
|
||||
return generations
|
||||
|
||||
|
||||
def load_hf_lm_and_tokenizer(
|
||||
model_name_or_path,
|
||||
tokenizer_name_or_path=None,
|
||||
device_map="auto",
|
||||
load_in_8bit=False,
|
||||
load_in_half=True,
|
||||
gptq_model=False,
|
||||
use_fast_tokenizer=False,
|
||||
padding_side="left",
|
||||
use_safetensors=False,
|
||||
):
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
if not tokenizer_name_or_path:
|
||||
tokenizer_name_or_path = model_name_or_path
|
||||
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=use_fast_tokenizer, padding_side=padding_side, trust_remote_code=True)
|
||||
# tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, legacy=False, use_fast=use_fast_tokenizer, padding_side=padding_side, trust_remote_code=True)
|
||||
|
||||
# set pad token to eos token if pad token is not set
|
||||
if tokenizer.pad_token is None:
|
||||
if tokenizer.unk_token:
|
||||
tokenizer.pad_token = tokenizer.unk_token
|
||||
tokenizer.pad_token_id = tokenizer.unk_token_id
|
||||
elif tokenizer.eos_token:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
tokenizer.pad_token_id = tokenizer.eos_token_id
|
||||
else:
|
||||
raise ValueError("You are using a new tokenizer without a pad token."
|
||||
"This is not supported by this script.")
|
||||
|
||||
# if tokenizer.pad_token is None:
|
||||
# tokenizer.pad_token = tokenizer.unk_token
|
||||
# tokenizer.pad_token_id = tokenizer.unk_token_id
|
||||
|
||||
if gptq_model:
|
||||
from auto_gptq import AutoGPTQForCausalLM
|
||||
model_wrapper = AutoGPTQForCausalLM.from_quantized(
|
||||
model_name_or_path, device="cuda:0", use_triton=True
|
||||
)
|
||||
model = model_wrapper.model
|
||||
elif load_in_8bit:
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name_or_path,
|
||||
device_map=device_map,
|
||||
load_in_8bit=True
|
||||
)
|
||||
else:
|
||||
# return "", tokenizer
|
||||
# defaul load in float16
|
||||
model = AutoModelForCausalLM.from_pretrained(model_name_or_path,
|
||||
torch_dtype=torch.float16,
|
||||
device_map=device_map,
|
||||
trust_remote_code=True,
|
||||
use_safetensors=use_safetensors)
|
||||
if torch.cuda.is_available():
|
||||
model = model.cuda()
|
||||
if load_in_half:
|
||||
model = model.half()
|
||||
model.eval()
|
||||
return model, tokenizer
|
||||
|
||||
|
||||
def _test_generate_completions():
|
||||
model_name_or_path = "../models/codellama_7b/v1-16k"
|
||||
llm, tokenizer = load_hf_lm_and_tokenizer(
|
||||
model_name_or_path=model_name_or_path,
|
||||
load_in_half=True,
|
||||
use_fast_tokenizer=True,
|
||||
use_safetensors=True,
|
||||
)
|
||||
# some math word problems
|
||||
prompts = [
|
||||
"---\n1+1=2\n---2+2=4\n---3+3=6\n---4+4=8\n---5+5=10\n---6+6=",
|
||||
"---\n1+1=2\n---12+12=24\n---3+3=6\n---12345+12345=",
|
||||
# "A train leaves Chicago at 7am and travels at 60mph. Another train leaves Chicago at 9am and travels at 80mph. When will the second train overtake the first?",
|
||||
# "The sum of two numbers is 10. The difference of the same two numbers is 4. What are the two numbers?",
|
||||
]
|
||||
|
||||
stop_sequences = ["\n\n\n", "---"]
|
||||
# Because many tokenizers will treat the word after space differently from the original word alone,
|
||||
# to be consistent, we add a space before tokenization and remove it after tokenization.
|
||||
# stop_id_sequences = [tokenizer.encode(" " + x, add_special_tokens=False)[1:] for x in stop_sequences]
|
||||
outputs = generate_completions(
|
||||
model=llm,
|
||||
tokenizer=tokenizer,
|
||||
prompts=prompts,
|
||||
max_new_tokens=128,
|
||||
batch_size=16,
|
||||
# stop_id_sequences=stop_id_sequences,
|
||||
stop_id_sequences=stop_sequences,
|
||||
)
|
||||
print(outputs)
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test_generate_completions()
|
||||
Executable
+760
@@ -0,0 +1,760 @@
|
||||
import random
|
||||
import regex
|
||||
import re
|
||||
import sympy
|
||||
from latex2sympy2 import latex2sympy
|
||||
from typing import TypeVar, Iterable, List, Union, Any, Dict
|
||||
from word2number import w2n
|
||||
from data.qwen25math.utils import *
|
||||
|
||||
|
||||
def _fix_fracs(string):
|
||||
substrs = string.split("\\frac")
|
||||
new_str = substrs[0]
|
||||
if len(substrs) > 1:
|
||||
substrs = substrs[1:]
|
||||
for substr in substrs:
|
||||
new_str += "\\frac"
|
||||
if len(substr) > 0 and substr[0] == "{":
|
||||
new_str += substr
|
||||
else:
|
||||
try:
|
||||
assert len(substr) >= 2
|
||||
except:
|
||||
return string
|
||||
a = substr[0]
|
||||
b = substr[1]
|
||||
if b != "{":
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}{" + b + "}" + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}{" + b + "}"
|
||||
else:
|
||||
if len(substr) > 2:
|
||||
post_substr = substr[2:]
|
||||
new_str += "{" + a + "}" + b + post_substr
|
||||
else:
|
||||
new_str += "{" + a + "}" + b
|
||||
string = new_str
|
||||
return string
|
||||
|
||||
|
||||
def _fix_a_slash_b(string):
|
||||
if len(string.split("/")) != 2:
|
||||
return string
|
||||
a = string.split("/")[0]
|
||||
b = string.split("/")[1]
|
||||
try:
|
||||
if "sqrt" not in a:
|
||||
a = int(a)
|
||||
if "sqrt" not in b:
|
||||
b = int(b)
|
||||
assert string == "{}/{}".format(a, b)
|
||||
new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
|
||||
return new_string
|
||||
except:
|
||||
return string
|
||||
|
||||
|
||||
def _fix_sqrt(string):
|
||||
_string = re.sub(r"\\sqrt(\w+)", r"\\sqrt{\1}", string)
|
||||
return _string
|
||||
|
||||
|
||||
def convert_word_number(text: str) -> str:
|
||||
try:
|
||||
text = str(w2n.word_to_num(text))
|
||||
except:
|
||||
pass
|
||||
return text
|
||||
|
||||
|
||||
# units mainly from MathQA
|
||||
unit_texts = [
|
||||
"east",
|
||||
"degree",
|
||||
"mph",
|
||||
"kmph",
|
||||
"ft",
|
||||
"m sqaure",
|
||||
" m east",
|
||||
"sq m",
|
||||
"deg",
|
||||
"mile",
|
||||
"q .",
|
||||
"monkey",
|
||||
"prime",
|
||||
"ratio",
|
||||
"profit of rs",
|
||||
"rd",
|
||||
"o",
|
||||
"gm",
|
||||
"p . m",
|
||||
"lb",
|
||||
"tile",
|
||||
"per",
|
||||
"dm",
|
||||
"lt",
|
||||
"gain",
|
||||
"ab",
|
||||
"way",
|
||||
"west",
|
||||
"a .",
|
||||
"b .",
|
||||
"c .",
|
||||
"d .",
|
||||
"e .",
|
||||
"f .",
|
||||
"g .",
|
||||
"h .",
|
||||
"t",
|
||||
"a",
|
||||
"h",
|
||||
"no change",
|
||||
"men",
|
||||
"soldier",
|
||||
"pie",
|
||||
"bc",
|
||||
"excess",
|
||||
"st",
|
||||
"inches",
|
||||
"noon",
|
||||
"percent",
|
||||
"by",
|
||||
"gal",
|
||||
"kmh",
|
||||
"c",
|
||||
"acre",
|
||||
"rise",
|
||||
"a . m",
|
||||
"th",
|
||||
"π r 2",
|
||||
"sq",
|
||||
"mark",
|
||||
"l",
|
||||
"toy",
|
||||
"coin",
|
||||
"sq . m",
|
||||
"gallon",
|
||||
"° f",
|
||||
"profit",
|
||||
"minw",
|
||||
"yr",
|
||||
"women",
|
||||
"feet",
|
||||
"am",
|
||||
"pm",
|
||||
"hr",
|
||||
"cu cm",
|
||||
"square",
|
||||
"v â € ™",
|
||||
"are",
|
||||
"rupee",
|
||||
"rounds",
|
||||
"cubic",
|
||||
"cc",
|
||||
"mtr",
|
||||
"s",
|
||||
"ohm",
|
||||
"number",
|
||||
"kmph",
|
||||
"day",
|
||||
"hour",
|
||||
"minute",
|
||||
"min",
|
||||
"second",
|
||||
"man",
|
||||
"woman",
|
||||
"sec",
|
||||
"cube",
|
||||
"mt",
|
||||
"sq inch",
|
||||
"mp",
|
||||
"∏ cm ³",
|
||||
"hectare",
|
||||
"more",
|
||||
"sec",
|
||||
"unit",
|
||||
"cu . m",
|
||||
"cm 2",
|
||||
"rs .",
|
||||
"rs",
|
||||
"kg",
|
||||
"g",
|
||||
"month",
|
||||
"km",
|
||||
"m",
|
||||
"cm",
|
||||
"mm",
|
||||
"apple",
|
||||
"liter",
|
||||
"loss",
|
||||
"yard",
|
||||
"pure",
|
||||
"year",
|
||||
"increase",
|
||||
"decrease",
|
||||
"d",
|
||||
"less",
|
||||
"Surface",
|
||||
"litre",
|
||||
"pi sq m",
|
||||
"s .",
|
||||
"metre",
|
||||
"meter",
|
||||
"inch",
|
||||
]
|
||||
|
||||
unit_texts.extend([t + "s" for t in unit_texts])
|
||||
|
||||
|
||||
def strip_string(string, skip_unit=False):
|
||||
string = str(string).strip()
|
||||
# linebreaks
|
||||
string = string.replace("\n", "")
|
||||
|
||||
# right "."
|
||||
string = string.rstrip(".")
|
||||
|
||||
# remove inverse spaces
|
||||
# replace \\ with \
|
||||
string = string.replace("\\!", "")
|
||||
# string = string.replace("\\ ", "")
|
||||
# string = string.replace("\\\\", "\\")
|
||||
|
||||
# matrix
|
||||
string = re.sub(r"\\begin\{array\}\{.*?\}", r"\\begin{pmatrix}", string)
|
||||
string = re.sub(r"\\end\{array\}", r"\\end{pmatrix}", string)
|
||||
string = string.replace("bmatrix", "pmatrix")
|
||||
|
||||
# replace tfrac and dfrac with frac
|
||||
string = string.replace("tfrac", "frac")
|
||||
string = string.replace("dfrac", "frac")
|
||||
string = (
|
||||
string.replace("\\neq", "\\ne")
|
||||
.replace("\\leq", "\\le")
|
||||
.replace("\\geq", "\\ge")
|
||||
)
|
||||
|
||||
# remove \left and \right
|
||||
string = string.replace("\\left", "")
|
||||
string = string.replace("\\right", "")
|
||||
string = string.replace("\\{", "{")
|
||||
string = string.replace("\\}", "}")
|
||||
|
||||
# Remove unit: miles, dollars if after is not none
|
||||
_string = re.sub(r"\\text{.*?}$", "", string).strip()
|
||||
if _string != "" and _string != string:
|
||||
# print("Warning: unit not removed: '{}' -> '{}'".format(string, _string))
|
||||
string = _string
|
||||
|
||||
if not skip_unit:
|
||||
# Remove unit: texts
|
||||
for _ in range(2):
|
||||
for unit_text in unit_texts:
|
||||
# use regex, the prefix should be either the start of the string or a non-alphanumeric character
|
||||
# the suffix should be either the end of the string or a non-alphanumeric character
|
||||
_string = re.sub(r"(^|\W)" + unit_text + r"($|\W)", r"\1\2", string)
|
||||
if _string != "":
|
||||
string = _string
|
||||
|
||||
# Remove circ (degrees)
|
||||
string = string.replace("^{\\circ}", "")
|
||||
string = string.replace("^\\circ", "")
|
||||
|
||||
# remove dollar signs
|
||||
string = string.replace("\\$", "")
|
||||
string = string.replace("$", "")
|
||||
string = string.replace("\\(", "").replace("\\)", "")
|
||||
|
||||
# convert word number to digit
|
||||
string = convert_word_number(string)
|
||||
|
||||
# replace "\\text{...}" to "..."
|
||||
string = re.sub(r"\\text\{(.*?)\}", r"\1", string)
|
||||
for key in ["x=", "y=", "z=", "x\\in", "y\\in", "z\\in", "x\\to", "y\\to", "z\\to"]:
|
||||
string = string.replace(key, "")
|
||||
string = string.replace("\\emptyset", r"{}")
|
||||
string = string.replace("(-\\infty,\\infty)", "\\mathbb{R}")
|
||||
|
||||
# remove percentage
|
||||
string = string.replace("\\%", "")
|
||||
string = string.replace("\%", "")
|
||||
string = string.replace("%", "")
|
||||
|
||||
# " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
|
||||
string = string.replace(" .", " 0.")
|
||||
string = string.replace("{.", "{0.")
|
||||
|
||||
# cdot
|
||||
# string = string.replace("\\cdot", "")
|
||||
if (
|
||||
string.startswith("{")
|
||||
and string.endswith("}")
|
||||
and string.isalnum()
|
||||
or string.startswith("(")
|
||||
and string.endswith(")")
|
||||
and string.isalnum()
|
||||
or string.startswith("[")
|
||||
and string.endswith("]")
|
||||
and string.isalnum()
|
||||
):
|
||||
string = string[1:-1]
|
||||
|
||||
# inf
|
||||
string = string.replace("infinity", "\\infty")
|
||||
if "\\infty" not in string:
|
||||
string = string.replace("inf", "\\infty")
|
||||
string = string.replace("+\\inity", "\\infty")
|
||||
|
||||
# and
|
||||
string = string.replace("and", "")
|
||||
string = string.replace("\\mathbf", "")
|
||||
|
||||
# use regex to remove \mbox{...}
|
||||
string = re.sub(r"\\mbox{.*?}", "", string)
|
||||
|
||||
# quote
|
||||
string.replace("'", "")
|
||||
string.replace('"', "")
|
||||
|
||||
# i, j
|
||||
if "j" in string and "i" not in string:
|
||||
string = string.replace("j", "i")
|
||||
|
||||
# replace a.000b where b is not number or b is end, with ab, use regex
|
||||
string = re.sub(r"(\d+)\.0*([^\d])", r"\1\2", string)
|
||||
string = re.sub(r"(\d+)\.0*$", r"\1", string)
|
||||
|
||||
# if empty, return empty string
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if string[0] == ".":
|
||||
string = "0" + string
|
||||
|
||||
# to consider: get rid of e.g. "k = " or "q = " at beginning
|
||||
if len(string.split("=")) == 2:
|
||||
if len(string.split("=")[0]) <= 2:
|
||||
string = string.split("=")[1]
|
||||
|
||||
string = _fix_sqrt(string)
|
||||
string = string.replace(" ", "")
|
||||
|
||||
# \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
|
||||
string = _fix_fracs(string)
|
||||
|
||||
# NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
|
||||
string = _fix_a_slash_b(string)
|
||||
|
||||
return string
|
||||
|
||||
|
||||
def extract_multi_choice_answer(pred_str):
|
||||
# TODO: SFT models
|
||||
if "Problem:" in pred_str:
|
||||
pred_str = pred_str.split("Problem:", 1)[0]
|
||||
pred_str = pred_str.replace("choice is", "answer is")
|
||||
patt = regex.search(r"answer is \(?(?P<ans>[abcde])\)?", pred_str.lower())
|
||||
if patt is not None:
|
||||
return patt.group("ans").upper()
|
||||
return "placeholder"
|
||||
|
||||
|
||||
direct_answer_trigger_for_fewshot = ("choice is", "answer is")
|
||||
|
||||
|
||||
def choice_answer_clean(pred: str):
|
||||
pred = pred.strip("\n")
|
||||
|
||||
# Determine if this is ICL, if so, use \n\n to split the first chunk.
|
||||
ICL = False
|
||||
for trigger in direct_answer_trigger_for_fewshot:
|
||||
if pred.count(trigger) > 1:
|
||||
ICL = True
|
||||
if ICL:
|
||||
pred = pred.split("\n\n")[0]
|
||||
|
||||
# Split the trigger to find the answer.
|
||||
preds = re.split("|".join(direct_answer_trigger_for_fewshot), pred)
|
||||
if len(preds) > 1:
|
||||
answer_flag = True
|
||||
pred = preds[-1]
|
||||
else:
|
||||
answer_flag = False
|
||||
|
||||
pred = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":")
|
||||
|
||||
# Clean the answer based on the dataset
|
||||
tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper())
|
||||
if tmp:
|
||||
pred = tmp
|
||||
else:
|
||||
pred = [pred.strip().strip(".")]
|
||||
|
||||
if len(pred) == 0:
|
||||
pred = ""
|
||||
else:
|
||||
if answer_flag:
|
||||
# choose the first element in list ...
|
||||
pred = pred[0]
|
||||
else:
|
||||
# choose the last e
|
||||
pred = pred[-1]
|
||||
|
||||
# Remove the period at the end, again!
|
||||
pred = pred.rstrip(".").rstrip("/")
|
||||
|
||||
return pred
|
||||
|
||||
|
||||
def find_box(pred_str: str):
|
||||
ans = pred_str.split("boxed")[-1]
|
||||
if not ans:
|
||||
return ""
|
||||
if ans[0] == "{":
|
||||
stack = 1
|
||||
a = ""
|
||||
for c in ans[1:]:
|
||||
if c == "{":
|
||||
stack += 1
|
||||
a += c
|
||||
elif c == "}":
|
||||
stack -= 1
|
||||
if stack == 0:
|
||||
break
|
||||
a += c
|
||||
else:
|
||||
a += c
|
||||
else:
|
||||
a = ans.split("$")[0].strip()
|
||||
return a
|
||||
|
||||
|
||||
def clean_units(pred_str: str):
|
||||
"""Clean the units in the number."""
|
||||
|
||||
def convert_pi_to_number(code_string):
|
||||
code_string = code_string.replace("\\pi", "π")
|
||||
# Replace \pi or π not preceded by a digit or } with 3.14
|
||||
code_string = re.sub(r"(?<![\d}])\\?π", "3.14", code_string)
|
||||
# Replace instances where π is preceded by a digit but without a multiplication symbol, e.g., "3π" -> "3*3.14"
|
||||
code_string = re.sub(r"(\d)(\\?π)", r"\1*3.14", code_string)
|
||||
# Handle cases where π is within braces or followed by a multiplication symbol
|
||||
# This replaces "{π}" with "3.14" directly and "3*π" with "3*3.14"
|
||||
code_string = re.sub(r"\{(\\?π)\}", "3.14", code_string)
|
||||
code_string = re.sub(r"\*(\\?π)", "*3.14", code_string)
|
||||
return code_string
|
||||
|
||||
pred_str = convert_pi_to_number(pred_str)
|
||||
pred_str = pred_str.replace("%", "/100")
|
||||
pred_str = pred_str.replace("$", "")
|
||||
pred_str = pred_str.replace("¥", "")
|
||||
pred_str = pred_str.replace("°C", "")
|
||||
pred_str = pred_str.replace(" C", "")
|
||||
pred_str = pred_str.replace("°", "")
|
||||
return pred_str
|
||||
|
||||
|
||||
def extract_theoremqa_answer(pred: str, answer_flag: bool = True):
|
||||
if any([option in pred.lower() for option in ["yes", "true"]]):
|
||||
pred = "True"
|
||||
elif any([option in pred.lower() for option in ["no", "false"]]):
|
||||
pred = "False"
|
||||
elif any(
|
||||
[
|
||||
option in pred.lower()
|
||||
for option in ["(a)", "(b)", "(c)", "(d)", "(e)", "(f)"]
|
||||
]
|
||||
):
|
||||
pass
|
||||
else:
|
||||
# Some of the models somehow get used to boxed output from pre-training
|
||||
if "boxed" in pred:
|
||||
pred = find_box(pred)
|
||||
|
||||
if answer_flag:
|
||||
# Extract the numbers out of the string
|
||||
pred = pred.split("=")[-1].strip()
|
||||
pred = clean_units(pred)
|
||||
try:
|
||||
tmp = str(latex2sympy(pred))
|
||||
pred = str(eval(tmp))
|
||||
except Exception:
|
||||
if re.match(r"-?[\d\.]+\s\D+$", pred):
|
||||
pred = pred.split(" ")[0]
|
||||
elif re.match(r"-?[\d\.]+\s[^\s]+$", pred):
|
||||
pred = pred.split(" ")[0]
|
||||
else:
|
||||
# desparate search over the last number
|
||||
preds = re.findall(r"-?\d*\.?\d+", pred)
|
||||
if len(preds) >= 1:
|
||||
pred = preds[-1]
|
||||
else:
|
||||
pred = ""
|
||||
|
||||
return pred
|
||||
|
||||
|
||||
def extract_answer(pred_str, data_name, use_last_number=True):
|
||||
pred_str = pred_str.replace("\u043a\u0438", "")
|
||||
if data_name in ["mmlu_stem", "sat_math", "aqua", "gaokao2023"]:
|
||||
# TODO check multiple choice
|
||||
return choice_answer_clean(pred_str)
|
||||
|
||||
if "final answer is $" in pred_str and "$. I hope" in pred_str:
|
||||
# minerva_math
|
||||
tmp = pred_str.split("final answer is $", 1)[1]
|
||||
pred = tmp.split("$. I hope", 1)[0].strip()
|
||||
elif "boxed" in pred_str:
|
||||
ans = pred_str.split("boxed")[-1]
|
||||
if len(ans) == 0:
|
||||
return ""
|
||||
elif ans[0] == "{":
|
||||
stack = 1
|
||||
a = ""
|
||||
for c in ans[1:]:
|
||||
if c == "{":
|
||||
stack += 1
|
||||
a += c
|
||||
elif c == "}":
|
||||
stack -= 1
|
||||
if stack == 0:
|
||||
break
|
||||
a += c
|
||||
else:
|
||||
a += c
|
||||
else:
|
||||
a = ans.split("$")[0].strip()
|
||||
pred = a
|
||||
elif "he answer is" in pred_str:
|
||||
pred = pred_str.split("he answer is")[-1].strip()
|
||||
elif "final answer is" in pred_str:
|
||||
pred = pred_str.split("final answer is")[-1].strip()
|
||||
elif "答案是" in pred_str:
|
||||
# Handle Chinese few-shot multiple choice problem answer extraction
|
||||
pred = pred_str.split("答案是")[1].strip().split("\n\n")[0].strip()
|
||||
else: # use the last number
|
||||
if use_last_number:
|
||||
pattern = "-?\d*\.?\d+"
|
||||
pred = re.findall(pattern, pred_str.replace(",", ""))
|
||||
if len(pred) >= 1:
|
||||
pred = pred[-1]
|
||||
else:
|
||||
pred = ""
|
||||
else:
|
||||
pred = ""
|
||||
|
||||
# choice answer
|
||||
if (
|
||||
data_name in ["sat_math", "aqua"]
|
||||
or "mmlu" in data_name
|
||||
):
|
||||
tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper())
|
||||
if tmp:
|
||||
pred = tmp[-1]
|
||||
else:
|
||||
pred = pred.strip().strip(".")
|
||||
|
||||
# multiple line
|
||||
# pred = pred.split("\n")[0]
|
||||
pred = re.sub(r"\n\s*", "", pred)
|
||||
if pred != "" and pred[0] == ":":
|
||||
pred = pred[1:]
|
||||
if pred != "" and pred[-1] == ".":
|
||||
pred = pred[:-1]
|
||||
if pred != "" and pred[-1] == "/":
|
||||
pred = pred[:-1]
|
||||
pred = strip_string(pred, skip_unit=data_name in ["carp_en", "minerva_math"])
|
||||
return pred
|
||||
|
||||
|
||||
STRIP_EXCEPTIONS = ["carp_en", "minerva_math"]
|
||||
|
||||
|
||||
def parse_ground_truth(example: Dict[str, Any], data_name):
|
||||
if "gt_cot" in example and "gt" in example:
|
||||
if data_name in ["math"]:
|
||||
gt_ans = extract_answer(example["gt_cot"], data_name)
|
||||
elif data_name in STRIP_EXCEPTIONS:
|
||||
gt_ans = example["gt"]
|
||||
else:
|
||||
gt_ans = strip_string(example["gt"])
|
||||
return example["gt_cot"], gt_ans
|
||||
|
||||
# parse ground truth
|
||||
if data_name in ["math", "minerva_math"]:
|
||||
gt_cot = example["solution"]
|
||||
gt_ans = extract_answer(gt_cot, data_name)
|
||||
elif data_name == "gsm8k":
|
||||
gt_cot, gt_ans = example["answer"].split("####")
|
||||
elif data_name == "svamp":
|
||||
gt_cot, gt_ans = example["Equation"], example["Answer"]
|
||||
elif data_name == "asdiv":
|
||||
gt_cot = example["formula"]
|
||||
gt_ans = re.sub(r"\(.*?\)", "", example["answer"])
|
||||
elif data_name == "mawps":
|
||||
gt_cot, gt_ans = None, example["target"]
|
||||
elif data_name == "tabmwp":
|
||||
gt_cot = example["solution"]
|
||||
gt_ans = example["answer"]
|
||||
if example["ans_type"] in ["integer_number", "decimal_number"]:
|
||||
if "/" in gt_ans:
|
||||
gt_ans = int(gt_ans.split("/")[0]) / int(gt_ans.split("/")[1])
|
||||
elif "," in gt_ans:
|
||||
gt_ans = float(gt_ans.replace(",", ""))
|
||||
elif "%" in gt_ans:
|
||||
gt_ans = float(gt_ans.split("%")[0]) / 100
|
||||
else:
|
||||
gt_ans = float(gt_ans)
|
||||
elif data_name == "carp_en":
|
||||
gt_cot, gt_ans = example["steps"], example["answer"]
|
||||
elif data_name == "mmlu_stem":
|
||||
abcd = "ABCD"
|
||||
gt_cot, gt_ans = None, abcd[example["answer"]]
|
||||
elif data_name == "sat_math":
|
||||
gt_cot, gt_ans = None, example["Answer"]
|
||||
elif data_name == "aqua":
|
||||
gt_cot, gt_ans = None, example["correct"]
|
||||
elif data_name in ["gaokao2023en", "college_math", "gaokao_math_cloze"]:
|
||||
gt_cot, gt_ans = None, example["answer"].replace("$", "").strip()
|
||||
elif data_name == "gaokao_math_qa":
|
||||
gt_cot, gt_ans = None, example["label"]
|
||||
elif data_name in ["gaokao2024_mix", "cn_middle_school"]:
|
||||
if len(example["choice_answer"]) > 0:
|
||||
gt_cot, gt_ans = None, example["choice_answer"]
|
||||
else:
|
||||
gt_cot, gt_ans = None, example["answer"]
|
||||
elif data_name == "olympiadbench":
|
||||
gt_cot, gt_ans = None, example["final_answer"][0].strip("$")
|
||||
elif data_name in [
|
||||
"aime24",
|
||||
"amc23",
|
||||
"cmath",
|
||||
"gaokao2024_I",
|
||||
"gaokao2024_II",
|
||||
"imo2024",
|
||||
]:
|
||||
gt_cot, gt_ans = None, example["answer"]
|
||||
else:
|
||||
raise NotImplementedError(f"`{data_name}`")
|
||||
# post process
|
||||
gt_cot = str(gt_cot).strip()
|
||||
if data_name not in STRIP_EXCEPTIONS:
|
||||
gt_ans = strip_string(gt_ans, skip_unit=data_name == "carp_en")
|
||||
else:
|
||||
gt_ans = (
|
||||
gt_ans.replace("\\neq", "\\ne")
|
||||
.replace("\\leq", "\\le")
|
||||
.replace("\\geq", "\\ge")
|
||||
)
|
||||
return gt_cot, gt_ans
|
||||
|
||||
|
||||
def parse_question(example, data_name):
|
||||
question = ""
|
||||
if data_name == "asdiv":
|
||||
question = f"{example['body'].strip()} {example['question'].strip()}"
|
||||
elif data_name == "svamp":
|
||||
body = example["Body"].strip()
|
||||
if not body.endswith("."):
|
||||
body = body + "."
|
||||
question = f'{body} {example["Question"].strip()}'
|
||||
elif data_name == "tabmwp":
|
||||
title_str = (
|
||||
f'regarding "{example["table_title"]}" ' if example["table_title"] else ""
|
||||
)
|
||||
question = f"Read the following table {title_str}and answer a question:\n"
|
||||
question += f'{example["table"]}\n{example["question"]}'
|
||||
if example["choices"]:
|
||||
question += (
|
||||
f' Please select from the following options: {example["choices"]}'
|
||||
)
|
||||
elif data_name == "carp_en":
|
||||
question = example["content"]
|
||||
elif data_name == "mmlu_stem":
|
||||
options = example["choices"]
|
||||
assert len(options) == 4
|
||||
for i, (label, option) in enumerate(zip("ABCD", options)):
|
||||
options[i] = f"({label}) {str(option).strip()}"
|
||||
options = " ".join(options)
|
||||
# question = f"{example['question'].strip()}\nWhat of the following is the right choice? Explain your answer.\n{options}"
|
||||
question = f"{example['question'].strip()}\nAnswer Choices: {options}"
|
||||
elif data_name == "sat_math":
|
||||
options = example["options"].strip()
|
||||
assert "A" == options[0]
|
||||
options = "(" + options
|
||||
for ch in "BCD":
|
||||
if f" {ch}) " in options:
|
||||
options = regex.sub(f" {ch}\) ", f" ({ch}) ", options)
|
||||
# question = f"{example['question'].strip()}\nWhat of the following is the right choice? Explain your answer.\n{options.strip()}"
|
||||
question = f"{example['question'].strip()}\nAnswer Choices: {options}"
|
||||
elif "aqua" in data_name:
|
||||
options = example["options"]
|
||||
choice = "(" + "(".join(options)
|
||||
choice = choice.replace("(", " (").replace(")", ") ").strip()
|
||||
choice = "\nAnswer Choices: " + choice
|
||||
question = example["question"].strip() + choice
|
||||
elif data_name == "gaokao_math_qa":
|
||||
options_dict = example["options"]
|
||||
options = []
|
||||
for key in options_dict:
|
||||
options.append(f"({key}) {options_dict[key]}")
|
||||
options = " ".join(options)
|
||||
question = f"{example['question'].strip()}\n选项: {options}"
|
||||
else:
|
||||
for key in ["question", "problem", "Question", "input"]:
|
||||
if key in example:
|
||||
question = example[key]
|
||||
break
|
||||
# assert question != ""
|
||||
# Yes or No question
|
||||
_, gt_ans = parse_ground_truth(example, data_name)
|
||||
if isinstance(gt_ans, str):
|
||||
gt_lower = gt_ans.lower()
|
||||
if gt_lower in ["true", "false"]:
|
||||
question += " (True or False)"
|
||||
if gt_lower in ["yes", "no"]:
|
||||
question += " (Yes or No)"
|
||||
return question.strip()
|
||||
|
||||
|
||||
def run_execute(executor, result, prompt_type, data_name, execute=False):
|
||||
if not result or result == "error":
|
||||
return None, None
|
||||
report = None
|
||||
|
||||
if "program_only" in prompt_type:
|
||||
prediction = extract_program_output(result)
|
||||
elif prompt_type in ["pot", "pal"] and execute:
|
||||
code = extract_program(result)
|
||||
prediction, report = executor.apply(code)
|
||||
else:
|
||||
prediction = extract_answer(result, data_name)
|
||||
|
||||
# prediction = strip_string(prediction, skip_unit=data_name == "carp_en")
|
||||
prediction = strip_string(prediction, skip_unit=data_name in STRIP_EXCEPTIONS)
|
||||
return prediction, report
|
||||
|
||||
|
||||
def _test_extract_answer():
|
||||
text = """
|
||||
This is still not equal to $0$, so we must have made another mistake.
|
||||
|
||||
When we subtracted $7$ from $\frac{386}{64}$, we should have subtracted $7 \cdot 64$ from $386$, not the other way around. Let's correct that:
|
||||
|
||||
\[\frac{386}{64} - 7 = \frac{386}{64} - \frac{7 \cdot 64}{1 \cdot 64} = \frac{386 - 448}{64} = \frac{-62}{64}.\]
|
||||
|
||||
This is still not equal to $0$, so we must have made another mistake.
|
||||
|
||||
When we subtracted $7$ from $\frac{386}{64}$, we should have subtracted $7 \cdot 64$ from $386$, not the other way around. Let's correct that:
|
||||
|
||||
\[\frac{386}{64}
|
||||
"""
|
||||
print(extract_answer(text, "math-oai", use_last_number=False))
|
||||
print(choice_answer_clean("\mathrm{(D)\}1,008,016"))
|
||||
# should output a dict
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test_extract_answer()
|
||||
Executable
+230
@@ -0,0 +1,230 @@
|
||||
import os
|
||||
import io
|
||||
import regex
|
||||
import pickle
|
||||
import traceback
|
||||
import copy
|
||||
import datetime
|
||||
import dateutil.relativedelta
|
||||
import multiprocess
|
||||
from multiprocess import Pool
|
||||
from typing import Any, Dict, Optional
|
||||
from pebble import ProcessPool
|
||||
from tqdm import tqdm
|
||||
from concurrent.futures import TimeoutError
|
||||
from functools import partial
|
||||
from timeout_decorator import timeout
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
|
||||
class GenericRuntime:
|
||||
GLOBAL_DICT = {}
|
||||
LOCAL_DICT = None
|
||||
HEADERS = []
|
||||
def __init__(self):
|
||||
self._global_vars = copy.copy(self.GLOBAL_DICT)
|
||||
self._local_vars = copy.copy(self.LOCAL_DICT) if self.LOCAL_DICT else None
|
||||
|
||||
for c in self.HEADERS:
|
||||
self.exec_code(c)
|
||||
|
||||
def exec_code(self, code_piece: str) -> None:
|
||||
if regex.search(r'(\s|^)?input\(', code_piece):
|
||||
# regex.search(r'(\s|^)?os.', code_piece):
|
||||
raise RuntimeError()
|
||||
exec(code_piece, self._global_vars)
|
||||
|
||||
# TODO: use: https://github.com/shroominic/codebox-api
|
||||
# @high safe exec in sandbox
|
||||
# byte_code = compile_restricted(
|
||||
# code_piece,
|
||||
# filename='<inline code>',
|
||||
# mode='exec'
|
||||
# )
|
||||
# print("global vars:", self._global_vars)
|
||||
# _print_ = PrintCollector
|
||||
# exec(byte_code, {'__builtins__': utility_builtins}, None)
|
||||
|
||||
def eval_code(self, expr: str) -> Any:
|
||||
return eval(expr, self._global_vars)
|
||||
|
||||
def inject(self, var_dict: Dict[str, Any]) -> None:
|
||||
for k, v in var_dict.items():
|
||||
self._global_vars[k] = v
|
||||
|
||||
@property
|
||||
def answer(self):
|
||||
return self._global_vars['answer']
|
||||
|
||||
class DateRuntime(GenericRuntime):
|
||||
GLOBAL_DICT = {
|
||||
'datetime': datetime.datetime,
|
||||
'timedelta': dateutil.relativedelta.relativedelta,
|
||||
'relativedelta': dateutil.relativedelta.relativedelta
|
||||
}
|
||||
|
||||
|
||||
class CustomDict(dict):
|
||||
def __iter__(self):
|
||||
return list(super().__iter__()).__iter__()
|
||||
|
||||
class ColorObjectRuntime(GenericRuntime):
|
||||
GLOBAL_DICT = {'dict': CustomDict}
|
||||
|
||||
|
||||
class PythonExecutor:
|
||||
def __init__(
|
||||
self,
|
||||
runtime: Optional[Any] = None,
|
||||
get_answer_symbol: Optional[str] = None,
|
||||
get_answer_expr: Optional[str] = None,
|
||||
get_answer_from_stdout: bool = False,
|
||||
timeout_length: int = 5,
|
||||
) -> None:
|
||||
self.runtime = runtime if runtime else GenericRuntime()
|
||||
self.answer_symbol = get_answer_symbol
|
||||
self.answer_expr = get_answer_expr
|
||||
self.get_answer_from_stdout = get_answer_from_stdout
|
||||
self.pool = Pool(multiprocess.cpu_count())
|
||||
self.timeout_length = timeout_length
|
||||
|
||||
def process_generation_to_code(self, gens: str):
|
||||
return [g.strip().split('\n') for g in gens]
|
||||
|
||||
@staticmethod
|
||||
def execute(
|
||||
code,
|
||||
get_answer_from_stdout = None,
|
||||
runtime = None,
|
||||
answer_symbol = None,
|
||||
answer_expr = None,
|
||||
timeout_length = 10,
|
||||
auto_mode=False
|
||||
):
|
||||
try:
|
||||
if auto_mode:
|
||||
if "print(" in code[-1]:
|
||||
program_io = io.StringIO()
|
||||
with redirect_stdout(program_io):
|
||||
timeout(timeout_length)(runtime.exec_code)('\n'.join(code))
|
||||
program_io.seek(0)
|
||||
result = program_io.read()
|
||||
else:
|
||||
print(code)
|
||||
timeout(timeout_length)(runtime.exec_code)('\n'.join(code[:-1]))
|
||||
result = timeout(timeout_length)(runtime.eval_code)(code[-1])
|
||||
else:
|
||||
if get_answer_from_stdout:
|
||||
program_io = io.StringIO()
|
||||
with redirect_stdout(program_io):
|
||||
timeout(timeout_length)(runtime.exec_code)('\n'.join(code))
|
||||
program_io.seek(0)
|
||||
result = program_io.read()
|
||||
elif answer_symbol:
|
||||
timeout(timeout_length)(runtime.exec_code)('\n'.join(code))
|
||||
result = runtime._global_vars[answer_symbol]
|
||||
elif answer_expr:
|
||||
timeout(timeout_length)(runtime.exec_code)('\n'.join(code))
|
||||
result = timeout(timeout_length)(runtime.eval_code)(answer_expr)
|
||||
else:
|
||||
timeout(timeout_length)(runtime.exec_code)('\n'.join(code[:-1]))
|
||||
result = timeout(timeout_length)(runtime.eval_code)(code[-1])
|
||||
report = "Done"
|
||||
str(result)
|
||||
pickle.dumps(result) # serialization check
|
||||
except:
|
||||
result = ''
|
||||
report = traceback.format_exc().split('\n')[-2]
|
||||
return result, report
|
||||
|
||||
def apply(self, code):
|
||||
return self.batch_apply([code])[0]
|
||||
|
||||
@staticmethod
|
||||
def truncate(s, max_length=400):
|
||||
half = max_length // 2
|
||||
if len(s) > max_length:
|
||||
s = s[:half] + "..." + s[-half:]
|
||||
return s
|
||||
|
||||
def batch_apply(self, batch_code):
|
||||
all_code_snippets = self.process_generation_to_code(batch_code)
|
||||
|
||||
timeout_cnt = 0
|
||||
all_exec_results = []
|
||||
# with ProcessPool(max_workers=min(len(all_code_snippets), os.cpu_count())) as pool:
|
||||
with ProcessPool(max_workers=min(len(all_code_snippets), 1)) as pool:
|
||||
executor = partial(
|
||||
self.execute,
|
||||
get_answer_from_stdout=self.get_answer_from_stdout,
|
||||
runtime=self.runtime,
|
||||
answer_symbol=self.answer_symbol,
|
||||
answer_expr=self.answer_expr,
|
||||
timeout_length=self.timeout_length, # this timeout not work
|
||||
auto_mode=True
|
||||
)
|
||||
future = pool.map(executor, all_code_snippets, timeout=self.timeout_length)
|
||||
iterator = future.result()
|
||||
|
||||
if len(all_code_snippets) > 100:
|
||||
progress_bar = tqdm(total=len(all_code_snippets), desc="Execute")
|
||||
else:
|
||||
progress_bar = None
|
||||
|
||||
while True:
|
||||
try:
|
||||
result = next(iterator)
|
||||
all_exec_results.append(result)
|
||||
except StopIteration:
|
||||
break
|
||||
except TimeoutError as error:
|
||||
print(error)
|
||||
all_exec_results.append(("", "Timeout Error"))
|
||||
timeout_cnt += 1
|
||||
except Exception as error:
|
||||
print(error)
|
||||
exit()
|
||||
if progress_bar is not None:
|
||||
progress_bar.update(1)
|
||||
|
||||
if progress_bar is not None:
|
||||
progress_bar.close()
|
||||
|
||||
batch_results = []
|
||||
for code, (res, report) in zip(all_code_snippets, all_exec_results):
|
||||
# post processing
|
||||
res, report = str(res).strip(), str(report).strip()
|
||||
res, report = self.truncate(res), self.truncate(report)
|
||||
batch_results.append((res, report))
|
||||
return batch_results
|
||||
|
||||
|
||||
def _test():
|
||||
batch_code = [
|
||||
"""
|
||||
from sympy import Matrix
|
||||
|
||||
def null_space_basis():
|
||||
# Define the matrix
|
||||
A = Matrix([[3, 3, -1, -6], [9, -1, -8, -1], [7, 4, -2, -9]])
|
||||
|
||||
# Compute the basis for the null space
|
||||
basis = A.nullspace()
|
||||
|
||||
# Round the elements of the basis vectors to three decimal places
|
||||
basis_rounded = [v.evalf(3) for v in basis]
|
||||
|
||||
return basis_rounded
|
||||
|
||||
result = null_space_basis()
|
||||
print(result)
|
||||
"""
|
||||
]
|
||||
|
||||
executor = PythonExecutor(get_answer_from_stdout=True)
|
||||
predictions = executor.apply(batch_code[0])
|
||||
print(predictions)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
_test()
|
||||
@@ -0,0 +1,15 @@
|
||||
# common
|
||||
vllm
|
||||
tqdm
|
||||
datasets
|
||||
torch
|
||||
transformers
|
||||
python_dateutil
|
||||
flash_attn
|
||||
|
||||
# math_eval
|
||||
sympy==1.12
|
||||
antlr4-python3-runtime==4.11.1 # ! The version needs to be compatible with sympy.
|
||||
word2number
|
||||
Pebble
|
||||
timeout-decorator
|
||||
Executable
+196
@@ -0,0 +1,196 @@
|
||||
import re
|
||||
"""
|
||||
trajcectory:
|
||||
[
|
||||
{"role": "rationale", "content": "..."},
|
||||
{"role": "program", "content": "..."},
|
||||
{"role": "output", "content": "..."},
|
||||
{"role": "rationale", "content": "..."},
|
||||
...
|
||||
]
|
||||
"""
|
||||
|
||||
def text_to_trajectory(traj_str: str) -> None:
|
||||
"""
|
||||
"""
|
||||
# parse the above interleaved string of raionale, program, output, raionale, program, output, ...
|
||||
# output a list of dict
|
||||
trajectory = []
|
||||
cur_role = "rationale"
|
||||
cur_content = ""
|
||||
|
||||
# print(traj_str)
|
||||
for i, line in enumerate(traj_str.split("\n")):
|
||||
if line == "```python": # program begin
|
||||
assert cur_role == "rationale"
|
||||
if cur_content:
|
||||
trajectory.append({"role": cur_role, "content": cur_content})
|
||||
cur_content = ""
|
||||
cur_role = "program"
|
||||
elif cur_role == "program" and line == "```": # program end
|
||||
assert cur_content
|
||||
trajectory.append({"role": cur_role, "content": cur_content})
|
||||
cur_content = ""
|
||||
cur_role = "output"
|
||||
elif cur_role == "output" and line.startswith("```output"): # output begin
|
||||
assert cur_content == ""
|
||||
elif cur_role == "output" and line == "```": # output end
|
||||
trajectory.append({"role": cur_role, "content": cur_content})
|
||||
cur_content = ""
|
||||
cur_role = "rationale"
|
||||
else: # content
|
||||
cur_content += line
|
||||
if i < len(traj_str.split("\n")) - 1:
|
||||
cur_content += "\n"
|
||||
# the last content
|
||||
if cur_content:
|
||||
trajectory.append({"role": cur_role, "content": cur_content})
|
||||
return trajectory
|
||||
|
||||
|
||||
def trajectory_to_text(trajectory: list) -> str:
|
||||
text = ""
|
||||
for item in trajectory:
|
||||
content = item["content"]
|
||||
if item["role"] == "program":
|
||||
content = f"```python\n{content}```\n"
|
||||
elif item["role"] == "output":
|
||||
content = f"```output\n{content}```\n"
|
||||
text += content
|
||||
return text
|
||||
|
||||
|
||||
def is_execution_success(output):
|
||||
error_key_words = ["error", "exception", "no algorithms", "no algorithms", "cannot", "nan", "..."]
|
||||
success = all([k not in output.lower() for k in error_key_words])
|
||||
return success
|
||||
|
||||
|
||||
def extract_program(text:str=None, trajectory:list=None, last_only=False) -> str:
|
||||
assert text is not None or trajectory is not None, "Either text or trajectory should be provided."
|
||||
if trajectory is None:
|
||||
try:
|
||||
trajectory = text_to_trajectory(text)
|
||||
except:
|
||||
return "raise ValueError('Invalid trajectory')"
|
||||
|
||||
program_list = []
|
||||
import_lines = []
|
||||
for i, item in enumerate(trajectory):
|
||||
if item["role"] == "program":
|
||||
cur_program = item["content"]
|
||||
if i < len(trajectory) - 1:
|
||||
assert trajectory[i+1]["role"] == "output"
|
||||
output = trajectory[i+1]["content"].strip()
|
||||
if is_execution_success(output):
|
||||
program_list.append(cur_program)
|
||||
else:
|
||||
# extract import lines only
|
||||
for line in cur_program.split("\n"):
|
||||
if line.startswith("import") or line.startswith("from"):
|
||||
import_lines.append(line)
|
||||
else:
|
||||
program_list.append(cur_program)
|
||||
# add import lines to the first program
|
||||
if len(program_list) == 0:
|
||||
program_list.append("")
|
||||
if len(import_lines) > 0:
|
||||
program_list[0] = "\n".join(import_lines) + "\n" + program_list[0]
|
||||
for i, program in enumerate(program_list[:-1]):
|
||||
program_list[i] = "\n".join([line for line in program.split("\n") if not line.strip().startswith("print(")])
|
||||
|
||||
if last_only:
|
||||
program = program_list[-1]
|
||||
else:
|
||||
program = "\n".join(program_list)
|
||||
return program
|
||||
|
||||
|
||||
def extract_program_output(pred_str, last_only=True):
|
||||
"""
|
||||
extract output between ```output\n...\n```, use regex, there might be multiple outputs, each output may have multiple lines
|
||||
"""
|
||||
outputs = re.findall(r"```output\n(.*?)\n```", pred_str, re.DOTALL)
|
||||
if last_only:
|
||||
return outputs[-1] if len(outputs) > 0 else ""
|
||||
else:
|
||||
return outputs
|
||||
|
||||
|
||||
def _test_str_to_trajectory():
|
||||
traj_text = """We can use the Chinese Remainder Theorem to solve this system of congruences. The Chinese Remainder Theorem states that if $n_1, n_2, \ldots, n_k$ are pairwise coprime (i.e., the greatest common divisor of any two of them is 1), and if $a_1, a_2, \ldots, a_k$ are any integers, then there is a number $x$ that satisfies the system of congruences:
|
||||
|
||||
$x \equiv a_1 \pmod{n_1}$
|
||||
$x \equiv a_2 \pmod{n_2}$
|
||||
$\vdots$
|
||||
$x \equiv a_k \pmod{n_k}$
|
||||
|
||||
Moreover, any two solutions of this system are congruent modulo the product $n_1n_2 \cdots n_k$.
|
||||
|
||||
Let's write a function to solve this:
|
||||
|
||||
```python
|
||||
from sympy import symbols, solve
|
||||
|
||||
def smallest_x():
|
||||
x = symbols('x')
|
||||
# Solve the system of congruences
|
||||
solution = solve([x - 7, x - 17], x, domain='Z')
|
||||
# Return the smallest positive solution
|
||||
return min([sol for sol in solution if sol >= 0])
|
||||
|
||||
result = smallest_x()
|
||||
print(result)
|
||||
```
|
||||
```output
|
||||
ValueError: min() arg is an empty sequence
|
||||
```
|
||||
The error in the last example is due to the incorrect usage of the `solve` function. The `solve` function is used to solve equations, not congruences. To solve a system of congruences, we can use the `solve_congruence` function from the `sympy.ntheory.modular` module.
|
||||
|
||||
Here is the corrected code:
|
||||
|
||||
```python
|
||||
from sympy.ntheory.modular import solve_congruence
|
||||
|
||||
def smallest_x():
|
||||
# Solve the system of congruences
|
||||
solution = solve_congruence(('x', 7, 12), ('x', 17, 11))
|
||||
# Return the smallest positive solution
|
||||
return solution[0][0]
|
||||
|
||||
result = smallest_x()
|
||||
print(result)
|
||||
```
|
||||
```output
|
||||
good
|
||||
```
|
||||
ValueError: too many values to unpack (expected 2)
|
||||
The error in the last example is due to the incorrect usage of the `solve_congruence` function. The `solve_congruence` function returns a tuple of two elements: the solution and the modulus. We need to unpack these two values correctly.
|
||||
|
||||
Here is the corrected code:
|
||||
|
||||
```python
|
||||
from sympy.ntheory.modular import solve_congruence
|
||||
|
||||
def smallest_x():
|
||||
# Solve the system of congruences
|
||||
solution, modulus = solve_congruence(('x', 7, 12), ('x', 17, 11))
|
||||
# Return the smallest positive solution
|
||||
return solution
|
||||
|
||||
result = smallest_x()
|
||||
print(result)
|
||||
```"""
|
||||
|
||||
import pprint
|
||||
trajectory = text_to_trajectory(traj_text)
|
||||
pprint.pprint(trajectory)
|
||||
|
||||
text = trajectory_to_text(trajectory)
|
||||
assert text == traj_text
|
||||
|
||||
# print(extract_program(traj_text))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test_str_to_trajectory()
|
||||
Executable
+273
@@ -0,0 +1,273 @@
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Union, Any
|
||||
|
||||
from data.qwen25math.examples import get_examples
|
||||
|
||||
|
||||
def set_seed(seed: int = 42) -> None:
|
||||
np.random.seed(seed)
|
||||
random.seed(seed)
|
||||
os.environ["PYTHONHASHSEED"] = str(seed)
|
||||
print(f"Random seed set as {seed}")
|
||||
|
||||
|
||||
def load_jsonl(file: Union[str, Path]) -> Iterable[Any]:
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
yield json.loads(line)
|
||||
except:
|
||||
print("Error in loading:", line)
|
||||
exit()
|
||||
|
||||
|
||||
def save_jsonl(samples, save_path):
|
||||
# ensure path
|
||||
folder = os.path.dirname(save_path)
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
|
||||
with open(save_path, "w", encoding="utf-8") as f:
|
||||
for sample in samples:
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
print("Saved to", save_path)
|
||||
|
||||
|
||||
def lower_keys(example):
|
||||
new_example = {}
|
||||
for key, value in example.items():
|
||||
if key != key.lower():
|
||||
new_key = key.lower()
|
||||
new_example[new_key] = value
|
||||
else:
|
||||
new_example[key] = value
|
||||
return new_example
|
||||
|
||||
|
||||
EXAMPLES = get_examples()
|
||||
|
||||
|
||||
def load_prompt(data_name, prompt_type, num_shots):
|
||||
if not num_shots:
|
||||
return []
|
||||
|
||||
if data_name in ["gsm_hard", "svamp", "tabmwp", "asdiv", "mawps"]:
|
||||
data_name = "gsm8k"
|
||||
if data_name in ["math_oai", "hungarian_exam", "math-oai", "aime24", "amc23"]:
|
||||
data_name = "math"
|
||||
if data_name in ["sat_math"]:
|
||||
data_name = "mmlu_stem"
|
||||
if data_name in [
|
||||
"gaokao2024_I",
|
||||
"gaokao2024_II",
|
||||
"gaokao_math_qa",
|
||||
"gaokao2024_mix",
|
||||
"cn_middle_school",
|
||||
]:
|
||||
data_name = "gaokao"
|
||||
|
||||
if prompt_type in ["tool-integrated"]:
|
||||
prompt_type = "tora"
|
||||
|
||||
return EXAMPLES[data_name][:num_shots]
|
||||
|
||||
|
||||
PROMPT_TEMPLATES = {
|
||||
"direct": ("Question: {input}\nAnswer: ", "{output}", "\n\n"),
|
||||
"cot": ("Question: {input}\nAnswer: ", "{output}", "\n\n\n"),
|
||||
"pal": ("Question: {input}\n\n", "{output}", "\n---\n"),
|
||||
"tool-integrated": ("Question: {input}\n\nSolution:\n", "{output}", "\n---\n"),
|
||||
"self-instruct": ("<|user|>\n{input}\n<|assistant|>\n", "{output}", "\n"),
|
||||
"tora": ("<|user|>\n{input}\n<|assistant|>\n", "{output}", "\n"),
|
||||
"wizard_zs": (
|
||||
"### Instruction:\n{input}\n\n### Response: Let's think step by step.",
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"platypus_fs": (
|
||||
"### Instruction:\n{input}\n\n### Response:\n",
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"deepseek-math": (
|
||||
"User: {input}\nPlease reason step by step, "
|
||||
"and put your final answer within \\boxed{{}}.\n\nAssistant:",
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"kpmath": (
|
||||
"User: Please reason step by step and put your final answer at the end "
|
||||
'with "The answer is: ".\n\n{input}\n\nAssistant:',
|
||||
"{output}",
|
||||
),
|
||||
"jiuzhang": (
|
||||
"## Question\n{input}\n\n## Solution\n",
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"jiuzhang_tora": (
|
||||
"## Question\n{input}\n\n## Code Solution\n",
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"jiuzhang_nl": (
|
||||
"## Question\n{input}\n\n## Natural Language Solution\n",
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"mmiqc": (
|
||||
'Please solve the following problem and put your answer at the end with "The answer is: ".\n\n{input}\n\n',
|
||||
"{output}",
|
||||
"\n\n\n",
|
||||
),
|
||||
"abel": (
|
||||
"Question:\n{input}\nAnswer:\nLet's think step by step.\n",
|
||||
"{output}",
|
||||
"\n\n",
|
||||
),
|
||||
"shepherd": ("{input}\n", "{output}", "\n\n\n"),
|
||||
"qwen-boxed": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n{input}\nPlease reason step by step, and put your final answer within \\boxed{{}}.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n",
|
||||
"{output}",
|
||||
"\n\n",
|
||||
),
|
||||
"qwen25-math-cot": (
|
||||
"<|im_start|>system\nPlease reason step by step, and put your final answer within \\boxed{{}}.<|im_end|>\n"
|
||||
"<|im_start|>user\n{input}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n",
|
||||
"{output}",
|
||||
"\n\n",
|
||||
),
|
||||
"mathstral": (
|
||||
"{input}\nPlease reason step by step, and put your final answer within \\boxed{{}}.",
|
||||
"{output}",
|
||||
"\n\n",
|
||||
),
|
||||
"internlm-math-fs": ("Question:{input}\nAnswer:", "{output}", "\n"),
|
||||
"internlm-math-chat": (
|
||||
"<|im_start|>user\n{input}<|im_end|>\n" "<|im_start|>assistant\n",
|
||||
"{output}",
|
||||
"\n\n",
|
||||
),
|
||||
"mistral": (
|
||||
"[INST] {input}[/INST]",
|
||||
"{output}",
|
||||
"\n\n",
|
||||
),
|
||||
"numina": ("### Problem: {input}\n### Solution:", " {output}", "\n\n"),
|
||||
}
|
||||
|
||||
|
||||
def construct_prompt(example, data_name, args):
|
||||
if args.adapt_few_shot and data_name in [
|
||||
"gaokao2024_I",
|
||||
"gaokao2024_II",
|
||||
"gaokao_math_qa",
|
||||
"gaokao2024_mix",
|
||||
"cn_middle_school",
|
||||
]:
|
||||
demos = load_prompt(data_name, args.prompt_type, 5)
|
||||
else:
|
||||
demos = load_prompt(data_name, args.prompt_type, args.num_shots)
|
||||
prompt_type = args.prompt_type
|
||||
if prompt_type == "platypus_fs":
|
||||
prompt_type = "cot"
|
||||
if prompt_type == "tool-integrated":
|
||||
prompt_type = "tora"
|
||||
|
||||
prompt_temp = PROMPT_TEMPLATES[args.prompt_type]
|
||||
|
||||
splitter = prompt_temp[2]
|
||||
input_template, output_template, splitter = (
|
||||
prompt_temp[0],
|
||||
prompt_temp[1],
|
||||
prompt_temp[2],
|
||||
)
|
||||
if args.prompt_type == "qwen25-math-cot":
|
||||
# Hotfix to support putting all demos into a single turn
|
||||
demo_prompt = splitter.join([q + "\n" + a for q, a in demos])
|
||||
else:
|
||||
demo_prompt = splitter.join(
|
||||
[
|
||||
input_template.format(input=q) + output_template.format(output=a)
|
||||
for q, a in demos
|
||||
]
|
||||
)
|
||||
context = input_template.format(input=example["question"])
|
||||
if len(demo_prompt) == 0 or (
|
||||
args.adapt_few_shot and example["gt_ans"] not in ["A", "B", "C", "D", "E"]
|
||||
):
|
||||
full_prompt = context
|
||||
else:
|
||||
if args.prompt_type == "qwen25-math-cot":
|
||||
# Hotfix to supportting put all demos into a single turn
|
||||
full_prompt = demo_prompt + splitter + example["question"]
|
||||
full_prompt = input_template.format(input=full_prompt)
|
||||
else:
|
||||
full_prompt = demo_prompt + splitter + context
|
||||
|
||||
if args.prompt_type == "platypus_fs":
|
||||
full_prompt_temp = (
|
||||
"Below is an instruction that describes a task. "
|
||||
"Write a response that appropriately completes the request.\n\n"
|
||||
"### Instruction:\n{instruction}\n\n### Response:\n"
|
||||
)
|
||||
full_prompt = full_prompt_temp.format(instruction=full_prompt)
|
||||
|
||||
if prompt_type == "tora":
|
||||
full_prompt = (
|
||||
"""Integrate step-by-step reasoning and Python code to solve math problems using the following guidelines:
|
||||
|
||||
- Analyze the question and write functions to solve the problem; the function should not take any arguments.
|
||||
- Present the final result in LaTeX using a `\boxed{}` without any units.
|
||||
- Utilize the `pi` symbol and `Rational`` from Sympy for $\pi$ and fractions, and simplify all fractions and square roots without converting them to decimal values.
|
||||
|
||||
Here are some examples you may refer to:
|
||||
|
||||
---
|
||||
|
||||
"""
|
||||
+ full_prompt
|
||||
)
|
||||
|
||||
return full_prompt.strip(" ") # important!
|
||||
|
||||
|
||||
key_map = {
|
||||
"gt": "Ground Truth",
|
||||
"pred": "Prediction",
|
||||
"gt_cot": "Reference CoT",
|
||||
"score": "Score",
|
||||
}
|
||||
|
||||
|
||||
def show_sample(sample, print_all_preds=False):
|
||||
print("==" * 20)
|
||||
for key in ["idx", "type", "level", "dataset"]:
|
||||
if key in sample:
|
||||
# capitalize
|
||||
print("{}: {}".format(key[0].upper() + key[1:], sample[key]))
|
||||
print("Question:", repr(sample["question"]))
|
||||
if "code" in sample:
|
||||
if print_all_preds:
|
||||
for code in sample["code"]:
|
||||
print("-" * 20)
|
||||
print("code:", code)
|
||||
print("Execution:", sample["report"])
|
||||
else:
|
||||
print("Solution:\n", sample["code"][0])
|
||||
print("Execution:", sample["report"][0])
|
||||
if "pred" in sample:
|
||||
print("Prediction:", repr(sample["pred"][0]))
|
||||
for key in ["gt", "score", "unit", "gt_cot"]:
|
||||
if key in sample:
|
||||
_key = key_map.get(key, key)
|
||||
print("{}: {}".format(_key, repr(sample[key])))
|
||||
print()
|
||||
@@ -0,0 +1,147 @@
|
||||
import requests
|
||||
import json
|
||||
from typing import List, Iterable, Tuple, Union
|
||||
from omegaconf.listconfig import ListConfig
|
||||
|
||||
from general_util.logger import get_child_logger
|
||||
|
||||
logger = get_child_logger("VLLM")
|
||||
|
||||
|
||||
def post_http_request(api_url: str,
|
||||
n: int = 1,
|
||||
max_tokens: int = 16,
|
||||
temperature: float = 0.0,
|
||||
use_beam_search: bool = False,
|
||||
stream: bool = False,
|
||||
stop: List[str] = ["</s>"],
|
||||
**kwargs) -> requests.Response:
|
||||
headers = {"User-Agent": "MERIt Test Client"}
|
||||
# headers = {
|
||||
# "accept": "application/json",
|
||||
# "content-type": "application/json",
|
||||
#
|
||||
# }
|
||||
if "TOGETHER_API_KEY" in kwargs:
|
||||
headers["authorization"] = f"Bearer {kwargs.pop('TOGETHER_API_KEY')}"
|
||||
p_load = {
|
||||
"n": n,
|
||||
"use_beam_search": use_beam_search,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": stream,
|
||||
"stop": stop,
|
||||
}
|
||||
p_load.update(kwargs)
|
||||
response = requests.post(api_url, headers=headers, json=p_load, stream=True)
|
||||
return response
|
||||
|
||||
|
||||
def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:
|
||||
for chunk in response.iter_lines(chunk_size=8192,
|
||||
decode_unicode=False,
|
||||
delimiter=b"\0"):
|
||||
if chunk:
|
||||
data = json.loads(chunk.decode("utf-8"))
|
||||
output = data["text"]
|
||||
yield output
|
||||
|
||||
|
||||
def get_response(response: requests.Response) -> Tuple[str, str]:
|
||||
data = json.loads(response.content)
|
||||
output = data["text"]
|
||||
return output, data
|
||||
|
||||
|
||||
class VLLMRequestGenerator:
|
||||
def __init__(self, api_url: str, n: int = 1, max_tokens: int = 1024, use_beam_search: bool = False, stream: bool = False,
|
||||
temperature: float = 0.0, stop: Union[List[str], ListConfig] = ["</s>"], **kwargs):
|
||||
self.api_url = api_url
|
||||
self.n = n
|
||||
self.max_tokens = max_tokens
|
||||
self.use_beam_search = use_beam_search
|
||||
self.stream = stream
|
||||
self.stop = stop if isinstance(stop, list) else list(stop)
|
||||
self.temperature = temperature
|
||||
self.kwargs = kwargs
|
||||
self.max_retry = 10
|
||||
print(self.kwargs)
|
||||
|
||||
def __call__(self, prompt: str) -> str:
|
||||
re_try = 0
|
||||
max_tokens = self.max_tokens
|
||||
while True:
|
||||
if "completions" not in self.api_url:
|
||||
response, content = get_response(post_http_request(prompt=prompt,
|
||||
api_url=self.api_url,
|
||||
n=self.n,
|
||||
max_tokens=max_tokens,
|
||||
temperature=self.temperature,
|
||||
use_beam_search=self.use_beam_search,
|
||||
stream=self.stream,
|
||||
stop=self.stop,
|
||||
**self.kwargs))[0]
|
||||
response = response.replace(prompt, "")
|
||||
# TODO: Add error processing here (need to remove `get_response` method)
|
||||
elif "/chat/completions" in self.api_url:
|
||||
if not isinstance(prompt, list):
|
||||
prompt = [{"role": "user", "content": prompt}]
|
||||
|
||||
response = post_http_request(messages=prompt,
|
||||
api_url=self.api_url,
|
||||
n=self.n,
|
||||
max_tokens=max_tokens,
|
||||
temperature=self.temperature,
|
||||
use_beam_search=self.use_beam_search,
|
||||
stream=self.stream,
|
||||
stop=self.stop,
|
||||
**self.kwargs)
|
||||
if response.status_code != 200:
|
||||
logger.warning(response.json())
|
||||
response = response.content.decode('utf-8')
|
||||
if "maximum context length" in response:
|
||||
max_tokens -= 100
|
||||
re_try += 1
|
||||
logger.warning("max_tokens reduced to: {}".format(max_tokens))
|
||||
response = ""
|
||||
else:
|
||||
re_try = 0
|
||||
outputs = []
|
||||
for item in json.loads(response.content)["choices"]:
|
||||
outputs.append(item["message"]["content"].replace(prompt[-1]["content"], ""))
|
||||
if len(outputs) == 1:
|
||||
response = outputs[0]
|
||||
else:
|
||||
response = outputs
|
||||
else:
|
||||
response = post_http_request(prompt=prompt,
|
||||
api_url=self.api_url,
|
||||
n=self.n,
|
||||
max_tokens=max_tokens,
|
||||
temperature=self.temperature,
|
||||
use_beam_search=self.use_beam_search,
|
||||
stream=self.stream,
|
||||
stop=self.stop,
|
||||
**self.kwargs)
|
||||
if response.status_code != 200:
|
||||
logger.warning(response.json())
|
||||
response = response.content.decode('utf-8')
|
||||
if "maximum context length" in response:
|
||||
max_tokens -= 100
|
||||
re_try += 1
|
||||
logger.warning("max_tokens reduced to: {}".format(max_tokens))
|
||||
response = ""
|
||||
else:
|
||||
re_try = 0
|
||||
outputs = []
|
||||
for item in json.loads(response.content)["choices"]:
|
||||
outputs.append(item["text"].replace(prompt, ""))
|
||||
if len(outputs) == 1:
|
||||
response = outputs[0]
|
||||
else:
|
||||
response = outputs
|
||||
|
||||
if re_try == 0 or re_try > self.max_retry:
|
||||
break
|
||||
|
||||
return response
|
||||
Reference in New Issue
Block a user