mirrors/microsoft--unilm
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:24:13 +08:00
commit 1037506f2e
6050 changed files with 1731598 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mathscale/MWPBench/eval_vllm/driver.py
+209
View File
@@ -0,0 +1,209 @@
import argparse
import json
import os
import re
import sys
import eval_vllm.util as util
from vllm import LLM, SamplingParams
from tqdm import tqdm
MAX_INT = sys.maxsize
TEMPLATE_DICT = {
"none": (
"{instruction}"
),
"alpaca": (
"Below is an instruction that describes a task. "
"Write a response that appropriately completes the request.\n\n"
"### Instruction:\n{instruction}\n\n### Response:"
),
"alpaca_force_ans": (
"Below is an instruction that describes a task. "
"Write a response that appropriately completes the request.\n\n"
"### Instruction:\n{instruction}\n\nTry to conclude your response with 'The answer is ...'.\n### Response:"
),
"alpaca_cot": (
"Below is an instruction that describes a task. "
"Write a response that appropriately completes the request.\n\n"
"### Instruction:\n{instruction}\n\n### Response: Let's think step by step."
)
}
def batch_data(data_list, batch_size=1):
n = len(data_list) // batch_size
batch_data = []
for i in range(n-1):
start = i * batch_size
end = (i+1)*batch_size
batch_data.append(data_list[start:end])
last_start = (n-1) * batch_size
last_end = MAX_INT
batch_data.append(data_list[last_start:last_end])
return batch_data
def evaluate_one_task(args, model, sampling_params, prompt_template, task_name, sample):
math_ins = []
math_answers = []
for item in sample:
question = item["question"]
answer = item["answer"]
temp_instr = prompt_template.format(instruction=question)
math_ins.append(temp_instr)
math_answers.append(answer)
batch_math_ins = batch_data(math_ins, batch_size=args.batch_size)
res_completions = []
for batch_prompt in batch_math_ins:
completions = model.generate(batch_prompt, sampling_params)
for output in completions:
prompt_temp = output.prompt
generated_text = output.outputs[0].text
res_completions.append(generated_text)
fw = open(os.path.join(args.save_dir, task_name.strip(".") + ".prediction.json"), "w")
results = []
for idx, (example, completion, answer) in enumerate(zip(sample, res_completions, math_answers)):
res, clean_prediction_ans, clean_reference_ans = util.is_correct(completion, answer, verbose=args.verbose)
results.append(res)
dump = {
"question": example["question"],
"answer": answer,
"completion": completion,
'clean_reference_ans': clean_reference_ans,
'clean_prediction_ans': clean_prediction_ans,
"judge": res
}
dump = json.dumps(dump, ensure_ascii=False)
fw.write(dump + "\n")
fw.close()
acc = sum(results) / len(results)
fw = open(os.path.join(args.save_dir, task_name.strip(".") + ".metric.json"), "w")
metric = {
"task_name": task_name,
"test_size": len(results),
"accuracy": acc,
}
print(metric)
print(f"evaluate task done.")
metric = json.dump(metric, fw, ensure_ascii=False)
fw.close()
return acc
def main(args):
if args.save_dir is None:
args.save_dir = os.path.join("results", args.model_name_or_path.replace("/", ".").strip(".") + f".{args.prompt_template}")
os.makedirs(args.save_dir, exist_ok=True)
# Load data
task2sample = {}
with open(args.data_file) as fd:
for line in tqdm(fd, desc="load data..."):
example = json.loads(line)
task = example["data_topic"]
if args.target_tasks is not None:
if task not in args.target_tasks:
continue
if task not in task2sample:
task2sample[task] = []
task2sample[task].append(example)
if args.max_num_examples_per_task != -1:
task2sample_t = {}
for task_name, sample in task2sample.items():
task2sample_t[task_name] = sample[:args.max_num_examples_per_task]
task2sample = task2sample_t
print("load data done.")
for task_name, sample in task2sample.items():
print(f"evaluating task name: {task_name}; sample size: {len(sample)}")
prompt_template = TEMPLATE_DICT[args.prompt_template]
print(f"using prompt template: {args.prompt_template}\n{prompt_template}")
# Init model
model = LLM(model=args.model_name_or_path, tensor_parallel_size=args.tensor_parallel_size)
print("init model done.")
stop_tokens = ["Question:", "Question", "USER:", "USER", "ASSISTANT:", "ASSISTANT", "Instruction:", "Instruction", "Response:", "Response", "</s>"]
sampling_params = SamplingParams(temperature=0, top_p=1, max_tokens=2048, stop=stop_tokens)
print(f"init sampling params done: {sampling_params}")
# evaluate tasks
layer_MATH_task2acc = {}
layer_college_math_task2acc = {}
layer_top_task2acc = {}
full_MATH_size = 0
full_college_math_size = 0
full_size = 0
for task_name, sample in task2sample.items():
try:
acc = evaluate_one_task(args, model, sampling_params, prompt_template, task_name, sample)
test_size = len(sample)
full_size += test_size
if task_name.startswith("MATH."):
layer_MATH_task2acc[task_name] = {"accuracy": acc, "test_size": test_size}
full_MATH_size += test_size
elif task_name.startswith("college_math."):
layer_college_math_task2acc[task_name] = {"accuracy": acc, "test_size": test_size}
full_college_math_size += test_size
else:
layer_top_task2acc[task_name] = {"accuracy": acc, "test_size": test_size}
except Exception as e:
print(e)
continue
# compute MATH acc
MATH_acc = 0
for task_name, task_metric in layer_MATH_task2acc.items():
acc = task_metric["accuracy"]
test_size = task_metric["test_size"]
weight = test_size / full_MATH_size
MATH_acc += weight * acc
layer_top_task2acc["MATH"] = {"accuracy": MATH_acc, "test_size": full_MATH_size, "subset_metric": layer_MATH_task2acc}
# compute college_math acc
college_math_acc = 0
for task_name, task_metric in layer_college_math_task2acc.items():
acc = task_metric["accuracy"]
test_size = task_metric["test_size"]
weight = test_size / full_college_math_size
college_math_acc += weight * acc
layer_top_task2acc["college_math"] = {"accuracy": college_math_acc, "test_size": full_college_math_size, "subset_metric": layer_college_math_task2acc}
# compute micro & macro avg
micro_acc = 0
macro_acc = 0
for task_name, task_metric in layer_top_task2acc.items():
acc = task_metric["accuracy"]
test_size = task_metric["test_size"]
weight = test_size / full_size
micro_acc += weight * acc
macro_acc += acc
macro_acc /= len(layer_top_task2acc)
layer_top_task2acc["micro_average_accuracy"] = micro_acc
layer_top_task2acc["macro_average_accuracy"] = macro_acc
print("evaluate all done.")
print(json.dumps(layer_top_task2acc, indent=4))
fw = open(os.path.join(args.save_dir, "all.metric.json"), "w")
layer_top_task2acc = json.dump(layer_top_task2acc, fw, ensure_ascii=False)
fw.close()
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--model_name_or_path", type=str, default=None) # model path
parser.add_argument("--data_file", type=str, default='data/full_test.json') # data path
parser.add_argument("--target_tasks", type=str, default=None) # choose from gsm8k,MATH.Algebra,MATH.Counting_&_Probability,MATH.Geometry,MATH.Intermediate_Algebra,MATH.Number_Theory,MATH.Prealgebra,MATH.Precalculus,college_math.algebra,college_math.precalculus,college_math.calculus,college_math.vector_calculus,college_math.probability,college_math.linear_algebra,college_math.differential_equation,tal,gaokao_bench_math_en,math23k_en,ape210k_en,agieval.gaokao-math-en,agieval.math,agieval.sat-math
parser.add_argument("--save_dir", type=str, default=None) # data path
parser.add_argument("--max_num_examples_per_task", type=int, default=2000) # max_num_examples_per_task, set -1 to disable it
parser.add_argument("--batch_size", type=int, default=60) # batch_size
parser.add_argument("--tensor_parallel_size", type=int, default=4) # num_gpus
parser.add_argument("--prompt_template", type=str, default="alpaca") # choose from [none, alpaca, alpaca_force_ans, alpaca_cot]
parser.add_argument("--verbose", action="store_true")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
main(args)
mathscale/MWPBench/eval_vllm/util.py
+341
View File
@@ -0,0 +1,341 @@
import re
def last_boxed_only(sample):
q, a = sample
a = last_boxed_only_string(a)
if a == None:
return None
return (q, a)
def last_boxed_only_string(string):
idx = string.rfind("\\boxed")
if idx < 0:
idx = string.rfind("\\fbox")
if idx < 0:
return None
i = idx
right_brace_idx = None
num_left_braces_open = 0
while i < len(string):
if string[i] == "{":
num_left_braces_open += 1
if string[i] == "}":
num_left_braces_open -= 1
if num_left_braces_open == 0:
right_brace_idx = i
break
i += 1
if right_brace_idx == None:
retval = None
else:
retval = string[idx:right_brace_idx + 1]
return retval
def only_until_first_boxed_from_tokens(string, tokens):
idx = string.find("\\boxed")
if idx < 0:
idx = string.find("\\fbox")
if idx < 0:
return None
cum_length = 0
for i, t in enumerate(tokens):
cum_length += len(t)
if cum_length >= idx:
break
return tokens[:i]
def fix_fracs(string):
substrs = string.split("\\frac")
new_str = substrs[0]
if len(substrs) > 1:
substrs = substrs[1:]
for substr in substrs:
new_str += "\\frac"
if substr[0] == "{":
new_str += substr
else:
try:
assert len(substr) >= 2
except AssertionError:
return string
a = substr[0]
b = substr[1]
if b != "{":
if len(substr) > 2:
post_substr = substr[2:]
new_str += "{" + a + "}{" + b + "}" + post_substr
else:
new_str += "{" + a + "}{" + b + "}"
else:
if len(substr) > 2:
post_substr = substr[2:]
new_str += "{" + a + "}" + b + post_substr
else:
new_str += "{" + a + "}" + b
string = new_str
return string
def fix_a_slash_b(string):
if len(string.split("/")) != 2:
return string
a = string.split("/")[0]
b = string.split("/")[1]
try:
a = int(a)
b = int(b)
assert string == "{}/{}".format(a, b)
new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
return new_string
except Exception as e:
return string
def remove_right_units(string):
# "\\text{ " only ever occurs (at least in the val set) when describing units
if "\\text{ " in string:
splits = string.split("\\text{ ")
assert len(splits) == 2
return splits[0]
else:
return string
def fix_sqrt(string):
if "\\sqrt" not in string:
return string
splits = string.split("\\sqrt")
new_string = splits[0]
for split in splits[1:]:
if split[0] != "{":
a = split[0]
new_substr = "\\sqrt{" + a + "}" + split[1:]
else:
new_substr = "\\sqrt" + split
new_string += new_substr
return new_string
def unbox_and_extract(text):
start_indices = [m.start() for m in re.finditer(r'\\boxed{', text)]
extracted_contents = []
for start in start_indices:
brace_count = 0
for i, char in enumerate(text[start:]):
if char == '{':
brace_count += 1
elif char == '}':
brace_count -= 1
if brace_count == 0:
end = start + i + 1
extracted_contents.append(text[start+7:end-1]) # +7 to skip '\\boxed{'
break
# Replace '\\boxed{...}' with the content inside it
unboxed_text = re.sub(r'\\boxed{(.*?)}', r'\1', text)
return unboxed_text, extracted_contents
def convert_to_latex_fraction(text: str) -> str:
# Use regex to find all occurrences of ((num)/(denom))
pattern = re.compile(r"\(\(([\d]+)\)/\(([\d]+)\)\)")
matches = pattern.findall(text)
for match in matches:
num, denom = match
latex_frac = f"\\\\frac{{{num}}}{{{denom}}}"
# Replace the old expression with the LaTeX fraction
text = text.replace(f"(({num})/({denom}))", latex_frac)
return text
def strip_string(string):
# convert ((3)/(4)) -> \\frac{3}{4}
string = convert_to_latex_fraction(string)
# remove ,
string = string.replace(",", "")
# linebreaks
string = string.replace("\n", "")
# remove inverse spaces
string = string.replace("\\!", "")
# replace \\ with \
string = string.replace("\\\\", "\\")
# replace tfrac and dfrac with frac
string = string.replace("tfrac", "frac")
string = string.replace("dfrac", "frac")
# remove \left and \right
string = string.replace("\\left", "")
string = string.replace("\\right", "")
# Remove circ (degrees)
string = string.replace("^{\\circ}", "")
string = string.replace("^\\circ", "")
# remove dollar signs
string = string.replace("\\$", "")
# remove units (on the right)
string = remove_right_units(string)
# remove percentage
string = string.replace("\\%", "")
string = string.replace("\%", "") # noqa: W605
# " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
string = string.replace(" .", " 0.")
string = string.replace("{.", "{0.")
# if empty, return empty string
if len(string) == 0:
return string
if string[0] == ".":
string = "0" + string
# to consider: get rid of e.g. "k = " or "q = " at beginning
if len(string.split("=")) == 2:
if len(string.split("=")[0]) <= 2:
string = string.split("=")[1]
# fix sqrt3 --> sqrt{3}
string = fix_sqrt(string)
# My own
string = string.replace("\\quad", " ")
# remove spaces
string = string.replace(" ", "")
# \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
string = fix_fracs(string)
# manually change 0.5 --> \frac{1}{2}
if string == "0.5":
string = "\\frac{1}{2}"
# NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
string = fix_a_slash_b(string)
return string
def is_number(s):
s = s.strip("$")
try:
# Try to convert the string to an integer
int(s)
return True
except ValueError:
try:
# Try to convert the string to a float
float(s)
return True
except ValueError:
return False
def is_single_inline_math(expression: str) -> bool:
# Use regex to check for a pattern that starts and ends with dollar signs,
# and contains no other dollar signs in between.
pattern = re.compile(r"^\$[^$]+\$$")
match = pattern.match(expression)
return bool(match)
def is_equiv(prediction_ans, reference_ans, verbose=False):
if prediction_ans is None and reference_ans is None:
print("WARNING: Both None")
return True, prediction_ans, reference_ans
if prediction_ans is None or reference_ans is None:
return False, prediction_ans, reference_ans
try:
clean_prediction_ans = strip_string(prediction_ans)
clean_reference_ans = strip_string(reference_ans)
if is_number(clean_prediction_ans) and is_number(clean_reference_ans):
judge = float(clean_prediction_ans.strip("$")) == float(clean_reference_ans.strip("$"))
# print(f"1 judge: {judge}")
elif is_single_inline_math(clean_reference_ans):
judge = (clean_reference_ans.strip("$") in clean_prediction_ans.strip("$"))
# print(f"2 judge: {judge}")
elif (len(clean_prediction_ans) >= 3) and (not is_number(clean_prediction_ans)) and (not clean_prediction_ans.startswith("-")) and (not clean_reference_ans.startswith("-")) and (clean_prediction_ans in clean_reference_ans):
judge = True
# print(f"3 judge: {judge}")
elif (len(clean_reference_ans) >= 3) and (not is_number(clean_reference_ans)) and (not clean_prediction_ans.startswith("-")) and (not clean_reference_ans.startswith("-")) and (clean_reference_ans in clean_prediction_ans):
judge = True
# print(f"4 judge: {judge}")
else:
judge = clean_prediction_ans == clean_reference_ans
# print(f"5 judge: {judge}")
if verbose:
print(f"clean_prediction_ans: {clean_prediction_ans} | clean_reference_ans: {clean_reference_ans} | judge: {judge}")
return judge, clean_prediction_ans, clean_reference_ans
except Exception as e:
print(e)
return prediction_ans == reference_ans, prediction_ans, reference_ans
def is_correct(completion, answer, verbose=False):
completion = completion.lower()
answer = answer.lower()
# Extract short answer from completion
extract_ans = None
clean_reference_ans = strip_string(answer)
is_reference_ans_number = is_number(clean_reference_ans)
# First extract boxed answer
unbox_long_answer, box_short_answers = unbox_and_extract(completion)
if box_short_answers != []:
extract_ans = box_short_answers[-1].strip()
# print(f"1 extract_ans: {extract_ans}")
# extract the last number answer
elif is_reference_ans_number:
numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion)
if numbers:
extract_ans = numbers[-1]
# print(f"2 extract_ans: {extract_ans}")
# extract "the answer is ..." answer
elif ("answer is" in completion) or ("solution is" in completion):
if "answer is" in completion:
split_ans = completion.split('answer is')
else:
split_ans = completion.split('solution is')
ans = split_ans[-1].strip().lstrip(":").strip()
extract_ans_temp = ans.split('.\n')[0]
extract_ans_temp = extract_ans_temp.strip()
extract_ans_temp = extract_ans_temp.strip('.')
if len(extract_ans_temp)>0 and extract_ans_temp[-1] == '.':
extract_ans = extract_ans_temp[0:-1]
else:
extract_ans = extract_ans_temp
extract_ans = extract_ans.strip()
# print(f"3 extract_ans: {extract_ans}")
# extract "therefore xx is xxx" answer
elif "is" in completion:
pos = completion.rfind("is")
ans = completion[pos+2:].strip().lstrip(":").strip()
extract_ans_temp = ans.split('.\n')[0]
extract_ans_temp = extract_ans_temp.strip()
extract_ans_temp = extract_ans_temp.strip('.')
if len(extract_ans_temp)>0 and extract_ans_temp[-1] == '.':
extract_ans = extract_ans_temp[0:-1]
else:
extract_ans = extract_ans_temp
extract_ans = extract_ans.strip()
# print(f"4 extract_ans: {extract_ans}")
else:
return False, f"failed extracting answer from completion", clean_reference_ans
judge, clean_prediction_ans, clean_reference_ans = is_equiv(extract_ans, answer, verbose=verbose)
return judge, clean_prediction_ans, clean_reference_ans
if __name__ == "__main__":
reference_ans = "$2$"
prediction_ans = "To find the value of $a$, we need to evaluate the function $f$ at $f(\\sqrt{6})$ and set it equal to 3.\n\nFirst, let's find the value of $f(\\sqrt{6})$. Since $\\sqrt{6}$ is not in the domain of the first piece of the function, we move on to the second piece. \n\nFor $x \\leq 2$, the function becomes $f(x) = |x-3| + a$. Plugging in $\\sqrt{6}$, we have $f(\\sqrt{6}) = |(\\sqrt{6})-3| + a$.\n\nNext, we set $f(\\sqrt{6})$ equal to 3 and solve for $a$. \n\n$|(\\sqrt{6})-3| + a = 3$\n\nSince we are looking for a real number value of $a$, we can ignore the absolute value and solve for $a$.\n\n$(\\sqrt{6})-3 + a = 3$\n\n$\\sqrt{6} - 3 + a = 3$\n\n$\\sqrt{6} + a = 3 + 3$\n\n$\\sqrt{6} + a = 6$\n\nSubtracting 6 from both sides, we get:\n\n$a = 6 - \\sqrt{6}$\n\nTherefore, the value of $a$ is $6 - \\sqrt{6}$.\n\nThe answer is $a = 6 - \\sqrt{6}$."
print(is_correct(prediction_ans, reference_ans, verbose=True))