wehub-resource-sync
341 lines
12 KiB
Python
341 lines
12 KiB
Python
import re
|
|
|
|
def last_boxed_only(sample):
|
|
q, a = sample
|
|
a = last_boxed_only_string(a)
|
|
if a == None:
|
|
return None
|
|
return (q, a)
|
|
|
|
def last_boxed_only_string(string):
|
|
idx = string.rfind("\\boxed")
|
|
if idx < 0:
|
|
idx = string.rfind("\\fbox")
|
|
if idx < 0:
|
|
return None
|
|
|
|
i = idx
|
|
right_brace_idx = None
|
|
num_left_braces_open = 0
|
|
while i < len(string):
|
|
if string[i] == "{":
|
|
num_left_braces_open += 1
|
|
if string[i] == "}":
|
|
num_left_braces_open -= 1
|
|
if num_left_braces_open == 0:
|
|
right_brace_idx = i
|
|
break
|
|
i += 1
|
|
|
|
if right_brace_idx == None:
|
|
retval = None
|
|
else:
|
|
retval = string[idx:right_brace_idx + 1]
|
|
|
|
return retval
|
|
|
|
def only_until_first_boxed_from_tokens(string, tokens):
|
|
idx = string.find("\\boxed")
|
|
if idx < 0:
|
|
idx = string.find("\\fbox")
|
|
if idx < 0:
|
|
return None
|
|
|
|
cum_length = 0
|
|
for i, t in enumerate(tokens):
|
|
cum_length += len(t)
|
|
if cum_length >= idx:
|
|
break
|
|
|
|
return tokens[:i]
|
|
|
|
def fix_fracs(string):
|
|
substrs = string.split("\\frac")
|
|
new_str = substrs[0]
|
|
if len(substrs) > 1:
|
|
substrs = substrs[1:]
|
|
for substr in substrs:
|
|
new_str += "\\frac"
|
|
if substr[0] == "{":
|
|
new_str += substr
|
|
else:
|
|
try:
|
|
assert len(substr) >= 2
|
|
except AssertionError:
|
|
return string
|
|
a = substr[0]
|
|
b = substr[1]
|
|
if b != "{":
|
|
if len(substr) > 2:
|
|
post_substr = substr[2:]
|
|
new_str += "{" + a + "}{" + b + "}" + post_substr
|
|
else:
|
|
new_str += "{" + a + "}{" + b + "}"
|
|
else:
|
|
if len(substr) > 2:
|
|
post_substr = substr[2:]
|
|
new_str += "{" + a + "}" + b + post_substr
|
|
else:
|
|
new_str += "{" + a + "}" + b
|
|
string = new_str
|
|
return string
|
|
|
|
def fix_a_slash_b(string):
|
|
if len(string.split("/")) != 2:
|
|
return string
|
|
a = string.split("/")[0]
|
|
b = string.split("/")[1]
|
|
try:
|
|
a = int(a)
|
|
b = int(b)
|
|
assert string == "{}/{}".format(a, b)
|
|
new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
|
|
return new_string
|
|
except Exception as e:
|
|
return string
|
|
|
|
def remove_right_units(string):
|
|
# "\\text{ " only ever occurs (at least in the val set) when describing units
|
|
if "\\text{ " in string:
|
|
splits = string.split("\\text{ ")
|
|
assert len(splits) == 2
|
|
return splits[0]
|
|
else:
|
|
return string
|
|
|
|
def fix_sqrt(string):
|
|
if "\\sqrt" not in string:
|
|
return string
|
|
splits = string.split("\\sqrt")
|
|
new_string = splits[0]
|
|
for split in splits[1:]:
|
|
if split[0] != "{":
|
|
a = split[0]
|
|
new_substr = "\\sqrt{" + a + "}" + split[1:]
|
|
else:
|
|
new_substr = "\\sqrt" + split
|
|
new_string += new_substr
|
|
return new_string
|
|
|
|
def unbox_and_extract(text):
|
|
start_indices = [m.start() for m in re.finditer(r'\\boxed{', text)]
|
|
extracted_contents = []
|
|
for start in start_indices:
|
|
brace_count = 0
|
|
for i, char in enumerate(text[start:]):
|
|
if char == '{':
|
|
brace_count += 1
|
|
elif char == '}':
|
|
brace_count -= 1
|
|
if brace_count == 0:
|
|
end = start + i + 1
|
|
extracted_contents.append(text[start+7:end-1]) # +7 to skip '\\boxed{'
|
|
break
|
|
# Replace '\\boxed{...}' with the content inside it
|
|
unboxed_text = re.sub(r'\\boxed{(.*?)}', r'\1', text)
|
|
return unboxed_text, extracted_contents
|
|
|
|
def convert_to_latex_fraction(text: str) -> str:
|
|
# Use regex to find all occurrences of ((num)/(denom))
|
|
pattern = re.compile(r"\(\(([\d]+)\)/\(([\d]+)\)\)")
|
|
|
|
matches = pattern.findall(text)
|
|
|
|
for match in matches:
|
|
num, denom = match
|
|
latex_frac = f"\\\\frac{{{num}}}{{{denom}}}"
|
|
|
|
# Replace the old expression with the LaTeX fraction
|
|
text = text.replace(f"(({num})/({denom}))", latex_frac)
|
|
|
|
return text
|
|
|
|
def strip_string(string):
|
|
# convert ((3)/(4)) -> \\frac{3}{4}
|
|
string = convert_to_latex_fraction(string)
|
|
|
|
# remove ,
|
|
string = string.replace(",", "")
|
|
|
|
# linebreaks
|
|
string = string.replace("\n", "")
|
|
|
|
# remove inverse spaces
|
|
string = string.replace("\\!", "")
|
|
|
|
# replace \\ with \
|
|
string = string.replace("\\\\", "\\")
|
|
|
|
# replace tfrac and dfrac with frac
|
|
string = string.replace("tfrac", "frac")
|
|
string = string.replace("dfrac", "frac")
|
|
|
|
# remove \left and \right
|
|
string = string.replace("\\left", "")
|
|
string = string.replace("\\right", "")
|
|
|
|
# Remove circ (degrees)
|
|
string = string.replace("^{\\circ}", "")
|
|
string = string.replace("^\\circ", "")
|
|
|
|
# remove dollar signs
|
|
string = string.replace("\\$", "")
|
|
|
|
# remove units (on the right)
|
|
string = remove_right_units(string)
|
|
|
|
# remove percentage
|
|
string = string.replace("\\%", "")
|
|
string = string.replace("\%", "") # noqa: W605
|
|
|
|
# " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
|
|
string = string.replace(" .", " 0.")
|
|
string = string.replace("{.", "{0.")
|
|
# if empty, return empty string
|
|
if len(string) == 0:
|
|
return string
|
|
if string[0] == ".":
|
|
string = "0" + string
|
|
|
|
# to consider: get rid of e.g. "k = " or "q = " at beginning
|
|
if len(string.split("=")) == 2:
|
|
if len(string.split("=")[0]) <= 2:
|
|
string = string.split("=")[1]
|
|
|
|
# fix sqrt3 --> sqrt{3}
|
|
string = fix_sqrt(string)
|
|
|
|
# My own
|
|
string = string.replace("\\quad", " ")
|
|
# remove spaces
|
|
string = string.replace(" ", "")
|
|
|
|
# \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
|
|
string = fix_fracs(string)
|
|
|
|
# manually change 0.5 --> \frac{1}{2}
|
|
if string == "0.5":
|
|
string = "\\frac{1}{2}"
|
|
|
|
# NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
|
|
string = fix_a_slash_b(string)
|
|
|
|
return string
|
|
|
|
def is_number(s):
|
|
s = s.strip("$")
|
|
|
|
try:
|
|
# Try to convert the string to an integer
|
|
int(s)
|
|
return True
|
|
except ValueError:
|
|
try:
|
|
# Try to convert the string to a float
|
|
float(s)
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
def is_single_inline_math(expression: str) -> bool:
|
|
# Use regex to check for a pattern that starts and ends with dollar signs,
|
|
# and contains no other dollar signs in between.
|
|
pattern = re.compile(r"^\$[^$]+\$$")
|
|
|
|
match = pattern.match(expression)
|
|
|
|
return bool(match)
|
|
|
|
def is_equiv(prediction_ans, reference_ans, verbose=False):
|
|
if prediction_ans is None and reference_ans is None:
|
|
print("WARNING: Both None")
|
|
return True, prediction_ans, reference_ans
|
|
if prediction_ans is None or reference_ans is None:
|
|
return False, prediction_ans, reference_ans
|
|
|
|
try:
|
|
clean_prediction_ans = strip_string(prediction_ans)
|
|
clean_reference_ans = strip_string(reference_ans)
|
|
|
|
if is_number(clean_prediction_ans) and is_number(clean_reference_ans):
|
|
judge = float(clean_prediction_ans.strip("$")) == float(clean_reference_ans.strip("$"))
|
|
# print(f"1 judge: {judge}")
|
|
elif is_single_inline_math(clean_reference_ans):
|
|
judge = (clean_reference_ans.strip("$") in clean_prediction_ans.strip("$"))
|
|
# print(f"2 judge: {judge}")
|
|
elif (len(clean_prediction_ans) >= 3) and (not is_number(clean_prediction_ans)) and (not clean_prediction_ans.startswith("-")) and (not clean_reference_ans.startswith("-")) and (clean_prediction_ans in clean_reference_ans):
|
|
judge = True
|
|
# print(f"3 judge: {judge}")
|
|
elif (len(clean_reference_ans) >= 3) and (not is_number(clean_reference_ans)) and (not clean_prediction_ans.startswith("-")) and (not clean_reference_ans.startswith("-")) and (clean_reference_ans in clean_prediction_ans):
|
|
judge = True
|
|
# print(f"4 judge: {judge}")
|
|
else:
|
|
judge = clean_prediction_ans == clean_reference_ans
|
|
# print(f"5 judge: {judge}")
|
|
if verbose:
|
|
print(f"clean_prediction_ans: {clean_prediction_ans} | clean_reference_ans: {clean_reference_ans} | judge: {judge}")
|
|
return judge, clean_prediction_ans, clean_reference_ans
|
|
except Exception as e:
|
|
print(e)
|
|
return prediction_ans == reference_ans, prediction_ans, reference_ans
|
|
|
|
def is_correct(completion, answer, verbose=False):
|
|
completion = completion.lower()
|
|
answer = answer.lower()
|
|
|
|
# Extract short answer from completion
|
|
extract_ans = None
|
|
|
|
clean_reference_ans = strip_string(answer)
|
|
is_reference_ans_number = is_number(clean_reference_ans)
|
|
|
|
# First extract boxed answer
|
|
unbox_long_answer, box_short_answers = unbox_and_extract(completion)
|
|
if box_short_answers != []:
|
|
extract_ans = box_short_answers[-1].strip()
|
|
# print(f"1 extract_ans: {extract_ans}")
|
|
# extract the last number answer
|
|
elif is_reference_ans_number:
|
|
numbers = re.findall(r"[\-+]?\d*[\.,/]?\d+", completion)
|
|
if numbers:
|
|
extract_ans = numbers[-1]
|
|
# print(f"2 extract_ans: {extract_ans}")
|
|
# extract "the answer is ..." answer
|
|
elif ("answer is" in completion) or ("solution is" in completion):
|
|
if "answer is" in completion:
|
|
split_ans = completion.split('answer is')
|
|
else:
|
|
split_ans = completion.split('solution is')
|
|
ans = split_ans[-1].strip().lstrip(":").strip()
|
|
extract_ans_temp = ans.split('.\n')[0]
|
|
extract_ans_temp = extract_ans_temp.strip()
|
|
extract_ans_temp = extract_ans_temp.strip('.')
|
|
if len(extract_ans_temp)>0 and extract_ans_temp[-1] == '.':
|
|
extract_ans = extract_ans_temp[0:-1]
|
|
else:
|
|
extract_ans = extract_ans_temp
|
|
extract_ans = extract_ans.strip()
|
|
# print(f"3 extract_ans: {extract_ans}")
|
|
# extract "therefore xx is xxx" answer
|
|
elif "is" in completion:
|
|
pos = completion.rfind("is")
|
|
ans = completion[pos+2:].strip().lstrip(":").strip()
|
|
extract_ans_temp = ans.split('.\n')[0]
|
|
extract_ans_temp = extract_ans_temp.strip()
|
|
extract_ans_temp = extract_ans_temp.strip('.')
|
|
if len(extract_ans_temp)>0 and extract_ans_temp[-1] == '.':
|
|
extract_ans = extract_ans_temp[0:-1]
|
|
else:
|
|
extract_ans = extract_ans_temp
|
|
extract_ans = extract_ans.strip()
|
|
# print(f"4 extract_ans: {extract_ans}")
|
|
else:
|
|
return False, f"failed extracting answer from completion", clean_reference_ans
|
|
|
|
judge, clean_prediction_ans, clean_reference_ans = is_equiv(extract_ans, answer, verbose=verbose)
|
|
return judge, clean_prediction_ans, clean_reference_ans
|
|
|
|
if __name__ == "__main__":
|
|
reference_ans = "$2$"
|
|
prediction_ans = "To find the value of $a$, we need to evaluate the function $f$ at $f(\\sqrt{6})$ and set it equal to 3.\n\nFirst, let's find the value of $f(\\sqrt{6})$. Since $\\sqrt{6}$ is not in the domain of the first piece of the function, we move on to the second piece. \n\nFor $x \\leq 2$, the function becomes $f(x) = |x-3| + a$. Plugging in $\\sqrt{6}$, we have $f(\\sqrt{6}) = |(\\sqrt{6})-3| + a$.\n\nNext, we set $f(\\sqrt{6})$ equal to 3 and solve for $a$. \n\n$|(\\sqrt{6})-3| + a = 3$\n\nSince we are looking for a real number value of $a$, we can ignore the absolute value and solve for $a$.\n\n$(\\sqrt{6})-3 + a = 3$\n\n$\\sqrt{6} - 3 + a = 3$\n\n$\\sqrt{6} + a = 3 + 3$\n\n$\\sqrt{6} + a = 6$\n\nSubtracting 6 from both sides, we get:\n\n$a = 6 - \\sqrt{6}$\n\nTherefore, the value of $a$ is $6 - \\sqrt{6}$.\n\nThe answer is $a = 6 - \\sqrt{6}$."
|
|
print(is_correct(prediction_ans, reference_ans, verbose=True)) |