chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,82 @@
|
||||
import argparse
|
||||
import collections
|
||||
import os.path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from glob import glob
|
||||
import json
|
||||
|
||||
|
||||
def majority_voting_predict(preds):
|
||||
if isinstance(preds, str):
|
||||
return preds
|
||||
|
||||
preds = [pred for pred in preds if pred]
|
||||
if len(preds) == 0:
|
||||
return ""
|
||||
|
||||
assert isinstance(preds, list)
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
pred, freq = collections.Counter(tmp).most_common(1)[0]
|
||||
pred = eval(pred)
|
||||
elif isinstance(preds[0], str):
|
||||
pred, freq = collections.Counter(preds).most_common(1)[0]
|
||||
else:
|
||||
print(f"Unknown type {type(preds[0])}")
|
||||
pred = ""
|
||||
freq = 0
|
||||
return pred, freq
|
||||
|
||||
|
||||
def plot_histogram(data, bins=10, x_label="Value", y_label="Frequency", title="Histogram", output_file="histogram.png"):
|
||||
plt.hist(data, bins=bins, edgecolor='black', alpha=0.7)
|
||||
plt.xlabel(x_label)
|
||||
plt.ylabel(y_label)
|
||||
plt.title(title)
|
||||
plt.grid(True)
|
||||
# plt.show()
|
||||
plt.savefig(output_file)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
responses = json.load(open(args.input_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.input_file):
|
||||
responses += json.load(open(file))
|
||||
freq = {'correct': 0, 'incorrect': 0}
|
||||
num = {'correct': 0, 'incorrect': 0}
|
||||
correct_freqs = []
|
||||
incorrect_freqs = []
|
||||
for item in responses:
|
||||
sc_pred, f = majority_voting_predict(item["pred"])
|
||||
if item["sc_res"]:
|
||||
num['correct'] += 1
|
||||
freq['correct'] += f
|
||||
correct_freqs.append(f)
|
||||
else:
|
||||
num['incorrect'] += 1
|
||||
freq['incorrect'] += f
|
||||
incorrect_freqs.append(f)
|
||||
|
||||
print("Correct: ", num['correct'])
|
||||
print("Incorrect: ", num['incorrect'])
|
||||
print("Correct freq: ", freq['correct'])
|
||||
print("Incorrect freq: ", freq['incorrect'])
|
||||
print(f"Correct sc avg freq: {freq['correct'] / num['correct']}")
|
||||
print(f"Incorrect sc avg freq: {freq['incorrect'] / num['incorrect']}")
|
||||
|
||||
plot_histogram(correct_freqs, bins=50, title="Correct SC Prediction Frequency", output_file="correct_sc_freq.png")
|
||||
plot_histogram(incorrect_freqs, bins=50, title="Incorrect SC Prediction Frequency", output_file="incorrect_sc_freq.png")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,228 @@
|
||||
import collections
|
||||
import json
|
||||
import os.path
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
import random
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def sample_step(response: str, upper_step_ratio: float, sample_ratio: float):
|
||||
orig_lines = response.split("\n")
|
||||
lines = [(i, line) for i, line in enumerate(orig_lines)]
|
||||
lines = [(i, line) for i, line in lines if line.strip()]
|
||||
|
||||
if len(lines) < 5:
|
||||
return []
|
||||
|
||||
upper_step = int(len(lines) * upper_step_ratio)
|
||||
if upper_step == 0:
|
||||
return []
|
||||
|
||||
sample_step_num = int(upper_step * sample_ratio)
|
||||
if sample_step_num == 0:
|
||||
return []
|
||||
|
||||
sample_steps = random.sample(lines[:upper_step], sample_step_num)
|
||||
if len(sample_steps) == 0:
|
||||
return []
|
||||
|
||||
step_prefixes = []
|
||||
for i, line in sample_steps:
|
||||
step_prefixes.append("\n".join(orig_lines[:(i + 1)]))
|
||||
|
||||
return step_prefixes
|
||||
|
||||
|
||||
def get_pred_set(preds):
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = set(tmp)
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = set(preds)
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return tmp
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--upper_step_ratio", type=float, default=0.6)
|
||||
parser.add_argument("--sample_ratio", type=float, default=0.3)
|
||||
parser.add_argument("--filter_all_correct", action="store_true", default=False)
|
||||
parser.add_argument("--filter_all_same", action="store_true", default=False)
|
||||
parser.add_argument("--sample_over_p", default=-1, type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in sorted(glob(args.input_file)):
|
||||
if ".metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
# # First use self-consistency to construct pseudo labels
|
||||
# for item in data:
|
||||
# all_preds = collections.Counter()
|
||||
# for pred in item["pred"]:
|
||||
# if isinstance(pred, list):
|
||||
# all_preds.update(pred)
|
||||
# else:
|
||||
# all_preds[pred] += 1
|
||||
#
|
||||
# pseudo_label = all_preds.most_common(1)[0][0]
|
||||
# item["sc_label_0"] = pseudo_label
|
||||
|
||||
outputs = []
|
||||
num = 0
|
||||
for item in tqdm(data):
|
||||
if args.filter_all_correct and all(item["res"]):
|
||||
continue
|
||||
if args.filter_all_same and len(set(item["pred"])) == 1:
|
||||
continue
|
||||
|
||||
prefixes = []
|
||||
prefix_ids = []
|
||||
if not item["response"]:
|
||||
item["prefix"] = []
|
||||
continue
|
||||
|
||||
for resp_id, resp in enumerate(item["response"]):
|
||||
response_prefixes = sample_step(resp, args.upper_step_ratio, args.sample_ratio)
|
||||
prefixes.extend(response_prefixes)
|
||||
prefix_ids.extend([f"{item['id']}_{resp_id}_{i}" for i in range(len(response_prefixes))])
|
||||
|
||||
if args.sample_over_p > 0:
|
||||
if len(prefixes) > args.sample_over_p:
|
||||
prefixes = random.sample(prefixes, args.sample_over_p)
|
||||
prefix_ids = random.sample(prefix_ids, args.sample_over_p)
|
||||
item["prefix"] = prefixes
|
||||
item["prefix_id"] = prefix_ids
|
||||
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
item.pop("res")
|
||||
item.pop("sc_pred")
|
||||
item.pop("sc_res")
|
||||
|
||||
outputs.append(item)
|
||||
num += len(prefixes)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
print(f"Number of prefixes: {num}")
|
||||
print(len(outputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python ~/gpt-chat-examples/scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "math.test.v1.1.0shot.n10.tem1.0.p0.9.8-of-?.json" \
|
||||
--output_file math.test.v1.1.0shot.n10.tem1.0.p0.9.prefix.upper0.6.r0.3.json --upper_step_ratio 0.6 --sample_ratio 0.3
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py --input_file "../msranlpintern/share/models/deepseek-math-7b-instruct/meta_math/sub_math.cot.train.0shot.n10.tem1.0.p0.9.v1.0.?-of-24.json" --output_file "../msranlpintern/share/models/deepseek-math-7b-instruct/meta_math/sub_math.cot.train.0shot.n10.tem1.0.p0.9.v1.0.upper0.7.r0.3.inter_step.filter_all_true.json" --upper_step_ra
|
||||
tio 0.7 --sample_ratio 0.3 --filter_all_correct
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.[0-9]*-of-64.json" \
|
||||
--output_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.upper0.7.r0.3.inter_step.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.[0-9]*-of-64.json" \
|
||||
--output_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.upper0.7.r0.3.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.?-of-8.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same
|
||||
|
||||
Number of prefixes: 19521786
|
||||
309876
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.?-of-8.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 3098719
|
||||
309876
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-32.json" \
|
||||
--output_file "../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 2879063
|
||||
287913
|
||||
|
||||
############################################################ ITERATION 1 #######################################################
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 2389255
|
||||
238928
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 2389255
|
||||
238928
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample32.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 32
|
||||
|
||||
Number of prefixes: 7512389
|
||||
238928
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample32.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 32
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-[0-9]-of-10/cot.de_con.[0-9]-of-10.n8.tem1.0.p1.0.*-of-16.s0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/cot.de_con.n8.tem1.0.p1.0.s0.upper0.7.r0.3.sample16.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 16
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" \
|
||||
--output_file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample16.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 16
|
||||
|
||||
Number of prefixes: 5053462
|
||||
345626
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.*-of-8.s0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/cot.de_con.n8.tem1.0.p1.0.orig_0-of-8.s0.upper0.7.r0.3.sample32.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 32
|
||||
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
import collections
|
||||
from tqdm import tqdm
|
||||
|
||||
"""
|
||||
We sampled some intermediate steps from single response, and each intermediate state will be calculated with a value by counting the reached outcomes.
|
||||
We can rank the intermediate steps according to their distance to the origin, i.e., the prompt.
|
||||
The problem is how to adjust the value based on the values of its preceding states:
|
||||
|
||||
|
||||
Remember that if the outcome label is accurate, we can directly use the expected value as the reward since it can well indicate its importance.
|
||||
However, under self-consistency setting, we should always assume that, if the outcome is incorrect, then we should find the most distant state from the origin,
|
||||
and remains the largest probability that it is still possible to reach the correct answer.
|
||||
|
||||
TO maintain the prefixes with the least confidence over the pseudo label.
|
||||
|
||||
|
||||
TODO: Deepseek's extraction utils always return a list for MATH questions. Think about how to process this (for self-consistency).
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--top_k", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
|
||||
item_id2prefixes = collections.defaultdict(list)
|
||||
for i, prefix in tqdm(enumerate(data)):
|
||||
tmp = prefix["prefix_id"].split("_")
|
||||
# resp_id = int(resp_id) # -2
|
||||
# prefix_id = int(prefix_id) # -1
|
||||
item_id = "_".join(tmp[:-2])
|
||||
|
||||
last_iter_pseudo_label = prefix["sc_label_0"]
|
||||
|
||||
cnt = collections.Counter()
|
||||
for pred in prefix["pred"]:
|
||||
if isinstance(pred, list):
|
||||
cnt.update(pred)
|
||||
else:
|
||||
cnt[pred] += 1
|
||||
|
||||
if last_iter_pseudo_label not in cnt:
|
||||
v = 0
|
||||
else:
|
||||
v = cnt[last_iter_pseudo_label]
|
||||
|
||||
item_id2prefixes[item_id].append((i, v))
|
||||
|
||||
print(len(item_id2prefixes))
|
||||
|
||||
outputs = []
|
||||
for item_id, prefixes in item_id2prefixes.items():
|
||||
prefixes.sort(key=lambda x: x[1]) # Ascending order
|
||||
for i, v in prefixes[:args.top_k]:
|
||||
outputs.append(data[i])
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
from data.deepseek_math_utils import eval_script
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file_format", type=str)
|
||||
parser.add_argument("--seed_list", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = []
|
||||
for seed in eval(args.seed_list):
|
||||
_file = args.input_file_format.format(seed)
|
||||
if os.path.exists(_file):
|
||||
print(f"Loading: {_file}")
|
||||
data.extend(json.load(open(_file)))
|
||||
else:
|
||||
print(f"File not found: {_file}")
|
||||
|
||||
id2data = {}
|
||||
for item in tqdm(data):
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
|
||||
id2data[item["id"]]["response"] = [id2data[item["id"]]["response"]]
|
||||
id2data[item["id"]]["pred"] = [id2data[item["id"]]["pred"]]
|
||||
id2data[item["id"]]["res"] = [id2data[item["id"]]["res"]]
|
||||
else:
|
||||
assert item["text"] == id2data[item["id"]]["text"]
|
||||
|
||||
id2data[item["id"]]["response"].append(item["response"])
|
||||
id2data[item["id"]]["pred"].append(item["pred"])
|
||||
id2data[item["id"]]["res"].append(item["res"])
|
||||
|
||||
data = list(id2data.values())
|
||||
pass_at_k = 0
|
||||
maj_at_k = 0
|
||||
for item in tqdm(data):
|
||||
sc_pred = majority_voting_predict(item["pred"])
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
item["sc_pred"] = sc_pred
|
||||
item["sc_res"] = sc_res
|
||||
|
||||
if any(item["res"]):
|
||||
pass_at_k += 1
|
||||
if sc_res:
|
||||
maj_at_k += 1
|
||||
|
||||
print("Pass at k: ", pass_at_k / len(data))
|
||||
print("Maj at k: ", maj_at_k / len(data))
|
||||
json.dump(data, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
import collections
|
||||
import json
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
from glob import glob
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.deepseek_math_utils import eval_script, answer_extraction
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
|
||||
|
||||
def pred2str(pred):
|
||||
if isinstance(pred, str):
|
||||
return pred
|
||||
if isinstance(pred, list):
|
||||
pred = sorted(pred)
|
||||
pred = str(pred)
|
||||
return pred
|
||||
raise ValueError(f"Unknown type {type(pred)}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
parser.add_argument("--output_file")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
cnt = 0
|
||||
pass_at_k = 0
|
||||
sc = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], list):
|
||||
preds = item["pred"]
|
||||
else:
|
||||
preds = [item["pred"]]
|
||||
|
||||
if "res" not in item:
|
||||
mul_pass = 0
|
||||
if len(preds) > 0:
|
||||
res = []
|
||||
for pred in preds:
|
||||
res.append(eval_script.eval_math({"prediction": pred, "answer": item["label"]}))
|
||||
if any(res):
|
||||
mul_pass = 1
|
||||
else:
|
||||
res = []
|
||||
|
||||
item["pass_at_k"] = mul_pass
|
||||
|
||||
if len(preds) == 1:
|
||||
res = res[0]
|
||||
item["res"] = res
|
||||
|
||||
if not isinstance(item["res"], list):
|
||||
res = [item["res"]]
|
||||
else:
|
||||
res = item["res"]
|
||||
|
||||
if any(res):
|
||||
pass_at_k += 1
|
||||
if res[0]:
|
||||
cnt += 1
|
||||
|
||||
# str_preds = [pred2str(item) for item in preds]
|
||||
# counter = collections.Counter(str_preds)
|
||||
# sc_pred = eval(counter.most_common(1)[0][0])
|
||||
# sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
|
||||
# if "sc_res" in item:
|
||||
# sc_res = item["sc_res"]
|
||||
# else:
|
||||
# preds = [x for x in preds if x]
|
||||
# if len(preds):
|
||||
# sc_pred = majority_voting_predict(preds)
|
||||
# try:
|
||||
# sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
# except Exception as e:
|
||||
# print(f"Error in {item['id']} during evaluation: {e}")
|
||||
# sc_res = False
|
||||
# else:
|
||||
# sc_res = False
|
||||
# if sc_res:
|
||||
# sc += 1
|
||||
|
||||
print(f"Pass at k: {pass_at_k}/{len(data)} = {pass_at_k / len(data)}")
|
||||
print(f"Correct at k: {cnt}/{len(data)} = {cnt / len(data)}")
|
||||
print(f"Self-consistency: {sc}/{len(data)} = {sc / len(data)}")
|
||||
if args.output_file:
|
||||
json.dump(data, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
path=$1
|
||||
|
||||
shift 1 # Shift the first 5 arguments, so $1 now refers to the 6th argument
|
||||
|
||||
for step in "$@"; do
|
||||
echo "Merging predictions for step ================================== $step"
|
||||
python scripts/math_scale/merge_dp_predictions.py --input_file "$path/mwpbench/checkpoint-$step/train_wo_gsm.2k.v1.0.0shot.n1.tem0.0.p1.0.?-of-8.json" \
|
||||
--output_file "$path/mwpbench/checkpoint-$step/train_wo_gsm.2k.v1.0.0shot.n1.tem0.0.p1.0.json"
|
||||
python scripts/math/merge_dp_predictions.py --input_file "$path/math/checkpoint-$step/math.test.v1.1.0shot.n1.tem0.0.p1.0.8-of-?.json" \
|
||||
--output_file "$path/math/checkpoint-$step/math.test.v1.1.0shot.n1.tem0.0.p1.0.json"
|
||||
echo "Done merging predictions for step ================================== $step"
|
||||
echo
|
||||
done
|
||||
@@ -0,0 +1,83 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import collections
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.deepseek_math_utils import eval_script, answer_extraction
|
||||
|
||||
|
||||
def pred2str(pred):
|
||||
if isinstance(pred, str):
|
||||
return pred
|
||||
if isinstance(pred, list):
|
||||
pred = sorted(pred)
|
||||
pred = str(pred)
|
||||
return pred
|
||||
raise ValueError(f"Unknown type {type(pred)}")
|
||||
|
||||
|
||||
def load_data(file_path):
|
||||
if not file_path:
|
||||
return []
|
||||
if os.path.exists(file_path):
|
||||
data = json.load(open(file_path))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
data += json.load(open(file))
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file1", type=str)
|
||||
parser.add_argument("--input_file2", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--n", type=int, default=-1)
|
||||
args = parser.parse_args()
|
||||
|
||||
data1 = load_data(args.input_file1)
|
||||
data2 = load_data(args.input_file2)
|
||||
|
||||
item_id2preds = {}
|
||||
sc = 0
|
||||
for item in tqdm(data1 + data2):
|
||||
if "prefix_id" in item:
|
||||
prefix_id = item["prefix_id"]
|
||||
tmp = prefix_id.split("_")
|
||||
item_id = "_".join(tmp[:-2])
|
||||
else:
|
||||
item_id = item["id"]
|
||||
|
||||
if item_id not in item_id2preds:
|
||||
item_id2preds[item_id] = {
|
||||
"response": [],
|
||||
"pred": [],
|
||||
"label": item["label"],
|
||||
}
|
||||
|
||||
item_id2preds[item_id]["response"].extend(item["response"])
|
||||
item_id2preds[item_id]["pred"].extend(item["pred"])
|
||||
|
||||
for item, responses in tqdm(item_id2preds.items()):
|
||||
preds = responses["pred"]
|
||||
if args.n != -1:
|
||||
preds = preds[:args.n]
|
||||
str_preds = [pred2str(item) for item in preds]
|
||||
counter = collections.Counter(str_preds)
|
||||
sc_pred = eval(counter.most_common(1)[0][0])
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": responses["label"]})
|
||||
if sc_res:
|
||||
sc += 1
|
||||
|
||||
print(f"Self-consistency: {sc}/{len(item_id2preds)} = {sc / len(item_id2preds)}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file_format", type=str)
|
||||
parser.add_argument("--seed_list", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
seed_list = eval(args.seed_list)
|
||||
rewards = []
|
||||
for seed in seed_list:
|
||||
# _file = args.response_file_format.format(seed)
|
||||
_file = args.response_file_format.replace("[[seed]]", str(seed))
|
||||
if os.path.exists(_file):
|
||||
print(f"Loading: {_file}")
|
||||
data = json.load(open(_file))
|
||||
for item in data:
|
||||
item["index"] = f"{item['index']}_{seed}"
|
||||
rewards.extend(data)
|
||||
else:
|
||||
print(f"File not found: {_file}")
|
||||
|
||||
json.dump(rewards, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from functools import partial
|
||||
import torch
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
from data.deepseek_math_utils import eval_script
|
||||
|
||||
|
||||
def load_rewards(reward_file):
|
||||
if os.path.exists(reward_file):
|
||||
rewards = json.load(open(reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob(reward_file):
|
||||
print(file)
|
||||
rewards.extend(json.load(open(file)))
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def _init(id2reward):
|
||||
global _id2reward
|
||||
_id2reward = id2reward
|
||||
|
||||
|
||||
def _worker(item, sc_top_k=None):
|
||||
sorted_results = []
|
||||
if not item["response"] or not item["pred"] or not item["res"]:
|
||||
return {
|
||||
"missing": 1,
|
||||
"reward_missing": 0,
|
||||
"pred_missing": 0,
|
||||
"seq_too_long": 0,
|
||||
"sorted_results": [],
|
||||
"sc_res": False
|
||||
}
|
||||
reward_missing = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
resp_id = f"{item['id']}_{i}"
|
||||
if resp_id not in _id2reward:
|
||||
reward_missing += 1
|
||||
continue
|
||||
if not pred:
|
||||
pred_missing += 1
|
||||
continue
|
||||
|
||||
reward = _id2reward[resp_id]
|
||||
sorted_results.append((resp, pred, r, reward))
|
||||
sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
|
||||
sc_top_k_res = {}
|
||||
if sc_top_k and sorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in sorted_results[:k]]
|
||||
sc_pred = majority_voting_predict(preds)
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
sc_top_k_res[k] = sc_res
|
||||
|
||||
return {
|
||||
"missing": 0,
|
||||
"reward_missing": reward_missing,
|
||||
"pred_missing": pred_missing,
|
||||
"seq_too_long": seq_too_long,
|
||||
"sorted_results": sorted_results,
|
||||
"sc_res": item["sc_res"],
|
||||
"sc_top_k_res": sc_top_k_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.response_file):
|
||||
responses = json.load(open(args.response_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.response_file):
|
||||
print(file)
|
||||
responses.extend(json.load(open(file)))
|
||||
|
||||
rewards = load_rewards(args.reward_file)
|
||||
id2reward = {item["index"]: item["reward"][1] for item in rewards}
|
||||
|
||||
_k = [1, 3, 5]
|
||||
orm_pass_at_k = {k: 0 for k in _k}
|
||||
missing = 0
|
||||
missing_reward = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
sc_cnt = 0
|
||||
# for item in tqdm(responses):
|
||||
# sorted_results = []
|
||||
# if not item["response"] or not item["pred"] or not item["res"]:
|
||||
# missing += 1
|
||||
# continue
|
||||
# for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
# resp_id = f"{item['id']}_{i}"
|
||||
# if resp_id not in id2reward:
|
||||
# missing_reward += 1
|
||||
# continue
|
||||
#
|
||||
# reward = id2reward[resp_id]
|
||||
# sorted_results.append((resp, pred, r, reward))
|
||||
#
|
||||
# if not sorted_results:
|
||||
# continue
|
||||
# sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
# for k in _k:
|
||||
# if any([r[2] for r in sorted_results[:k]]):
|
||||
# orm_pass_at_k[k] += 1
|
||||
#
|
||||
# if item["sc_res"]:
|
||||
# sc_cnt += 1
|
||||
with Pool(args.num_workers, initializer=_init, initargs=(id2reward,)) as pool:
|
||||
annotate = partial(_worker, sc_top_k=(5, 10))
|
||||
results = list(tqdm(pool.imap_unordered(annotate, responses), total=len(responses)))
|
||||
|
||||
sc_top_k = {k: 0 for k in (5, 10)}
|
||||
for item in results:
|
||||
missing += item["missing"]
|
||||
missing_reward += item["reward_missing"]
|
||||
pred_missing += item["pred_missing"]
|
||||
seq_too_long += item["seq_too_long"]
|
||||
sorted_results = item["sorted_results"]
|
||||
|
||||
if item["sc_res"]:
|
||||
sc_cnt += 1
|
||||
|
||||
if not sorted_results:
|
||||
continue
|
||||
|
||||
# ultimate_results.append(sorted_results)
|
||||
for k in _k:
|
||||
if any([r[2] for r in sorted_results[:k]]):
|
||||
orm_pass_at_k[k] += 1
|
||||
|
||||
for k, v in item["sc_top_k_res"].items():
|
||||
if v:
|
||||
sc_top_k[k] += 1
|
||||
|
||||
print(f"Total: {len(responses)}")
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing reward: {missing_reward}")
|
||||
print(f"Missing pred: {pred_missing}")
|
||||
print(f"Seq too long: {seq_too_long}")
|
||||
print(f"SC: {sc_cnt}")
|
||||
for k, v in orm_pass_at_k.items():
|
||||
print(f"PRM pass at {k}: {v}")
|
||||
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
for k, v in sc_top_k.items():
|
||||
print(f"SC pass at {k}: {v}")
|
||||
print(f"SC pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,272 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
from multiprocessing import Pool
|
||||
import torch
|
||||
import sys
|
||||
from functools import partial
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
from data.deepseek_math_utils import eval_script
|
||||
|
||||
|
||||
def load_rewards(reward_file, re_index):
|
||||
if os.path.exists(reward_file):
|
||||
rewards = json.load(open(reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for i, file in enumerate(sorted(glob(reward_file))):
|
||||
print(file)
|
||||
sub_rewards = json.load(open(file))
|
||||
if re_index:
|
||||
for item in sub_rewards:
|
||||
item["index"] = f"{item['index']}_{i}"
|
||||
rewards.extend(sub_rewards)
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def reward_reduction(ending_logits, reduction: str = "min", norm: bool = True):
|
||||
if norm:
|
||||
ending_logits = torch.tensor(ending_logits)
|
||||
ending_probs = torch.softmax(ending_logits, dim=-1).tolist()
|
||||
step_rewards = [step[1] for step in ending_probs]
|
||||
else:
|
||||
step_rewards = [step[1] for step in ending_logits]
|
||||
if reduction == "min":
|
||||
reward = min(step_rewards)
|
||||
elif reduction == "product":
|
||||
reward = 1
|
||||
for prob in step_rewards:
|
||||
reward *= prob
|
||||
elif reduction == "sum":
|
||||
reward = sum(step_rewards)
|
||||
else:
|
||||
raise ValueError(f"Invalid reduction method: {reduction}")
|
||||
return reward
|
||||
|
||||
|
||||
def weighted_majority_voting_predict(preds, weights):
|
||||
pred2weight = {}
|
||||
for pred, weight in zip(preds, weights):
|
||||
if pred not in pred2weight:
|
||||
pred2weight[pred] = 0
|
||||
pred2weight[pred] += weight
|
||||
return max(pred2weight, key=pred2weight.get)
|
||||
|
||||
|
||||
def _init(id2reward):
|
||||
global _id2reward
|
||||
_id2reward = id2reward
|
||||
|
||||
|
||||
def _worker(item, reduction, norm, sc_top_k=None):
|
||||
if not item["response"] or not item["pred"] or not item["res"]:
|
||||
return {
|
||||
"missing": 1,
|
||||
"reward_missing": 0,
|
||||
"pred_missing": 0,
|
||||
"seq_too_long": 0,
|
||||
"sorted_results": [],
|
||||
"sc_res": False
|
||||
}
|
||||
unsorted_results = []
|
||||
reward_missing = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
resp_id = f"{item['id']}_{i}"
|
||||
if resp_id not in _id2reward:
|
||||
reward_missing += 1
|
||||
continue
|
||||
if not pred:
|
||||
pred_missing += 1
|
||||
continue
|
||||
|
||||
process_rewards = _id2reward[resp_id]
|
||||
if len(process_rewards["ending_logits"]) == 0:
|
||||
seq_too_long += 1
|
||||
continue
|
||||
|
||||
assert resp == process_rewards["response"], f"{resp} \n\n {process_rewards['response']} \n\n ========="
|
||||
|
||||
reward = reward_reduction(process_rewards["ending_logits"], reduction, norm)
|
||||
unsorted_results.append((resp, pred, r, reward))
|
||||
sorted_results = sorted(unsorted_results, key=lambda x: x[-1], reverse=True)
|
||||
|
||||
sc_top_k_res = {}
|
||||
if sc_top_k and sorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in sorted_results[:k]]
|
||||
sc_pred = majority_voting_predict(preds)
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
sc_top_k_res[k] = sc_res
|
||||
|
||||
sc_k_res = {}
|
||||
if sc_top_k and unsorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in unsorted_results[:k]]
|
||||
sc_pred = majority_voting_predict(preds)
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
sc_k_res[k] = sc_res
|
||||
|
||||
weighted_best_of_k = {}
|
||||
if sc_top_k and unsorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in unsorted_results[:k]]
|
||||
weights = torch.softmax(torch.tensor([r[-1] for r in unsorted_results]), dim=0).tolist()
|
||||
best_of_k_pred = weighted_majority_voting_predict(preds, weights)
|
||||
if best_of_k_pred != "":
|
||||
best_of_k_res = eval_script.eval_math({"prediction": best_of_k_pred, "answer": item["label"]})
|
||||
else:
|
||||
best_of_k_res = False
|
||||
weighted_best_of_k[k] = best_of_k_res
|
||||
|
||||
return {
|
||||
"missing": 0,
|
||||
"reward_missing": reward_missing,
|
||||
"pred_missing": pred_missing,
|
||||
"seq_too_long": seq_too_long,
|
||||
"sorted_results": sorted_results,
|
||||
"sc_res": item["sc_res"],
|
||||
"sc_top_k_res": sc_top_k_res,
|
||||
"weighted_best_of_k": weighted_best_of_k,
|
||||
"sc_k_res": sc_k_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
parser.add_argument("--reduction", type=str, default="min")
|
||||
parser.add_argument("--raw_logits", action="store_true", default=False)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--re_index", action="store_true", default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.response_file):
|
||||
responses = json.load(open(args.response_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.response_file):
|
||||
print(file)
|
||||
responses.extend(json.load(open(file)))
|
||||
|
||||
print(len(responses[0]["response"]), len(responses[0]["pred"]), len(responses[0]["res"]))
|
||||
print(responses[0]["id"])
|
||||
|
||||
rewards = load_rewards(args.reward_file, args.re_index)
|
||||
print(rewards[0]['index'])
|
||||
id2reward = {item["index"]: item for item in rewards}
|
||||
|
||||
# _k = [1, 3, 5]
|
||||
_k = [3, 5, 4, 8, 16, 32, 64, 128]
|
||||
prm_pass_at_k = {k: 0 for k in _k}
|
||||
missing = 0
|
||||
missing_reward = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
sc_cnt = 0
|
||||
ultimate_results = []
|
||||
# for item in tqdm(responses):
|
||||
# sorted_results = []
|
||||
# if not item["response"] or not item["pred"] or not item["res"]:
|
||||
# missing += 1
|
||||
# continue
|
||||
# for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
# resp_id = f"{item['id']}_{i}"
|
||||
# if resp_id not in id2reward:
|
||||
# missing_reward += 1
|
||||
# continue
|
||||
# process_rewards = id2reward[resp_id]
|
||||
# if len(process_rewards["ending_logits"]) == 0:
|
||||
# continue
|
||||
#
|
||||
# assert resp == process_rewards["response"], f"{resp} \n\n {process_rewards['response']} \n\n ========="
|
||||
#
|
||||
# reward = reward_reduction(process_rewards["ending_logits"], args.reduction, not args.raw_logits)
|
||||
# sorted_results.append((resp, pred, r, reward))
|
||||
#
|
||||
# if not sorted_results:
|
||||
# continue
|
||||
# sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
# ultimate_results.append(sorted_results)
|
||||
# for k in _k:
|
||||
# if any([r[2] for r in sorted_results[:k]]):
|
||||
# prm_pass_at_k[k] += 1
|
||||
#
|
||||
# if item["sc_res"]:
|
||||
# sc_cnt += 1
|
||||
with Pool(args.num_workers, initializer=_init, initargs=(id2reward,)) as pool:
|
||||
annotate = partial(_worker, reduction=args.reduction, norm=(not args.raw_logits), sc_top_k=_k)
|
||||
results = list(tqdm(pool.imap_unordered(annotate, responses), total=len(responses)))
|
||||
|
||||
sc_top_k = {k: 0 for k in _k}
|
||||
weighted_best_of_k = {k: 0 for k in _k}
|
||||
sc_k = {k: 0 for k in _k}
|
||||
for item in results:
|
||||
missing += item["missing"]
|
||||
missing_reward += item["reward_missing"]
|
||||
pred_missing += item["pred_missing"]
|
||||
seq_too_long += item["seq_too_long"]
|
||||
sorted_results = item["sorted_results"]
|
||||
|
||||
if item["sc_res"]:
|
||||
sc_cnt += 1
|
||||
|
||||
if not sorted_results:
|
||||
continue
|
||||
|
||||
ultimate_results.append(sorted_results)
|
||||
for k in _k:
|
||||
if any([r[2] for r in sorted_results[:k]]):
|
||||
prm_pass_at_k[k] += 1
|
||||
|
||||
for k, v in item["sc_top_k_res"].items():
|
||||
if v:
|
||||
sc_top_k[k] += 1
|
||||
|
||||
for k, v in item["weighted_best_of_k"].items():
|
||||
if v:
|
||||
weighted_best_of_k[k] += 1
|
||||
|
||||
for k, v in item["sc_k_res"].items():
|
||||
if v:
|
||||
sc_k[k] += 1
|
||||
|
||||
print(f"Total: {len(responses)}")
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing reward: {missing_reward}")
|
||||
print(f"Missing pred: {pred_missing}")
|
||||
print(f"Seq too long: {seq_too_long}")
|
||||
print(f"SC: {sc_cnt}")
|
||||
for k, v in prm_pass_at_k.items():
|
||||
print(f"PRM pass at {k}: {v}")
|
||||
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
for k, v in sc_top_k.items():
|
||||
print(f"SC pass at {k}: {v}")
|
||||
print(f"SC pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
||||
for k, v in weighted_best_of_k.items():
|
||||
print(f"Weighted best of k pass at {k}: {v}")
|
||||
print(f"Weighted best of k pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
for k, v in sc_k.items():
|
||||
print(f"SC k pass at {k}: {v}")
|
||||
print(f"SC k pass at {k} rate {v / len(responses) * 100:.2f}%")
|
||||
|
||||
json.dump(ultimate_results[:100], open("debug.json", "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,177 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import collections
|
||||
import torch
|
||||
|
||||
|
||||
def load_rewards(reward_file):
|
||||
if os.path.exists(reward_file):
|
||||
rewards = json.load(open(reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob(reward_file):
|
||||
print(file)
|
||||
rewards.extend(json.load(open(file)))
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def reward_reduction(ending_logits, reduction: str = "min", norm: bool = True):
|
||||
if norm:
|
||||
ending_logits = torch.tensor(ending_logits)
|
||||
ending_probs = torch.softmax(ending_logits, dim=-1).tolist()
|
||||
step_rewards = [step[1] for step in ending_probs]
|
||||
else:
|
||||
step_rewards = [step[1] for step in ending_logits]
|
||||
if reduction == "min":
|
||||
reward = min(step_rewards)
|
||||
elif reduction == "product":
|
||||
reward = 1
|
||||
for prob in step_rewards:
|
||||
reward *= prob
|
||||
elif reduction == "sum":
|
||||
reward = sum(step_rewards)
|
||||
else:
|
||||
raise ValueError(f"Invalid reduction method: {reduction}")
|
||||
return reward
|
||||
|
||||
|
||||
def merge_rewards(group_logits, weights, reduction: str = "min", norm: bool = True, group_reduction: str = "min"):
|
||||
# rewards = []
|
||||
# for g_id, item in group_logits.items():
|
||||
# if len(item["ending_logits"]) == 0:
|
||||
# continue
|
||||
# r = reward_reduction(item["ending_logits"], reduction, norm)
|
||||
# rewards.append(weights[g_id] * r)
|
||||
# if len(rewards) == 0:
|
||||
# return 0
|
||||
# if group_reduction == "min":
|
||||
# reward = min(rewards)
|
||||
# elif group_reduction == "product":
|
||||
# reward = 1
|
||||
# for r in rewards:
|
||||
# reward *= r
|
||||
# elif group_reduction == "sum":
|
||||
# reward = sum(rewards)
|
||||
# else:
|
||||
# raise ValueError(f"Invalid group reduction method: {group_reduction}")
|
||||
ending_logits = []
|
||||
if len(group_logits) == 0:
|
||||
return 0
|
||||
_len = len(group_logits[0]["ending_logits"])
|
||||
if _len == 0:
|
||||
return 0
|
||||
for g_id, item in group_logits.items():
|
||||
assert len(item["ending_logits"]) == _len
|
||||
ending_logits.append(item["ending_logits"])
|
||||
ending_logits = torch.tensor(ending_logits)
|
||||
assert ending_logits.size() == (len(group_logits), _len, 2), ending_logits.size()
|
||||
|
||||
if norm:
|
||||
ending_logits = torch.softmax(ending_logits, dim=-1)
|
||||
ending_logits = ending_logits[:, :, 1]
|
||||
else:
|
||||
ending_logits = ending_logits[:, :, 1]
|
||||
|
||||
ending_logits = ending_logits * torch.tensor(weights).view(-1, 1)
|
||||
|
||||
if group_reduction == "min":
|
||||
ending_logits = torch.min(ending_logits, dim=0)[0]
|
||||
elif group_reduction == "product":
|
||||
ending_logits = torch.prod(ending_logits, dim=0)
|
||||
elif group_reduction == "sum":
|
||||
ending_logits = torch.sum(ending_logits, dim=0)
|
||||
else:
|
||||
raise ValueError(f"Invalid group reduction method: {group_reduction}")
|
||||
|
||||
if reduction == "min":
|
||||
reward = torch.min(ending_logits).item()
|
||||
elif reduction == "product":
|
||||
reward = torch.prod(ending_logits).item()
|
||||
elif reduction == "sum":
|
||||
reward = torch.sum(ending_logits).item()
|
||||
else:
|
||||
raise ValueError(f"Invalid reduction method: {reduction}")
|
||||
|
||||
return reward
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True, nargs='+')
|
||||
parser.add_argument("--weights", type=float, nargs='+', default=[1.0])
|
||||
parser.add_argument("--reduction", type=str, default="min")
|
||||
parser.add_argument("--raw_logits", action="store_true", default=False)
|
||||
parser.add_argument("--group_reduction", type=str, default="min")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.response_file):
|
||||
responses = json.load(open(args.response_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.response_file):
|
||||
print(file)
|
||||
responses.extend(json.load(open(file)))
|
||||
|
||||
id2reward = collections.defaultdict(dict)
|
||||
assert len(args.reward_file) == len(args.weights)
|
||||
print(args.reward_file)
|
||||
print(args.weights)
|
||||
for _g_id, _reward_group in enumerate(args.reward_file):
|
||||
rewards = load_rewards(_reward_group)
|
||||
for r in rewards:
|
||||
id2reward[r["index"]][_g_id] = r
|
||||
|
||||
_k = [1, 3, 5]
|
||||
prm_pass_at_k = {k: 0 for k in _k}
|
||||
missing = 0
|
||||
missing_reward = 0
|
||||
sc_cnt = 0
|
||||
ultimate_results = []
|
||||
for item in tqdm(responses):
|
||||
sorted_results = []
|
||||
if not item["response"] or not item["pred"] or not item["res"]:
|
||||
missing += 1
|
||||
continue
|
||||
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
resp_id = f"{item['id']}_{i}"
|
||||
if resp_id not in id2reward:
|
||||
missing_reward += 1
|
||||
continue
|
||||
process_rewards = id2reward[resp_id]
|
||||
# if len(process_rewards["ending_logits"]) == 0:
|
||||
# continue
|
||||
|
||||
# reward = merge_rewards(process_rewards["ending_logits"], args.reduction, not args.raw_logits)
|
||||
reward = merge_rewards(process_rewards, args.weights, args.reduction, not args.raw_logits, args.group_reduction)
|
||||
sorted_results.append((resp, pred, r, reward))
|
||||
|
||||
if not sorted_results:
|
||||
continue
|
||||
sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
ultimate_results.append(sorted_results)
|
||||
for k in _k:
|
||||
if any([r[2] for r in sorted_results[:k]]):
|
||||
prm_pass_at_k[k] += 1
|
||||
|
||||
if item["sc_res"]:
|
||||
sc_cnt += 1
|
||||
|
||||
print(f"Total: {len(responses)}")
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing reward: {missing_reward}")
|
||||
print(f"SC: {sc_cnt}")
|
||||
for k, v in prm_pass_at_k.items():
|
||||
print(f"PRM pass at {k}: {v}")
|
||||
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
||||
|
||||
json.dump(ultimate_results[:100], open("debug.json", "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user