wehub-resource-sync
340 lines
14 KiB
Python
340 lines
14 KiB
Python
import argparse
|
|
import json
|
|
import sys
|
|
import os
|
|
from glob import glob
|
|
import collections
|
|
from tqdm import tqdm
|
|
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
|
|
|
from data.mathscale.util import mathscale_is_equiv
|
|
|
|
|
|
def evaluate_by_sc(item, external_sc: str = None):
|
|
if not item["response"]:
|
|
return []
|
|
|
|
if isinstance(item["response"], list):
|
|
preds = item["pred"]
|
|
else:
|
|
preds = [item["pred"]]
|
|
|
|
assert len(preds) > 1, f"Self-consistency requires at least 2 predictions, but got {len(preds)}."
|
|
|
|
if external_sc:
|
|
sc_pred = external_sc
|
|
else:
|
|
sc_pred = item["sc_pred"]
|
|
res = [mathscale_is_equiv(p, sc_pred)[0] for p in preds]
|
|
|
|
return res
|
|
|
|
|
|
def majority_voting_frequency(preds):
|
|
assert isinstance(preds, list)
|
|
if isinstance(preds[0], list):
|
|
tmp = []
|
|
for pred in preds:
|
|
tmp.append(str(sorted(pred)))
|
|
tmp = collections.Counter(tmp)
|
|
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
|
sorted_preds = [(eval(pred), fre) for pred, fre in tmp]
|
|
elif isinstance(preds[0], str):
|
|
tmp = collections.Counter(preds)
|
|
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
|
sorted_preds = [(pred, fre) for pred, fre in tmp]
|
|
else:
|
|
raise ValueError(f"Unknown type {type(preds[0])}")
|
|
return sorted_preds
|
|
|
|
|
|
def get_pred_frequency(preds, target_pred):
|
|
assert isinstance(preds, list)
|
|
if isinstance(preds[0], list):
|
|
tmp = []
|
|
for pred in preds:
|
|
tmp.append(str(sorted(pred)))
|
|
tmp = collections.Counter(tmp)
|
|
elif isinstance(preds[0], str):
|
|
tmp = collections.Counter(preds)
|
|
else:
|
|
raise ValueError(f"Unknown type {type(preds[0])}")
|
|
if isinstance(target_pred, list):
|
|
target_pred = str(sorted(target_pred))
|
|
return tmp.get(target_pred, 0) / len(preds)
|
|
|
|
|
|
def merge_key(item, value):
|
|
assert isinstance(item, list)
|
|
if isinstance(value, list):
|
|
item = item + value
|
|
else:
|
|
item.append(value)
|
|
return item
|
|
|
|
|
|
def merge_seed_sampled_data(data):
|
|
id2data = {}
|
|
for item in data:
|
|
if item["id"] not in id2data:
|
|
id2data[item["id"]] = item
|
|
continue
|
|
|
|
tmp = id2data[item["id"]]
|
|
if isinstance(tmp["response"], str):
|
|
tmp["response"] = [tmp["response"]]
|
|
if not isinstance(tmp["res"], list):
|
|
tmp["res"] = [tmp["res"]]
|
|
if not isinstance(tmp["pred"], list):
|
|
tmp["pred"] = [tmp["pred"]]
|
|
|
|
tmp["response"] = merge_key(tmp["response"], item["response"])
|
|
tmp["res"] = merge_key(tmp["res"], item["res"])
|
|
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
|
assert isinstance(tmp["pred"], list), tmp["pred"]
|
|
id2data[item["id"]] = tmp
|
|
|
|
return list(id2data.values())
|
|
|
|
|
|
def load_data(file_path):
|
|
if os.path.exists(file_path):
|
|
data = json.load(open(file_path))
|
|
else:
|
|
data = []
|
|
for file in glob(file_path, recursive=True):
|
|
if ".metrics" in file:
|
|
continue
|
|
print(file)
|
|
f = open(file, "r")
|
|
try:
|
|
data += json.load(f)
|
|
except:
|
|
print(f"Error in file {file}")
|
|
new_file = file.replace(".json", ".jsonl")
|
|
lines = open(new_file, "r").readlines()
|
|
for line in lines:
|
|
try:
|
|
data.append(json.loads(line))
|
|
except:
|
|
print(f"Error in line: {line}")
|
|
return data
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--input_file", type=str)
|
|
parser.add_argument("--output_file", type=str)
|
|
parser.add_argument("--external_file_for_sc", type=str, default=None)
|
|
parser.add_argument("--top_p", type=float, default=0.0)
|
|
args = parser.parse_args()
|
|
|
|
data = load_data(args.input_file)
|
|
data = merge_seed_sampled_data(data)
|
|
|
|
filtered = 0
|
|
if args.external_file_for_sc:
|
|
external_data = load_data(args.external_file_for_sc)
|
|
# id2external_sc = {item["id"]: item["sc_pred"] for item in external_data}
|
|
id2sc = {}
|
|
for item in tqdm(external_data):
|
|
sc_pred = item["sc_pred"]
|
|
|
|
if item["pred"] == "":
|
|
continue
|
|
|
|
if item["pred"] == "failed extracting answer from completion":
|
|
continue
|
|
|
|
freq = get_pred_frequency(item["pred"], sc_pred)
|
|
if freq > args.top_p:
|
|
id2sc[item["id"]] = sc_pred
|
|
else:
|
|
filtered += 1
|
|
else:
|
|
id2sc = {}
|
|
for item in tqdm(data):
|
|
sc_pred = item["sc_pred"]
|
|
|
|
if item["pred"] == "":
|
|
continue
|
|
|
|
if item["pred"] == "failed extracting answer from completion":
|
|
continue
|
|
|
|
freq = get_pred_frequency(item["pred"], sc_pred)
|
|
if freq > args.top_p:
|
|
id2sc[item["id"]] = sc_pred
|
|
else:
|
|
filtered += 1
|
|
|
|
print(f"Filtered {filtered} samples.")
|
|
|
|
outputs = []
|
|
cnt = 0
|
|
pass_at_k = 0
|
|
num_pairs = 0
|
|
pos_missing = 0
|
|
neg_missing = 0
|
|
full_positive_samples = []
|
|
full_negative_samples = []
|
|
for item in data:
|
|
if item["id"] in id2sc:
|
|
res_by_sc = evaluate_by_sc(item, id2sc[item["id"]])
|
|
else:
|
|
continue
|
|
|
|
if len(res_by_sc) == 0:
|
|
continue
|
|
|
|
if res_by_sc[0]:
|
|
cnt += 1
|
|
if any(res_by_sc):
|
|
pass_at_k += 1
|
|
|
|
pos = []
|
|
neg = []
|
|
for resp, r in zip(item["response"], res_by_sc):
|
|
if r:
|
|
pos.append(resp)
|
|
else:
|
|
neg.append(resp)
|
|
|
|
if len(pos) == 0:
|
|
full_negative_samples.append(item)
|
|
pos_missing += 1
|
|
if len(neg) == 0:
|
|
neg_missing += 1
|
|
full_positive_samples.append(item)
|
|
if len(pos) == 0 or len(neg) == 0:
|
|
continue
|
|
|
|
item["pos"] = pos
|
|
item["neg"] = neg
|
|
num_pairs += len(pos) * len(neg)
|
|
outputs.append(item)
|
|
|
|
print(f"Total number of items: {len(data)}")
|
|
print(f"Acc: {cnt / (len(data) - filtered)}")
|
|
print(f"Pass at k: {pass_at_k / len(data)}")
|
|
print(f"No positive solutions: {pos_missing} / {len(data)}")
|
|
print(f"No negative solutions: {neg_missing} / {len(data)}")
|
|
print(f"Num pairs: {num_pairs}")
|
|
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
|
json.dump(full_positive_samples[:100], open(args.output_file.replace(".json", ".pos.sample.json"), "w"), indent=2)
|
|
json.dump(full_negative_samples[:100], open(args.output_file.replace(".json", ".neg.sample.json"), "w"), indent=2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
"""
|
|
########################################## ITERATION 1 ###########################################################
|
|
|
|
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
|
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.json
|
|
|
|
|
|
|
|
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
|
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc.json
|
|
# Total number of items: 491733
|
|
# Acc: 0.6936853943095135
|
|
# Pass at k: 0.8353761085792493
|
|
# No positive solutions: 80951 / 491733
|
|
# No negative solutions: 255617 / 491733
|
|
# Num pairs: 2550984
|
|
Total number of items: 491733
|
|
Acc: 0.85128718227168
|
|
Pass at k: 1.0
|
|
No positive solutions: 0 / 491733
|
|
No negative solutions: 275526 / 491733
|
|
Num pairs: 3958764
|
|
|
|
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "${OUTPUT_PATH_PREFIX}/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.*-of-8.s0.json" \
|
|
--output_file $OUTPUT_PATH_PREFIX/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.s0.prefer_pair.by_sc.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.0-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.1-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.2-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.3-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.4-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.5-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.6-of-8.s0.json
|
|
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.7-of-8.s0.json
|
|
Total number of items: 82642
|
|
Acc: 0.706493066479514
|
|
Pass at k: 1.0
|
|
No positive solutions: 0 / 82642
|
|
No negative solutions: 18718 / 82642
|
|
Num pairs: 786166
|
|
|
|
|
|
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "${OUTPUT_PATH_PREFIX}/experiments//mathstral.mathscale4o.sc-prm.raft.iter1.H100.dp8.v1.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.s[01].json" \
|
|
--output_file $OUTPUT_PATH_PREFIX/experiments//mathstral.mathscale4o.sc-prm.raft.iter1.H100.dp8.v1.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n20.tem1.0.p0.9.v0.1.prefer_pair.by_sc.json
|
|
|
|
Total number of items: 491733
|
|
Acc: 0.8624497440684273
|
|
Pass at k: 1.0
|
|
No positive solutions: 0 / 491733
|
|
No negative solutions: 264522 / 491733
|
|
Num pairs: 14647115
|
|
|
|
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
|
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc_p0.6.json
|
|
--top_p 0.6
|
|
|
|
Filtered 123792 samples.
|
|
Total number of items: 491733
|
|
Acc: 0.7133363024242831
|
|
Pass at k: 0.7482536254430758
|
|
No positive solutions: 0 / 491733
|
|
No negative solutions: 272161 / 491733
|
|
Num pairs: 1350704
|
|
|
|
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n90.tem1.0.p0.9.json" \
|
|
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n90.tem1.0.p0.9.prefer_pair.by_sc_p0.6.json \
|
|
--top_p 0.6
|
|
|
|
Filtered 0 samples.
|
|
Total number of items: 491733
|
|
Acc: 0.8171100983663899
|
|
Pass at k: 1.0
|
|
No positive solutions: 0 / 491733
|
|
No negative solutions: 199144 / 491733
|
|
Num pairs: 333282827
|
|
|
|
|
|
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" \
|
|
--output_file ${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc_p0.0.json
|
|
|
|
Filtered 0 samples.
|
|
Total number of items: 491337
|
|
Acc: 0.7944180877890328
|
|
Pass at k: 1.0
|
|
No positive solutions: 0 / 491337
|
|
No negative solutions: 160871 / 491337
|
|
Num pairs: 5961085
|
|
|
|
|
|
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
|
--input_file "${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" \
|
|
--output_file ${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc_p0.0.json --top_p 0.5
|
|
|
|
Filtered 164514 samples.
|
|
Total number of items: 491337
|
|
Acc: 0.6043285972764111
|
|
Pass at k: 0.6651707483865453
|
|
No positive solutions: 0 / 491337
|
|
No negative solutions: 159681 / 491337
|
|
Num pairs: 2672191
|
|
|
|
"""
|