wehub-resource-sync
273 lines
9.4 KiB
Python
273 lines
9.4 KiB
Python
import json
|
|
import argparse
|
|
import os.path
|
|
from glob import glob
|
|
from tqdm import tqdm
|
|
from multiprocessing import Pool
|
|
import torch
|
|
import sys
|
|
from functools import partial
|
|
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
|
|
|
from post_processors.openai_api_callback import majority_voting_predict
|
|
from data.deepseek_math_utils import eval_script
|
|
|
|
|
|
def load_rewards(reward_file, re_index):
|
|
if os.path.exists(reward_file):
|
|
rewards = json.load(open(reward_file))
|
|
else:
|
|
rewards = []
|
|
for i, file in enumerate(sorted(glob(reward_file))):
|
|
print(file)
|
|
sub_rewards = json.load(open(file))
|
|
if re_index:
|
|
for item in sub_rewards:
|
|
item["index"] = f"{item['index']}_{i}"
|
|
rewards.extend(sub_rewards)
|
|
|
|
return rewards
|
|
|
|
|
|
def reward_reduction(ending_logits, reduction: str = "min", norm: bool = True):
|
|
if norm:
|
|
ending_logits = torch.tensor(ending_logits)
|
|
ending_probs = torch.softmax(ending_logits, dim=-1).tolist()
|
|
step_rewards = [step[1] for step in ending_probs]
|
|
else:
|
|
step_rewards = [step[1] for step in ending_logits]
|
|
if reduction == "min":
|
|
reward = min(step_rewards)
|
|
elif reduction == "product":
|
|
reward = 1
|
|
for prob in step_rewards:
|
|
reward *= prob
|
|
elif reduction == "sum":
|
|
reward = sum(step_rewards)
|
|
else:
|
|
raise ValueError(f"Invalid reduction method: {reduction}")
|
|
return reward
|
|
|
|
|
|
def weighted_majority_voting_predict(preds, weights):
|
|
pred2weight = {}
|
|
for pred, weight in zip(preds, weights):
|
|
if pred not in pred2weight:
|
|
pred2weight[pred] = 0
|
|
pred2weight[pred] += weight
|
|
return max(pred2weight, key=pred2weight.get)
|
|
|
|
|
|
def _init(id2reward):
|
|
global _id2reward
|
|
_id2reward = id2reward
|
|
|
|
|
|
def _worker(item, reduction, norm, sc_top_k=None):
|
|
if not item["response"] or not item["pred"] or not item["res"]:
|
|
return {
|
|
"missing": 1,
|
|
"reward_missing": 0,
|
|
"pred_missing": 0,
|
|
"seq_too_long": 0,
|
|
"sorted_results": [],
|
|
"sc_res": False
|
|
}
|
|
unsorted_results = []
|
|
reward_missing = 0
|
|
pred_missing = 0
|
|
seq_too_long = 0
|
|
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
|
resp_id = f"{item['id']}_{i}"
|
|
if resp_id not in _id2reward:
|
|
reward_missing += 1
|
|
continue
|
|
if not pred:
|
|
pred_missing += 1
|
|
continue
|
|
|
|
process_rewards = _id2reward[resp_id]
|
|
if len(process_rewards["ending_logits"]) == 0:
|
|
seq_too_long += 1
|
|
continue
|
|
|
|
assert resp == process_rewards["response"], f"{resp} \n\n {process_rewards['response']} \n\n ========="
|
|
|
|
reward = reward_reduction(process_rewards["ending_logits"], reduction, norm)
|
|
unsorted_results.append((resp, pred, r, reward))
|
|
sorted_results = sorted(unsorted_results, key=lambda x: x[-1], reverse=True)
|
|
|
|
sc_top_k_res = {}
|
|
if sc_top_k and sorted_results:
|
|
for k in sc_top_k:
|
|
preds = [r[1] for r in sorted_results[:k]]
|
|
sc_pred = majority_voting_predict(preds)
|
|
if sc_pred != "":
|
|
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
|
else:
|
|
sc_res = False
|
|
sc_top_k_res[k] = sc_res
|
|
|
|
sc_k_res = {}
|
|
if sc_top_k and unsorted_results:
|
|
for k in sc_top_k:
|
|
preds = [r[1] for r in unsorted_results[:k]]
|
|
sc_pred = majority_voting_predict(preds)
|
|
if sc_pred != "":
|
|
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
|
else:
|
|
sc_res = False
|
|
sc_k_res[k] = sc_res
|
|
|
|
weighted_best_of_k = {}
|
|
if sc_top_k and unsorted_results:
|
|
for k in sc_top_k:
|
|
preds = [r[1] for r in unsorted_results[:k]]
|
|
weights = torch.softmax(torch.tensor([r[-1] for r in unsorted_results]), dim=0).tolist()
|
|
best_of_k_pred = weighted_majority_voting_predict(preds, weights)
|
|
if best_of_k_pred != "":
|
|
best_of_k_res = eval_script.eval_math({"prediction": best_of_k_pred, "answer": item["label"]})
|
|
else:
|
|
best_of_k_res = False
|
|
weighted_best_of_k[k] = best_of_k_res
|
|
|
|
return {
|
|
"missing": 0,
|
|
"reward_missing": reward_missing,
|
|
"pred_missing": pred_missing,
|
|
"seq_too_long": seq_too_long,
|
|
"sorted_results": sorted_results,
|
|
"sc_res": item["sc_res"],
|
|
"sc_top_k_res": sc_top_k_res,
|
|
"weighted_best_of_k": weighted_best_of_k,
|
|
"sc_k_res": sc_k_res,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--response_file", type=str, required=True)
|
|
parser.add_argument("--reward_file", type=str, required=True)
|
|
parser.add_argument("--reduction", type=str, default="min")
|
|
parser.add_argument("--raw_logits", action="store_true", default=False)
|
|
parser.add_argument("--num_workers", type=int, default=4)
|
|
parser.add_argument("--re_index", action="store_true", default=False)
|
|
args = parser.parse_args()
|
|
|
|
if os.path.exists(args.response_file):
|
|
responses = json.load(open(args.response_file))
|
|
else:
|
|
responses = []
|
|
for file in glob(args.response_file):
|
|
print(file)
|
|
responses.extend(json.load(open(file)))
|
|
|
|
print(len(responses[0]["response"]), len(responses[0]["pred"]), len(responses[0]["res"]))
|
|
print(responses[0]["id"])
|
|
|
|
rewards = load_rewards(args.reward_file, args.re_index)
|
|
print(rewards[0]['index'])
|
|
id2reward = {item["index"]: item for item in rewards}
|
|
|
|
# _k = [1, 3, 5]
|
|
_k = [3, 5, 4, 8, 16, 32, 64, 128]
|
|
prm_pass_at_k = {k: 0 for k in _k}
|
|
missing = 0
|
|
missing_reward = 0
|
|
pred_missing = 0
|
|
seq_too_long = 0
|
|
sc_cnt = 0
|
|
ultimate_results = []
|
|
# for item in tqdm(responses):
|
|
# sorted_results = []
|
|
# if not item["response"] or not item["pred"] or not item["res"]:
|
|
# missing += 1
|
|
# continue
|
|
# for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
|
# resp_id = f"{item['id']}_{i}"
|
|
# if resp_id not in id2reward:
|
|
# missing_reward += 1
|
|
# continue
|
|
# process_rewards = id2reward[resp_id]
|
|
# if len(process_rewards["ending_logits"]) == 0:
|
|
# continue
|
|
#
|
|
# assert resp == process_rewards["response"], f"{resp} \n\n {process_rewards['response']} \n\n ========="
|
|
#
|
|
# reward = reward_reduction(process_rewards["ending_logits"], args.reduction, not args.raw_logits)
|
|
# sorted_results.append((resp, pred, r, reward))
|
|
#
|
|
# if not sorted_results:
|
|
# continue
|
|
# sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
|
# ultimate_results.append(sorted_results)
|
|
# for k in _k:
|
|
# if any([r[2] for r in sorted_results[:k]]):
|
|
# prm_pass_at_k[k] += 1
|
|
#
|
|
# if item["sc_res"]:
|
|
# sc_cnt += 1
|
|
with Pool(args.num_workers, initializer=_init, initargs=(id2reward,)) as pool:
|
|
annotate = partial(_worker, reduction=args.reduction, norm=(not args.raw_logits), sc_top_k=_k)
|
|
results = list(tqdm(pool.imap_unordered(annotate, responses), total=len(responses)))
|
|
|
|
sc_top_k = {k: 0 for k in _k}
|
|
weighted_best_of_k = {k: 0 for k in _k}
|
|
sc_k = {k: 0 for k in _k}
|
|
for item in results:
|
|
missing += item["missing"]
|
|
missing_reward += item["reward_missing"]
|
|
pred_missing += item["pred_missing"]
|
|
seq_too_long += item["seq_too_long"]
|
|
sorted_results = item["sorted_results"]
|
|
|
|
if item["sc_res"]:
|
|
sc_cnt += 1
|
|
|
|
if not sorted_results:
|
|
continue
|
|
|
|
ultimate_results.append(sorted_results)
|
|
for k in _k:
|
|
if any([r[2] for r in sorted_results[:k]]):
|
|
prm_pass_at_k[k] += 1
|
|
|
|
for k, v in item["sc_top_k_res"].items():
|
|
if v:
|
|
sc_top_k[k] += 1
|
|
|
|
for k, v in item["weighted_best_of_k"].items():
|
|
if v:
|
|
weighted_best_of_k[k] += 1
|
|
|
|
for k, v in item["sc_k_res"].items():
|
|
if v:
|
|
sc_k[k] += 1
|
|
|
|
print(f"Total: {len(responses)}")
|
|
print(f"Missing: {missing}")
|
|
print(f"Missing reward: {missing_reward}")
|
|
print(f"Missing pred: {pred_missing}")
|
|
print(f"Seq too long: {seq_too_long}")
|
|
print(f"SC: {sc_cnt}")
|
|
for k, v in prm_pass_at_k.items():
|
|
print(f"PRM pass at {k}: {v}")
|
|
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
|
for k, v in sc_top_k.items():
|
|
print(f"SC pass at {k}: {v}")
|
|
print(f"SC pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
|
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
|
for k, v in weighted_best_of_k.items():
|
|
print(f"Weighted best of k pass at {k}: {v}")
|
|
print(f"Weighted best of k pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
|
for k, v in sc_k.items():
|
|
print(f"SC k pass at {k}: {v}")
|
|
print(f"SC k pass at {k} rate {v / len(responses) * 100:.2f}%")
|
|
|
|
json.dump(ultimate_results[:100], open("debug.json", "w"), indent=2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|