chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,140 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
colors = ['#FFEBAD', '#DCD7C1', '#BFB1D0', '#A7C0DE', '#6C91C2', '#F46F43']
|
||||
|
||||
|
||||
def draw_double_histogram(correct_data, incorrect_data, file_path):
|
||||
# Create histogram
|
||||
plt.figure(figsize=(5, 4))
|
||||
plt.hist(correct_data, bins=10, alpha=0.7, label='Correct', color='blue')
|
||||
plt.hist(incorrect_data, bins=10, alpha=0.7, label='Incorrect', color='red')
|
||||
|
||||
# Add labels and title
|
||||
plt.xlabel('Top-1 Frequency Averaged by Number of Test Cases')
|
||||
plt.ylabel('No. of Data Points')
|
||||
# plt.title('Frequency Distribution of Correct and Incorrect Data')
|
||||
plt.legend(loc='upper right')
|
||||
|
||||
# Show the plot
|
||||
# plt.show()
|
||||
plt.savefig(file_path)
|
||||
|
||||
|
||||
def draw_histogram(data, labels, file_path):
|
||||
# Create bar chart for the single group of data
|
||||
# Create a bar chart for the three metrics
|
||||
plt.figure(figsize=(4, 4))
|
||||
# bars = plt.bar(labels, data, color='#5ea69c', width=0.4)
|
||||
# bars = plt.bar(labels, data, color=colors[5], width=0.4)
|
||||
bars = plt.bar(labels, data, color="#1E90FF", width=0.4)
|
||||
# Add text labels above each bar to indicate the values
|
||||
for bar in bars:
|
||||
yval = bar.get_height()
|
||||
plt.text(bar.get_x() + bar.get_width() / 2, yval + 0.005, round(yval, 2), ha='center', va='bottom', fontsize=16)
|
||||
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
|
||||
# Add labels and title
|
||||
# plt.xlabel('Category')
|
||||
plt.ylabel('Pass@1', fontsize=16)
|
||||
# plt.title('Accuracy Comparison Across Different Metrics')
|
||||
|
||||
plt.ylim(bottom=15)
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def draw_line_plot(correct_data, incorrect_data, file_path):
|
||||
# Compute histogram data
|
||||
correct_hist, correct_bins = np.histogram(correct_data, bins=10)
|
||||
incorrect_hist, incorrect_bins = np.histogram(incorrect_data, bins=10)
|
||||
|
||||
# Get the center of each bin for plotting
|
||||
correct_bin_centers = 0.5 * (correct_bins[1:] + correct_bins[:-1])
|
||||
incorrect_bin_centers = 0.5 * (incorrect_bins[1:] + incorrect_bins[:-1])
|
||||
|
||||
# Create the line plot
|
||||
plt.figure(figsize=(4, 4))
|
||||
# plt.plot(correct_bin_centers, correct_hist, label='Correct', color='#658873', marker='.', linestyle='-')
|
||||
# plt.plot(incorrect_bin_centers, incorrect_hist, label='Incorrect', color='#d2bfa5', marker='.', linestyle='-')
|
||||
plt.plot(correct_bin_centers, correct_hist, label='Correct', color=colors[2], marker='.', linestyle='-')
|
||||
plt.plot(incorrect_bin_centers, incorrect_hist, label='Incorrect', color=colors[4], marker='.', linestyle='-')
|
||||
|
||||
# Change the size of the font in the legent
|
||||
plt.legend(prop={"size": 15})
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
|
||||
# Add labels and title
|
||||
# plt.xlabel('Top-1 Averaged Frequency', fontsize=16)
|
||||
plt.ylabel('No. of Data Points', fontsize=16)
|
||||
plt.legend(loc='upper left')
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--freq_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.freq_file))
|
||||
|
||||
pos_freqs = []
|
||||
neg_freqs = []
|
||||
difficulties = set([item["difficulty"] for item in data])
|
||||
diff_freqs = {
|
||||
diff: {"pos": [], "neg": []} for diff in difficulties
|
||||
}
|
||||
for item in data:
|
||||
if item["prog_sc_res"]:
|
||||
pos_freqs.append(item["tot_freq"])
|
||||
diff_freqs[item["difficulty"]]["pos"].append(item["tot_freq"])
|
||||
else:
|
||||
neg_freqs.append(item["tot_freq"])
|
||||
diff_freqs[item["difficulty"]]["neg"].append(item["tot_freq"])
|
||||
|
||||
# paint(pos_freqs, neg_freqs, args.output_file.replace(".png", "_all.png"))
|
||||
draw_line_plot(pos_freqs, neg_freqs, args.output_file.replace(".png", "_all.png"))
|
||||
for diff, freqs in diff_freqs.items():
|
||||
# paint(freqs["pos"], freqs["neg"], args.output_file.replace(".png", f"_{diff}.png"))
|
||||
draw_line_plot(freqs["pos"], freqs["neg"], args.output_file.replace(".png", f"_{diff}.png"))
|
||||
|
||||
sc = 0
|
||||
prog_sc = 0
|
||||
first_res = 0
|
||||
diff_res = {
|
||||
diff: {"sc": 0, "prog_sc": 0, "first_res": 0} for diff in difficulties
|
||||
}
|
||||
for item in data:
|
||||
if item["sc_res"]:
|
||||
sc += 1
|
||||
diff_res[item["difficulty"]]["sc"] += 1
|
||||
if item["prog_sc_res"]:
|
||||
prog_sc += 1
|
||||
diff_res[item["difficulty"]]["prog_sc"] += 1
|
||||
if item["res"]:
|
||||
first_res += 1
|
||||
diff_res[item["difficulty"]]["first_res"] += 1
|
||||
|
||||
print(f"Self-consistency: {sc}/{len(data)} = {sc / len(data)}")
|
||||
print(f"Program self-consistency: {prog_sc}/{len(data)} = {prog_sc / len(data)}")
|
||||
print(f"First res: {first_res}/{len(data)} = {first_res / len(data)}")
|
||||
for diff, res in diff_res.items():
|
||||
print(f"Difficulty {diff}:")
|
||||
print(f"Self-consistency: {res['sc']}/{len(data)} = {res['sc'] / len(data)}")
|
||||
print(f"Program self-consistency: {res['prog_sc']}/{len(data)} = {res['prog_sc'] / len(data)}")
|
||||
print(f"First res: {res['first_res']}/{len(data)} = {res['first_res'] / len(data)}")
|
||||
|
||||
draw_histogram([19.24, prog_sc / len(data) * 100, sc / len(data) * 100],
|
||||
["G.D.", "S.C. - P", "S.C. - T"], args.output_file.replace(".png", "_res.png"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,263 @@
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datasets import load_dataset
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def worker(x):
|
||||
item, orig_item = x
|
||||
input_output = item["test_cases"]
|
||||
inputs = input_output["inputs"]
|
||||
|
||||
num_test_cases = len(input_output["inputs"])
|
||||
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
output2res = [
|
||||
{} for _ in range(len(inputs))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
# if not str(case_o):
|
||||
# continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
# assert case_j < len(orig_item["full_res"][resp_id]), (resp_id, case_j, orig_item["full_res"][resp_id])
|
||||
if len(orig_item["full_res"][resp_id]) <= case_j:
|
||||
assert all(x == -1 for x in orig_item["full_res"][resp_id])
|
||||
output_res = -1
|
||||
else:
|
||||
output_res = orig_item["full_res"][resp_id][case_j]
|
||||
output2res[case_j][str(case_o)] = output_res
|
||||
|
||||
# Get self-consistency
|
||||
sc_res = True
|
||||
sc_outputs = []
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if len(output_cnt) == 0:
|
||||
sc_res = False
|
||||
sc_outputs.append(None)
|
||||
continue
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_res = output2res[case_j][sc_o]
|
||||
if not sc_o_res:
|
||||
sc_res = False
|
||||
sc_outputs.append(sc_o)
|
||||
|
||||
# Estimate the corresponding frequency rates
|
||||
tot_freq = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if len(output_cnt) == 0:
|
||||
continue
|
||||
|
||||
max_freq = output_cnt.most_common(1)[0][1]
|
||||
tot_freq += max_freq
|
||||
|
||||
tot_freq /= num_test_cases
|
||||
|
||||
# Confirm if there is some program solution meets all self-consistency outputs
|
||||
sc_match_res = []
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
resp_match_res = True
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
# resp_match_res = False
|
||||
break
|
||||
if case_r != 0:
|
||||
resp_match_res = False
|
||||
break
|
||||
# if not str(case_o):
|
||||
# resp_match_res = False
|
||||
# break
|
||||
|
||||
if sc_outputs[case_j] is None:
|
||||
resp_match_res = False
|
||||
break
|
||||
|
||||
if str(case_o) != sc_outputs[case_j]:
|
||||
resp_match_res = False
|
||||
if orig_item["full_res"][resp_id][case_j] and output2res[case_j][sc_outputs[case_j]]:
|
||||
print(f"warning", str(case_o), sc_outputs[case_j], orig_item["test_cases"]["outputs"][case_j])
|
||||
break
|
||||
|
||||
sc_match_res.append(resp_match_res)
|
||||
|
||||
if sc_res and any(
|
||||
sc_match_res): # If the self-consistency determined group is correct and there is one program match all predictions with self-consistency, then it is a correct solution
|
||||
prog_sc_res = True
|
||||
else:
|
||||
prog_sc_res = False
|
||||
|
||||
return {
|
||||
"sc_res": sc_res,
|
||||
"sc_outputs": sc_outputs,
|
||||
"tot_freq": tot_freq,
|
||||
"prog_sc_res": prog_sc_res,
|
||||
"sc_match_res": sc_match_res,
|
||||
"res": orig_item["res"][0],
|
||||
"id": item["id"]
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
if not isinstance(tmp["full_res"], list):
|
||||
tmp["full_res"] = [tmp["full_res"]]
|
||||
if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--exec_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
parser.add_argument("--split", default="test")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(f'apps_difficulty_{args.split}.json'):
|
||||
_dataset = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
problem_id2difficulty = {item["problem_id"]: item["difficulty"] for item in _dataset}
|
||||
all_difficulties = collections.Counter(problem_id2difficulty.values())
|
||||
json.dump(problem_id2difficulty, open(f'apps_difficulty_{args.split}.json', "w"), ensure_ascii=False)
|
||||
json.dump(all_difficulties, open(f'apps_difficulty_{args.split}_all.json', "w"), ensure_ascii=False)
|
||||
else:
|
||||
problem_id2difficulty = json.load(open(f'apps_difficulty_{args.split}.json'))
|
||||
all_difficulties = json.load(open(f'apps_difficulty_{args.split}_all.json'))
|
||||
problem_id2difficulty = {int(k): v for k, v in problem_id2difficulty.items()}
|
||||
|
||||
completions = load_files(args.completion_file)
|
||||
completions = merge_seed_sampled_data(completions)
|
||||
execs = load_files(args.exec_file)
|
||||
execs = merge_seed_sampled_data(execs)
|
||||
print(len(completions), len(execs))
|
||||
|
||||
id2completion = {item["id"]: item for item in completions}
|
||||
id2exec = {item["id"]: item for item in execs}
|
||||
print(len(id2completion), len(id2exec))
|
||||
|
||||
commons = set(id2completion.keys()) & set(id2exec.keys())
|
||||
print(f"Found {len(commons)} common items.")
|
||||
|
||||
inputs = []
|
||||
for _id in commons:
|
||||
inputs.append((id2exec[_id], id2completion[_id]))
|
||||
|
||||
pbar = tqdm(inputs)
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = worker
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
result = future.result()
|
||||
result["difficulty"] = problem_id2difficulty[result["id"]]
|
||||
outputs.append(result)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"))
|
||||
|
||||
sc = 0
|
||||
prog_sc = 0
|
||||
first_res = 0
|
||||
num_prog = 0
|
||||
for item in outputs:
|
||||
if item["sc_res"]:
|
||||
sc += 1
|
||||
if item["prog_sc_res"]:
|
||||
prog_sc += 1
|
||||
if item["res"]:
|
||||
first_res += 1
|
||||
num_prog += len(item["sc_match_res"])
|
||||
|
||||
print(f"Self-consistency: {sc}/{len(completions)} = {sc / len(completions)}")
|
||||
print(f"Program self-consistency: {prog_sc}/{len(completions)} = {prog_sc / len(completions)}")
|
||||
print(f"First res: {first_res}/{len(completions)} = {first_res / len(completions)}")
|
||||
print(f"Programs: {num_prog}/{len(completions)} = {num_prog / len(completions)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,26 @@
|
||||
exp_dir="../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.dpo.H100.dp8.v1.0.s42/apps/checkpoint-2400"
|
||||
|
||||
#python scripts/apps/analyze/get_output_frequency.py \
|
||||
# --completion_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.json" \
|
||||
# --exec_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.run_outputs.json" \
|
||||
# --output_file ${exp_dir}/test.0shot.tem1.0.n8.v2.0.run_outputs.analyze_freq.json --num_workers 16
|
||||
|
||||
|
||||
#python scripts/apps/analyze/freq2image.py \
|
||||
# --freq_file ${exp_dir}/test.0shot.tem1.0.n8.v2.0.run_outputs.analyze_freq.json \
|
||||
# --output_file ./apps.test.0shot.tem1.0.n8.v2.0.analyze_freq.png
|
||||
|
||||
|
||||
#python scripts/apps/analyze/get_output_frequency.py \
|
||||
# --completion_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.s?.json" \
|
||||
# --exec_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.s?.run_outputs.json" \
|
||||
# --output_file ${exp_dir}/test.0shot.tem1.0.n64.v2.0.run_outputs.analyze_freq.json --num_workers 16
|
||||
|
||||
#Self-consistency: 1067/5000 = 0.2134
|
||||
#Program self-consistency: 1053/5000 = 0.2106
|
||||
#First res: 883/5000 = 0.1766
|
||||
#Programs: 320000/5000 = 64.0
|
||||
|
||||
python scripts/apps/analyze/freq2image.py \
|
||||
--freq_file ${exp_dir}/test.0shot.tem1.0.n64.v2.0.run_outputs.analyze_freq.json \
|
||||
--output_file ./apps.test.0shot.tem1.0.n64.v2.0.analyze_freq.png
|
||||
@@ -0,0 +1,15 @@
|
||||
python scripts/apps/worsen_gpt4_combine.py \
|
||||
--worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_problem_id_field id \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f100k.json
|
||||
|
||||
|
||||
python scripts/apps/rerank_code_rm.py \
|
||||
--completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_dpo.A100.dp8.v3.0.s42/apps/checkpoint-200/test.0shot.tem1.0.n20.v1.1.json \
|
||||
--reward_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.pair-rm.gpt4o-worsen.A100.w8.v1.0.s42/r2c.dpo.step-200.test.n20/test-checkpoint-50/eval_predictions_rank?.json"
|
||||
|
||||
|
||||
python scripts/apps/rerank_code_rm.py \
|
||||
--completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_dpo.A100.dp8.v3.0.s42/apps/checkpoint-200/test.0shot.tem1.0.n20.v1.1.json \
|
||||
--reward_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.pair-rm.r2c-policy-data.A100.w8.v1.0.s42/r2c.dpo.step-200.test.n20/test-checkpoint-50/eval_predictions_rank?.json"
|
||||
@@ -0,0 +1,114 @@
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--response_field", type=str, default="response")
|
||||
parser.add_argument("--test_case_field", type=str, default="input_output")
|
||||
args = parser.parse_args()
|
||||
|
||||
cnt = 0
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
print(len(data))
|
||||
|
||||
for item in tqdm(data):
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
if isinstance(item[args.response_field], str): # We cannot make pairs if there is only one response.
|
||||
item["pos"] = []
|
||||
item["pos_code"] = []
|
||||
item["neg"] = []
|
||||
item["neg_code"] = []
|
||||
continue
|
||||
|
||||
if "res" in item and "full_res" in item and item[args.test_case_field]: # If there is no test-cases, we cannot determine the correctness
|
||||
assert len(item["res"]) == len(item["full_res"]) == len(item[args.response_field]), (len(item["res"]),
|
||||
len(item["full_res"]),
|
||||
len(item[args.response_field]),
|
||||
item[args.response_field])
|
||||
for i, r in enumerate(item["res"]):
|
||||
if r is True:
|
||||
assert item[args.response_field][i]
|
||||
assert item["pred"][i], item["pred"]
|
||||
if item[args.response_field][i]:
|
||||
pos.append(item[args.response_field][i])
|
||||
pos_code.append(item["pred"][i])
|
||||
else:
|
||||
if item[args.response_field][i]:
|
||||
neg.append(item[args.response_field][i])
|
||||
if item["pred"][i] and item["pred"][i].strip():
|
||||
neg_code.append(item["pred"][i])
|
||||
item["pos"] = pos
|
||||
item["neg"] = neg
|
||||
item["pos_code"] = pos_code
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos) * len(neg)
|
||||
|
||||
if args.response_field != "response":
|
||||
item["response"] = item.pop(args.response_field)
|
||||
if args.test_case_field != "input_output":
|
||||
item["input_output"] = item.pop(args.test_case_field)
|
||||
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
print(len(data), cnt, cnt / len(data))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.dpo_v1.0.json --test_case_field test_cases
|
||||
|
||||
4223
|
||||
100%|████████████████████████████████████████████████████████████████████████████████████| 4223/4223 [00:00<00:00, 148189.90it/s]
|
||||
4223 18383 4.353066540374142
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec_dpo.json --test_case_field test_cases
|
||||
|
||||
4715
|
||||
100%|██████████████████████████████████████████████████████████| 4715/4715 [00:00<00:00, 107036.93it/s]
|
||||
4715 15921 3.3766702014846235
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/apps/construct_prefer_pair.py --input_file "train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file "train.0shot.tem1.0.n10.v2.0.dpo_v1.0.json" --test_case_field test_cases
|
||||
|
||||
4500
|
||||
100%|██████████████████████| 4500/4500 [00:00<00:00, 142921.59it/s]
|
||||
4500 42550 9.455555555555556
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.dpo_v1.0.json \
|
||||
--test_case_field test_cases
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
import argparse
|
||||
import sys
|
||||
import collections
|
||||
import glob
|
||||
import os
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
parser.add_argument("--lower_bound", type=float, default=0.0)
|
||||
parser.add_argument("--margin", type=float, default=1.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
solutions = json.load(open(args.completion_file))
|
||||
if "difficulty" in solutions[0]:
|
||||
all_diff = collections.Counter([item["difficulty"] for item in solutions])
|
||||
all_diff_success = {k: 0 for k in all_diff}
|
||||
else:
|
||||
all_diff = None
|
||||
all_diff_success = None
|
||||
|
||||
if os.path.exists(args.reward_file):
|
||||
rewards = json.load(open(args.reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob.glob(args.reward_file):
|
||||
rewards += json.load(open(file))
|
||||
|
||||
id2reward = collections.defaultdict(dict)
|
||||
for item in rewards:
|
||||
index = item["index"]
|
||||
orig_id, pred_id = list(map(int, index.split("_")))
|
||||
if isinstance(item["reward"], list):
|
||||
id2reward[orig_id][pred_id] = item["reward"][0]
|
||||
else:
|
||||
id2reward[orig_id][pred_id] = item["reward"]
|
||||
|
||||
outputs = []
|
||||
success = 0
|
||||
num_pairs = 0
|
||||
for item in solutions:
|
||||
problem_id = item["id"]
|
||||
|
||||
if not item["response"]:
|
||||
continue
|
||||
if not item["pred"]:
|
||||
continue
|
||||
|
||||
assert isinstance(item["response"], list)
|
||||
assert isinstance(item["pred"], list)
|
||||
assert len(item["response"]) == len(item["pred"])
|
||||
|
||||
tuples = []
|
||||
for j, (resp, pred) in enumerate(zip(item["response"], item["pred"])):
|
||||
if pred:
|
||||
tuples.append((j, resp, pred, id2reward[problem_id][j]))
|
||||
|
||||
if not tuples:
|
||||
continue
|
||||
|
||||
tuples = sorted(tuples, key=lambda x: x[3], reverse=True)
|
||||
# Construct preference pairs: collect pairs with margin greater than args.margin
|
||||
pos = []
|
||||
neg = []
|
||||
for i in range(len(tuples)):
|
||||
if tuples[i][3] < args.lower_bound:
|
||||
break
|
||||
for j in range(i + 1, len(tuples)):
|
||||
if tuples[i][3] - tuples[j][3] < args.margin:
|
||||
continue
|
||||
pos.append(tuples[i][1])
|
||||
neg.append(tuples[j][1])
|
||||
outputs.append({
|
||||
"id": problem_id,
|
||||
"pos": pos,
|
||||
"neg": neg
|
||||
})
|
||||
num_pairs += len(pos)
|
||||
|
||||
_pred_id = tuples[0][0]
|
||||
if item["res"] and item["res"][_pred_id]:
|
||||
success += 1
|
||||
if all_diff is not None:
|
||||
all_diff_success[item["difficulty"]] += 1
|
||||
|
||||
print(f"Success rate: {success / len(solutions)}")
|
||||
if all_diff is not None:
|
||||
for k, v in all_diff.items():
|
||||
print(f"Difficulty {k}: {all_diff_success[k] / v}")
|
||||
|
||||
json.dump(outputs, open(args.completion_file.replace(".json", f".lower-{args.lower_bound}.mar-{args.margin}.json"), "w"), ensure_ascii=False, indent=2)
|
||||
print(f"Number of pairs: {num_pairs}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/construct_prefer_pair_rm.py --completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.json --reward_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.pair-rm.gpt4o-worsen.A100.w8.v1.2.s42/r2c.sft.step-400.train.n10/test-checkpoint-50/eval_predictions_rank0.json --lower_bound 0.0 --margin 1.0
|
||||
|
||||
Success rate: 0.3566
|
||||
Number of pairs: 29584
|
||||
"""
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import collections
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Soft version of constructing preference pair. This would be useful for teacher-generated pseudo test cases.
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--response_field", type=str, default="response")
|
||||
parser.add_argument("--test_case_field", type=str, default="input_output")
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
args = parser.parse_args()
|
||||
|
||||
cnt = 0
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
print(len(data))
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
if isinstance(item[args.response_field], str): # We cannot make pairs if there is only one response.
|
||||
item["pos"] = []
|
||||
item["pos_code"] = []
|
||||
item["neg"] = []
|
||||
item["neg_code"] = []
|
||||
continue
|
||||
|
||||
if len(item[args.test_case_field]["inputs"]) == 0:
|
||||
item["pos"] = []
|
||||
item["pos_code"] = []
|
||||
item["neg"] = []
|
||||
item["neg_code"] = []
|
||||
continue
|
||||
|
||||
if "res" in item and "full_res" in item and item[args.test_case_field]: # If there is no test-cases, we cannot determine the correctness
|
||||
assert len(item["res"]) == len(item["full_res"]) == len(item[args.response_field]), (len(item["res"]),
|
||||
len(item["full_res"]),
|
||||
len(item[args.response_field]),
|
||||
item[args.response_field])
|
||||
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
num_test_cases = len(item[args.test_case_field]["inputs"])
|
||||
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["neg"] = neg
|
||||
item["pos_code"] = pos_code
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
if args.response_field != "response":
|
||||
item["response"] = item.pop(args.response_field)
|
||||
if args.test_case_field != "input_output":
|
||||
item["input_output"] = item.pop(args.test_case_field)
|
||||
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
print(len(data), cnt, cnt / len(data))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.dpo_v1.0.json --test_case_field test_cases
|
||||
|
||||
4223
|
||||
100%|████████████████████████████████████████████████████████████████████████████████████| 4223/4223 [00:00<00:00, 148189.90it/s]
|
||||
4223 18383 4.353066540374142
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec_dpo.json --test_case_field test_cases
|
||||
|
||||
4715
|
||||
100%|██████████████████████████████████████████████████████████| 4715/4715 [00:00<00:00, 107036.93it/s]
|
||||
4715 15921 3.3766702014846235
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/apps/construct_prefer_pair.py --input_file "train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file "train.0shot.tem1.0.n10.v2.0.dpo_v1.0.json" --test_case_field test_cases
|
||||
|
||||
4500
|
||||
100%|██████████████████████| 4500/4500 [00:00<00:00, 142921.59it/s]
|
||||
4500 42550 9.455555555555556
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.dpo_v1.0.json \
|
||||
--test_case_field test_cases
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,35 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.code.code import APPsEvaluator
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
outputs = []
|
||||
with open(args.input_file, "r", encoding="utf-8")as f:
|
||||
for line in f:
|
||||
item = json.loads(line)
|
||||
pred = standard_cleaner_default(item["completion"])
|
||||
item["pred"] = pred
|
||||
item["test_cases"] = item.pop("input_output")
|
||||
outputs.append(item)
|
||||
|
||||
evaluator = APPsEvaluator()
|
||||
predictions, metrics = evaluator(outputs, num_workers=24)
|
||||
|
||||
print(metrics)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,153 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def extract_solution_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'<BEGIN>(.*?)<END>'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def _worker(item):
|
||||
solution = item["solutions"][0]
|
||||
|
||||
all_results = check_correctness(item["input_output"], solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
# res = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
result = True
|
||||
else:
|
||||
result = False
|
||||
|
||||
return result, res
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--test_case_field", type=str)
|
||||
parser.add_argument("--test_case_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train", trust_remote_code=True).to_list()
|
||||
print(f"Total number of items before: {len(data)}")
|
||||
train_sub_val_ids = set(json.load(open("apps_train_sub_val_ids.json")))
|
||||
|
||||
data = [item for item in data if item["problem_id"] not in train_sub_val_ids]
|
||||
print(len(data))
|
||||
|
||||
test_cases = json.load(open(args.test_case_file))
|
||||
# id2test_cases = {item[args.test_case_id_field]: item[args.test_case_field] for item in test_cases}
|
||||
missing = 0
|
||||
repeat = 0
|
||||
id2test_cases = {}
|
||||
for item in test_cases:
|
||||
if item[args.test_case_id_field] in id2test_cases:
|
||||
repeat += 1
|
||||
if args.test_case_field not in item:
|
||||
missing += 1
|
||||
continue
|
||||
id2test_cases[item[args.test_case_id_field]] = item[args.test_case_field]
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Repeat: {repeat}")
|
||||
|
||||
missing_solutions = 0
|
||||
missing_test_cases = 0
|
||||
new_data = []
|
||||
for item in data:
|
||||
if item["solutions"]:
|
||||
item["solutions"] = json.loads(item["solutions"])
|
||||
assert isinstance(item["solutions"], list)
|
||||
if len(item["solutions"]) > 0:
|
||||
if item["problem_id"] in id2test_cases:
|
||||
item["input_output"] = id2test_cases[item["problem_id"]]
|
||||
new_data.append(item)
|
||||
elif f"apps-train-{item['problem_id']}" in id2test_cases:
|
||||
item["input_output"] = id2test_cases[f"apps-train-{item['problem_id']}"]
|
||||
new_data.append(item)
|
||||
else:
|
||||
missing_test_cases += 1
|
||||
else:
|
||||
missing_solutions += 1
|
||||
else:
|
||||
missing_solutions += 1
|
||||
|
||||
print(f"Missing solutions: {missing_solutions}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
micro_corr = 0
|
||||
micro_all = 0
|
||||
pbar = tqdm(data)
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
result, full_res = item
|
||||
if result:
|
||||
corr += 1
|
||||
for r in full_res:
|
||||
if r:
|
||||
micro_corr += 1
|
||||
micro_all += 1
|
||||
|
||||
print(f"Missing: {missing} / {len(outputs)} = {missing / len(outputs)}")
|
||||
print(f"Correct: {corr} / {len(outputs)} = {corr / len(outputs)}")
|
||||
print(f"Micro Correct: {micro_corr} / {micro_all} = {micro_corr / micro_all}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
export OUTPUT_PATH_PREFIX="/mnt/fangkai_blob/reward_modeling/"
|
||||
|
||||
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
# --test_case_field "pseudo_test_cases" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#Total number of items before: 5000
|
||||
#4500
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 264
|
||||
#Total number of items: 4236
|
||||
#Missing: 0 / 4236 = 0.0
|
||||
#Correct: 1831 / 4236 = 0.43224740321057603
|
||||
#Micro Correct: 26256 / 41889 = 0.6267993984100838
|
||||
|
||||
# deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
# --test_case_field "input_output" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#Total number of items before: 5000
|
||||
#4500
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 377
|
||||
#Total number of items: 4123
|
||||
#Missing: 0 / 4123 = 0.0
|
||||
#Correct: 2066 / 4123 = 0.5010914382731021
|
||||
#Micro Correct: 26954 / 40349 = 0.6680215123051376
|
||||
|
||||
|
||||
#deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.dpo.H100.dp8.v1.0.s42
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
# --test_case_field "input_output" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#4500
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 331
|
||||
#Total number of items: 4169
|
||||
#Missing: 0 / 4169 = 0.0
|
||||
#Correct: 2077 / 4169 = 0.49820100743583595
|
||||
#Micro Correct: 27217 / 40775 = 0.6674923359901901
|
||||
|
||||
|
||||
#deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
# --test_case_field "input_output" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#4500
|
||||
#Missing: 1
|
||||
#Repeat: 0
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 600
|
||||
#Total number of items: 3900
|
||||
#Missing: 0 / 3900 = 0.0
|
||||
#Correct: 511 / 3900 = 0.13102564102564102
|
||||
#Micro Correct: 50202 / 60914 = 0.8241455166300029
|
||||
|
||||
# The sc-version of 4o-based test cases
|
||||
python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
--test_case_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.json \
|
||||
--test_case_field "input_output" --test_case_id_field "problem_id" \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.gd_sol_exec.json \
|
||||
--num_workers 128
|
||||
#Total number of items before: 5000
|
||||
#4500
|
||||
#Missing: 0
|
||||
#Repeat: 0
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 248
|
||||
#Total number of items: 4252
|
||||
#Collecting results: 100%|██████████| 4252/4252 [00:54<00:00, 78.74it/s]
|
||||
#Missing: 0 / 4252 = 0.0
|
||||
#Correct: 2697 / 4252 = 0.6342897460018815
|
||||
#Micro Correct: 32348 / 42080 = 0.7687262357414448
|
||||
@@ -0,0 +1,110 @@
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--use_sc", default=False, action="store_true")
|
||||
parser.add_argument("--problem_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--test_case_field", type=str, default="input_output")
|
||||
parser.add_argument("--cover", default=False, action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
pseudo_test_cases = []
|
||||
unaligned = 0
|
||||
total = 0
|
||||
for item in data:
|
||||
problem_id = item[args.problem_id_field]
|
||||
|
||||
if not item[args.test_case_field]:
|
||||
continue
|
||||
|
||||
if "res" not in item or (not item["res"]):
|
||||
continue
|
||||
|
||||
if len(item["res"]) != len(item["outputs"]):
|
||||
unaligned += 1
|
||||
continue
|
||||
|
||||
input_outputs = collections.defaultdict(list)
|
||||
for i, (r, outputs) in enumerate(zip(item["res"], item["outputs"])):
|
||||
if r in [-1, -2]:
|
||||
continue
|
||||
|
||||
for j, o in enumerate(outputs):
|
||||
if o:
|
||||
# _input = item["input_output"]["inputs"][j]
|
||||
input_outputs[j].append(o)
|
||||
|
||||
if args.use_sc:
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
"outputs": []
|
||||
}
|
||||
for _input_index, outputs in input_outputs.items():
|
||||
pairs = {str(o): o for o in outputs}
|
||||
cnt = collections.Counter(list(pairs.keys()))
|
||||
cnt = sorted(cnt.items(), key=lambda x: x[1], reverse=True)
|
||||
_output = cnt[0][0]
|
||||
_output = pairs[_output]
|
||||
test_cases["inputs"].append(item[args.test_case_field]["inputs"][_input_index])
|
||||
test_cases["outputs"].append(_output)
|
||||
else:
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
for _input, outputs in input_outputs.items():
|
||||
test_cases["inputs"].append(_input)
|
||||
test_cases["outputs"].append(outputs[0])
|
||||
|
||||
total += len(test_cases["inputs"])
|
||||
if len(test_cases["inputs"]) == 0:
|
||||
continue
|
||||
if args.cover:
|
||||
item.pop("res")
|
||||
item.pop("outputs")
|
||||
item.pop("full_res")
|
||||
item.pop("errors")
|
||||
if "fn_name" in item[args.test_case_field]:
|
||||
test_cases["fn_name"] = item[args.test_case_field]["fn_name"]
|
||||
item["input_output"] = test_cases
|
||||
pseudo_test_cases.append(item)
|
||||
else:
|
||||
pseudo_test_cases.append({
|
||||
"problem_id": problem_id,
|
||||
"input_output": test_cases
|
||||
})
|
||||
|
||||
print(f"Unaligned: {unaligned}")
|
||||
print(f"Total number of pseudo test cases: {len(pseudo_test_cases)}")
|
||||
print(f"Average number of test cases: {total} / {len(pseudo_test_cases)} = {total / len(pseudo_test_cases)}")
|
||||
json.dump(pseudo_test_cases, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/extract_pseudo_outputs_as_label.py --input_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pure_outputs.json --output_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pseudo_test_cases.json --use_sc
|
||||
|
||||
Unaligned: 582
|
||||
Total number of pseudo test cases: 4223
|
||||
Average number of test cases: 13674 / 4223 = 3.2379824769121477
|
||||
|
||||
|
||||
>>> python scripts/apps/extract_pseudo_outputs_as_label.py --input_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.json --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.pseudo_cases.sc.json --use_sc --problem_id_field id --test_case_field test_cases
|
||||
|
||||
Unaligned: 87
|
||||
Total number of pseudo test cases: 4715
|
||||
Average number of test cases: 17551 / 4715 = 3.72237539766702
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,112 @@
|
||||
import copy
|
||||
import json
|
||||
import argparse
|
||||
import collections
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
missed = 0
|
||||
averaged_rank = 0
|
||||
case_cnt = 0
|
||||
|
||||
loose_avg_rank = 0
|
||||
loose_missed = 0
|
||||
loose_case_cnt = 0
|
||||
|
||||
tmp_cnt = 0
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print("No outputs")
|
||||
continue
|
||||
|
||||
test_cases = item["input_output"]
|
||||
if not test_cases:
|
||||
print("No ground truth test cases")
|
||||
continue
|
||||
|
||||
test_case_outputs = collections.defaultdict(collections.Counter)
|
||||
case_level_res = collections.defaultdict(dict)
|
||||
for i, sol_outputs in enumerate(item["outputs"]):
|
||||
for j, o in enumerate(sol_outputs):
|
||||
if o is not None:
|
||||
test_case_outputs[j][o] += 1
|
||||
if len(item["full_res"][i]) < len(sol_outputs):
|
||||
# print("Full res is not complete")
|
||||
tmp_cnt += 1
|
||||
continue
|
||||
case_level_res[j][o] = item["full_res"][i][j]
|
||||
|
||||
for i, o in enumerate(test_cases["outputs"]):
|
||||
o = str(o)
|
||||
cnt = test_case_outputs[i]
|
||||
sorted_cnt = sorted(cnt.items(), key=lambda x: x[1], reverse=True)
|
||||
o2rank = {k: i for i, (k, v) in enumerate(sorted_cnt)}
|
||||
|
||||
if o not in o2rank:
|
||||
missed += 1
|
||||
else:
|
||||
rank = o2rank[o]
|
||||
averaged_rank += rank
|
||||
case_cnt += 1
|
||||
|
||||
pseudo_cnt = 0
|
||||
loose_cnt = copy.deepcopy(cnt)
|
||||
loose_tuples = copy.deepcopy(list(loose_cnt.items()))
|
||||
for real_o, real_o_res in loose_tuples:
|
||||
if real_o_res:
|
||||
pseudo_cnt += test_case_outputs[i][real_o]
|
||||
loose_cnt.pop(real_o)
|
||||
|
||||
if pseudo_cnt == 0:
|
||||
loose_missed += 1
|
||||
continue
|
||||
|
||||
loose_cnt["$$LOOSE_POS$$"] = pseudo_cnt
|
||||
loose_sorted_cnt = sorted(loose_cnt.items(), key=lambda x: x[1], reverse=True)
|
||||
loose_o2rank = {k: i for i, (k, v) in enumerate(loose_sorted_cnt)}
|
||||
loose_rank = loose_o2rank.get("$$LOOSE_POS$$", -1)
|
||||
assert loose_rank != -1
|
||||
loose_avg_rank += loose_rank
|
||||
loose_case_cnt += 1
|
||||
|
||||
print(f"Missed: {missed}")
|
||||
print(f"Averaged rank: {averaged_rank / case_cnt}")
|
||||
print(f"Case count: {case_cnt}")
|
||||
print(f"Total count: {len(data)}")
|
||||
|
||||
print(f"Loose missed: {loose_missed}")
|
||||
print(f"Loose averaged rank: {loose_avg_rank / loose_case_cnt}")
|
||||
print(f"Loose case count: {loose_case_cnt}")
|
||||
print(f"Loose total count: {len(data)}")
|
||||
|
||||
print(f"tmp_cnt: {tmp_cnt}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/get_output_frequency.py --input_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.exec.w_outputs.json
|
||||
|
||||
Missed: 24502
|
||||
Averaged rank: 0.20820668693009117
|
||||
Case count: 1316
|
||||
Total count: 5000
|
||||
Loose missed: 8206
|
||||
Loose averaged rank: 0.0
|
||||
Loose case count: 17612
|
||||
Loose total count: 5000
|
||||
tmp_cnt: 3077
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
outputs = []
|
||||
with open(args.input_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
item = json.loads(line)
|
||||
pred = standard_cleaner_default(item["completion"])
|
||||
item["pred"] = pred
|
||||
# item["test_cases"] = item.pop("input_output")
|
||||
item["response"] = item.pop("completion")
|
||||
item["id"] = item["problem_id"]
|
||||
outputs.append(item)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
from glob import glob
|
||||
from collections import Counter
|
||||
from datasets import load_dataset
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--split", type=str, default="test")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(f'apps_difficulty_{args.split}.json'):
|
||||
_dataset = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
problem_id2difficulty = {item["problem_id"]: item["difficulty"] for item in _dataset}
|
||||
all_difficulties = Counter(problem_id2difficulty.values())
|
||||
json.dump(problem_id2difficulty, open(f'apps_difficulty_{args.split}.json', "w"), ensure_ascii=False)
|
||||
json.dump(all_difficulties, open(f'apps_difficulty_{args.split}_all.json', "w"), ensure_ascii=False)
|
||||
else:
|
||||
problem_id2difficulty = json.load(open(f'apps_difficulty_{args.split}.json'))
|
||||
all_difficulties = json.load(open(f'apps_difficulty_{args.split}_all.json'))
|
||||
problem_id2difficulty = {int(k): v for k, v in problem_id2difficulty.items()}
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
if file.endswith(".json"):
|
||||
# data += json.load(open(file))
|
||||
f = open(file, "r")
|
||||
data += json.load(f)
|
||||
else:
|
||||
# data += [json.loads(line) for line in open(file).readlines()]
|
||||
lines = list(open(file).readlines())
|
||||
for line in lines:
|
||||
data.append(json.loads(line))
|
||||
|
||||
success = 0
|
||||
success_at_k = 0
|
||||
successes_at_difficulty = {difficulty: 0 for difficulty in all_difficulties}
|
||||
successes_at_k_at_difficulty = {difficulty: 0 for difficulty in all_difficulties}
|
||||
for item in data:
|
||||
p_id = item["id"]
|
||||
if isinstance(item["res"], bool):
|
||||
if item["res"]:
|
||||
success += 1
|
||||
success_at_k += 1
|
||||
successes_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
successes_at_k_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
else:
|
||||
if len(item["res"]):
|
||||
if any(item["res"]):
|
||||
success_at_k += 1
|
||||
successes_at_k_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
if item["res"][0]:
|
||||
success += 1
|
||||
successes_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
|
||||
metrics = {"acc": success / len(data), "pass@k": success_at_k / len(data), "correct": success, "total": len(data)}
|
||||
for difficulty in all_difficulties:
|
||||
metrics[f"acc_{difficulty}"] = successes_at_difficulty[difficulty] / all_difficulties[difficulty]
|
||||
metrics[f"pass@k_{difficulty}"] = successes_at_k_at_difficulty[difficulty] / all_difficulties[difficulty]
|
||||
metrics[f"correct_{difficulty}"] = successes_at_difficulty[difficulty]
|
||||
metrics[f"total_{difficulty}"] = all_difficulties[difficulty]
|
||||
print(json.dumps(metrics, indent=2))
|
||||
|
||||
if args.output_file:
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
json.dump(metrics, open(args.output_file.replace(".json", ".metrics"), "w"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
path=$1
|
||||
|
||||
shift 1 # Shift the first 5 arguments, so $1 now refers to the 6th argument
|
||||
|
||||
for step in "$@"; do
|
||||
echo "Merging predictions for checkpoint $step"
|
||||
python scripts/apps/merge_dp_predictions.py --input_file "$path/apps/checkpoint-$step/sub_dev.0shot.tem0.0.n1.?-of-8.v2.0.json" \
|
||||
--output_file "$path/apps/checkpoint-$step/sub_dev.0shot.tem0.0.n1.v2.0.json" --split train
|
||||
python scripts/apps/merge_dp_predictions.py --input_file "$path/apps/checkpoint-$step/test.0shot.tem0.0.n1.?-of-8.v2.0.json" \
|
||||
--output_file "$path/apps/checkpoint-$step/test.0shot.tem0.0.n1.v2.0.json" --split test
|
||||
# cat $path/apps/checkpoint-$step/sub_dev.0shot.tem0.0.n1.0-of-1.v2.0.metrics.json
|
||||
# cat $path/apps/checkpoint-$step/test.0shot.tem0.0.n1.0-of-1.v2.0.metrics.json
|
||||
echo "Done merging predictions for checkpoint $step"
|
||||
echo ""
|
||||
done
|
||||
@@ -0,0 +1,74 @@
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.code.clean import tag_cleaner
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str) # Use the dpo results file
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--difficulty", type=str, default="introductory|interview")
|
||||
parser.add_argument("--prompt_file", type=str, )
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
difficulty = args.difficulty.split("|")
|
||||
|
||||
prompt_template = open(args.prompt_file).read()
|
||||
|
||||
parent_dir = os.path.dirname(args.output_file)
|
||||
if not os.path.exists(parent_dir):
|
||||
os.makedirs(parent_dir)
|
||||
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
if item["difficulty"] not in difficulty:
|
||||
continue
|
||||
|
||||
for i, neg in enumerate(item["neg"]):
|
||||
question = item["question"]
|
||||
if item["starter_code"]:
|
||||
question = question + "\n\n" + item["starter_code"]
|
||||
|
||||
neg_code = tag_cleaner(neg)
|
||||
if not neg_code:
|
||||
continue
|
||||
|
||||
assert neg_code, (neg, item["question"])
|
||||
|
||||
prompt = prompt_template.format(question=question, code=neg_code)
|
||||
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["id"] = f"{item['problem_id']}_neg{i}"
|
||||
new_item["prompt"] = prompt
|
||||
new_item["neg_code"] = neg_code
|
||||
outputs.append(new_item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pp_critique_difficulty.py \
|
||||
--input_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.exec.dpo_v1.0.json \
|
||||
--output_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.inputs.intro.inter.jsonl \
|
||||
--prompt_file prompts/apps/critique_0shot_v1.0.txt
|
||||
|
||||
>>> python azure/gpt_crawler_mp.py \
|
||||
--prompt_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.jsonl \
|
||||
--outfile ../gpt-chat-examples/outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--model gpt-4o --max_gen_tokens 4096 --temperature 1.0 --num_processes 24 --seed 42 --n 1 --response_format json_object
|
||||
"""
|
||||
@@ -0,0 +1,40 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.apps import APPsWithFunctionName
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
# reader = APPsWithFunctionName(split="test", use_starter_code=True)
|
||||
reader = APPsWithFunctionName(split="train", train_sub_split="train", use_starter_code=True)
|
||||
|
||||
data = reader("codeparrot/apps")
|
||||
|
||||
# prompt_template = "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
prompt_template = "{question}\n\nPlease write a Python program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
prompt = prompt_template.format(question=item["question"])
|
||||
item["prompt"] = prompt
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w", encoding="utf-8") as f:
|
||||
for item in inputs:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
print(len(inputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.apps import PseudoInputsWithFunctionNameFixStarterCode
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, default="../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1.json")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
# reader = APPsWithFunctionName(split="test", use_starter_code=True)
|
||||
# reader = APPsWithFunctionName(split="train", train_sub_split="train", use_starter_code=True)
|
||||
reader = PseudoInputsWithFunctionNameFixStarterCode(train_sub_split="train", use_starter_code=True)
|
||||
|
||||
data = reader(args.input_file)
|
||||
|
||||
prompt_template = "{question}\n\nPlease write a Python program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
prompt = prompt_template.format(question=item["question"])
|
||||
item["prompt"] = prompt
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w", encoding="utf-8") as f:
|
||||
for item in inputs:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
print(len(inputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
--------
|
||||
{}
|
||||
--------
|
||||
Please carefully comprehend the requirements in the problem, and write down the solution program to pass it under the given time and memory constraints.
|
||||
|
||||
**REMEMBER** to strictly follow the steps below to help reduce the potential flaws:
|
||||
(1) According to the input scale and the time/memory constraints, think about the time complexity and space complexity of your solution.
|
||||
(2) Think **step-by-step** to design the algorithm.
|
||||
(3) Translate your thoughts into Python program to solve it.
|
||||
|
||||
Besides, your Python solution program should be located between <BEGIN> and <END> tags. For example:
|
||||
<BEGIN>
|
||||
t = int(input())
|
||||
...
|
||||
print(ans)
|
||||
<END>
|
||||
|
||||
Now, remember my instruction and let's get started:
|
||||
"""
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(data, desc="Writing prompts"):
|
||||
prompt = PROMPTS[args.prompt_type].format(item["question"])
|
||||
item["prompt"] = prompt
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
from scripts.apps.utils_execute import run_test
|
||||
|
||||
|
||||
def extract_content_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'\[BEGIN\](.*?)\[END\]'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--test_case_inputs", type=str, required=True, help="The completion file.")
|
||||
parser.add_argument("--test_case_outputs", type=str, required=True, help="The completion file.")
|
||||
# parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
# for item in data: # Currently, we do not require the solutions and input_output fields
|
||||
# if item["solutions"]:
|
||||
# item["solutions"] = json.loads(item["solutions"])
|
||||
# if item["input_output"]:
|
||||
# item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
case_inputs = [json.loads(line) for line in open(args.test_case_inputs).readlines()]
|
||||
case_outputs = [json.loads(line) for line in open(args.test_case_outputs).readlines()]
|
||||
|
||||
problem_id2case = defaultdict(list)
|
||||
for item in case_inputs:
|
||||
cases = extract_test_cases(item["completion"])
|
||||
if len(cases):
|
||||
for i, case in enumerate(cases):
|
||||
problem_id2case[item["problem_id"]].append({
|
||||
"input": case,
|
||||
"case_id": i,
|
||||
})
|
||||
|
||||
for item in case_outputs:
|
||||
output = extract_content_between_tags(item["completion"])
|
||||
if output:
|
||||
assert isinstance(output, str), output
|
||||
case_id = int(item["case_id"].split("_")[1])
|
||||
assert case_id == problem_id2case[item["problem_id"]][case_id]["case_id"]
|
||||
problem_id2case[item["problem_id"]][case_id]["output"] = output
|
||||
|
||||
# Currently, we can directly verify
|
||||
outputs = []
|
||||
for item in data:
|
||||
if item["solutions"]:
|
||||
item["solutions"] = json.loads(item["solutions"])
|
||||
|
||||
problem_id = item["problem_id"]
|
||||
cases = {
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
for case in problem_id2case[problem_id]:
|
||||
if "output" in case:
|
||||
cases["inputs"].append(case["input"])
|
||||
cases["outputs"].append(case["output"])
|
||||
assert isinstance(case["input"], str), case["input"]
|
||||
assert isinstance(case["output"], str), case["output"]
|
||||
if cases["inputs"]:
|
||||
item["gen_test_cases"] = cases
|
||||
outputs.append(item)
|
||||
|
||||
# print(outputs[0]["gen_test_cases"])
|
||||
|
||||
cnt = 0
|
||||
num_program = 0
|
||||
for i, item in tqdm(enumerate(outputs)):
|
||||
if i in [7]:
|
||||
continue # Some bad cases.
|
||||
print(item["gen_test_cases"])
|
||||
for solution in item["solutions"]:
|
||||
try:
|
||||
res = run_test(item["gen_test_cases"], solution)
|
||||
pass_num = sum([1 for item in res if item is True])
|
||||
cnt += pass_num
|
||||
except:
|
||||
pass
|
||||
num_program += 1
|
||||
|
||||
print(f"Pass rate: {cnt / num_program}")
|
||||
|
||||
# print(json.dumps(data[0], indent=2))
|
||||
# print(data[0]["question"])
|
||||
# print(data[0]["solutions"][0])
|
||||
# print(data[0]["input_output"]["inputs"][0])
|
||||
# print(data[0]["input_output"]["outputs"][0])
|
||||
|
||||
# with open(args.output_file, "w") as f:
|
||||
# for item in tqdm(outputs, desc="Writing prompts"):
|
||||
# f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
import json
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
{}
|
||||
|
||||
Please carefully comprehend the requirements in the problem, and generate **10** more test cases for me. You can find some examples in the problem description.
|
||||
|
||||
**REMEMBER** (1) I only need the inputs, please do not show me the outputs. (2) Please follow the format below to provide the test cases:
|
||||
|
||||
<TEST INPUT 1>
|
||||
input 1
|
||||
</TEST INPUT 1>
|
||||
<TEST INPUT 2>
|
||||
input 2
|
||||
</TEST INPUT 2>
|
||||
...
|
||||
<TEST INPUT 10>
|
||||
input 10
|
||||
</TEST INPUT 10>
|
||||
"""
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
# for item in data: # Currently, we do not require the solutions and input_output fields
|
||||
# if item["solutions"]:
|
||||
# item["solutions"] = json.loads(item["solutions"])
|
||||
# if item["input_output"]:
|
||||
# item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
# print(json.dumps(data[0], indent=2))
|
||||
# print(data[0]["question"])
|
||||
# print(data[0]["solutions"][0])
|
||||
# print(data[0]["input_output"]["inputs"][0])
|
||||
# print(data[0]["input_output"]["outputs"][0])
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(data, desc="Writing prompts"):
|
||||
prompt = PROMPTS[args.prompt_type].format(item["question"])
|
||||
item["prompt"] = prompt
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/test_input_gen_2shot_v2.0.txt")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--function_only", default=False, action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
|
||||
prompt_template = open(args.prompt_file).read()
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(data, desc="Writing prompts"):
|
||||
question = item["question"]
|
||||
if item["input_output"]:
|
||||
input_output = json.loads(item["input_output"])
|
||||
if "fn_name" in input_output:
|
||||
question += f"\n\n{item['starter_code']}"
|
||||
else:
|
||||
input_output = {}
|
||||
|
||||
prompt = prompt_template.replace("[[Question]]", question)
|
||||
item["prompt"] = prompt
|
||||
if args.function_only:
|
||||
if "fn_name" in input_output:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
else:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.0.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.0.inputs.jsonl
|
||||
|
||||
>>> python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.1.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only.inputs.jsonl --function_only
|
||||
"""
|
||||
@@ -0,0 +1,113 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and simulate it **step-by-step** to obtain the expected outputs the above test case inputs.
|
||||
|
||||
**REMEMBER** to follow the format below to provide the outputs:
|
||||
[BEGIN]
|
||||
outputs
|
||||
[END]
|
||||
"""
|
||||
|
||||
|
||||
_vanilla_prompt_2 = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and derive the expected outputs the above test case inputs. You should follow the steps below:
|
||||
(1) Simulate the algorithm/process described in the problem with the given inputs **step-by-step**, and record the step-level intermediate results.
|
||||
(2) Derive the correct outcome.
|
||||
(3) Put the final outputs in the format below:
|
||||
|
||||
[BEGIN]
|
||||
*put your outputs here*
|
||||
[END]
|
||||
|
||||
and return back it to me.
|
||||
"""
|
||||
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
"vanilla_2": _vanilla_prompt_2,
|
||||
}
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
cases = extract_test_cases(item["completion"])
|
||||
if len(cases):
|
||||
for i, case in enumerate(cases):
|
||||
new_item = copy.deepcopy(item)
|
||||
new_id = f"{item['problem_id']}_{i}"
|
||||
|
||||
prompt = PROMPTS[args.prompt_type].format(item["question"], case)
|
||||
|
||||
new_item["prompt"] = prompt
|
||||
new_item["case_id"] = new_id
|
||||
outputs.append(new_item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(outputs, desc="Writing prompts"):
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
python scripts/apps/pp_test_case_gen_outputs.py --input_file outputs/apps/apps.train.test_case_inputs.500.outputs.vanilla.v1.0.gpt-35-turbo.s42.jsonl --output_file outputs/apps/apps.train.test_case_outputs.500.vanilla.gpt-35-turbo.s42.v1.0.jsonl
|
||||
|
||||
python scripts/apps/pp_test_case_gen_outputs.py --input_file outputs/apps/apps.train.test_case_inputs.500.outputs.vanilla.v1.0.gpt-35-turbo.s42.jsonl --output_file outputs/apps/apps.train.test_case_outputs.500.vanilla.gpt-35-turbo.s42.v1.0.jsonl --prompt_type vanilla_2
|
||||
|
||||
python scripts/apps/pp_test_case_gen_outputs.py --input_file outputs/apps/apps.train.test_case_inputs.500.outputs.vanilla.v1.0.gpt-35-turbo.s42.jsonl --output_file outputs/apps/apps.train.test_case_outputs.500.vanilla.gpt-35-turbo.s42.v1.1.jsonl --prompt_type vanilla_2
|
||||
"""
|
||||
@@ -0,0 +1,116 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
import json
|
||||
from datasets import load_dataset
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and simulate it **step-by-step** to obtain the expected outputs the above test case inputs.
|
||||
|
||||
**REMEMBER** to follow the format below to provide the outputs:
|
||||
[BEGIN]
|
||||
outputs
|
||||
[END]
|
||||
"""
|
||||
|
||||
_vanilla_prompt_2 = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and derive the expected outputs the above test case inputs. You should follow the steps below:
|
||||
(1) Simulate the algorithm/process described in the problem with the given inputs **step-by-step**, and record the step-level intermediate results.
|
||||
(2) Derive the correct outcome.
|
||||
(3) Put the final outputs in the format below:
|
||||
|
||||
[BEGIN]
|
||||
*put your outputs here*
|
||||
[END]
|
||||
|
||||
and return back it to me.
|
||||
"""
|
||||
|
||||
_vanilla_prompt_3 = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and derive the expected outputs the above test case inputs. You should follow the steps below:
|
||||
(1) Simulate the algorithm/process described in the problem with the given inputs **step-by-step**, and record the step-level intermediate results.
|
||||
(2) Derive the correct outcome.
|
||||
|
||||
Response format:
|
||||
[BEGIN]
|
||||
*The expected outputs*
|
||||
[END]
|
||||
"""
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
"vanilla_2": _vanilla_prompt_2,
|
||||
"vanilla_3": _vanilla_prompt_3,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
print(len(data))
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(inputs, total=len(inputs)):
|
||||
problem = item["question"]
|
||||
for case in item["input_output"]["inputs"]:
|
||||
new_item = copy.deepcopy(item)
|
||||
prompt = PROMPTS[args.prompt_type].format(problem, case)
|
||||
new_item["prompt"] = prompt
|
||||
f.write(json.dumps(new_item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
"""
|
||||
@@ -0,0 +1,47 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
import json
|
||||
from datasets import load_dataset
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--prompt_file", type=str, default="vanilla")
|
||||
args = parser.parse_args()
|
||||
|
||||
_prompt_template = open(args.prompt_file).read()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
print(len(data))
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(inputs, total=len(inputs)):
|
||||
problem = item["question"]
|
||||
for case in item["input_output"]["inputs"]:
|
||||
# assert isinstance(case, str), (problem, case, json.loads(item["solutions"])[0]) # For leetcode-format, this is not applied.
|
||||
new_item = copy.deepcopy(item)
|
||||
prompt = _prompt_template.replace("<<PROBLEM>>", problem).replace("<<TEST INPUTS>>", str(case))
|
||||
new_item["prompt"] = prompt
|
||||
f.write(json.dumps(new_item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
"""
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import run_test, check_correctness
|
||||
|
||||
|
||||
def extract_content_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'\[BEGIN\](.*?)\[END\]'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
# return match.group(1).strip()
|
||||
return match.group(1)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
print(len(data))
|
||||
|
||||
flag = "Now, let's get started with the following problem:"
|
||||
|
||||
problem_pass_cnt = Counter()
|
||||
problem_case_cnt = Counter()
|
||||
difficulty_cnt = Counter()
|
||||
for item in tqdm(data, total=len(data)):
|
||||
# if item["problem_id"] > 2000:
|
||||
# continue
|
||||
# if item["difficulty"] not in ["introductory", "interview"]:
|
||||
# continue
|
||||
|
||||
difficulty_cnt[item["difficulty"]] += 1
|
||||
print(f"=========================={item['problem_id']}")
|
||||
# if item['problem_id'] in [160, 379, 438, 4096]:
|
||||
# print(f"Skip problem {item['problem_id']}")
|
||||
# continue
|
||||
prompt = item["prompt"]
|
||||
assert flag in prompt, prompt
|
||||
prompt = prompt.split(flag)[1].strip()
|
||||
|
||||
if "## TEST INPUTS\n\n" not in prompt:
|
||||
continue
|
||||
if "\n\n## SIMULATE" not in prompt:
|
||||
continue
|
||||
|
||||
inputs_s = prompt.index("## TEST INPUTS") + len("## TEST INPUTS\n\n")
|
||||
inputs_e = prompt.index("\n\n## SIMULATE")
|
||||
# inputs = prompt[inputs_s:inputs_e].strip()
|
||||
inputs = prompt[inputs_s:inputs_e]
|
||||
# print(inputs)
|
||||
|
||||
outputs = extract_content_between_tags(item["completion"])
|
||||
problem_case_cnt[item["problem_id"]] += 1
|
||||
|
||||
if outputs is None:
|
||||
continue
|
||||
|
||||
mapping = {str(_input): _input for i, _input in enumerate(item["input_output"]["inputs"])}
|
||||
|
||||
# if "fn_name" in item["input_output"]:
|
||||
# _cases = {
|
||||
# "inputs": []
|
||||
# }
|
||||
assert inputs in mapping, (inputs, json.dumps(mapping))
|
||||
_cases = {
|
||||
# "inputs": [inputs],
|
||||
# "outputs": [outputs]
|
||||
"inputs": [mapping[inputs]],
|
||||
"outputs": [outputs]
|
||||
}
|
||||
if "fn_name" in item["input_output"]:
|
||||
_cases["fn_name"] = item["input_output"]["fn_name"]
|
||||
|
||||
solutions = json.loads(item["solutions"])
|
||||
for solution in solutions[:1]: # I think one positive solution is enough
|
||||
try:
|
||||
# res = run_test(_cases, solution)
|
||||
res = check_correctness(_cases, solution, timeout=10, debug=False)
|
||||
pass_num = sum([1 for item in res if item is True])
|
||||
problem_pass_cnt[item["problem_id"]] += pass_num
|
||||
except:
|
||||
pass
|
||||
|
||||
print(difficulty_cnt)
|
||||
|
||||
print(f"Averaged pass cnt: {sum(problem_pass_cnt.values()) / sum(problem_case_cnt.values())}")
|
||||
print(f"Absolute pass rate over all generated cases: {sum(problem_pass_cnt.values()) / sum(problem_case_cnt.values())}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
"""
|
||||
@@ -0,0 +1,147 @@
|
||||
import json
|
||||
from collections import defaultdict, Counter
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from glob import glob
|
||||
import copy
|
||||
from tqdm import tqdm
|
||||
from datasets import load_dataset
|
||||
import random
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
# from post_processors.code.clean import tag_cleaner
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--critique_exec_file", type=str,
|
||||
help="The file contains completion from the teacher model for critique, as well as the execution results."
|
||||
"The inputs for this file are generated by `pp_critique_difficulty` script.")
|
||||
parser.add_argument("--completion_file", type=str,
|
||||
help="The file contains the completion for each query.")
|
||||
parser.add_argument("--completion_response_field", type=str, default="completion")
|
||||
parser.add_argument("--completion_problem_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/worsen_from_feedback_0shot_v1.0.txt")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--sample_num", type=int, default=2000)
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
args = parser.parse_args()
|
||||
|
||||
random.seed(args.seed)
|
||||
|
||||
prompt_template = open(args.prompt_file).read()
|
||||
|
||||
_dataset = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
p_id2item = {item["problem_id"]: item for item in _dataset}
|
||||
|
||||
critiques = json.load(open(args.critique_exec_file))
|
||||
correct_critiques = []
|
||||
for item in critiques:
|
||||
if isinstance(item["completion"], str) or isinstance(item["completion"], dict):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
preds = item["pred"]
|
||||
|
||||
for i, (comp, p, r) in enumerate(zip(completions, preds, item["res"])):
|
||||
if r:
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["completion"] = comp
|
||||
new_item["pred"] = p
|
||||
new_item["res"] = r
|
||||
correct_critiques.append(new_item)
|
||||
|
||||
print(f"Total correct critiques: {len(correct_critiques)}")
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
data += json.load(open(file))
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
problem_id = item[args.completion_problem_id_field]
|
||||
critique = random.choice(correct_critiques)
|
||||
while critique["problem_id"] == problem_id:
|
||||
critique = random.choice(correct_critiques)
|
||||
|
||||
preds = item["pred"]
|
||||
if not preds:
|
||||
continue
|
||||
if isinstance(preds, str):
|
||||
preds = [preds]
|
||||
|
||||
for i, pred in enumerate(preds):
|
||||
if pred:
|
||||
prompt = prompt_template.format(
|
||||
example_question=critique["question"],
|
||||
example_code=critique["neg_code"],
|
||||
feedback=critique["completion"]["feedback"],
|
||||
corrected_program=critique["completion"]["corrected_program"],
|
||||
question=p_id2item[problem_id]["question"],
|
||||
code=pred,
|
||||
)
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["id"] = f"{item[args.completion_problem_id_field]}_neg{i}"
|
||||
new_item["prompt"] = prompt
|
||||
if args.completion_response_field != "response":
|
||||
new_item["response"] = new_item.pop(args.completion_response_field)
|
||||
outputs.append(new_item)
|
||||
|
||||
if len(outputs) >= args.sample_num:
|
||||
break
|
||||
if len(outputs) >= args.sample_num:
|
||||
break
|
||||
|
||||
print(f"Total number of items: {len(outputs)}")
|
||||
# json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.0-of-8.v1.1.json" \
|
||||
--completion_response_field response \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f2000.jsonl \
|
||||
--sample_num 2000 --seed 42 --completion_problem_id_field id
|
||||
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_response_field response \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f10k.jsonl \
|
||||
--sample_num 10000 --seed 42 --completion_problem_id_field id
|
||||
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_response_field response
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.jsonl \
|
||||
--sample_num 100000 --seed 42 --completion_problem_id_field id
|
||||
|
||||
Total correct critiques: 7205
|
||||
100%|██████████████████████████████████████████████████████████████████| 5000/5000 [00:04<00:00, 1070.68it/s]
|
||||
Total number of items: 49868
|
||||
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_response_field response \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o.s42.f10k.jsonl \
|
||||
--sample_num 10000 --seed 42 --completion_problem_id_field id --prompt_file prompts/apps/worsen_0shot_v1.0.txt
|
||||
|
||||
python azure/gpt_crawler_mp.py --prompt_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o.s42.f10k.jsonl \
|
||||
--outfile ../gpt-chat-examples/outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o.s42.f10k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--model gpt-4o --max_gen_tokens 4096 --temperature 1.0 --num_processes 24 --seed 42 --n 1 --response_format json_object
|
||||
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from functools import partial
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
For self-consistency based input-output pairs, first copy the pseudo test cases into the prefix data, run `prefix_fail_extract_pseudo_label.py`,
|
||||
and the run this script.
|
||||
|
||||
This script is incorrect for pseudo test cases since we cannot ensure the correctness of each test case, but it is appropriate for ground-truth test cases,
|
||||
serving as hard limit, i.e., if there is one completion for some prefix has passed all test cases, then it is a gold prefix.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def counting_partial_response_value(res):
|
||||
return sum([1 if x else 0 for x in res])
|
||||
|
||||
|
||||
def parse_value(v, binary: bool):
|
||||
if binary:
|
||||
return 1 if v > 0 else 0
|
||||
return v
|
||||
|
||||
|
||||
def _process_trajectories_worker(item, top_k: int, binary: bool):
|
||||
item_id, trajectories = item
|
||||
outputs = {
|
||||
"idx": item_id
|
||||
}
|
||||
for i in range(top_k):
|
||||
level_trajectories = [(traj["vs"][i], traj["prefix"]) for traj in trajectories if len(traj["vs"]) > i]
|
||||
if len(level_trajectories) == 0:
|
||||
continue
|
||||
prefix_vis = set()
|
||||
level_values = []
|
||||
level_prefixes = []
|
||||
for v, prefix in level_trajectories:
|
||||
if prefix in prefix_vis:
|
||||
continue
|
||||
level_values.append(parse_value(v, binary))
|
||||
level_prefixes.append(prefix)
|
||||
prefix_vis.add(prefix)
|
||||
outputs[f"traj_level_{i}_values"] = level_values
|
||||
outputs[f"traj_level_{i}_prefixes"] = level_prefixes
|
||||
return outputs
|
||||
|
||||
|
||||
def _annotate(file):
|
||||
return json.load(open(file, encoding="utf-8"))
|
||||
|
||||
|
||||
def multiprocessing_loading(files, num_workers: int = 8):
|
||||
with Pool(num_workers) as p:
|
||||
data = list(tqdm(p.imap(_annotate, files), total=len(files)))
|
||||
all_data = []
|
||||
for d in data:
|
||||
all_data.extend(d)
|
||||
return all_data
|
||||
# data = []
|
||||
# for file in tqdm(files):
|
||||
# data += json.load(open(file, encoding="utf-8"))
|
||||
# return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--binary", default=False, action="store_true")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Collecting data...")
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
# data = []
|
||||
# for file in glob(args.input_file):
|
||||
# print(file)
|
||||
# data += json.load(open(file))
|
||||
files = glob(args.input_file)
|
||||
files = sorted(files)
|
||||
print(len(files))
|
||||
print(files)
|
||||
data = multiprocessing_loading(files)
|
||||
|
||||
print(len(data))
|
||||
|
||||
num_prefixes = 0
|
||||
val_cnt = collections.Counter()
|
||||
outputs = []
|
||||
preference_pairs = dict()
|
||||
missing = 0
|
||||
for item in tqdm(data):
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
prefix = item["prefix"]
|
||||
problem_id = int(problem_id)
|
||||
|
||||
if "res" not in item:
|
||||
missing += 1
|
||||
continue
|
||||
v = counting_partial_response_value(item["res"])
|
||||
v = parse_value(v, args.binary)
|
||||
|
||||
outputs.append({
|
||||
"problem_id": problem_id,
|
||||
"prefix": prefix,
|
||||
"value": v,
|
||||
})
|
||||
num_prefixes += 1
|
||||
val_cnt[v] += 1
|
||||
|
||||
if problem_id not in preference_pairs:
|
||||
preference_pairs[problem_id] = {
|
||||
"pos": [],
|
||||
"neg": [],
|
||||
}
|
||||
|
||||
if v > 0:
|
||||
preference_pairs[problem_id]["pos"].append(prefix)
|
||||
else:
|
||||
preference_pairs[problem_id]["neg"].append(prefix)
|
||||
|
||||
preference_pairs = [
|
||||
{
|
||||
"problem_id": problem_id,
|
||||
"pos": pair["pos"],
|
||||
"neg": pair["neg"],
|
||||
}
|
||||
for problem_id, pair in preference_pairs.items()
|
||||
]
|
||||
|
||||
print(f"Missing: {missing}")
|
||||
print(val_cnt)
|
||||
print(f"Processed {num_prefixes} prefixes.")
|
||||
print(f"Averaged {num_prefixes / len(data)} prefixes per problem.")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
json.dump(preference_pairs, open(args.output_file.replace(".json", "_pairs.json"), "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>>
|
||||
"""
|
||||
@@ -0,0 +1,189 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from functools import partial
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
For self-consistency based input-output pairs, first copy the pseudo test cases into the prefix data, run `prefix_fail_extract_pseudo_label.py`,
|
||||
and the run this script.
|
||||
|
||||
|
||||
Note:
|
||||
1. Sometimes the pseudo test cases can include some extremely large cases, leading to out-of-memory error.
|
||||
"""
|
||||
|
||||
|
||||
def counting_partial_response_value(full_res):
|
||||
pass_num = []
|
||||
for pred_res in full_res:
|
||||
if len(pred_res) == 0:
|
||||
pass_num.append(0)
|
||||
# elif pred_res[0] == -1: # FIXME: This line is commented since 2024/09/25. Seems no difference.
|
||||
# pass_num.append(0)
|
||||
else:
|
||||
pass_num.append(sum([1 for x in pred_res if x is True]))
|
||||
|
||||
return pass_num
|
||||
|
||||
|
||||
def annotate(file, exclude: str = ""):
|
||||
exclude = exclude.split(",")
|
||||
if any([e and e in file for e in exclude]):
|
||||
print(f"Excluding {file}")
|
||||
return []
|
||||
return json.load(open(file, encoding="utf-8"))
|
||||
|
||||
|
||||
def multiprocessing_loading(files, exclude: str = "", num_workers: int = 8):
|
||||
_annotate = partial(annotate, exclude=exclude)
|
||||
|
||||
# with Pool(num_workers) as p:
|
||||
# data = list(tqdm(p.imap(_annotate, files), total=len(files)))
|
||||
# all_data = []
|
||||
# for d in data:
|
||||
# all_data.extend(d)
|
||||
# return all_data
|
||||
data = []
|
||||
for file in tqdm(files):
|
||||
data += _annotate(file)
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_input_output")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
parser.add_argument("--reduction", type=str, default="max")
|
||||
parser.add_argument("--exclude", type=str, default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Collecting data...")
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
files = glob(args.input_file)
|
||||
files = sorted(files)
|
||||
print(len(files))
|
||||
print(files)
|
||||
data = multiprocessing_loading(files, args.exclude)
|
||||
|
||||
print(len(data))
|
||||
|
||||
num_prefixes = 0
|
||||
val_cnt = collections.Counter()
|
||||
outputs = []
|
||||
p_id2prefixes = collections.defaultdict(list)
|
||||
preference_pairs = []
|
||||
missing = 0
|
||||
missing_test_cases = 0
|
||||
for item in tqdm(data):
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
prefix = item["prefix"]
|
||||
# problem_id = int(problem_id)
|
||||
|
||||
if "res" not in item:
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
if args.test_case_field not in item or (not isinstance(item[args.test_case_field], dict)) or "inputs" not in item[args.test_case_field]:
|
||||
missing_test_cases += 1
|
||||
continue
|
||||
|
||||
test_case_num = len(item[args.test_case_field]["inputs"])
|
||||
if test_case_num == 0:
|
||||
missing_test_cases += 1
|
||||
continue
|
||||
|
||||
pass_num = counting_partial_response_value(item["full_res"])
|
||||
if args.reduction == "max":
|
||||
max_pass_num = max(pass_num)
|
||||
elif args.reduction == "avg":
|
||||
max_pass_num = sum(pass_num) / len(pass_num)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
outputs.append({
|
||||
"problem_id": problem_id,
|
||||
"prefix": prefix,
|
||||
"pass_num": pass_num,
|
||||
"max_pass_num": max_pass_num,
|
||||
"test_case_num": test_case_num,
|
||||
})
|
||||
num_prefixes += 1
|
||||
val_cnt.update(pass_num)
|
||||
|
||||
p_id2prefixes[problem_id].append(outputs[-1])
|
||||
|
||||
for problem_id, all_prefixes in tqdm(p_id2prefixes.items()):
|
||||
max_pass_num2prefixes = collections.defaultdict(list)
|
||||
for prefix in all_prefixes:
|
||||
max_pass_num2prefixes[prefix["max_pass_num"]].append(prefix)
|
||||
max_pass_num2prefixes = sorted(max_pass_num2prefixes.items(), key=lambda x: x[0])
|
||||
|
||||
pos_prefixes = []
|
||||
neg_prefixes = []
|
||||
for p in all_prefixes:
|
||||
pass_ratio = p["max_pass_num"] / p["test_case_num"]
|
||||
if pass_ratio < args.pass_case_lower_bound:
|
||||
continue
|
||||
neg_upper_pass_num = p["max_pass_num"] - args.pass_case_margin
|
||||
|
||||
target_neg = []
|
||||
for pass_num, prefixes in max_pass_num2prefixes:
|
||||
if pass_num < neg_upper_pass_num:
|
||||
target_neg.extend([x["prefix"] for x in prefixes])
|
||||
else:
|
||||
break
|
||||
|
||||
pos_prefixes.append(p["prefix"])
|
||||
neg_prefixes.append(target_neg)
|
||||
|
||||
preference_pairs.append({
|
||||
"problem_id": problem_id,
|
||||
"pos": pos_prefixes,
|
||||
"neg": neg_prefixes
|
||||
})
|
||||
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
print(val_cnt)
|
||||
print(f"Processed {num_prefixes} prefixes.")
|
||||
print(f"Averaged {num_prefixes / len(data)} prefixes per problem.")
|
||||
print(f"Processed {len(preference_pairs)} problems.")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
json.dump(preference_pairs, open(args.output_file.replace(".json", f"_low{args.pass_case_lower_bound}_m{args.pass_case_margin}_{args.reduction}.json"),
|
||||
"w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "/mnt/fangkai_blob/reward_modeling//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.[0-9]*-of-256.pseudo_test_case.exec.json" \
|
||||
--output_file /mnt/fangkai_blob/reward_modeling//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.pseudo_test_case.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.8 --pass_case_margin 4 --test_case_field pseudo_input_output
|
||||
|
||||
|
||||
>>> python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.*-of-256.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output --exclude "204-of-256,225-of-256"
|
||||
|
||||
Missing: 0
|
||||
Missing test cases: 0
|
||||
Counter({10: 448824, 0: 153053, 9: 25420, 1: 23469, 8: 16718, 5: 15115, 2: 14803, 3: 12821, 4: 12253, 7: 12053, 6: 11947, 11: 724, 20: 112, 16: 3}) # TODO: This should be a problem. Why there are some problems have more than 10 test cases?
|
||||
Processed 249105 prefixes.
|
||||
Averaged 1.0 prefixes per problem.
|
||||
"""
|
||||
@@ -0,0 +1,152 @@
|
||||
import json
|
||||
import argparse
|
||||
import os
|
||||
from glob import glob
|
||||
import sys
|
||||
import random
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
from post_processors.code.clean import get as get_code_cleaner
|
||||
|
||||
|
||||
def sample_step(response: str, code: str, upper_step_ratio: float, sample_ratio: float):
|
||||
if code:
|
||||
assert code in response, (code, "\n\n========================\n\n", response)
|
||||
end_of_response = response.find(code) + len(code)
|
||||
response = response[:end_of_response]
|
||||
|
||||
orig_lines = response.split("\n")
|
||||
lines = [(i, line) for i, line in enumerate(orig_lines)]
|
||||
lines = [(i, line) for i, line in lines if line.strip()]
|
||||
|
||||
if len(lines) < 5:
|
||||
return []
|
||||
|
||||
upper_step = int(len(lines) * upper_step_ratio)
|
||||
if upper_step == 0:
|
||||
return []
|
||||
|
||||
sample_step_num = int(upper_step * sample_ratio)
|
||||
if sample_step_num == 0:
|
||||
return []
|
||||
|
||||
sample_steps = random.sample(lines[:upper_step], sample_step_num)
|
||||
if len(sample_steps) == 0:
|
||||
return []
|
||||
|
||||
step_prefixes = []
|
||||
for i, line in sample_steps:
|
||||
step_prefixes.append("\n".join(orig_lines[:(i + 1)]))
|
||||
|
||||
return step_prefixes
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
large_mem = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
print(f"Too large outputs: {large_mem}")
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, )
|
||||
parser.add_argument("--output_file", type=str, )
|
||||
parser.add_argument("--upper_step_ratio", type=float)
|
||||
parser.add_argument("--sample_ratio", type=float)
|
||||
parser.add_argument("--filter_all_correct", action="store_true")
|
||||
parser.add_argument("--sample_over_p", type=int, default=-1)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.input_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
|
||||
outputs = []
|
||||
num = 0
|
||||
for item in data:
|
||||
if args.filter_all_correct and all(item["res"]):
|
||||
continue
|
||||
|
||||
prefixes = []
|
||||
prefix_ids = []
|
||||
if not item["response"]:
|
||||
item["prefix"] = []
|
||||
continue
|
||||
|
||||
for resp_id, resp in enumerate(item["response"]):
|
||||
response_prefixes = sample_step(resp, item["pred"][resp_id], args.upper_step_ratio, args.sample_ratio)
|
||||
prefixes.extend(response_prefixes)
|
||||
prefix_ids.extend([f"{item['id']}_{resp_id}_{i}" for i in range(len(response_prefixes))])
|
||||
|
||||
if args.sample_over_p > 0:
|
||||
if len(prefixes) > args.sample_over_p:
|
||||
prefixes = random.sample(prefixes, args.sample_over_p)
|
||||
prefix_ids = random.sample(prefix_ids, args.sample_over_p)
|
||||
|
||||
item["prefix"] = prefixes
|
||||
item["prefix_id"] = prefix_ids
|
||||
outputs.append(item)
|
||||
num += len(prefixes)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
print(f"Total number of samples: {num}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.prefix.upper0.8.r0.3.json
|
||||
|
||||
Total number of samples: 470010
|
||||
"""
|
||||
@@ -0,0 +1,68 @@
|
||||
|
||||
# Ground-truth test case inputs
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.run_outputs.json" \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/extract_pseudo_outputs_as_label.py \
|
||||
--input_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.run_outputs.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--use_sc
|
||||
|
||||
# ================================================================
|
||||
# Synthetic test case inputs
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json" \
|
||||
--num_workers 64 --completion_test_field "input_output" \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
# =================================================
|
||||
|
||||
# Synthetic test cases inputs with GPT-4o for output
|
||||
python scripts/apps/pseudo_test_cases/combine_gpt_raw_requests.py \
|
||||
--input_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1_gpt_inputs.shuf.jsonl \
|
||||
--raw_output_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1_gpt_inputs.shuf.bing.x5.jsonl_out \
|
||||
--output_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--num_workers 64 --id_field problem_id --test_case_field input_output
|
||||
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ${DATA_PREFIX_PATH}/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.json \
|
||||
--test_case_field "input_output"
|
||||
#19795
|
||||
#0
|
||||
#15
|
||||
#9.866784541550897
|
||||
#9.013993432685021
|
||||
#Counter()
|
||||
|
||||
# Non-sc-version
|
||||
python vllm_inference.py fp16_bfloat16=True seed=42 split_id={split_id} \
|
||||
exp_name=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42 \
|
||||
eval_sub_path=checkpoint-100 -cp conf/api/vllm/apps/deepseek_coder/r2c -cn general_combine_train_v2_0_4o_non_sc
|
||||
|
||||
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.prefer_pair.low0.5.m6.json" \
|
||||
--response_field response --test_case_field input_output_non_sc --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
# SC-version
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "/mnt/fangkai_blob/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.json" \
|
||||
--output_file "/mnt/fangkai_blob/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.json" \
|
||||
--num_workers 64 --id_field id --test_case_field input_output
|
||||
# bash scripts/apps/pseudo_test_cases/run_outputs_local.sh split_id
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.prefer_pair.low0.5.m6.4o-sc.json" \
|
||||
--test_case_field "input_output" --pass_case_margin 6 --pass_case_lower_bound 0.5 --construct_prefer_pair
|
||||
@@ -0,0 +1,224 @@
|
||||
### Synthesis Test Case Inputs
|
||||
|
||||
Prepare inputs for prompting GPT-4o:
|
||||
|
||||
```bash
|
||||
python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.0.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.0.inputs.jsonl
|
||||
|
||||
python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.1.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only.inputs.jsonl --function_only
|
||||
```
|
||||
|
||||
Extract the corresponding outputs from GPT-4o and combine them to obtain the following file:
|
||||
|
||||
`../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json`
|
||||
|
||||
For post-processing to align the format with oss data:
|
||||
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/combine_pseudo_test_inputs.py \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/apps-train.4o-test-func.json \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
```
|
||||
|
||||
For directly processing on APPs:
|
||||
|
||||
```bash
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--pseudo_test_case outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json --num_workers 128
|
||||
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json" \
|
||||
--num_workers 64 --completion_test_field "input_output" \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
```
|
||||
|
||||
Collect pseudo outputs and construct DPO training pairs
|
||||
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
```
|
||||
|
||||
Either, run execution again on existing pseudo test cases.
|
||||
|
||||
First, extract the pseudo label:
|
||||
```bash
|
||||
python scripts/apps/extract_pseudo_outputs_as_label.py \
|
||||
--input_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--use_sc --problem_id_field problem_id --test_case_field pseudo_test_cases
|
||||
#Unaligned: 0
|
||||
#Total number of pseudo test cases: 3901
|
||||
#Average number of test cases: 36333 / 3901 = 9.313765701102282
|
||||
```
|
||||
|
||||
Secondly, run execution on pseudo test cases:
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
#Missing: 0.0002563445270443476
|
||||
#Correct: 0.08203024865419123
|
||||
#Correct at k: 0.1579082286593181
|
||||
#Counter({False: 36311, True: 2689})
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.prefer_pair.json \
|
||||
--test_case_field pseudo_input_output
|
||||
# 3901 9165 2.3493975903614457
|
||||
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
# 3901 12851 3.294283517046911
|
||||
```
|
||||
|
||||
Run process-DPO prefix execution
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.4o_pseudo_test_case.exec.json" \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--test_case_field input_output --id_field problem_id
|
||||
```
|
||||
|
||||
Construct Process-DPO preference pair
|
||||
```bash
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.*-of-256.4o_pseudo_test_case.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.4o_pseudo_test_case.prm_prefer_pair.json \
|
||||
--pass_case_margin 4 --pass_case_lower_bound 0.5 --test_case_field pseudo_input_output --num_workers 24 --reduction avg
|
||||
#Missing: 0
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 1440638, 10: 108458, 1: 80861, 2: 34440, 3: 24141, 4: 18476, 5: 15235, 9: 14017, 6: 13494, 7: 12309, 8: 11546})
|
||||
#Processed 354723 prefixes.
|
||||
#Averaged 1.0 prefixes per problem.
|
||||
#Processed 3900 problems.
|
||||
```
|
||||
---------------------------------------------------------------
|
||||
|
||||
Based on DPO-Iter-0 model to construct new test cases on magicoder
|
||||
|
||||
Run outputs on newly generated oss-apps-combine data
|
||||
```bash
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
```
|
||||
|
||||
Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#11925
|
||||
#11925
|
||||
#117006
|
||||
#0
|
||||
#10.004442649434571
|
||||
#9.823396226415094
|
||||
#Counter({10: 73857, 0: 23639, 9: 3901, 1: 3222, 8: 2589, 5: 2251, 2: 2103, 7: 1976, 4: 1865, 6: 1865, 3: 1852, 11: 105, 20: 25})
|
||||
```
|
||||
We need to reuse the pre-synthesized pseudo test cases for APPs dataset for training.
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.apps_only.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
3901
|
||||
3901 11364 2.913099205331966
|
||||
```
|
||||
|
||||
---------------------------------------------------------------
|
||||
|
||||
Based on DPO-Iter-1 model to construct new test cases on magicoder and xcode
|
||||
|
||||
```bash
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
```
|
||||
|
||||
Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
# we have skipped split id 20 since it uses too much time.
|
||||
#18608
|
||||
#18608
|
||||
#193834
|
||||
#0
|
||||
#10.003605287755375
|
||||
#9.80116079105761
|
||||
#Counter({10: 99051, 0: 37603, 1: 8459, 9: 7563, 2: 5656, 8: 5528, 3: 4714, 5: 4698, 7: 4306, 4: 4260, 6: 4070, 11: 136, 20: 34, 12: 2})
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.all.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
# Added the skipped split id 20
|
||||
#19051
|
||||
#19051
|
||||
#197647
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.800902839745945
|
||||
#Counter({10: 102049, 0: 38432, 1: 8511, 9: 7700, 2: 5688, 8: 5591, 3: 4783, 5: 4776, 7: 4358, 4: 4329, 6: 4119, 11: 136, 20: 34, 12: 4})
|
||||
```
|
||||
|
||||
We need to reuse the pre-synthesized pseudo test cases for APPs dataset for training.
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300//train.0shot.tem1.0.n10.apps_only.v2.1.*-of-9.s42.4o_pseudo_test_cases.exec.v1.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.apps_only.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
|
||||
```
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
|
||||
def load_file(file_path):
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
else:
|
||||
return [json.loads(line) for line in open(file_path).readlines()]
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file, encoding="utf-8"))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
return data
|
||||
|
||||
|
||||
def extract_json(completion: str):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'```(.*?)```'
|
||||
|
||||
match = re.search(pattern, completion, re.DOTALL)
|
||||
if match:
|
||||
item = match.group(1).strip()
|
||||
if item.startswith("json"):
|
||||
item = item[5:]
|
||||
item = item.replace(",\n}", "\n}").replace(",}", "}").replace("None", "null")
|
||||
item = item.replace("(", "[").replace(")", "]").replace("\'", "\"")
|
||||
item = item.replace("True", "true").replace("False", "false")
|
||||
return json.loads(item)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_test_cases(item):
|
||||
try:
|
||||
response = extract_json(item["response"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["response"])
|
||||
return {}
|
||||
|
||||
if not response:
|
||||
return {}
|
||||
|
||||
if not isinstance(response, dict):
|
||||
return {}
|
||||
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
}
|
||||
for k, v in response.items():
|
||||
if not k.startswith("test_case_"):
|
||||
continue
|
||||
test_cases["inputs"].append(v)
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def load_func_head(item) -> str:
|
||||
try:
|
||||
response = extract_json(item["response"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["response"])
|
||||
return ""
|
||||
|
||||
if not response:
|
||||
return ""
|
||||
|
||||
if "func_head" not in response:
|
||||
return ""
|
||||
|
||||
return response["func_head"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_file", type=str)
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--func_head_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_file(args.data_file)
|
||||
|
||||
test_cases = load_file(args.test_case_file)
|
||||
id2test_cases = {}
|
||||
test_num = 0
|
||||
for item in test_cases:
|
||||
tmp = load_test_cases(item)
|
||||
if tmp:
|
||||
id2test_cases[item["id"]] = tmp
|
||||
test_num += 1
|
||||
|
||||
func_heads = load_file(args.func_head_file)
|
||||
id2func_head = {}
|
||||
func_num = 0
|
||||
for item in func_heads:
|
||||
tmp = load_func_head(item)
|
||||
if tmp:
|
||||
id2func_head[item["id"]] = tmp
|
||||
func_num += 1
|
||||
|
||||
print(f"Test cases: {test_num} / {len(test_cases)}")
|
||||
print(f"Func heads: {func_num} / {len(func_heads)}")
|
||||
|
||||
outputs = []
|
||||
for item in data:
|
||||
item_id = item["index"]
|
||||
if item_id in id2test_cases:
|
||||
test_inputs = id2test_cases[item_id]
|
||||
else:
|
||||
continue
|
||||
|
||||
if item_id in id2func_head:
|
||||
func_head = id2func_head[item_id]
|
||||
if not func_head:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
# item["input_output"] = test_cases
|
||||
# item["input_output"]["fn_name"] = func_head
|
||||
# outputs.append(item)
|
||||
test_inputs["fn_name"] = func_head
|
||||
outputs.append({
|
||||
"input_output": test_inputs,
|
||||
"question": item["problem"],
|
||||
"problem_id": f"oss-instruct-{item_id}",
|
||||
})
|
||||
|
||||
print(len(outputs))
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/clean_oss_mistral_data.py \
|
||||
--data_file ../msranlpintern/share/dataset/magicoder/data-oss_instruct-decontaminated-python.json \
|
||||
--test_case_file "../msranlpintern/share/models/Mistral-Large-Instruct-2407/apps-test-inputs-gen/sub_dev.0shot.tem0.0.n1.*-of-16.v1.0.json" \
|
||||
--func_head_file "../msranlpintern/share/models/Mistral-Large-Instruct-2407/apps-test-inputs-gen/oss_instruct_python.func_head_extract.tem0.0.n1.*-of-16.v1.0.json" \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/data-oss_instruct-decontaminated-python.mistral-large-test-func.json
|
||||
|
||||
Test cases: 31134
|
||||
Func heads: 23807
|
||||
18214
|
||||
|
||||
Test cases: 32066
|
||||
Func heads: 23807
|
||||
18969
|
||||
|
||||
Test cases: 32185
|
||||
Func heads: 23807
|
||||
19280
|
||||
|
||||
|
||||
Test cases: 33760
|
||||
Func heads: 23807
|
||||
20342
|
||||
|
||||
Test cases: 33760 / 38284
|
||||
Func heads: 23807 / 38284
|
||||
20342
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,131 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
|
||||
def load_file(file_path):
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
else:
|
||||
return [json.loads(line) for line in open(file_path).readlines()]
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file, encoding="utf-8"))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
return data
|
||||
|
||||
|
||||
def load_test_cases(item):
|
||||
try:
|
||||
input_output = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["completion"])
|
||||
return []
|
||||
|
||||
inputs = list(input_output.values())
|
||||
return inputs
|
||||
|
||||
|
||||
x_code_eval_template_v1 = """{description}
|
||||
|
||||
Input: {input_from}
|
||||
Output: {output_to}
|
||||
|
||||
Time limit: {time_limit}
|
||||
Memory limit: {memory_limit}
|
||||
|
||||
#### Input Format
|
||||
|
||||
{input_spec}
|
||||
|
||||
#### Output Format
|
||||
|
||||
{output_spec}
|
||||
|
||||
#### Notes
|
||||
|
||||
{notes}
|
||||
|
||||
#### Example Input-Output
|
||||
|
||||
{sample_input_output}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_file", type=str)
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--test_file", type=str)
|
||||
parser.add_argument("--val_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_file(args.data_file)
|
||||
|
||||
test_data = load_file(args.test_file)
|
||||
test_ids = set([item["src_uid"] for item in test_data])
|
||||
|
||||
val_data = load_file(args.val_file)
|
||||
test_ids.update([item["src_uid"] for item in val_data])
|
||||
|
||||
test_cases = load_file(args.test_case_file)
|
||||
id2test_cases = {}
|
||||
test_num = 0
|
||||
for item in test_cases:
|
||||
tmp = load_test_cases(item)
|
||||
if tmp:
|
||||
id2test_cases[item["src_uid"]] = tmp
|
||||
test_num += 1
|
||||
|
||||
print(f"Test cases: {test_num} / {len(test_cases)}")
|
||||
|
||||
outputs = []
|
||||
in_out_template = "Input\n```{}```\n\nOutput\n```{}```"
|
||||
test_held = 0
|
||||
for item in data:
|
||||
if item["input_from"] != 'standard input':
|
||||
continue
|
||||
|
||||
if item["src_uid"] in test_ids:
|
||||
test_held += 1
|
||||
continue
|
||||
|
||||
item_id = item["src_uid"]
|
||||
if item_id in id2test_cases:
|
||||
test_inputs = id2test_cases[item_id]
|
||||
else:
|
||||
continue
|
||||
|
||||
sample_input_output = []
|
||||
for _in, _out in zip(item["sample_inputs"], item["sample_outputs"]):
|
||||
sample_input_output.append(in_out_template.format(_in, _out))
|
||||
|
||||
item["sample_input_output"] = "\n\n".join(sample_input_output)
|
||||
|
||||
question = x_code_eval_template_v1.format(**item)
|
||||
|
||||
test_inputs = {
|
||||
"inputs": test_inputs,
|
||||
}
|
||||
outputs.append({
|
||||
"input_output": test_inputs,
|
||||
"question": question,
|
||||
"problem_id": f"xcode-eval-{item['src_uid']}",
|
||||
})
|
||||
|
||||
print(len(outputs))
|
||||
print(f"No test: {test_held}")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,252 @@
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def worker(item, pseudo_test_case_field):
|
||||
inputs = item[pseudo_test_case_field]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_inputs_non_sc = []
|
||||
new_outputs_non_sc = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
# Non-sc output
|
||||
if case_j in resp2outputs[0]:
|
||||
new_inputs_non_sc.append(inputs[case_j])
|
||||
new_outputs_non_sc.append(output_str2orig_pred[case_j][resp2outputs[0][case_j]])
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"inputs_non_sc": new_inputs_non_sc,
|
||||
"outputs_non_sc": new_outputs_non_sc,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_test_cases")
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.pseudo_test_case_file))
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
missing_predictions = 0
|
||||
avg_test_case_num = 0
|
||||
avg_non_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
result = worker(item, args.test_case_field)
|
||||
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item[args.test_case_field]["inputs"] = result["inputs"]
|
||||
item[args.test_case_field]["outputs"] = result["outputs"]
|
||||
item[args.test_case_field]["output_meta"] = result["output_meta"]
|
||||
item[f"{args.test_case_field}_non_sc"] = copy.deepcopy(item[args.test_case_field])
|
||||
item[f"{args.test_case_field}_non_sc"]["inputs"] = result["inputs_non_sc"]
|
||||
item[f"{args.test_case_field}_non_sc"]["outputs"] = result["outputs_non_sc"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
avg_non_test_case_num += len(result["inputs_non_sc"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item[args.test_case_field]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(avg_non_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 3 --pass_case_lower_bound 0.5
|
||||
|
||||
93%|██████████████████ | 4095/4418 [00:03<00:00, 629.19it/s]None
|
||||
100%|█████████████████████████████| 4418/4418 [00:03<00:00, 1323.18it/s]
|
||||
~~4299
|
||||
~~82365
|
||||
~~1
|
||||
~~9.953477552919283
|
||||
~~Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
4299
|
||||
72724
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 2 --pass_case_lower_bound 0.5
|
||||
|
||||
~~4299
|
||||
~~82365
|
||||
~~1
|
||||
~~9.953477552919283
|
||||
~~Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
4299
|
||||
77024
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
4299
|
||||
59226
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
4712
|
||||
59606
|
||||
2
|
||||
9.955220713073006
|
||||
Counter({10: 17718, 0: 16091, 1: 2515, 2: 1841, 9: 1807, 3: 1364, 8: 1291, 6: 1172, 4: 1125, 5: 1115, 7: 1081})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
4712
|
||||
0
|
||||
2
|
||||
9.955220713073006
|
||||
Counter()
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
import json
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--raw_output_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
input_data = [json.loads(line) for line in open(args.input_file, encoding="utf-8").readlines()]
|
||||
|
||||
raw_outputs = [json.loads(line) for line in open(args.raw_output_file, encoding="utf-8").readlines()]
|
||||
for o in tqdm(raw_outputs):
|
||||
request_meta = o["request_metadata"]
|
||||
s_id = int(request_meta["sid"].split("@")[1])
|
||||
input_item = input_data[s_id]
|
||||
assert o["request"]["messages"][1]["content"] in input_item["prompt"]
|
||||
|
||||
if "response" not in input_item:
|
||||
input_item["response"] = []
|
||||
if "content" not in o["response"]["choices"][0]["message"]:
|
||||
print(o)
|
||||
continue
|
||||
input_item["response"].append(o['response']['choices'][0]['message']['content'])
|
||||
|
||||
missing_completion = 0
|
||||
num_completions = 0
|
||||
for item in input_data:
|
||||
if "response" not in item:
|
||||
missing_completion += 1
|
||||
item["response"] = ""
|
||||
item["pred"] = None
|
||||
else:
|
||||
preds = [standard_cleaner_default(resp) for resp in item["response"]]
|
||||
item["pred"] = preds
|
||||
num_completions += len(preds)
|
||||
|
||||
json.dump(input_data, open(args.output_file, "w", encoding="utf-8"))
|
||||
print(f"Averaged number of completions: {num_completions / len(input_data)}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
def extract_test_case_inputs(item):
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
if "fn_name" in item["input_output"]:
|
||||
function_call = True
|
||||
else:
|
||||
function_call = False
|
||||
|
||||
full_inputs = []
|
||||
if function_call:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
|
||||
for k, v in response.items():
|
||||
if not isinstance(v, list):
|
||||
assert isinstance(v, str), v
|
||||
v = [v]
|
||||
full_inputs.append(v)
|
||||
else:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
for k, v in response.items():
|
||||
inputs = v
|
||||
full_inputs.append(inputs)
|
||||
|
||||
return full_inputs
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--pseudo_test_case", type=str)
|
||||
parser.add_argument("--completion_test_field", type=str, default="input_output")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
|
||||
ps_test_cases = []
|
||||
cnt = 0
|
||||
if args.pseudo_test_case.endswith(".json"):
|
||||
test_cases = json.load(open(args.pseudo_test_case))
|
||||
for item in test_cases:
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
else:
|
||||
with open(args.pseudo_test_case) as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except:
|
||||
print(f"Cannot load {line}")
|
||||
cnt += 1
|
||||
pass
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
print(cnt)
|
||||
print(f"Total number of pseudo test cases: {len(ps_test_cases)}")
|
||||
|
||||
id2item = {item["problem_id"]: item for item in ps_test_cases}
|
||||
outputs = []
|
||||
for item in data:
|
||||
if item["problem_id"] not in id2item:
|
||||
continue
|
||||
test_cases = {
|
||||
"inputs": id2item[item["problem_id"]]["pseudo_inputs"],
|
||||
}
|
||||
if item[args.completion_test_field]:
|
||||
if not isinstance(item[args.completion_test_field], dict):
|
||||
item[args.completion_test_field] = json.loads(item[args.completion_test_field])
|
||||
if "fn_name" in item[args.completion_test_field]:
|
||||
test_cases["fn_name"] = item[args.completion_test_field]["fn_name"]
|
||||
|
||||
# This is to align with oss data format.
|
||||
new_item = {
|
||||
"input_output": test_cases,
|
||||
"question": item["question"],
|
||||
"problem_id": f"apps-train-{item['problem_id']}",
|
||||
"starter_code": item["starter_code"],
|
||||
}
|
||||
outputs.append(new_item)
|
||||
|
||||
print(f"Total number of items: {len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/combine_pseudo_test_inputs.py \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/apps-train.4o-test-func.json \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
#export exp_dir=deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.pdpo.V100.tp8dp32.v1.3.s42/oss-apps-xcode-combine/checkpoint-300/
|
||||
export exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.pdpo.H100.dp8.v1.2.s42/oss-apps-xcode-combine/checkpoint-500/
|
||||
|
||||
p=$1
|
||||
|
||||
echo "Constructing process_rm sample for split 0"
|
||||
echo "p" $p
|
||||
|
||||
# Just check the outputs on the previously generated data
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_takes_extra.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output_by_n64.v1.0.dpo_m6_low0.5_min5_p$p.json \
|
||||
# --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_p$p.json \
|
||||
# --construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
## Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
# --construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
--construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
@@ -0,0 +1,10 @@
|
||||
OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
split_id=$1
|
||||
|
||||
python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label_align_ts_num.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.pseudo_test_case.exec.ctr_ts_num.v1.0.json" \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.*-of-256.pseudo_test_case.exec.ctr_ts_num.v1.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.pseudo_test_case.ctr_ts_num.prefix_pass_num.fix_low0.5_m4.0_avg.json" \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--num_workers 64 --id_field problem_id --test_case_field input_output
|
||||
@@ -0,0 +1,230 @@
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# if not len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]): # TODO: Figure it out why this happens
|
||||
# print(len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.pseudo_test_case_file):
|
||||
print(f"Loading pseudo test cases from {args.pseudo_test_case_file}")
|
||||
data = json.load(open(args.pseudo_test_case_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.pseudo_test_case_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
|
||||
outputs = []
|
||||
before_test_num = 0
|
||||
cnt = 0
|
||||
missing_predictions = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
|
||||
result = worker(item, min_success_test_num=args.min_success_test_num)
|
||||
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item["input_output"]["inputs"] = result["inputs"]
|
||||
item["input_output"]["outputs"] = result["outputs"]
|
||||
item["input_output"]["output_meta"] = result["output_meta"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,382 @@
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# from pympler import asizeof
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# if not len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]): # TODO: Figure it out why this happens
|
||||
# print(len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / len(item["full_res"])
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
# size_in_bytes = asizeof.asizeof(new_outputs)
|
||||
# if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
# "output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item['id']} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
preds = []
|
||||
for _ in item["response"]:
|
||||
preds.append("")
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["pred"] = preds
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
# if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=16)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# if os.path.exists(args.pseudo_test_case_file):
|
||||
# print(f"Loading pseudo test cases from {args.pseudo_test_case_file}")
|
||||
# data = json.load(open(args.pseudo_test_case_file))
|
||||
# else:
|
||||
# data = []
|
||||
# for file in glob(args.pseudo_test_case_file):
|
||||
# print(file)
|
||||
# if file.endswith(".json"):
|
||||
# tmp = json.load(open(file))
|
||||
# else:
|
||||
# tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
# data.extend(tmp)
|
||||
data = load_files(args.pseudo_test_case_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2data = {item["id"]: item for item in data}
|
||||
|
||||
missing_predictions = 0
|
||||
before_test_num = 0
|
||||
|
||||
_mp_inputs = []
|
||||
for item in tqdm(data, desc="Preparing Inputs"):
|
||||
# TODO: Here we do not consider those bad solutions without even complete programs. These programs should also be penalized
|
||||
# but we have already removed them here.
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append(item)
|
||||
|
||||
pbar = tqdm(_mp_inputs, desc="Submitting tasks")
|
||||
_mp_outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
sc_test_cases_outputs = []
|
||||
prefer_pair_outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in tqdm(_mp_outputs):
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2data[result["id"]]
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
|
||||
test_case_item = {
|
||||
"problem_id": item["problem_id"],
|
||||
"input_output": {
|
||||
"inputs": result["inputs"],
|
||||
"outputs": result["outputs"],
|
||||
},
|
||||
"sc_full_res": result["sc_match_res"],
|
||||
}
|
||||
sc_test_cases_outputs.append(test_case_item)
|
||||
|
||||
# test_case_item["input_output"]["inputs"] = result["inputs"]
|
||||
# test_case_item["input_output"]["outputs"] = result["outputs"]
|
||||
# item["input_output"]["output_meta"] = result["output_meta"]
|
||||
# test_case_item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(test_case_item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(test_case_item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(test_case_item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"]), (len(pred_pass_cnt), len(item["response"]), len(item["pred"]))
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
|
||||
prefer_pair_outputs.append(item)
|
||||
|
||||
print(len(prefer_pair_outputs))
|
||||
print(len(sc_test_cases_outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(sc_test_cases_outputs) if sc_test_cases_outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(sc_test_cases_outputs, open(args.output_file.replace(".json", ".sc_test_cases.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
if args.construct_prefer_pair:
|
||||
json.dump(prefer_pair_outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# from pympler import asizeof
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
|
||||
A different saving format. One positive response is paired with a list of negative candidates.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / len(item["full_res"])
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
# "output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item['id']} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
preds = []
|
||||
for _ in item["response"]:
|
||||
preds.append("")
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["pred"] = preds
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
# if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=16)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.pseudo_test_case_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2data = {item["id"]: item for item in data}
|
||||
|
||||
missing_predictions = 0
|
||||
before_test_num = 0
|
||||
|
||||
_mp_inputs = []
|
||||
for item in tqdm(data, desc="Preparing Inputs"):
|
||||
# TODO: Here we do not consider those bad solutions without even complete programs. These programs should also be penalized
|
||||
# but we have already removed them here.
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append(item)
|
||||
|
||||
pbar = tqdm(_mp_inputs, desc="Submitting tasks")
|
||||
_mp_outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
sc_test_cases_outputs = []
|
||||
prefer_pair_outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in tqdm(_mp_outputs):
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2data[result["id"]]
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
|
||||
test_case_item = {
|
||||
"problem_id": item["problem_id"],
|
||||
"input_output": {
|
||||
"inputs": result["inputs"],
|
||||
"outputs": result["outputs"],
|
||||
},
|
||||
"sc_full_res": result["sc_match_res"],
|
||||
}
|
||||
sc_test_cases_outputs.append(test_case_item)
|
||||
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(test_case_item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(test_case_item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(test_case_item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"]), (len(pred_pass_cnt), len(item["response"]), len(item["pred"]))
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
|
||||
pos2neg = []
|
||||
pos2neg_code = []
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
# pos.append(resp_i)
|
||||
# pos_code.append(prog_i)
|
||||
# neg.append(resp_j)
|
||||
# neg_code.append(prog_j)
|
||||
pos2neg.append(resp_j)
|
||||
pos2neg_code.append(prog_j)
|
||||
|
||||
if pos2neg:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(pos2neg)
|
||||
neg_code.append(pos2neg_code)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
|
||||
prefer_pair_outputs.append(item)
|
||||
|
||||
print(len(prefer_pair_outputs))
|
||||
print(len(sc_test_cases_outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(sc_test_cases_outputs) if sc_test_cases_outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(sc_test_cases_outputs, open(args.output_file.replace(".json", ".sc_test_cases.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
if args.construct_prefer_pair:
|
||||
json.dump(prefer_pair_outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
import collections
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pympler import asizeof
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py`.
|
||||
|
||||
This script can accept external execution files for performing self-consistency over test cases.
|
||||
"""
|
||||
|
||||
|
||||
def worker(_input, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
item, sc_item = _input
|
||||
inputs = sc_item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
assert len(inputs) == len(item["input_output"]["inputs"]), (len(inputs), len(item["input_output"]["inputs"]))
|
||||
# assert len(sc_item["outputs"]) == item["outputs"], (len(sc_item["outputs"]), len(item["outputs"]))
|
||||
|
||||
assert len(sc_item["full_res"]) == len(sc_item["outputs"]) == len(sc_item["pred"]), (len(sc_item["full_res"]), len(sc_item["outputs"]),
|
||||
len(sc_item["pred"]))
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
|
||||
sc_prog_num = len(sc_item["full_res"])
|
||||
|
||||
# We use the external item to perform self-consistency over test cases.
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(sc_item["full_res"], sc_item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
|
||||
# We then record the target item's outputs
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / sc_prog_num
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
sc_o, sc_o_freq = output_cnt.most_common(1)[0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
large_mem = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "outputs" in item:
|
||||
size_in_bytes = asizeof.asizeof(item["outputs"])
|
||||
if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
for _ in item["response"]:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
print(f"Too large outputs: {large_mem}")
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str)
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=24)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
external_ps_test_cases = load_files(args.pseudo_test_case_file)
|
||||
external_ps_test_cases = merge_seed_sampled_data(external_ps_test_cases)
|
||||
id2external_ps_test_cases = {item["id"]: item for item in external_ps_test_cases}
|
||||
|
||||
data = load_files(args.completion_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2item = {item["id"]: item for item in data}
|
||||
|
||||
before_test_num = 0
|
||||
missing_predictions = 0
|
||||
_mp_inputs = []
|
||||
_mp_outputs = []
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append((item, id2external_ps_test_cases[item["id"]]))
|
||||
|
||||
pbar = tqdm(_mp_inputs)
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in _mp_outputs:
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2item[result["id"]]
|
||||
|
||||
item["input_output"]["inputs"] = result["inputs"]
|
||||
item["input_output"]["outputs"] = result["outputs"]
|
||||
item["input_output"]["output_meta"] = result["output_meta"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,179 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
if any(x and y in x for x in item['pred'] for y in ["os.makedirs", "views.contact", "extract_metadata.py"]):
|
||||
print(f"======================================================== item {item['problem_id']} ====================================================")
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
try:
|
||||
res = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
except Exception as e:
|
||||
print(f"======================================================== item {item['problem_id']} ====================================================")
|
||||
print(e)
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
new_res = []
|
||||
# res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
for tmp in res:
|
||||
try:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
new_res.append(bool(tmp))
|
||||
else:
|
||||
new_res.append(tmp)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
new_res.append(False)
|
||||
res = new_res
|
||||
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
|
||||
depreciated_ids = {"oss-instruct-9410"}
|
||||
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["problem_id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
# problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["input_output"] = pseudo_test_cases[problem_id]["input_output"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
data = [item for item in data if item["problem_id"] not in depreciated_ids]
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,154 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[problem_id]["pseudo_test_cases"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,34 @@
|
||||
#python scripts/apps/solution_run_outputs_local.py --completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.?-of-8.v2.0.json" --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.v2.0.run_outputs.json --num_workers 24 --id_field "problem_id" --test_case_field "input_output"
|
||||
python scripts/apps/solution_run_outputs_local.py --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.json" --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.{split_id}-of-32.run_outputs.json --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
# Iter - 1: PRM
|
||||
# Sample prefix
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.prefix.upper0.8.r0.3.v2.0.json
|
||||
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 20 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample20_per.json
|
||||
|
||||
|
||||
# Execute the prefix completions
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.*-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json
|
||||
|
||||
# Construct preference pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "$OUTPUT_PATH_PREFIX/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.*-of-256.json" \
|
||||
--output_file $OUTPUT_PATH_PREFIX/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/magicoder_cls_2shot.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file, encoding='utf-8')]
|
||||
prompt_template = open(args.prompt_file, 'r').read()
|
||||
|
||||
for i, item in enumerate(data):
|
||||
prompt = prompt_template.replace("[[Question]]", item["instruction"])
|
||||
prompt = prompt.replace("[[Solution]]", item["response"])
|
||||
data[i]["prompt"] = prompt
|
||||
data[i]["id"] = i
|
||||
|
||||
with open(args.output_file, 'w', encoding='utf-8') as f:
|
||||
for item in data:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/magicoder_cls_2shot.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file, encoding="utf-8"))
|
||||
prompt_template = open(args.prompt_file, 'r').read()
|
||||
|
||||
for i, item in enumerate(data):
|
||||
prompt = prompt_template.replace("[[Question]]", item["problem"])
|
||||
prompt = prompt.replace("[[Solution]]", item["solution"])
|
||||
data[i]["prompt"] = prompt
|
||||
data[i]["id"] = i
|
||||
|
||||
with open(args.output_file, 'w', encoding='utf-8') as f:
|
||||
for item in data:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,159 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
print(sys.path)
|
||||
|
||||
|
||||
from apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_test_cases")
|
||||
parser.add_argument("--id_field", type=str, default="id")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item[args.id_field]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[problem_id][args.test_case_field]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,174 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import random
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
"""
|
||||
This script is used to control the amount of pseudo test cases are not more than the amount of ground truth test cases.
|
||||
"""
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
parser.add_argument("--gd_test_case_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
num_test_cases = 0
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
ps_test_cases = pseudo_test_cases[problem_id]["pseudo_test_cases"]
|
||||
if not item[args.gd_test_case_field]:
|
||||
continue
|
||||
if len(ps_test_cases["inputs"]) > len(item[args.gd_test_case_field]["inputs"]):
|
||||
idx = list(range(len(ps_test_cases["inputs"])))
|
||||
random.shuffle(idx)
|
||||
num_test_cases = len(item[args.gd_test_case_field]["inputs"])
|
||||
ps_test_cases["inputs"] = [ps_test_cases["inputs"][i] for i in idx[:num_test_cases]]
|
||||
ps_test_cases["outputs"] = [ps_test_cases["outputs"][i] for i in idx[:num_test_cases]]
|
||||
item["pseudo_input_output"] = ps_test_cases
|
||||
num_test_cases += len(item[args.gd_test_case_field]["inputs"])
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
if len(data) == 0:
|
||||
print("No data to process.")
|
||||
return
|
||||
print(f"Average number of test cases: {num_test_cases / len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,44 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
split_id=$1
|
||||
|
||||
echo "Constructing process_rm sample for split $split_id"
|
||||
|
||||
#export exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/${exp_dir}/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/${exp_dir}/train.0shot.tem1.0.n10.v2.1.${split_id}-of-32.s42.run_outputs.json" \
|
||||
# --num_workers 64 --id_field id --test_case_field input_output
|
||||
|
||||
# Run 4o-based prm executing
|
||||
#python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.4o_pseudo_test_case.exec.json" \
|
||||
# --num_workers 128 \
|
||||
# --pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
# --test_case_field input_output --id_field problem_id
|
||||
|
||||
# 4o-DPO-Iter-0 run outputs
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.run_outputs.json" \
|
||||
# --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# 4o-DPO-Iter-0 run solutions on previous 4o-synthesized test cases (APPs only)
|
||||
#python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
# --pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
|
||||
# 4o-DPO-iter-1 run outputs
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.run_outputs.json" \
|
||||
# --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# 4o-DPO-Iter-1 run solutions on previous 4o-synthesized test cases (APPs only)
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.${split_id}-of-9.v2.1.s42.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.v2.1.${split_id}-of-9.s42.4o_pseudo_test_cases.exec.v1.0.json" \
|
||||
--pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
export OUTPUT_PREFIX_PATH=../msranlpintern/reward_modeling/
|
||||
|
||||
python scripts/apps/pseudo_test_cases/clean_xcode_4o_test_inputs_data.py \
|
||||
--data_file ../msranlpintern/share/xCodeEval/problem_descriptions.jsonl \
|
||||
--test_case_file ../gpt_crawler/outputs/xcode_test_case_inputs_gen_v2.1.4o.tem0.0.n1.json_obj.jsonl \
|
||||
--output_file ../msranlpintern/share/xCodeEval/xcode_train_4o_test_inputs_v1.json \
|
||||
--test_file "../msranlpintern/share/xCodeEval/**/test/*.jsonl" \
|
||||
--val_file "../msranlpintern/share/xCodeEval/**/validation/*.jsonl"
|
||||
#Test cases: 7588 / 7635
|
||||
#6163
|
||||
#No test: 1372
|
||||
|
||||
# Run outputs on newly generated data
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#5844
|
||||
#79260
|
||||
#0
|
||||
#10.0
|
||||
#9.7298083504449
|
||||
#Counter({0: 15165, 10: 14577, 1: 5613, 9: 3884, 2: 3674, 8: 3100, 3: 2896, 4: 2561, 7: 2376, 5: 2350, 6: 2244})
|
||||
|
||||
|
||||
# Run outputs on newly generated oss-apps-combine data
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-32.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#14107
|
||||
#106950
|
||||
#1
|
||||
#10.004038772213248
|
||||
#9.794782731977033
|
||||
#Counter({10: 96073, 0: 20843, 9: 4661, 1: 3399, 8: 3067, 5: 2755, 2: 2179, 7: 2116, 6: 2004, 3: 1940, 4: 1870, 11: 142, 20: 21})
|
||||
|
||||
|
||||
# Sample steps for xcode
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 61630
|
||||
|
||||
# Sample steps for new oss-apps
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 247580
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_rm_large_meta.json
|
||||
|
||||
|
||||
# Construct prm pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.*-of-32.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.*-of-32.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.6 --pass_case_margin 6 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
# Run outputs on apps-magicoder-xcode-combine newly generated data w/ n=64
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
# Just check the outputs on the previously generated data
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_takes_extra.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output_by_n64.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#21790
|
||||
#188616
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.875906379072969
|
||||
#Counter({10: 109418, 0: 55905, 1: 9618, 9: 8233, 2: 6063, 8: 5869, 5: 4976, 3: 4861, 4: 4392, 7: 4323, 6: 4073, 11: 146, 20: 23}) # TODO: Why there is 20 and 11????
|
||||
|
||||
#21790
|
||||
#188616
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.875906379072969
|
||||
#Counter({10: 109418, 0: 55905, 1: 9618, 9: 8233, 2: 6063, 8: 5869, 5: 4976, 3: 4861, 4: 4392, 7: 4323, 6: 4073, 11: 146, 20: 23})
|
||||
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.v2.0.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.pseudo_input_output.exec.json \
|
||||
# --num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_rm_large_meta.json
|
||||
#
|
||||
|
||||
# Starting from model w/ magicoder & apps - pdpo, repeat the process again.
|
||||
# Run outputs on newly generated data
|
||||
exp_dir=deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.pdpo.V100.tp8dp32.v1.3.s42/oss-apps-xcode-combine/checkpoint-300/
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.v2.0.s[0-8].json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.{split_id}-of-32.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Sample steps for xcode
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 61630
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#22303
|
||||
#22303
|
||||
#8588574
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.884141146930906
|
||||
#Counter({10: 767611, 0: 324963, 1: 57875, 9: 53845, 2: 37125, 8: 36508, 5: 31254, 3: 30650, 7: 28261, 4: 27218, 6: 26200, 11: 990, 20: 188})
|
||||
|
||||
# Run completion outputs
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-{global_split_id}-of-8.loc-{split_id}-of-64.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-{global_split_id}-of-8.loc-{split_id}-of-64.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5sc_test_cases.json
|
||||
|
||||
|
||||
# Construct prm pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-*-of-8.loc-*-of-64.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 890
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 268084, 10: 15225, 1: 7396, 9: 2466, 2: 2270, 3: 1795, 8: 1696, 5: 1385, 4: 1371, 6: 1290, 7: 1207})
|
||||
#Processed 101395 prefixes.
|
||||
#Averaged 0.9912988219191475 prefixes per problem.
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-*-of-8.loc-*-of-64.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.3 --pass_case_margin 3 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 890
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 268084, 10: 15225, 1: 7396, 9: 2466, 2: 2270, 3: 1795, 8: 1696, 5: 1385, 4: 1371, 6: 1290, 7: 1207})
|
||||
#Processed 101395 prefixes.
|
||||
#Averaged 0.9912988219191475 prefixes per problem.
|
||||
#Processed 10172 problems.
|
||||
|
||||
exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.pdpo.H100.dp8.v1.2.s42/oss-apps-xcode-combine/checkpoint-500/
|
||||
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
#Too large outputs: 0
|
||||
#Total number of samples: 309210
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 32 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.json
|
||||
#Too large outputs: 0
|
||||
#Total number of samples: 989473
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.v2.0.s{seed}.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.s{seed}.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
--construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
#p=0.0
|
||||
#21662
|
||||
#21662
|
||||
#608162
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.874849967685348
|
||||
#Counter({10: 679443, 0: 368401, 1: 61978, 9: 52376, 2: 39749, 8: 36629, 3: 31688, 5: 31673, 4: 28822, 7: 28171, 6: 26357, 11: 909, 20: 170, 12: 2})
|
||||
#p=0.2
|
||||
#18373
|
||||
#18373
|
||||
#585808
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.93577532248408
|
||||
#Counter({10: 668678, 0: 205613, 9: 50259, 1: 46523, 8: 34864, 2: 33257, 5: 29338, 3: 28297, 7: 26725, 4: 26387, 6: 24873, 11: 886, 20: 170, 12: 2})
|
||||
#p=0.4
|
||||
#14583
|
||||
#14583
|
||||
#506479
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.954878968662142
|
||||
#Counter({10: 634922, 0: 95150, 9: 41702, 8: 28477, 1: 22680, 5: 20957, 7: 20445, 6: 18610, 2: 17527, 4: 16296, 3: 15538, 11: 838, 20: 170})
|
||||
#=0.5
|
||||
#12903
|
||||
#12903
|
||||
#452734
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.960861815081763
|
||||
#Counter({10: 609930, 0: 62866, 9: 36059, 8: 23682, 7: 16213, 5: 15332, 6: 14783, 1: 14003, 2: 11235, 4: 10667, 3: 10087, 11: 791, 20: 144})
|
||||
|
||||
|
||||
# Run completion outputs
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-{global_split_id}-of-16.loc-{split_id}-of-64.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-{global_split_id}-of-16.loc-{split_id}-of-64.v2.0.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p0.0.sc_test_cases.json
|
||||
|
||||
# p=0.0
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-*-of-16.loc-*-of-64.v2.0.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 4235
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 1492601, 10: 211726, 1: 90982, 2: 45672, 9: 42604, 3: 34232, 8: 32685, 4: 29914, 5: 27977, 7: 26170, 6: 25720})
|
||||
#Processed 686761 prefixes.
|
||||
#Averaged 0.9938711656796856 prefixes per problem.
|
||||
#Processed 21533 problems.
|
||||
|
||||
# p=0.2
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-*-of-16.loc-*-of-64.v2.0.pseudo_input_output.p0.2.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.p0.2.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 2073
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 952288, 10: 159840, 1: 54560, 9: 29678, 2: 27823, 8: 23059, 3: 21733, 4: 20014, 5: 18776, 7: 17984, 6: 17690})
|
||||
#Processed 447815 prefixes.
|
||||
#Averaged 0.995392186499751 prefixes per problem.
|
||||
#Processed 14028 problems.
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
x_code_eval_template = """{description}
|
||||
|
||||
Input: {input_from}
|
||||
Output: {output_to}
|
||||
|
||||
Time limit: {time_limit}
|
||||
Memory limit: {memory_limit}
|
||||
|
||||
-----Input-----
|
||||
|
||||
{input_spec}
|
||||
|
||||
-----Output-----
|
||||
|
||||
{output_spec}
|
||||
|
||||
-----Notes-----
|
||||
|
||||
{notes}
|
||||
|
||||
-----Example-----
|
||||
|
||||
{input_output}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
parser.add_argument("--output_file")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/test_input_gen_2shot_v2.1.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
in_out_template = "Input\n```{}```\n\nOutput\n```{}```"
|
||||
prompt_tem = open(args.prompt_file).read()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
outputs = []
|
||||
for item in data:
|
||||
input_output = []
|
||||
for _in, _out in zip(item["sample_inputs"], item["sample_outputs"]):
|
||||
input_output.append(in_out_template.format(_in, _out))
|
||||
|
||||
item["input_output"] = "\n\n".join(input_output)
|
||||
|
||||
question = x_code_eval_template.format(**item)
|
||||
|
||||
prompt = prompt_tem.replace("[[Question]]", question)
|
||||
|
||||
item["prompt"] = prompt
|
||||
outputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
print(len(outputs))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
success = 0
|
||||
success_at_k = 0
|
||||
for item in data:
|
||||
if isinstance(item["res"], bool):
|
||||
assert isinstance(item["response"], str)
|
||||
if item["pred"] is None:
|
||||
item["res"] = False
|
||||
|
||||
if item["res"]:
|
||||
success += 1
|
||||
success_at_k += 1
|
||||
else:
|
||||
if len(item["res"]):
|
||||
assert isinstance(item["response"], list)
|
||||
for i in range(len(item["response"])):
|
||||
if item["pred"][i] is None:
|
||||
item["res"][i] = False
|
||||
item["full_res"][i] = [[False] * len(item["test_cases"]["inputs"]) if item["test_cases"] else 1]
|
||||
|
||||
if any(item["res"]):
|
||||
success_at_k += 1
|
||||
if item["res"][0]:
|
||||
success += 1
|
||||
|
||||
metrics = {"acc": success / len(data), "pass@k": success_at_k / len(data), "correct": success, "total": len(data)}
|
||||
print(json.dumps(metrics, indent=2))
|
||||
if args.output_file is not None:
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
json.dump(metrics, open(args.output_file.replace(".json", ".metrics"), "w"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
import json
|
||||
import argparse
|
||||
import sys
|
||||
import collections
|
||||
import glob
|
||||
import os
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
solutions = json.load(open(args.completion_file))
|
||||
if "difficulty" in solutions[0]:
|
||||
all_diff = collections.Counter([item["difficulty"] for item in solutions])
|
||||
all_diff_success = {k: 0 for k in all_diff}
|
||||
else:
|
||||
all_diff = None
|
||||
all_diff_success = None
|
||||
|
||||
if os.path.exists(args.reward_file):
|
||||
rewards = json.load(open(args.reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob.glob(args.reward_file):
|
||||
rewards += json.load(open(file))
|
||||
|
||||
id2reward = collections.defaultdict(dict)
|
||||
for item in rewards:
|
||||
index = item["index"]
|
||||
orig_id, pred_id = list(map(int, index.split("_")))
|
||||
# id2reward[orig_id][pred_id] = item["reward"]
|
||||
if isinstance(item["reward"], list):
|
||||
id2reward[orig_id][pred_id] = item["reward"][0]
|
||||
else:
|
||||
id2reward[orig_id][pred_id] = item["reward"]
|
||||
|
||||
success = 0
|
||||
for item in solutions:
|
||||
problem_id = item["id"]
|
||||
|
||||
if not item["response"]:
|
||||
continue
|
||||
if not item["pred"]:
|
||||
continue
|
||||
|
||||
tuples = []
|
||||
for j, (resp, pred) in enumerate(zip(item["response"], item["pred"])):
|
||||
if pred:
|
||||
tuples.append((j, id2reward[problem_id][j]))
|
||||
|
||||
if not tuples:
|
||||
continue
|
||||
|
||||
tuples = sorted(tuples, key=lambda x: x[1], reverse=True)
|
||||
pred_id = tuples[0][0]
|
||||
if item["res"][pred_id]:
|
||||
success += 1
|
||||
if all_diff is not None:
|
||||
all_diff_success[item["difficulty"]] += 1
|
||||
|
||||
print(f"Success rate: {success / len(solutions)}")
|
||||
if all_diff is not None:
|
||||
for k, v in all_diff.items():
|
||||
print(f"Difficulty {k}: {all_diff_success[k] / v}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,165 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def extract_solution_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'<BEGIN>(.*?)<END>'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if isinstance(item["completion"], str):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
solutions = []
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
for completion in completions:
|
||||
gen_solution = extract_solution_between_tags(completion)
|
||||
if gen_solution is None:
|
||||
solutions.append("")
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
solutions.append(gen_solution)
|
||||
|
||||
if not item["input_output"]:
|
||||
continue
|
||||
|
||||
all_results = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
# res = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
# assert len(res) == len(outputs) == len(errors), (len(res), len(outputs), len(errors))
|
||||
|
||||
item["pred"] = solutions if len(solutions) > 1 else solutions[0]
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["problem_id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["completion"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["completion"], list):
|
||||
data[item_id2data_id[item_id]]["completion"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["completion"] = [data[item_id2data_id[item_id]]["completion"]] + new_completions
|
||||
|
||||
for item in data:
|
||||
item["completion"] = list(set(item["completion"]))
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,160 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if isinstance(item["completion"], str) or isinstance(item["completion"], dict):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
# if item["input_output"]:
|
||||
# item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
solutions = []
|
||||
results = []
|
||||
full_results = []
|
||||
# all_outputs = []
|
||||
# all_errors = []
|
||||
for completion in completions:
|
||||
if "corrected_program" not in completion or completion["corrected_program"] is None:
|
||||
solutions.append("")
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
gen_solution = completion["corrected_program"]
|
||||
|
||||
solutions.append(gen_solution)
|
||||
|
||||
if not item["input_output"]:
|
||||
continue
|
||||
|
||||
all_results = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
# res, outputs, errors = all_results
|
||||
res = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
# all_outputs.append(outputs)
|
||||
# all_errors.append(errors)
|
||||
# assert len(res) == len(outputs) == len(errors), (len(res), len(outputs), len(errors))
|
||||
|
||||
item["pred"] = solutions if len(solutions) > 1 else solutions[0]
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
# item["outputs"] = all_outputs
|
||||
# item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This script takes the GPT-4 completion file with json object as the response format, where the returned json contains two fields:
|
||||
- feedback: the critique for the incorrect program.
|
||||
- corrected_program: the corrected program.
|
||||
|
||||
THe inputs file is generated by `pp_critique_difficulty` script.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["problem_id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["completion"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["completion"], list):
|
||||
data[item_id2data_id[item_id]]["completion"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["completion"] = [data[item_id2data_id[item_id]]["completion"]] + new_completions
|
||||
|
||||
new_data = []
|
||||
parsing_error = 0
|
||||
for item in tqdm(data):
|
||||
if isinstance(item["completion"], str):
|
||||
try:
|
||||
item["completion"] = json.loads(item["completion"])
|
||||
except:
|
||||
# print(f"Parsing error: {item['completion']}")
|
||||
parsing_error += 1
|
||||
if isinstance(item["completion"], dict):
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
print(f"Total number of items: {len(data)}")
|
||||
print(f"Total number of parsing error: {parsing_error}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,163 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict, Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
print(sys.path)
|
||||
|
||||
from apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
print(args.completion_file)
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["problem_id"]: item for item in pseudo_test_cases}
|
||||
print(len(pseudo_test_cases))
|
||||
print(list(pseudo_test_cases.keys())[:10])
|
||||
aux_ps_test_cases = {f"apps-train-{k}": v for k, v in pseudo_test_cases.items()}
|
||||
pseudo_test_cases.update(aux_ps_test_cases)
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
if item["id"] in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[item["id"]]["input_output"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.pseudo_test_case.exec.json \
|
||||
--num_workers 24 --pseudo_test_cases outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pseudo_test_cases.json
|
||||
|
||||
>>> python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v1.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.json \
|
||||
--num_workers 24 --pseudo_test_cases outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pseudo_test_cases.json
|
||||
|
||||
>>> python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.pseudo_cases.sc.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.37857900318133614
|
||||
Correct at k: 0.4721102863202545
|
||||
Counter({False: 30141, True: 17009})
|
||||
|
||||
>>>
|
||||
"""
|
||||
@@ -0,0 +1,173 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import run_inference_process
|
||||
|
||||
|
||||
def extract_solution_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'<BEGIN>(.*?)<END>'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if isinstance(item["completion"], str):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
solutions = []
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
for completion in completions:
|
||||
gen_solution = extract_solution_between_tags(completion)
|
||||
if gen_solution is None:
|
||||
solutions.append("")
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
solutions.append(gen_solution)
|
||||
|
||||
if not item["input_output"]:
|
||||
continue
|
||||
|
||||
all_results = run_inference_process(item["input_output"], gen_solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
try:
|
||||
json.dumps(outputs)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
except:
|
||||
print(f"Cannot dump outputs for {outputs}")
|
||||
all_outputs.append([])
|
||||
all_errors.append([])
|
||||
# assert len(res) == len(outputs) == len(errors), (len(res), len(outputs), len(errors))
|
||||
|
||||
item["pred"] = solutions if len(solutions) > 1 else solutions[0]
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["problem_id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["completion"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["completion"], list):
|
||||
data[item_id2data_id[item_id]]["completion"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["completion"] = [data[item_id2data_id[item_id]]["completion"]] + new_completions
|
||||
|
||||
for item in data:
|
||||
item["completion"] = list(set(item["completion"]))
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,235 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
import os
|
||||
from pympler import asizeof
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
print(sys.path)
|
||||
|
||||
from apps.utils_execute import run_inference_process
|
||||
|
||||
|
||||
def _worker(item, test_case_field: str):
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
return item
|
||||
|
||||
for pred in item["pred"]:
|
||||
gen_solution = pred
|
||||
if gen_solution is None:
|
||||
results.append(False)
|
||||
# full_results.append([False] * 3)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
continue
|
||||
|
||||
if "Hello, World!" in gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
continue
|
||||
|
||||
if not item[test_case_field]:
|
||||
continue
|
||||
|
||||
all_results = run_inference_process(item[test_case_field], gen_solution, timeout=1, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
try:
|
||||
json.dumps(outputs)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
except:
|
||||
print(f"Cannot dump outputs for {outputs}")
|
||||
all_outputs.append([])
|
||||
all_errors.append([])
|
||||
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
# Function to check if a string contains surrogate characters
|
||||
def has_surrogate_characters(text):
|
||||
return bool(re.search(r'[\ud800-\udfff]', text))
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data, id_field: str = "id"):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item[id_field]} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if item[id_field] not in id2data:
|
||||
id2data[item[id_field]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item[id_field]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item[id_field]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--id_field", type=str, default="id")
|
||||
parser.add_argument("--test_case_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.completion_file)
|
||||
data = merge_seed_sampled_data(data, args.id_field)
|
||||
# for item in data:
|
||||
# item["response"] = list(set(item["response"])) # No need to remove duplicates, since the `pred` field is aligned.
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = partial(_worker, test_case_field=args.test_case_field)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
large_mem = 0
|
||||
for item in outputs:
|
||||
if "res" in item and item["res"]:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
|
||||
try:
|
||||
size_in_bytes = asizeof.asizeof(item["outputs"])
|
||||
if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
except:
|
||||
print("failed to compute size. Still abandon the outputs.")
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
new_outputs = []
|
||||
for item in tqdm(outputs):
|
||||
tmp = json.dumps(item, ensure_ascii=False)
|
||||
if has_surrogate_characters(tmp):
|
||||
print(f"Surrogate characters found in {item[args.id_field]}")
|
||||
continue
|
||||
new_outputs.append(item)
|
||||
|
||||
outputs = new_outputs
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Large memory: {large_mem / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/solution_run_outputs_local.py --completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.json --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.json --num_workers 24
|
||||
|
||||
>>> python scripts/apps/solution_run_outputs_local.py --completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.?-of-8.v2.0.json --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.v2.0.run_outputs.json --num_workers 24 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,258 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import run_inference_process
|
||||
|
||||
|
||||
def extract_test_case_inputs(item):
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
if "fn_name" in item["input_output"]:
|
||||
function_call = True
|
||||
else:
|
||||
function_call = False
|
||||
|
||||
full_inputs = []
|
||||
if function_call:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
|
||||
for k, v in response.items():
|
||||
# if v.find(item["input_output"]["fn_name"]) != -1:
|
||||
# inputs = v.replace(item["input_output"]["fn_name"], "").strip()
|
||||
# inputs = inputs[1:-1] # Remove `(` and `)`.
|
||||
# try:
|
||||
# inputs = eval(f"[{inputs}]")
|
||||
# except Exception as e:
|
||||
# print(f"Cannot eval inputs for {inputs}\t{item['input_output']['fn_name']}\t{v}\t{item['input_output']['inputs']}")
|
||||
# print(e)
|
||||
# continue
|
||||
# full_inputs.append(inputs)
|
||||
if not isinstance(v, list):
|
||||
assert isinstance(v, str), v
|
||||
v = [v]
|
||||
full_inputs.append(v)
|
||||
else:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
for k, v in response.items():
|
||||
inputs = v
|
||||
full_inputs.append(inputs)
|
||||
|
||||
return full_inputs
|
||||
|
||||
|
||||
def _worker(item):
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
return item
|
||||
|
||||
for pred in item["pred"]:
|
||||
gen_solution = pred
|
||||
if gen_solution is None:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
continue
|
||||
|
||||
if not item["pseudo_test_cases"]:
|
||||
continue
|
||||
|
||||
all_results = run_inference_process(item["pseudo_test_cases"], gen_solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
try:
|
||||
json.dumps(outputs)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
except:
|
||||
print(f"Cannot dump outputs for {outputs}")
|
||||
all_outputs.append([])
|
||||
all_errors.append([])
|
||||
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_case", type=str)
|
||||
parser.add_argument("--completion_test_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["response"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["response"], list):
|
||||
data[item_id2data_id[item_id]]["response"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["response"] = [data[item_id2data_id[item_id]]["response"]] + new_completions
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
ps_test_cases = []
|
||||
cnt = 0
|
||||
if args.pseudo_test_case.endswith(".json"):
|
||||
test_cases = json.load(open(args.pseudo_test_case))
|
||||
for item in test_cases:
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
else:
|
||||
with open(args.pseudo_test_case) as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except:
|
||||
print(f"Cannot load {line}")
|
||||
cnt += 1
|
||||
pass
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
print(cnt)
|
||||
print(f"Total number of pseudo test cases: {len(ps_test_cases)}")
|
||||
|
||||
id2item = {item["problem_id"]: item for item in ps_test_cases}
|
||||
new_data = []
|
||||
for item in data:
|
||||
if not isinstance(item["response"], list):
|
||||
item["response"] = [item["response"]]
|
||||
item["pred"] = [item["pred"]]
|
||||
if item["id"] not in id2item:
|
||||
continue
|
||||
item["pseudo_test_cases"] = {
|
||||
"inputs": id2item[item["id"]]["pseudo_inputs"],
|
||||
"outputs": [],
|
||||
}
|
||||
if item[args.completion_test_field]:
|
||||
if not isinstance(item[args.completion_test_field], dict):
|
||||
item[args.completion_test_field] = json.loads(item[args.completion_test_field])
|
||||
if "fn_name" in item[args.completion_test_field]:
|
||||
item["pseudo_test_cases"]["fn_name"] = item[args.completion_test_field]["fn_name"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item and item["res"]:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--pseudo_test_case outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
|
||||
>>> python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json --num_workers 128
|
||||
"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
from collections import defaultdict, Counter
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from glob import glob
|
||||
import copy
|
||||
from tqdm import tqdm
|
||||
from datasets import load_dataset
|
||||
import random
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.apps import APPsWithFunctionName
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This script takes the completions from GPT4, the corresponding inputs for worsening code, and the original dataset, to construct a new combined dataset,
|
||||
which contains the solutions from the policy model, as well as the synthesized negative codes for each plausible solution.
|
||||
:return:
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--worsen_file", type=str,
|
||||
help="The file contains completion from the teacher model for worsening code"
|
||||
"The inputs for this file are generated by `pp_worsen_inputs.py` script.")
|
||||
parser.add_argument("--completion_file", type=str,
|
||||
help="The file contains the completion for each query.")
|
||||
# parser.add_argument("--completion_response_field", type=str, default="completion")
|
||||
parser.add_argument("--completion_problem_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
worsen_codes = [json.loads(line) for line in open(args.worsen_file).readlines()]
|
||||
print(f"Total number of worsen codes: {len(worsen_codes)}")
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
data += json.load(open(file))
|
||||
|
||||
p_id2response = {item[args.completion_problem_id_field]: item for item in data}
|
||||
|
||||
p_id2pairs = defaultdict(list)
|
||||
for item in worsen_codes:
|
||||
if item["completion"]:
|
||||
try:
|
||||
completion = json.loads(item["completion"])
|
||||
except:
|
||||
print(f"Json parsing error: {item['completion']}")
|
||||
continue
|
||||
neg_code = completion["incorrect_program"]
|
||||
# if isinstance(neg_code, dict):
|
||||
# print(json.dumps(completion, indent=2))
|
||||
|
||||
if not isinstance(neg_code, str):
|
||||
print(f"Bad format: {neg_code}.")
|
||||
continue
|
||||
|
||||
if not neg_code.strip():
|
||||
continue
|
||||
|
||||
p_id, pred_id = item["id"].split("_")
|
||||
pred_id = int(pred_id[3:])
|
||||
p_id = int(p_id)
|
||||
|
||||
response = p_id2response[p_id]
|
||||
pred = response["pred"][pred_id]
|
||||
assert pred in item["prompt"]
|
||||
|
||||
p_id2pairs[p_id].append((pred, neg_code))
|
||||
|
||||
outputs = []
|
||||
num_pairs = 0
|
||||
for p_id, pairs in p_id2pairs.items():
|
||||
pos = []
|
||||
neg = []
|
||||
for pred, neg_code in pairs:
|
||||
pos.append(pred)
|
||||
neg.append(neg_code)
|
||||
assert len(pos) == len(neg)
|
||||
num_pairs += len(pos)
|
||||
outputs.append({
|
||||
"problem_id": p_id,
|
||||
"pos": pos,
|
||||
"neg": neg,
|
||||
})
|
||||
|
||||
print(f"Total number of outputs: {len(outputs)}")
|
||||
print(f"Total number of pairs: {num_pairs}")
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/worsen_gpt4_combine.py --worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f2000.gpt4o.tem1.0.s42.n1.json_obj.jsonl ]
|
||||
--completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.json \
|
||||
--completion_problem_id_field id --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f2000.json
|
||||
|
||||
>>> python scripts/apps/worsen_gpt4_combine.py \
|
||||
--worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_problem_id_field id \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f100k.json
|
||||
>>> The above code is incorrect.
|
||||
|
||||
FIXME:
|
||||
>>> python scripts/apps/worsen_gpt4_combine.py \
|
||||
--worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.json" \
|
||||
--completion_problem_id_field id \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f100k.fix0708.json
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,129 @@
|
||||
import collections
|
||||
|
||||
from datasets import load_dataset
|
||||
import argparse
|
||||
import json
|
||||
from glob import glob
|
||||
from collections import Counter
|
||||
|
||||
|
||||
def process_completion(completion: str):
|
||||
if "[BEGIN]" in completion or "[ BEGIN ]" in completion:
|
||||
if "[BEGIN]" in completion:
|
||||
s = completion.index("[BEGIN]") + len("[BEGIN]")
|
||||
e = completion.index("[END]")
|
||||
else:
|
||||
s = completion.index("[ BEGIN ]") + len("[ BEGIN ]")
|
||||
e = completion.index("[ END ]")
|
||||
case = completion[s:e].strip()
|
||||
# outputs = case.split("==")[1].strip()
|
||||
outputs = case
|
||||
else:
|
||||
lines = completion.split()
|
||||
lines = [line for line in lines if line.startswith("assert")]
|
||||
if len(lines) == 0:
|
||||
outputs = None
|
||||
else:
|
||||
line = lines[-1]
|
||||
# outputs = line.split("==")[1].strip()
|
||||
outputs = line.strip()
|
||||
return outputs
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
# parser.add_argument("--sanitized", default=False, action="store_true")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
# parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
args = parser.parse_args()
|
||||
|
||||
# print(glob(args.input_file))
|
||||
# return
|
||||
|
||||
# if args.sanitized:
|
||||
# dataset = load_dataset("mbpp", "sanitized", split="test").to_list()
|
||||
# else:
|
||||
# dataset = load_dataset("mbpp", split="test").to_list()
|
||||
# prompt_key = "prompt" if args.sanitized else "text"
|
||||
|
||||
results = dict()
|
||||
missing = 0
|
||||
for file in glob(args.input_file):
|
||||
predictions = [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
for item in predictions:
|
||||
task_id = item["task_id"].split("_")[0]
|
||||
if task_id not in results:
|
||||
results[task_id] = {}
|
||||
|
||||
case_input = item["case_input"].strip()
|
||||
outputs = process_completion(item["completion"])
|
||||
|
||||
# ================ Original version =====================
|
||||
# if outputs is None or outputs.split("==")[0].strip() != case_input:
|
||||
# print(f"Task ID: {task_id}")
|
||||
# print(f"Case Input: {case_input}")
|
||||
# print(f"Outputs: {outputs}")
|
||||
# print("Skip due to mismatch prediction.")
|
||||
# missing += 1
|
||||
|
||||
if outputs is None:
|
||||
print(f"Task ID: {task_id}")
|
||||
print(f"Case Input: {case_input}")
|
||||
print(f"Outputs: {outputs}")
|
||||
print("Skip due to mismatch prediction.")
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
if case_input.lower() != outputs.split("==")[0].strip().lower():
|
||||
print(f"Task ID: {task_id}")
|
||||
print(f"Case Input: {case_input}")
|
||||
print(f"Outputs: {outputs}")
|
||||
print("Skip due to mismatch prediction.")
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
if case_input not in results[task_id]:
|
||||
results[task_id][case_input] = Counter()
|
||||
|
||||
if outputs.split("==")[0].strip() != case_input:
|
||||
tmp = outputs.split("==")
|
||||
outputs = case_input + " == " + tmp[1].strip()
|
||||
|
||||
results[task_id][case_input][outputs] += 1
|
||||
|
||||
outputs = {}
|
||||
cnt = 0
|
||||
for task_id, cases in results.items():
|
||||
outputs[task_id] = []
|
||||
for case_input in cases:
|
||||
times = sorted(cases[case_input].items(), key=lambda x: x[1], reverse=False)
|
||||
if times[0][1] < 2:
|
||||
continue
|
||||
outputs[task_id].append(times[0][0])
|
||||
cnt += 1
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"))
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Total: {cnt}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
"""
|
||||
python scripts/collect_mbpp_test_cases_outputs_sc_v1.0.py --input_file "outputs/mbpp_257_test_case_gen.gpt-4.v1.1.case_inputs.v1.1.outputs.gpt-*.tem1.0.s4[2456].jsonl" --output_file outputs/mbpp_257_test_case_gen.gpt-4.v1.1.case_inputs.v1.1.outputs.sc.0611.json
|
||||
Missing: 1980
|
||||
Total: 842
|
||||
|
||||
Missing: 1939
|
||||
Total: 1420 # Changed version
|
||||
|
||||
|
||||
python scripts/collect_mbpp_test_cases_outputs_sc_v1.0.py --input_file "outputs/mbpp_257_test_case_gen.gpt-4.v1.1.case_inputs.v1.1.outputs.gpt-*.tem1.0.s4[23456].jsonl" --output_file outputs/mbpp_257_test_case_gen.gpt-4.v1.1.case_inputs.v1.1.outputs.sc.0612.json
|
||||
Missing: 2175
|
||||
Total: 1502
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
|
||||
def mbpp_prediction_eval_judge(prediction):
|
||||
if "### The program is" not in prediction:
|
||||
return None
|
||||
|
||||
pattern = r"correct|incorrect"
|
||||
regrex = re.compile(pattern)
|
||||
|
||||
preds = re.findall(regrex, prediction.split("### The program is")[1])
|
||||
if len(preds) == 0 or len(preds) > 1:
|
||||
return None
|
||||
|
||||
if preds[0] == "correct":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def mbpp_judge(file_path, prompt2passed):
|
||||
data = [json.loads(line) for line in open(file_path, "r", encoding="utf8").readlines()]
|
||||
|
||||
missing = 0
|
||||
correct = 0
|
||||
true_samples = 0
|
||||
false_samples = 0
|
||||
for item in data:
|
||||
passed = prompt2passed[item["orig_prompt"]]
|
||||
|
||||
res = mbpp_prediction_eval_judge(item["completion"])
|
||||
if res is None:
|
||||
missing += 1
|
||||
continue
|
||||
if res == passed:
|
||||
correct += 1
|
||||
|
||||
print(f"Correct: {correct}, Missing: {missing}, Total: {len(data)}")
|
||||
print(f"Accuracy: {correct / len(data)}")
|
||||
print(f"Missing rate: {missing / len(data)}")
|
||||
print(f"True samples: {true_samples}, False samples: {false_samples}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Completion Judge')
|
||||
parser.add_argument("--judgement_file", type=str, required=True)
|
||||
parser.add_argument("--prediction_file", type=str, required=True)
|
||||
parser.add_argument("--result_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
results = json.load(open(args.result_file))
|
||||
predictions = json.load(open(args.prediction_file))
|
||||
assert len(results) == len(predictions)
|
||||
prompt2passed = {}
|
||||
for res, pred in zip(results.values(), predictions):
|
||||
prompt2passed[pred["prompt"]] = res[0][1]["passed"]
|
||||
|
||||
mbpp_judge(args.judgement_file, prompt2passed)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
|
||||
def mbpp_prediction_eval_judge(prediction):
|
||||
if "### The program is" not in prediction:
|
||||
return None
|
||||
|
||||
pattern = r"correct|incorrect"
|
||||
regrex = re.compile(pattern)
|
||||
|
||||
preds = re.findall(regrex, prediction.split("### The program is")[1])
|
||||
if len(preds) == 0 or len(preds) > 1:
|
||||
return None
|
||||
|
||||
if preds[0] == "correct":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def mbpp_judge(file_path):
|
||||
data = [json.loads(line) for line in open(file_path, "r", encoding="utf8").readlines()]
|
||||
|
||||
missing = 0
|
||||
correct = 0
|
||||
true_samples = 0
|
||||
false_samples = 0
|
||||
for item in data:
|
||||
# passed = prompt2passed[item["orig_prompt"]]
|
||||
passed = item["passed"]
|
||||
|
||||
res = mbpp_prediction_eval_judge(item["completion"])
|
||||
if res is None:
|
||||
missing += 1
|
||||
continue
|
||||
if res == passed:
|
||||
correct += 1
|
||||
|
||||
print(f"Correct: {correct}, Missing: {missing}, Total: {len(data)}")
|
||||
print(f"Accuracy: {correct / len(data)}")
|
||||
print(f"Missing rate: {missing / len(data)}")
|
||||
print(f"True samples: {true_samples}, False samples: {false_samples}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Completion Judge')
|
||||
parser.add_argument("--prediction_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
mbpp_judge(args.prediction_file)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,182 @@
|
||||
import json
|
||||
import argparse
|
||||
import io
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
|
||||
_vanilla_code_judge_template = """I will show you a programming problem. You will first be given the instruction about the program, and then I will show you the program implementation. Please determine if the completed program is correct or not. You can think step by step before determine its correctness.
|
||||
|
||||
After your reasoning process, **REMEMBER** to use **### The program is {}.** to repeat your judgement, where {} should only be **correct** or **incorrect**.
|
||||
|
||||
Instruction: {}
|
||||
|
||||
Program:
|
||||
```
|
||||
{}
|
||||
```
|
||||
"""
|
||||
|
||||
_running_code_judge_template_0 = """I will show you a programming problem. You will first be given the instruction about the program, and then I will show you the program implementation. Please determine if the completed program is correct or not. Besides, in order to reduce the difficulty, I will also provide you with a version that the same program is equipped with print statements and several test cases to check the intermediate running variables.
|
||||
|
||||
After your reasoning process, **REMEMBER** to use **### The program is {}.** to repeat your judgement, where {} should only be **correct** or **incorrect**.
|
||||
Instruction: {}
|
||||
|
||||
Program:
|
||||
```
|
||||
{}
|
||||
```
|
||||
|
||||
The program with necessary print statements and test cases to check intermediate variables:
|
||||
```
|
||||
{}
|
||||
```
|
||||
|
||||
The corresponding execution results of the program with print statements and test cases:
|
||||
```
|
||||
{}
|
||||
```
|
||||
|
||||
Now, please determine its correctness. You can think step by step, but **REMEMBER** to use **### The program is {}.** to repeat your judgement, where {} should only be **correct** or **incorrect**.
|
||||
"""
|
||||
|
||||
|
||||
_running_code_judge_template_1 = """I will show you a programming problem. You will first be given the instruction about the program, and then I will show you the program implementation. Please determine if the completed program is correct or not. Besides, in order to reduce the difficulty, I will also provide you with a version that the same program is equipped with print statements and several test cases to check the intermediate running variables. You can thus verify the correctness of the program by (1) implement your own program following the instruction; (2) Simulate the running process of your program with the given test cases, and then check if the expected results are consistent with the given values (including those of the intermediate variables).
|
||||
|
||||
After your reasoning process, **REMEMBER** to use **### The program is {}.** to repeat your judgement, where {} should only be **correct** or **incorrect**.
|
||||
|
||||
Instruction: {}
|
||||
|
||||
Program:
|
||||
```
|
||||
{}
|
||||
```
|
||||
|
||||
The program with necessary print statements and test cases to check intermediate variables:
|
||||
```
|
||||
{}
|
||||
```
|
||||
|
||||
The corresponding execution results of the program with print statements and test cases:
|
||||
```
|
||||
{}
|
||||
```
|
||||
|
||||
Now, please determine its correctness. You can think step by step, but **REMEMBER** to use **### The program is {}.** to repeat your judgement, where {} should only be **correct** or **incorrect**.
|
||||
"""
|
||||
|
||||
|
||||
def capture_print_output(code):
|
||||
# Create a StringIO stream to capture output
|
||||
output = io.StringIO()
|
||||
|
||||
# Save the current stdout (standard output)
|
||||
current_stdout = sys.stdout
|
||||
|
||||
# Set the stdout to the StringIO stream
|
||||
sys.stdout = output
|
||||
|
||||
try:
|
||||
exec_globals = {}
|
||||
# Run the function
|
||||
exec(code, exec_globals)
|
||||
except Exception as e:
|
||||
print("Error:", e)
|
||||
finally:
|
||||
# Restore stdout to its original setting
|
||||
sys.stdout = current_stdout
|
||||
|
||||
# Get the content of the StringIO stream
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def _process_item(item):
|
||||
completion = item["completion"]
|
||||
|
||||
program_s = completion.find("<NEW PROGRAM>")
|
||||
if program_s == -1:
|
||||
return False
|
||||
|
||||
program_s += len("<NEW PROGRAM>")
|
||||
program_e = completion.find("</NEW PROGRAM>")
|
||||
program = completion[program_s:program_e].strip()
|
||||
|
||||
test_cases = []
|
||||
case_s = completion.find("<TEST CASE>")
|
||||
while case_s != -1:
|
||||
case_s += len("<TEST CASE>")
|
||||
case_e = completion.find("</TEST CASE>", case_s)
|
||||
if case_e == -1:
|
||||
break
|
||||
case = completion[case_s:case_e].strip()
|
||||
test_cases.append(case)
|
||||
case_s = completion.find("<TEST CASE>", case_e)
|
||||
|
||||
if len(test_cases) == 0:
|
||||
return False
|
||||
|
||||
composed_program = "\n".join([program] + [f"print(\"Execution Results:\", {case})" for case in test_cases])
|
||||
|
||||
outputs = capture_print_output(composed_program)
|
||||
|
||||
return composed_program, outputs
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--prediction_file", type=str)
|
||||
parser.add_argument("--print_prediction_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_version", type=str, default="0")
|
||||
args = parser.parse_args()
|
||||
|
||||
_running_code_judge_template = {
|
||||
"0": _running_code_judge_template_0,
|
||||
"1": _running_code_judge_template_1,
|
||||
}[args.prompt_version]
|
||||
|
||||
predictions = json.load(open(args.prediction_file))
|
||||
task_id2pred = {str(item["task_id"]): item for item in predictions}
|
||||
|
||||
print_predictions = [json.loads(line) for line in open(args.print_prediction_file).readlines()]
|
||||
|
||||
for item in tqdm(print_predictions, total=len(print_predictions)):
|
||||
res = _process_item(item)
|
||||
|
||||
task_id = str(item["task_id"])
|
||||
orig_program = task_id2pred[task_id]["completion"]
|
||||
item["orig_program"] = orig_program
|
||||
item["print_completion"] = item.pop("completion")
|
||||
|
||||
if res:
|
||||
composed_program, outputs = res
|
||||
item["exec_outputs"] = outputs
|
||||
new_prompt = _running_code_judge_template.format("{}", "{}", item["orig_prompt"], orig_program, composed_program, outputs, "{}", "{}")
|
||||
item["prompt"] = new_prompt
|
||||
else:
|
||||
new_prompt = _vanilla_code_judge_template.format("{}", "{}", item["orig_prompt"], orig_program)
|
||||
item["prompt"] = new_prompt
|
||||
|
||||
with open(args.output_file, "w", encoding="utf8") as f:
|
||||
for item in print_predictions:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# outputs = [json.loads(line) for line in open("outputs/oss_magicoder_mbpp257_pred_print_inter_outputs.gpt4.outputs.jsonl").readlines()]
|
||||
#
|
||||
# for item in outputs[:10]:
|
||||
# res = _process_item(item)
|
||||
# if res:
|
||||
# print()
|
||||
# print("+++++++++++++++++++++++++++++++++++++++++++++")
|
||||
# print(res[0])
|
||||
# print("-----------------------------------------------")
|
||||
# print(res[1])
|
||||
# print("++++++++++++++++++++++++++++++++++++++++++++")
|
||||
# print()
|
||||
# else:
|
||||
# print("==========================================")
|
||||
# print(item["completion"])
|
||||
# print("=========================================")
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,267 @@
|
||||
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This code is adapted from OpenAI's release
|
||||
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
|
||||
|
||||
import contextlib
|
||||
import faulthandler
|
||||
import io
|
||||
import multiprocessing
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import tempfile
|
||||
|
||||
|
||||
def check_correctness(check_program, timeout, task_id, completion_id):
|
||||
"""
|
||||
Evaluates the functional correctness of a completion by running the test
|
||||
suite provided in the problem.
|
||||
|
||||
:param completion_id: an optional completion ID so we can match
|
||||
the results later even if execution finishes asynchronously.
|
||||
"""
|
||||
manager = multiprocessing.Manager()
|
||||
result = manager.list()
|
||||
|
||||
p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
|
||||
p.start()
|
||||
p.join(timeout=timeout + 1)
|
||||
if p.is_alive():
|
||||
p.kill()
|
||||
|
||||
if not result:
|
||||
result.append("timed out")
|
||||
|
||||
return dict(
|
||||
task_id=task_id,
|
||||
passed=result[0] == "passed",
|
||||
result=result[0],
|
||||
completion_id=completion_id,
|
||||
)
|
||||
|
||||
|
||||
import io
|
||||
import sys
|
||||
|
||||
|
||||
# Function to capture print output
|
||||
def capture_print_output(code):
|
||||
# Create a StringIO stream to capture output
|
||||
output = io.StringIO()
|
||||
|
||||
# Save the current stdout (standard output)
|
||||
current_stdout = sys.stdout
|
||||
|
||||
# Set the stdout to the StringIO stream
|
||||
sys.stdout = output
|
||||
|
||||
try:
|
||||
# Run the function
|
||||
exec(code)
|
||||
finally:
|
||||
# Restore stdout to its original setting
|
||||
sys.stdout = current_stdout
|
||||
|
||||
# Get the content of the StringIO stream
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
# Capture the output
|
||||
# output = capture_print_output(my_function)
|
||||
# print("Captured output:", output)
|
||||
|
||||
|
||||
def unsafe_execute(check_program, result, timeout):
|
||||
|
||||
with create_tempdir():
|
||||
|
||||
# These system calls are needed when cleaning up tempdir.
|
||||
import os
|
||||
import shutil
|
||||
|
||||
rmtree = shutil.rmtree
|
||||
rmdir = os.rmdir
|
||||
chdir = os.chdir
|
||||
|
||||
# Disable functionalities that can make destructive changes to the test.
|
||||
reliability_guard()
|
||||
|
||||
# Run program.
|
||||
try:
|
||||
exec_globals = {}
|
||||
with swallow_io():
|
||||
with time_limit(timeout):
|
||||
exec(check_program, exec_globals)
|
||||
result.append("passed")
|
||||
except TimeoutException:
|
||||
result.append("timed out")
|
||||
except BaseException as e:
|
||||
result.append(f"failed: {e}")
|
||||
|
||||
# Needed for cleaning up.
|
||||
shutil.rmtree = rmtree
|
||||
os.rmdir = rmdir
|
||||
os.chdir = chdir
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def time_limit(seconds):
|
||||
def signal_handler(signum, frame):
|
||||
raise TimeoutException("Timed out!")
|
||||
|
||||
signal.setitimer(signal.ITIMER_REAL, seconds)
|
||||
signal.signal(signal.SIGALRM, signal_handler)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def swallow_io():
|
||||
stream = WriteOnlyStringIO()
|
||||
with contextlib.redirect_stdout(stream):
|
||||
with contextlib.redirect_stderr(stream):
|
||||
with redirect_stdin(stream):
|
||||
yield
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def create_tempdir():
|
||||
with tempfile.TemporaryDirectory() as dirname:
|
||||
with chdir(dirname):
|
||||
yield dirname
|
||||
|
||||
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class WriteOnlyStringIO(io.StringIO):
|
||||
"""StringIO that throws an exception when it's read from"""
|
||||
|
||||
def read(self, *args, **kwargs):
|
||||
raise OSError
|
||||
|
||||
def readline(self, *args, **kwargs):
|
||||
raise OSError
|
||||
|
||||
def readlines(self, *args, **kwargs):
|
||||
raise OSError
|
||||
|
||||
def readable(self, *args, **kwargs):
|
||||
"""Returns True if the IO object can be read."""
|
||||
return False
|
||||
|
||||
|
||||
class redirect_stdin(contextlib._RedirectStream): # type: ignore
|
||||
_stream = "stdin"
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def chdir(root):
|
||||
if root == ".":
|
||||
yield
|
||||
return
|
||||
cwd = os.getcwd()
|
||||
os.chdir(root)
|
||||
try:
|
||||
yield
|
||||
except BaseException as exc:
|
||||
raise exc
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
|
||||
def reliability_guard(maximum_memory_bytes=None):
|
||||
"""
|
||||
This disables various destructive functions and prevents the generated code
|
||||
from interfering with the test (e.g. fork bomb, killing other processes,
|
||||
removing filesystem files, etc.)
|
||||
|
||||
WARNING
|
||||
This function is NOT a security sandbox. Untrusted code, including, model-
|
||||
generated code, should not be blindly executed outside of one. See the
|
||||
Codex paper for more information about OpenAI's code sandbox, and proceed
|
||||
with caution.
|
||||
"""
|
||||
|
||||
if maximum_memory_bytes is not None:
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
|
||||
resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
|
||||
if not platform.uname().system == "Darwin":
|
||||
resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
|
||||
|
||||
faulthandler.disable()
|
||||
|
||||
import builtins
|
||||
|
||||
builtins.exit = None
|
||||
builtins.quit = None
|
||||
|
||||
import os
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
|
||||
os.kill = None
|
||||
os.system = None
|
||||
os.putenv = None
|
||||
os.remove = None
|
||||
os.removedirs = None
|
||||
os.rmdir = None
|
||||
os.fchdir = None
|
||||
os.setuid = None
|
||||
os.fork = None
|
||||
os.forkpty = None
|
||||
os.killpg = None
|
||||
os.rename = None
|
||||
os.renames = None
|
||||
os.truncate = None
|
||||
os.replace = None
|
||||
os.unlink = None
|
||||
os.fchmod = None
|
||||
os.fchown = None
|
||||
os.chmod = None
|
||||
os.chown = None
|
||||
os.chroot = None
|
||||
os.fchdir = None
|
||||
os.lchflags = None
|
||||
os.lchmod = None
|
||||
os.lchown = None
|
||||
os.getcwd = None
|
||||
os.chdir = None
|
||||
|
||||
import shutil
|
||||
|
||||
shutil.rmtree = None
|
||||
shutil.move = None
|
||||
shutil.chown = None
|
||||
|
||||
import subprocess
|
||||
|
||||
subprocess.Popen = None # type: ignore
|
||||
|
||||
__builtins__["help"] = None
|
||||
|
||||
import sys
|
||||
|
||||
sys.modules["ipdb"] = None
|
||||
sys.modules["joblib"] = None
|
||||
sys.modules["resource"] = None
|
||||
sys.modules["psutil"] = None
|
||||
sys.modules["tkinter"] = None
|
||||
@@ -0,0 +1,17 @@
|
||||
cp=$1
|
||||
cn=$2
|
||||
exp_name=$3
|
||||
n_gpus=$4
|
||||
split_id_offset=$5
|
||||
|
||||
for ((i=6;i<=$#;i++)); do
|
||||
step=${!i}
|
||||
echo $step
|
||||
for j in {0..$(($n_gpus-1))}; do
|
||||
TOKENIZERS_PARALLELISM=false CUDA_VISIBLE_DEVICES=$j python vllm_inference.py eval_sub_path=checkpoint-$step output_path_prefix=/mnt/fangkai_blob/reward_modeling/ fp16_bfloat16=False split_id=$(($j+$split_id_offset)) exp_name=$exp_name -cp $cp -cn $cn &
|
||||
done
|
||||
wait
|
||||
echo "Step-$step Finished"
|
||||
done
|
||||
|
||||
echo "Finished"
|
||||
@@ -0,0 +1,82 @@
|
||||
import argparse
|
||||
import collections
|
||||
import os.path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from glob import glob
|
||||
import json
|
||||
|
||||
|
||||
def majority_voting_predict(preds):
|
||||
if isinstance(preds, str):
|
||||
return preds
|
||||
|
||||
preds = [pred for pred in preds if pred]
|
||||
if len(preds) == 0:
|
||||
return ""
|
||||
|
||||
assert isinstance(preds, list)
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
pred, freq = collections.Counter(tmp).most_common(1)[0]
|
||||
pred = eval(pred)
|
||||
elif isinstance(preds[0], str):
|
||||
pred, freq = collections.Counter(preds).most_common(1)[0]
|
||||
else:
|
||||
print(f"Unknown type {type(preds[0])}")
|
||||
pred = ""
|
||||
freq = 0
|
||||
return pred, freq
|
||||
|
||||
|
||||
def plot_histogram(data, bins=10, x_label="Value", y_label="Frequency", title="Histogram", output_file="histogram.png"):
|
||||
plt.hist(data, bins=bins, edgecolor='black', alpha=0.7)
|
||||
plt.xlabel(x_label)
|
||||
plt.ylabel(y_label)
|
||||
plt.title(title)
|
||||
plt.grid(True)
|
||||
# plt.show()
|
||||
plt.savefig(output_file)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
responses = json.load(open(args.input_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.input_file):
|
||||
responses += json.load(open(file))
|
||||
freq = {'correct': 0, 'incorrect': 0}
|
||||
num = {'correct': 0, 'incorrect': 0}
|
||||
correct_freqs = []
|
||||
incorrect_freqs = []
|
||||
for item in responses:
|
||||
sc_pred, f = majority_voting_predict(item["pred"])
|
||||
if item["sc_res"]:
|
||||
num['correct'] += 1
|
||||
freq['correct'] += f
|
||||
correct_freqs.append(f)
|
||||
else:
|
||||
num['incorrect'] += 1
|
||||
freq['incorrect'] += f
|
||||
incorrect_freqs.append(f)
|
||||
|
||||
print("Correct: ", num['correct'])
|
||||
print("Incorrect: ", num['incorrect'])
|
||||
print("Correct freq: ", freq['correct'])
|
||||
print("Incorrect freq: ", freq['incorrect'])
|
||||
print(f"Correct sc avg freq: {freq['correct'] / num['correct']}")
|
||||
print(f"Incorrect sc avg freq: {freq['incorrect'] / num['incorrect']}")
|
||||
|
||||
plot_histogram(correct_freqs, bins=50, title="Correct SC Prediction Frequency", output_file="correct_sc_freq.png")
|
||||
plot_histogram(incorrect_freqs, bins=50, title="Incorrect SC Prediction Frequency", output_file="incorrect_sc_freq.png")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,228 @@
|
||||
import collections
|
||||
import json
|
||||
import os.path
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
import random
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def sample_step(response: str, upper_step_ratio: float, sample_ratio: float):
|
||||
orig_lines = response.split("\n")
|
||||
lines = [(i, line) for i, line in enumerate(orig_lines)]
|
||||
lines = [(i, line) for i, line in lines if line.strip()]
|
||||
|
||||
if len(lines) < 5:
|
||||
return []
|
||||
|
||||
upper_step = int(len(lines) * upper_step_ratio)
|
||||
if upper_step == 0:
|
||||
return []
|
||||
|
||||
sample_step_num = int(upper_step * sample_ratio)
|
||||
if sample_step_num == 0:
|
||||
return []
|
||||
|
||||
sample_steps = random.sample(lines[:upper_step], sample_step_num)
|
||||
if len(sample_steps) == 0:
|
||||
return []
|
||||
|
||||
step_prefixes = []
|
||||
for i, line in sample_steps:
|
||||
step_prefixes.append("\n".join(orig_lines[:(i + 1)]))
|
||||
|
||||
return step_prefixes
|
||||
|
||||
|
||||
def get_pred_set(preds):
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = set(tmp)
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = set(preds)
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return tmp
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--upper_step_ratio", type=float, default=0.6)
|
||||
parser.add_argument("--sample_ratio", type=float, default=0.3)
|
||||
parser.add_argument("--filter_all_correct", action="store_true", default=False)
|
||||
parser.add_argument("--filter_all_same", action="store_true", default=False)
|
||||
parser.add_argument("--sample_over_p", default=-1, type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in sorted(glob(args.input_file)):
|
||||
if ".metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
# # First use self-consistency to construct pseudo labels
|
||||
# for item in data:
|
||||
# all_preds = collections.Counter()
|
||||
# for pred in item["pred"]:
|
||||
# if isinstance(pred, list):
|
||||
# all_preds.update(pred)
|
||||
# else:
|
||||
# all_preds[pred] += 1
|
||||
#
|
||||
# pseudo_label = all_preds.most_common(1)[0][0]
|
||||
# item["sc_label_0"] = pseudo_label
|
||||
|
||||
outputs = []
|
||||
num = 0
|
||||
for item in tqdm(data):
|
||||
if args.filter_all_correct and all(item["res"]):
|
||||
continue
|
||||
if args.filter_all_same and len(set(item["pred"])) == 1:
|
||||
continue
|
||||
|
||||
prefixes = []
|
||||
prefix_ids = []
|
||||
if not item["response"]:
|
||||
item["prefix"] = []
|
||||
continue
|
||||
|
||||
for resp_id, resp in enumerate(item["response"]):
|
||||
response_prefixes = sample_step(resp, args.upper_step_ratio, args.sample_ratio)
|
||||
prefixes.extend(response_prefixes)
|
||||
prefix_ids.extend([f"{item['id']}_{resp_id}_{i}" for i in range(len(response_prefixes))])
|
||||
|
||||
if args.sample_over_p > 0:
|
||||
if len(prefixes) > args.sample_over_p:
|
||||
prefixes = random.sample(prefixes, args.sample_over_p)
|
||||
prefix_ids = random.sample(prefix_ids, args.sample_over_p)
|
||||
item["prefix"] = prefixes
|
||||
item["prefix_id"] = prefix_ids
|
||||
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
item.pop("res")
|
||||
item.pop("sc_pred")
|
||||
item.pop("sc_res")
|
||||
|
||||
outputs.append(item)
|
||||
num += len(prefixes)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
print(f"Number of prefixes: {num}")
|
||||
print(len(outputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python ~/gpt-chat-examples/scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "math.test.v1.1.0shot.n10.tem1.0.p0.9.8-of-?.json" \
|
||||
--output_file math.test.v1.1.0shot.n10.tem1.0.p0.9.prefix.upper0.6.r0.3.json --upper_step_ratio 0.6 --sample_ratio 0.3
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py --input_file "../msranlpintern/share/models/deepseek-math-7b-instruct/meta_math/sub_math.cot.train.0shot.n10.tem1.0.p0.9.v1.0.?-of-24.json" --output_file "../msranlpintern/share/models/deepseek-math-7b-instruct/meta_math/sub_math.cot.train.0shot.n10.tem1.0.p0.9.v1.0.upper0.7.r0.3.inter_step.filter_all_true.json" --upper_step_ra
|
||||
tio 0.7 --sample_ratio 0.3 --filter_all_correct
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.[0-9]*-of-64.json" \
|
||||
--output_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.upper0.7.r0.3.inter_step.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.[0-9]*-of-64.json" \
|
||||
--output_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/split-0-of-11/train.330k.boxed.v1.0.0-of-11.0shot.n20.tem1.0.p0.9.upper0.7.r0.3.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.?-of-8.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same
|
||||
|
||||
Number of prefixes: 19521786
|
||||
309876
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.?-of-8.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 3098719
|
||||
309876
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-32.json" \
|
||||
--output_file "../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json" \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 2879063
|
||||
287913
|
||||
|
||||
############################################################ ITERATION 1 #######################################################
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 2389255
|
||||
238928
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 10
|
||||
|
||||
Number of prefixes: 2389255
|
||||
238928
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample32.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 32
|
||||
|
||||
Number of prefixes: 7512389
|
||||
238928
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample32.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 32
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-[0-9]-of-10/cot.de_con.[0-9]-of-10.n8.tem1.0.p1.0.*-of-16.s0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/cot.de_con.n8.tem1.0.p1.0.s0.upper0.7.r0.3.sample16.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 16
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" \
|
||||
--output_file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample16.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 16
|
||||
|
||||
Number of prefixes: 5053462
|
||||
345626
|
||||
|
||||
|
||||
>>> python scripts/math/deepseek_math_sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.*-of-8.s0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/cot.de_con.n8.tem1.0.p1.0.orig_0-of-8.s0.upper0.7.r0.3.sample32.filter_same.json \
|
||||
--upper_step_ratio 0.7 --sample_ratio 0.3 --filter_all_same --sample_over_p 32
|
||||
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
import collections
|
||||
from tqdm import tqdm
|
||||
|
||||
"""
|
||||
We sampled some intermediate steps from single response, and each intermediate state will be calculated with a value by counting the reached outcomes.
|
||||
We can rank the intermediate steps according to their distance to the origin, i.e., the prompt.
|
||||
The problem is how to adjust the value based on the values of its preceding states:
|
||||
|
||||
|
||||
Remember that if the outcome label is accurate, we can directly use the expected value as the reward since it can well indicate its importance.
|
||||
However, under self-consistency setting, we should always assume that, if the outcome is incorrect, then we should find the most distant state from the origin,
|
||||
and remains the largest probability that it is still possible to reach the correct answer.
|
||||
|
||||
TO maintain the prefixes with the least confidence over the pseudo label.
|
||||
|
||||
|
||||
TODO: Deepseek's extraction utils always return a list for MATH questions. Think about how to process this (for self-consistency).
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--top_k", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
|
||||
item_id2prefixes = collections.defaultdict(list)
|
||||
for i, prefix in tqdm(enumerate(data)):
|
||||
tmp = prefix["prefix_id"].split("_")
|
||||
# resp_id = int(resp_id) # -2
|
||||
# prefix_id = int(prefix_id) # -1
|
||||
item_id = "_".join(tmp[:-2])
|
||||
|
||||
last_iter_pseudo_label = prefix["sc_label_0"]
|
||||
|
||||
cnt = collections.Counter()
|
||||
for pred in prefix["pred"]:
|
||||
if isinstance(pred, list):
|
||||
cnt.update(pred)
|
||||
else:
|
||||
cnt[pred] += 1
|
||||
|
||||
if last_iter_pseudo_label not in cnt:
|
||||
v = 0
|
||||
else:
|
||||
v = cnt[last_iter_pseudo_label]
|
||||
|
||||
item_id2prefixes[item_id].append((i, v))
|
||||
|
||||
print(len(item_id2prefixes))
|
||||
|
||||
outputs = []
|
||||
for item_id, prefixes in item_id2prefixes.items():
|
||||
prefixes.sort(key=lambda x: x[1]) # Ascending order
|
||||
for i, v in prefixes[:args.top_k]:
|
||||
outputs.append(data[i])
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
from data.deepseek_math_utils import eval_script
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file_format", type=str)
|
||||
parser.add_argument("--seed_list", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = []
|
||||
for seed in eval(args.seed_list):
|
||||
_file = args.input_file_format.format(seed)
|
||||
if os.path.exists(_file):
|
||||
print(f"Loading: {_file}")
|
||||
data.extend(json.load(open(_file)))
|
||||
else:
|
||||
print(f"File not found: {_file}")
|
||||
|
||||
id2data = {}
|
||||
for item in tqdm(data):
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
|
||||
id2data[item["id"]]["response"] = [id2data[item["id"]]["response"]]
|
||||
id2data[item["id"]]["pred"] = [id2data[item["id"]]["pred"]]
|
||||
id2data[item["id"]]["res"] = [id2data[item["id"]]["res"]]
|
||||
else:
|
||||
assert item["text"] == id2data[item["id"]]["text"]
|
||||
|
||||
id2data[item["id"]]["response"].append(item["response"])
|
||||
id2data[item["id"]]["pred"].append(item["pred"])
|
||||
id2data[item["id"]]["res"].append(item["res"])
|
||||
|
||||
data = list(id2data.values())
|
||||
pass_at_k = 0
|
||||
maj_at_k = 0
|
||||
for item in tqdm(data):
|
||||
sc_pred = majority_voting_predict(item["pred"])
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
item["sc_pred"] = sc_pred
|
||||
item["sc_res"] = sc_res
|
||||
|
||||
if any(item["res"]):
|
||||
pass_at_k += 1
|
||||
if sc_res:
|
||||
maj_at_k += 1
|
||||
|
||||
print("Pass at k: ", pass_at_k / len(data))
|
||||
print("Maj at k: ", maj_at_k / len(data))
|
||||
json.dump(data, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
import collections
|
||||
import json
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
from glob import glob
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.deepseek_math_utils import eval_script, answer_extraction
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
|
||||
|
||||
def pred2str(pred):
|
||||
if isinstance(pred, str):
|
||||
return pred
|
||||
if isinstance(pred, list):
|
||||
pred = sorted(pred)
|
||||
pred = str(pred)
|
||||
return pred
|
||||
raise ValueError(f"Unknown type {type(pred)}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
parser.add_argument("--output_file")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
cnt = 0
|
||||
pass_at_k = 0
|
||||
sc = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], list):
|
||||
preds = item["pred"]
|
||||
else:
|
||||
preds = [item["pred"]]
|
||||
|
||||
if "res" not in item:
|
||||
mul_pass = 0
|
||||
if len(preds) > 0:
|
||||
res = []
|
||||
for pred in preds:
|
||||
res.append(eval_script.eval_math({"prediction": pred, "answer": item["label"]}))
|
||||
if any(res):
|
||||
mul_pass = 1
|
||||
else:
|
||||
res = []
|
||||
|
||||
item["pass_at_k"] = mul_pass
|
||||
|
||||
if len(preds) == 1:
|
||||
res = res[0]
|
||||
item["res"] = res
|
||||
|
||||
if not isinstance(item["res"], list):
|
||||
res = [item["res"]]
|
||||
else:
|
||||
res = item["res"]
|
||||
|
||||
if any(res):
|
||||
pass_at_k += 1
|
||||
if res[0]:
|
||||
cnt += 1
|
||||
|
||||
# str_preds = [pred2str(item) for item in preds]
|
||||
# counter = collections.Counter(str_preds)
|
||||
# sc_pred = eval(counter.most_common(1)[0][0])
|
||||
# sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
|
||||
# if "sc_res" in item:
|
||||
# sc_res = item["sc_res"]
|
||||
# else:
|
||||
# preds = [x for x in preds if x]
|
||||
# if len(preds):
|
||||
# sc_pred = majority_voting_predict(preds)
|
||||
# try:
|
||||
# sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
# except Exception as e:
|
||||
# print(f"Error in {item['id']} during evaluation: {e}")
|
||||
# sc_res = False
|
||||
# else:
|
||||
# sc_res = False
|
||||
# if sc_res:
|
||||
# sc += 1
|
||||
|
||||
print(f"Pass at k: {pass_at_k}/{len(data)} = {pass_at_k / len(data)}")
|
||||
print(f"Correct at k: {cnt}/{len(data)} = {cnt / len(data)}")
|
||||
print(f"Self-consistency: {sc}/{len(data)} = {sc / len(data)}")
|
||||
if args.output_file:
|
||||
json.dump(data, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
path=$1
|
||||
|
||||
shift 1 # Shift the first 5 arguments, so $1 now refers to the 6th argument
|
||||
|
||||
for step in "$@"; do
|
||||
echo "Merging predictions for step ================================== $step"
|
||||
python scripts/math_scale/merge_dp_predictions.py --input_file "$path/mwpbench/checkpoint-$step/train_wo_gsm.2k.v1.0.0shot.n1.tem0.0.p1.0.?-of-8.json" \
|
||||
--output_file "$path/mwpbench/checkpoint-$step/train_wo_gsm.2k.v1.0.0shot.n1.tem0.0.p1.0.json"
|
||||
python scripts/math/merge_dp_predictions.py --input_file "$path/math/checkpoint-$step/math.test.v1.1.0shot.n1.tem0.0.p1.0.8-of-?.json" \
|
||||
--output_file "$path/math/checkpoint-$step/math.test.v1.1.0shot.n1.tem0.0.p1.0.json"
|
||||
echo "Done merging predictions for step ================================== $step"
|
||||
echo
|
||||
done
|
||||
@@ -0,0 +1,83 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import collections
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.deepseek_math_utils import eval_script, answer_extraction
|
||||
|
||||
|
||||
def pred2str(pred):
|
||||
if isinstance(pred, str):
|
||||
return pred
|
||||
if isinstance(pred, list):
|
||||
pred = sorted(pred)
|
||||
pred = str(pred)
|
||||
return pred
|
||||
raise ValueError(f"Unknown type {type(pred)}")
|
||||
|
||||
|
||||
def load_data(file_path):
|
||||
if not file_path:
|
||||
return []
|
||||
if os.path.exists(file_path):
|
||||
data = json.load(open(file_path))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
data += json.load(open(file))
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file1", type=str)
|
||||
parser.add_argument("--input_file2", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--n", type=int, default=-1)
|
||||
args = parser.parse_args()
|
||||
|
||||
data1 = load_data(args.input_file1)
|
||||
data2 = load_data(args.input_file2)
|
||||
|
||||
item_id2preds = {}
|
||||
sc = 0
|
||||
for item in tqdm(data1 + data2):
|
||||
if "prefix_id" in item:
|
||||
prefix_id = item["prefix_id"]
|
||||
tmp = prefix_id.split("_")
|
||||
item_id = "_".join(tmp[:-2])
|
||||
else:
|
||||
item_id = item["id"]
|
||||
|
||||
if item_id not in item_id2preds:
|
||||
item_id2preds[item_id] = {
|
||||
"response": [],
|
||||
"pred": [],
|
||||
"label": item["label"],
|
||||
}
|
||||
|
||||
item_id2preds[item_id]["response"].extend(item["response"])
|
||||
item_id2preds[item_id]["pred"].extend(item["pred"])
|
||||
|
||||
for item, responses in tqdm(item_id2preds.items()):
|
||||
preds = responses["pred"]
|
||||
if args.n != -1:
|
||||
preds = preds[:args.n]
|
||||
str_preds = [pred2str(item) for item in preds]
|
||||
counter = collections.Counter(str_preds)
|
||||
sc_pred = eval(counter.most_common(1)[0][0])
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": responses["label"]})
|
||||
if sc_res:
|
||||
sc += 1
|
||||
|
||||
print(f"Self-consistency: {sc}/{len(item_id2preds)} = {sc / len(item_id2preds)}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file_format", type=str)
|
||||
parser.add_argument("--seed_list", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
seed_list = eval(args.seed_list)
|
||||
rewards = []
|
||||
for seed in seed_list:
|
||||
# _file = args.response_file_format.format(seed)
|
||||
_file = args.response_file_format.replace("[[seed]]", str(seed))
|
||||
if os.path.exists(_file):
|
||||
print(f"Loading: {_file}")
|
||||
data = json.load(open(_file))
|
||||
for item in data:
|
||||
item["index"] = f"{item['index']}_{seed}"
|
||||
rewards.extend(data)
|
||||
else:
|
||||
print(f"File not found: {_file}")
|
||||
|
||||
json.dump(rewards, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from functools import partial
|
||||
import torch
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
from data.deepseek_math_utils import eval_script
|
||||
|
||||
|
||||
def load_rewards(reward_file):
|
||||
if os.path.exists(reward_file):
|
||||
rewards = json.load(open(reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob(reward_file):
|
||||
print(file)
|
||||
rewards.extend(json.load(open(file)))
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def _init(id2reward):
|
||||
global _id2reward
|
||||
_id2reward = id2reward
|
||||
|
||||
|
||||
def _worker(item, sc_top_k=None):
|
||||
sorted_results = []
|
||||
if not item["response"] or not item["pred"] or not item["res"]:
|
||||
return {
|
||||
"missing": 1,
|
||||
"reward_missing": 0,
|
||||
"pred_missing": 0,
|
||||
"seq_too_long": 0,
|
||||
"sorted_results": [],
|
||||
"sc_res": False
|
||||
}
|
||||
reward_missing = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
resp_id = f"{item['id']}_{i}"
|
||||
if resp_id not in _id2reward:
|
||||
reward_missing += 1
|
||||
continue
|
||||
if not pred:
|
||||
pred_missing += 1
|
||||
continue
|
||||
|
||||
reward = _id2reward[resp_id]
|
||||
sorted_results.append((resp, pred, r, reward))
|
||||
sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
|
||||
sc_top_k_res = {}
|
||||
if sc_top_k and sorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in sorted_results[:k]]
|
||||
sc_pred = majority_voting_predict(preds)
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
sc_top_k_res[k] = sc_res
|
||||
|
||||
return {
|
||||
"missing": 0,
|
||||
"reward_missing": reward_missing,
|
||||
"pred_missing": pred_missing,
|
||||
"seq_too_long": seq_too_long,
|
||||
"sorted_results": sorted_results,
|
||||
"sc_res": item["sc_res"],
|
||||
"sc_top_k_res": sc_top_k_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.response_file):
|
||||
responses = json.load(open(args.response_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.response_file):
|
||||
print(file)
|
||||
responses.extend(json.load(open(file)))
|
||||
|
||||
rewards = load_rewards(args.reward_file)
|
||||
id2reward = {item["index"]: item["reward"][1] for item in rewards}
|
||||
|
||||
_k = [1, 3, 5]
|
||||
orm_pass_at_k = {k: 0 for k in _k}
|
||||
missing = 0
|
||||
missing_reward = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
sc_cnt = 0
|
||||
# for item in tqdm(responses):
|
||||
# sorted_results = []
|
||||
# if not item["response"] or not item["pred"] or not item["res"]:
|
||||
# missing += 1
|
||||
# continue
|
||||
# for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
# resp_id = f"{item['id']}_{i}"
|
||||
# if resp_id not in id2reward:
|
||||
# missing_reward += 1
|
||||
# continue
|
||||
#
|
||||
# reward = id2reward[resp_id]
|
||||
# sorted_results.append((resp, pred, r, reward))
|
||||
#
|
||||
# if not sorted_results:
|
||||
# continue
|
||||
# sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
# for k in _k:
|
||||
# if any([r[2] for r in sorted_results[:k]]):
|
||||
# orm_pass_at_k[k] += 1
|
||||
#
|
||||
# if item["sc_res"]:
|
||||
# sc_cnt += 1
|
||||
with Pool(args.num_workers, initializer=_init, initargs=(id2reward,)) as pool:
|
||||
annotate = partial(_worker, sc_top_k=(5, 10))
|
||||
results = list(tqdm(pool.imap_unordered(annotate, responses), total=len(responses)))
|
||||
|
||||
sc_top_k = {k: 0 for k in (5, 10)}
|
||||
for item in results:
|
||||
missing += item["missing"]
|
||||
missing_reward += item["reward_missing"]
|
||||
pred_missing += item["pred_missing"]
|
||||
seq_too_long += item["seq_too_long"]
|
||||
sorted_results = item["sorted_results"]
|
||||
|
||||
if item["sc_res"]:
|
||||
sc_cnt += 1
|
||||
|
||||
if not sorted_results:
|
||||
continue
|
||||
|
||||
# ultimate_results.append(sorted_results)
|
||||
for k in _k:
|
||||
if any([r[2] for r in sorted_results[:k]]):
|
||||
orm_pass_at_k[k] += 1
|
||||
|
||||
for k, v in item["sc_top_k_res"].items():
|
||||
if v:
|
||||
sc_top_k[k] += 1
|
||||
|
||||
print(f"Total: {len(responses)}")
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing reward: {missing_reward}")
|
||||
print(f"Missing pred: {pred_missing}")
|
||||
print(f"Seq too long: {seq_too_long}")
|
||||
print(f"SC: {sc_cnt}")
|
||||
for k, v in orm_pass_at_k.items():
|
||||
print(f"PRM pass at {k}: {v}")
|
||||
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
for k, v in sc_top_k.items():
|
||||
print(f"SC pass at {k}: {v}")
|
||||
print(f"SC pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,272 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
from multiprocessing import Pool
|
||||
import torch
|
||||
import sys
|
||||
from functools import partial
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.openai_api_callback import majority_voting_predict
|
||||
from data.deepseek_math_utils import eval_script
|
||||
|
||||
|
||||
def load_rewards(reward_file, re_index):
|
||||
if os.path.exists(reward_file):
|
||||
rewards = json.load(open(reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for i, file in enumerate(sorted(glob(reward_file))):
|
||||
print(file)
|
||||
sub_rewards = json.load(open(file))
|
||||
if re_index:
|
||||
for item in sub_rewards:
|
||||
item["index"] = f"{item['index']}_{i}"
|
||||
rewards.extend(sub_rewards)
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def reward_reduction(ending_logits, reduction: str = "min", norm: bool = True):
|
||||
if norm:
|
||||
ending_logits = torch.tensor(ending_logits)
|
||||
ending_probs = torch.softmax(ending_logits, dim=-1).tolist()
|
||||
step_rewards = [step[1] for step in ending_probs]
|
||||
else:
|
||||
step_rewards = [step[1] for step in ending_logits]
|
||||
if reduction == "min":
|
||||
reward = min(step_rewards)
|
||||
elif reduction == "product":
|
||||
reward = 1
|
||||
for prob in step_rewards:
|
||||
reward *= prob
|
||||
elif reduction == "sum":
|
||||
reward = sum(step_rewards)
|
||||
else:
|
||||
raise ValueError(f"Invalid reduction method: {reduction}")
|
||||
return reward
|
||||
|
||||
|
||||
def weighted_majority_voting_predict(preds, weights):
|
||||
pred2weight = {}
|
||||
for pred, weight in zip(preds, weights):
|
||||
if pred not in pred2weight:
|
||||
pred2weight[pred] = 0
|
||||
pred2weight[pred] += weight
|
||||
return max(pred2weight, key=pred2weight.get)
|
||||
|
||||
|
||||
def _init(id2reward):
|
||||
global _id2reward
|
||||
_id2reward = id2reward
|
||||
|
||||
|
||||
def _worker(item, reduction, norm, sc_top_k=None):
|
||||
if not item["response"] or not item["pred"] or not item["res"]:
|
||||
return {
|
||||
"missing": 1,
|
||||
"reward_missing": 0,
|
||||
"pred_missing": 0,
|
||||
"seq_too_long": 0,
|
||||
"sorted_results": [],
|
||||
"sc_res": False
|
||||
}
|
||||
unsorted_results = []
|
||||
reward_missing = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
resp_id = f"{item['id']}_{i}"
|
||||
if resp_id not in _id2reward:
|
||||
reward_missing += 1
|
||||
continue
|
||||
if not pred:
|
||||
pred_missing += 1
|
||||
continue
|
||||
|
||||
process_rewards = _id2reward[resp_id]
|
||||
if len(process_rewards["ending_logits"]) == 0:
|
||||
seq_too_long += 1
|
||||
continue
|
||||
|
||||
assert resp == process_rewards["response"], f"{resp} \n\n {process_rewards['response']} \n\n ========="
|
||||
|
||||
reward = reward_reduction(process_rewards["ending_logits"], reduction, norm)
|
||||
unsorted_results.append((resp, pred, r, reward))
|
||||
sorted_results = sorted(unsorted_results, key=lambda x: x[-1], reverse=True)
|
||||
|
||||
sc_top_k_res = {}
|
||||
if sc_top_k and sorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in sorted_results[:k]]
|
||||
sc_pred = majority_voting_predict(preds)
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
sc_top_k_res[k] = sc_res
|
||||
|
||||
sc_k_res = {}
|
||||
if sc_top_k and unsorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in unsorted_results[:k]]
|
||||
sc_pred = majority_voting_predict(preds)
|
||||
if sc_pred != "":
|
||||
sc_res = eval_script.eval_math({"prediction": sc_pred, "answer": item["label"]})
|
||||
else:
|
||||
sc_res = False
|
||||
sc_k_res[k] = sc_res
|
||||
|
||||
weighted_best_of_k = {}
|
||||
if sc_top_k and unsorted_results:
|
||||
for k in sc_top_k:
|
||||
preds = [r[1] for r in unsorted_results[:k]]
|
||||
weights = torch.softmax(torch.tensor([r[-1] for r in unsorted_results]), dim=0).tolist()
|
||||
best_of_k_pred = weighted_majority_voting_predict(preds, weights)
|
||||
if best_of_k_pred != "":
|
||||
best_of_k_res = eval_script.eval_math({"prediction": best_of_k_pred, "answer": item["label"]})
|
||||
else:
|
||||
best_of_k_res = False
|
||||
weighted_best_of_k[k] = best_of_k_res
|
||||
|
||||
return {
|
||||
"missing": 0,
|
||||
"reward_missing": reward_missing,
|
||||
"pred_missing": pred_missing,
|
||||
"seq_too_long": seq_too_long,
|
||||
"sorted_results": sorted_results,
|
||||
"sc_res": item["sc_res"],
|
||||
"sc_top_k_res": sc_top_k_res,
|
||||
"weighted_best_of_k": weighted_best_of_k,
|
||||
"sc_k_res": sc_k_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
parser.add_argument("--reduction", type=str, default="min")
|
||||
parser.add_argument("--raw_logits", action="store_true", default=False)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--re_index", action="store_true", default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.response_file):
|
||||
responses = json.load(open(args.response_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.response_file):
|
||||
print(file)
|
||||
responses.extend(json.load(open(file)))
|
||||
|
||||
print(len(responses[0]["response"]), len(responses[0]["pred"]), len(responses[0]["res"]))
|
||||
print(responses[0]["id"])
|
||||
|
||||
rewards = load_rewards(args.reward_file, args.re_index)
|
||||
print(rewards[0]['index'])
|
||||
id2reward = {item["index"]: item for item in rewards}
|
||||
|
||||
# _k = [1, 3, 5]
|
||||
_k = [3, 5, 4, 8, 16, 32, 64, 128]
|
||||
prm_pass_at_k = {k: 0 for k in _k}
|
||||
missing = 0
|
||||
missing_reward = 0
|
||||
pred_missing = 0
|
||||
seq_too_long = 0
|
||||
sc_cnt = 0
|
||||
ultimate_results = []
|
||||
# for item in tqdm(responses):
|
||||
# sorted_results = []
|
||||
# if not item["response"] or not item["pred"] or not item["res"]:
|
||||
# missing += 1
|
||||
# continue
|
||||
# for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
# resp_id = f"{item['id']}_{i}"
|
||||
# if resp_id not in id2reward:
|
||||
# missing_reward += 1
|
||||
# continue
|
||||
# process_rewards = id2reward[resp_id]
|
||||
# if len(process_rewards["ending_logits"]) == 0:
|
||||
# continue
|
||||
#
|
||||
# assert resp == process_rewards["response"], f"{resp} \n\n {process_rewards['response']} \n\n ========="
|
||||
#
|
||||
# reward = reward_reduction(process_rewards["ending_logits"], args.reduction, not args.raw_logits)
|
||||
# sorted_results.append((resp, pred, r, reward))
|
||||
#
|
||||
# if not sorted_results:
|
||||
# continue
|
||||
# sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
# ultimate_results.append(sorted_results)
|
||||
# for k in _k:
|
||||
# if any([r[2] for r in sorted_results[:k]]):
|
||||
# prm_pass_at_k[k] += 1
|
||||
#
|
||||
# if item["sc_res"]:
|
||||
# sc_cnt += 1
|
||||
with Pool(args.num_workers, initializer=_init, initargs=(id2reward,)) as pool:
|
||||
annotate = partial(_worker, reduction=args.reduction, norm=(not args.raw_logits), sc_top_k=_k)
|
||||
results = list(tqdm(pool.imap_unordered(annotate, responses), total=len(responses)))
|
||||
|
||||
sc_top_k = {k: 0 for k in _k}
|
||||
weighted_best_of_k = {k: 0 for k in _k}
|
||||
sc_k = {k: 0 for k in _k}
|
||||
for item in results:
|
||||
missing += item["missing"]
|
||||
missing_reward += item["reward_missing"]
|
||||
pred_missing += item["pred_missing"]
|
||||
seq_too_long += item["seq_too_long"]
|
||||
sorted_results = item["sorted_results"]
|
||||
|
||||
if item["sc_res"]:
|
||||
sc_cnt += 1
|
||||
|
||||
if not sorted_results:
|
||||
continue
|
||||
|
||||
ultimate_results.append(sorted_results)
|
||||
for k in _k:
|
||||
if any([r[2] for r in sorted_results[:k]]):
|
||||
prm_pass_at_k[k] += 1
|
||||
|
||||
for k, v in item["sc_top_k_res"].items():
|
||||
if v:
|
||||
sc_top_k[k] += 1
|
||||
|
||||
for k, v in item["weighted_best_of_k"].items():
|
||||
if v:
|
||||
weighted_best_of_k[k] += 1
|
||||
|
||||
for k, v in item["sc_k_res"].items():
|
||||
if v:
|
||||
sc_k[k] += 1
|
||||
|
||||
print(f"Total: {len(responses)}")
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing reward: {missing_reward}")
|
||||
print(f"Missing pred: {pred_missing}")
|
||||
print(f"Seq too long: {seq_too_long}")
|
||||
print(f"SC: {sc_cnt}")
|
||||
for k, v in prm_pass_at_k.items():
|
||||
print(f"PRM pass at {k}: {v}")
|
||||
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
for k, v in sc_top_k.items():
|
||||
print(f"SC pass at {k}: {v}")
|
||||
print(f"SC pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
||||
for k, v in weighted_best_of_k.items():
|
||||
print(f"Weighted best of k pass at {k}: {v}")
|
||||
print(f"Weighted best of k pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
for k, v in sc_k.items():
|
||||
print(f"SC k pass at {k}: {v}")
|
||||
print(f"SC k pass at {k} rate {v / len(responses) * 100:.2f}%")
|
||||
|
||||
json.dump(ultimate_results[:100], open("debug.json", "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,177 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import collections
|
||||
import torch
|
||||
|
||||
|
||||
def load_rewards(reward_file):
|
||||
if os.path.exists(reward_file):
|
||||
rewards = json.load(open(reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob(reward_file):
|
||||
print(file)
|
||||
rewards.extend(json.load(open(file)))
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def reward_reduction(ending_logits, reduction: str = "min", norm: bool = True):
|
||||
if norm:
|
||||
ending_logits = torch.tensor(ending_logits)
|
||||
ending_probs = torch.softmax(ending_logits, dim=-1).tolist()
|
||||
step_rewards = [step[1] for step in ending_probs]
|
||||
else:
|
||||
step_rewards = [step[1] for step in ending_logits]
|
||||
if reduction == "min":
|
||||
reward = min(step_rewards)
|
||||
elif reduction == "product":
|
||||
reward = 1
|
||||
for prob in step_rewards:
|
||||
reward *= prob
|
||||
elif reduction == "sum":
|
||||
reward = sum(step_rewards)
|
||||
else:
|
||||
raise ValueError(f"Invalid reduction method: {reduction}")
|
||||
return reward
|
||||
|
||||
|
||||
def merge_rewards(group_logits, weights, reduction: str = "min", norm: bool = True, group_reduction: str = "min"):
|
||||
# rewards = []
|
||||
# for g_id, item in group_logits.items():
|
||||
# if len(item["ending_logits"]) == 0:
|
||||
# continue
|
||||
# r = reward_reduction(item["ending_logits"], reduction, norm)
|
||||
# rewards.append(weights[g_id] * r)
|
||||
# if len(rewards) == 0:
|
||||
# return 0
|
||||
# if group_reduction == "min":
|
||||
# reward = min(rewards)
|
||||
# elif group_reduction == "product":
|
||||
# reward = 1
|
||||
# for r in rewards:
|
||||
# reward *= r
|
||||
# elif group_reduction == "sum":
|
||||
# reward = sum(rewards)
|
||||
# else:
|
||||
# raise ValueError(f"Invalid group reduction method: {group_reduction}")
|
||||
ending_logits = []
|
||||
if len(group_logits) == 0:
|
||||
return 0
|
||||
_len = len(group_logits[0]["ending_logits"])
|
||||
if _len == 0:
|
||||
return 0
|
||||
for g_id, item in group_logits.items():
|
||||
assert len(item["ending_logits"]) == _len
|
||||
ending_logits.append(item["ending_logits"])
|
||||
ending_logits = torch.tensor(ending_logits)
|
||||
assert ending_logits.size() == (len(group_logits), _len, 2), ending_logits.size()
|
||||
|
||||
if norm:
|
||||
ending_logits = torch.softmax(ending_logits, dim=-1)
|
||||
ending_logits = ending_logits[:, :, 1]
|
||||
else:
|
||||
ending_logits = ending_logits[:, :, 1]
|
||||
|
||||
ending_logits = ending_logits * torch.tensor(weights).view(-1, 1)
|
||||
|
||||
if group_reduction == "min":
|
||||
ending_logits = torch.min(ending_logits, dim=0)[0]
|
||||
elif group_reduction == "product":
|
||||
ending_logits = torch.prod(ending_logits, dim=0)
|
||||
elif group_reduction == "sum":
|
||||
ending_logits = torch.sum(ending_logits, dim=0)
|
||||
else:
|
||||
raise ValueError(f"Invalid group reduction method: {group_reduction}")
|
||||
|
||||
if reduction == "min":
|
||||
reward = torch.min(ending_logits).item()
|
||||
elif reduction == "product":
|
||||
reward = torch.prod(ending_logits).item()
|
||||
elif reduction == "sum":
|
||||
reward = torch.sum(ending_logits).item()
|
||||
else:
|
||||
raise ValueError(f"Invalid reduction method: {reduction}")
|
||||
|
||||
return reward
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--response_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True, nargs='+')
|
||||
parser.add_argument("--weights", type=float, nargs='+', default=[1.0])
|
||||
parser.add_argument("--reduction", type=str, default="min")
|
||||
parser.add_argument("--raw_logits", action="store_true", default=False)
|
||||
parser.add_argument("--group_reduction", type=str, default="min")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.response_file):
|
||||
responses = json.load(open(args.response_file))
|
||||
else:
|
||||
responses = []
|
||||
for file in glob(args.response_file):
|
||||
print(file)
|
||||
responses.extend(json.load(open(file)))
|
||||
|
||||
id2reward = collections.defaultdict(dict)
|
||||
assert len(args.reward_file) == len(args.weights)
|
||||
print(args.reward_file)
|
||||
print(args.weights)
|
||||
for _g_id, _reward_group in enumerate(args.reward_file):
|
||||
rewards = load_rewards(_reward_group)
|
||||
for r in rewards:
|
||||
id2reward[r["index"]][_g_id] = r
|
||||
|
||||
_k = [1, 3, 5]
|
||||
prm_pass_at_k = {k: 0 for k in _k}
|
||||
missing = 0
|
||||
missing_reward = 0
|
||||
sc_cnt = 0
|
||||
ultimate_results = []
|
||||
for item in tqdm(responses):
|
||||
sorted_results = []
|
||||
if not item["response"] or not item["pred"] or not item["res"]:
|
||||
missing += 1
|
||||
continue
|
||||
for i, (resp, pred, r) in enumerate(zip(item["response"], item["pred"], item["res"])):
|
||||
resp_id = f"{item['id']}_{i}"
|
||||
if resp_id not in id2reward:
|
||||
missing_reward += 1
|
||||
continue
|
||||
process_rewards = id2reward[resp_id]
|
||||
# if len(process_rewards["ending_logits"]) == 0:
|
||||
# continue
|
||||
|
||||
# reward = merge_rewards(process_rewards["ending_logits"], args.reduction, not args.raw_logits)
|
||||
reward = merge_rewards(process_rewards, args.weights, args.reduction, not args.raw_logits, args.group_reduction)
|
||||
sorted_results.append((resp, pred, r, reward))
|
||||
|
||||
if not sorted_results:
|
||||
continue
|
||||
sorted_results = sorted(sorted_results, key=lambda x: x[-1], reverse=True)
|
||||
ultimate_results.append(sorted_results)
|
||||
for k in _k:
|
||||
if any([r[2] for r in sorted_results[:k]]):
|
||||
prm_pass_at_k[k] += 1
|
||||
|
||||
if item["sc_res"]:
|
||||
sc_cnt += 1
|
||||
|
||||
print(f"Total: {len(responses)}")
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing reward: {missing_reward}")
|
||||
print(f"SC: {sc_cnt}")
|
||||
for k, v in prm_pass_at_k.items():
|
||||
print(f"PRM pass at {k}: {v}")
|
||||
print(f"PRM pass at {k} rate: {v / len(responses) * 100:.2f}%")
|
||||
print(f"SC rate: {sc_cnt / len(responses) * 100:.2f}%")
|
||||
|
||||
json.dump(ultimate_results[:100], open("debug.json", "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import collections
|
||||
from tqdm import tqdm
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from data.mathscale.util import mathscale_is_equiv
|
||||
|
||||
"""
|
||||
The output file should be processed by post_processors.openai_api_callback.MathScaleCallBack
|
||||
"""
|
||||
|
||||
|
||||
def majority_voting_frequency(preds):
|
||||
assert isinstance(preds, list), preds
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = collections.Counter(tmp)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(eval(pred), fre) for pred, fre in tmp]
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = collections.Counter(preds)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(pred, fre) for pred, fre in tmp]
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return sorted_preds
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--maj_ks", type=str, default="8,16,32,64,128")
|
||||
parser.add_argument("--id_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in sorted(glob(args.input_file)):
|
||||
if "metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
print(len(data))
|
||||
data = merge_seed_sampled_data(data)
|
||||
print(len(data))
|
||||
|
||||
id_data = json.load(open(args.id_file))
|
||||
all_ids = set([item["id"] for item in id_data])
|
||||
data = [item for item in data if item["id"] in all_ids]
|
||||
|
||||
maj_ks = list(map(int, args.maj_ks.split(",")))
|
||||
|
||||
cnt = 0
|
||||
pass_at_k = 0
|
||||
maj_at_k = {k: 0 for k in maj_ks}
|
||||
acc_data_topic = collections.Counter()
|
||||
cnt_data_topic = collections.Counter()
|
||||
avg_solution_num = 0
|
||||
for item in tqdm(data):
|
||||
if not isinstance(item["res"], list):
|
||||
res = [item["res"]]
|
||||
else:
|
||||
res = item["res"]
|
||||
|
||||
if res[0]:
|
||||
cnt += 1
|
||||
|
||||
if "data_topic" in item:
|
||||
acc_data_topic[item["data_topic"]] += int(res[0])
|
||||
cnt_data_topic[item["data_topic"]] += 1
|
||||
|
||||
if any(res):
|
||||
pass_at_k += 1
|
||||
|
||||
if isinstance(item["pred"], str):
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
pred2res = {pred: res[j] for j, pred in enumerate(item["pred"])}
|
||||
|
||||
avg_solution_num += len(item["pred"])
|
||||
|
||||
for _k in maj_ks:
|
||||
k_preds = item["pred"][:_k]
|
||||
k_sc_pred = majority_voting_frequency(k_preds)[0][0]
|
||||
k_sc_res = pred2res[k_sc_pred]
|
||||
if k_sc_res:
|
||||
maj_at_k[_k] += 1
|
||||
|
||||
sc_pred = majority_voting_frequency(item["pred"])[0][0]
|
||||
item["sc_pred"] = sc_pred
|
||||
item["sc_res"] = pred2res[sc_pred]
|
||||
|
||||
assert pass_at_k <= len(data)
|
||||
json.dump(data, open(args.output_file, "w"))
|
||||
|
||||
metrics = {"acc": cnt / len(data), "pass@k": pass_at_k / len(data), "correct": cnt, "total": len(data)}
|
||||
if len(acc_data_topic) > 0:
|
||||
for k, v in acc_data_topic.items():
|
||||
metrics[f"acc_{k}"] = v / cnt_data_topic[k]
|
||||
metrics[f"total_{k}"] = cnt_data_topic[k]
|
||||
metrics["avg_solution_num"] = avg_solution_num / len(data)
|
||||
for _k in maj_ks:
|
||||
metrics[f"maj@{_k}"] = maj_at_k[_k] / len(data)
|
||||
json.dump(metrics, open(args.output_file.replace(".json", ".metrics.json"), "w"), indent=2)
|
||||
|
||||
print(len(data))
|
||||
print(metrics)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
import collections
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import argparse
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def draw_line_plot(data, file_path, baseline, baseline_label):
|
||||
all_colors = ["#AFEEEE", "#4682B4", "#6A5ACD", "#BA55D3", "#3CB371"]
|
||||
|
||||
# Compute histogram data
|
||||
x_real = np.array([4, 8, 16, 32, 64, 128])
|
||||
x = np.arange(6)
|
||||
|
||||
# Create the line plot
|
||||
plt.figure(figsize=(4, 4))
|
||||
# plt.plot(x, data, color='#658873', marker='o', linestyle='-')
|
||||
for i, (label, values) in enumerate(data.items()):
|
||||
plt.plot(x, values, color=all_colors[i], marker='.', linestyle='-', label=label)
|
||||
|
||||
baseline = np.ones_like(list(data.values())[0]) * baseline
|
||||
plt.plot(x, baseline, linestyle='--', color=all_colors[-1], label=baseline_label)
|
||||
|
||||
# Change the size of the font in the legent
|
||||
plt.legend(prop={"size": 15})
|
||||
plt.xticks(x, x_real)
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
plt.grid(True)
|
||||
|
||||
# Add labels and title
|
||||
plt.xlabel('Sample@K', fontsize=16)
|
||||
plt.ylabel('Accuracy', fontsize=16)
|
||||
plt.legend(loc='center right')
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def main():
|
||||
# data = {
|
||||
# "self-consistency": [65.96, 69.30, 71.14, 72.30, 72.28, 72.52],
|
||||
# "BoN weighted": [66.98, 69.50, 71.32, 72.42, 72.54, 72.66],
|
||||
# }
|
||||
# greedy_decoding = 65.50
|
||||
#
|
||||
# draw_line_plot(data, "mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42.png", greedy_decoding, "greedy decoding")
|
||||
|
||||
# data = {
|
||||
# "self-consistency": [67.86, 70.26, 71.54, 72.38, 72.58, 72.88],
|
||||
# "BoN weighted": [68.84, 70.84, 71.66, 72.24, 72.70, 72.96],
|
||||
# }
|
||||
# greedy_decoding = 66.72
|
||||
#
|
||||
# draw_line_plot(data, "mathstral.mscale-v0.1-300k.process-dpo-sc.p0.5.iter1.V100.tp4dp32.v3.1.s42.png", greedy_decoding, "greedy decoding")
|
||||
|
||||
# data = {
|
||||
# "self-consistency": [68.76, 70.62, 71.72, 72.40, 72.50, 72.58],
|
||||
# "BoN weighted": [69.22, 70.90, 72.00, 72.36, 72.56, 72.66],
|
||||
# }
|
||||
# greedy_decoding = 67.50
|
||||
|
||||
data = {
|
||||
"self-consistency": [60.34, 65.92, 68.40, 69.66, 70.52, 71.02],
|
||||
"BoN weighted": [62.32, 66.70, 68.48, 69.68, 70.36, 71.20],
|
||||
}
|
||||
greedy_decoding = 61.42
|
||||
|
||||
draw_line_plot(data, "mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42.png", greedy_decoding, "greedy decoding")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,149 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import collections
|
||||
from tqdm import tqdm
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from data.mathscale.util import mathscale_is_equiv
|
||||
|
||||
"""
|
||||
The output file should be processed by post_processors.openai_api_callback.MathScaleCallBack
|
||||
"""
|
||||
|
||||
|
||||
def majority_voting_frequency(preds):
|
||||
assert isinstance(preds, list), preds
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = collections.Counter(tmp)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(eval(pred), fre) for pred, fre in tmp]
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = collections.Counter(preds)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(pred, fre) for pred, fre in tmp]
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return sorted_preds
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--maj_k", type=int, default=16)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in sorted(glob(args.input_file)):
|
||||
if "metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
print(len(data))
|
||||
data = merge_seed_sampled_data(data)
|
||||
print(len(data))
|
||||
|
||||
cnt = 0
|
||||
pass_at_k = 0
|
||||
maj_at_k = 0
|
||||
acc_data_topic = collections.Counter()
|
||||
cnt_data_topic = collections.Counter()
|
||||
avg_solution_num = 0
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
if not isinstance(item["res"], list):
|
||||
res = [item["res"]]
|
||||
else:
|
||||
res = item["res"]
|
||||
|
||||
if res[0]:
|
||||
cnt += 1
|
||||
|
||||
if "data_topic" in item:
|
||||
acc_data_topic[item["data_topic"]] += int(res[0])
|
||||
cnt_data_topic[item["data_topic"]] += 1
|
||||
|
||||
if any(res):
|
||||
pass_at_k += 1
|
||||
|
||||
if isinstance(item["pred"], str):
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
avg_solution_num += len(item["pred"])
|
||||
|
||||
k_preds = item["pred"][:args.maj_k]
|
||||
k_sc_pred = majority_voting_frequency(k_preds)[0][0]
|
||||
k_sc_res = mathscale_is_equiv(k_sc_pred, item["label"])[0]
|
||||
if k_sc_res:
|
||||
maj_at_k += 1
|
||||
else:
|
||||
outputs.append(item)
|
||||
|
||||
item[f"sc_pred@{args.maj_k}"] = k_sc_pred
|
||||
item[f"sc_res@{args.maj_k}"] = k_sc_res
|
||||
|
||||
assert pass_at_k <= len(data)
|
||||
json.dump(outputs, open(args.output_file, "w"))
|
||||
|
||||
metrics = {"acc": cnt / len(data), "pass@k": pass_at_k / len(data), "correct": cnt, "total": len(data)}
|
||||
if len(acc_data_topic) > 0:
|
||||
for k, v in acc_data_topic.items():
|
||||
metrics[f"acc_{k}"] = v / cnt_data_topic[k]
|
||||
metrics[f"total_{k}"] = cnt_data_topic[k]
|
||||
metrics["avg_solution_num"] = avg_solution_num / len(data)
|
||||
metrics[f"maj@{args.maj_k}"] = maj_at_k / len(data)
|
||||
json.dump(metrics, open(args.output_file.replace(".json", ".metrics.json"), "w"), indent=2)
|
||||
|
||||
print(len(data))
|
||||
print(len(outputs))
|
||||
print(metrics)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,110 @@
|
||||
import collections
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import argparse
|
||||
|
||||
|
||||
def plot_bar_chart(data, file_path):
|
||||
indices = list(range(len(data)))
|
||||
|
||||
# Create the bar chart
|
||||
plt.figure(figsize=(4, 4))
|
||||
bars = plt.bar(indices, data)
|
||||
|
||||
# Customize the chart
|
||||
plt.title('Bar Chart of Array Values')
|
||||
plt.xlabel('Frequency', fontsize=16)
|
||||
plt.ylabel('Amount Difference between Correct and Incorrect Prediction', fontsize=12)
|
||||
|
||||
# Add value labels on top of each bar
|
||||
# for i, bar in enumerate(bars):
|
||||
# height = bar.get_height()
|
||||
# plt.text(bar.get_x() + bar.get_width() / 2., height,
|
||||
# f'{data[i]}',
|
||||
# ha='center', va='bottom',
|
||||
# fontsize=12)
|
||||
|
||||
# Color positive and negative bars differently
|
||||
for i, value in enumerate(data):
|
||||
if value >= 0:
|
||||
bars[i].set_color('blue')
|
||||
else:
|
||||
bars[i].set_color('red')
|
||||
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def draw_line_plot(data, file_path):
|
||||
# Compute histogram data
|
||||
x = np.arange(0.1, 1.1, 0.1)
|
||||
|
||||
# Create the line plot
|
||||
plt.figure(figsize=(4, 4))
|
||||
plt.plot(x, data, color='#658873', marker='o', linestyle='-')
|
||||
|
||||
# Change the size of the font in the legent
|
||||
# plt.legend(prop={"size": 15})
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
plt.grid(True)
|
||||
|
||||
# Add labels and title
|
||||
# plt.xlabel('Top-1 Averaged Frequency', fontsize=16)
|
||||
plt.ylabel('Ratio of Correct Predictions', fontsize=16)
|
||||
# plt.legend(loc='upper left')
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--n", type=int, default=128)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
freq_pos = collections.Counter()
|
||||
freq_neg = collections.Counter()
|
||||
freq = collections.Counter()
|
||||
for item in data:
|
||||
if item["sc_res"]:
|
||||
freq_pos[item["sc_freq"]] += 1
|
||||
else:
|
||||
freq_neg[item["sc_freq"]] += 1
|
||||
freq[item["sc_freq"]] += 1
|
||||
|
||||
diffs = []
|
||||
for i in range(args.n + 1):
|
||||
tmp = 0
|
||||
if i in freq_pos:
|
||||
tmp += freq_pos[i]
|
||||
if i in freq_neg:
|
||||
tmp -= freq_neg[i]
|
||||
diffs.append(tmp)
|
||||
|
||||
plot_bar_chart(diffs, args.output_file)
|
||||
|
||||
ps = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
|
||||
reversed_acc_pos = collections.Counter()
|
||||
reversed_acc = collections.Counter()
|
||||
for p in ps:
|
||||
for key, value in freq_pos.items():
|
||||
if key / args.n >= p:
|
||||
reversed_acc_pos[p] += value
|
||||
for key, value in freq.items():
|
||||
if key / args.n >= p:
|
||||
reversed_acc[p] += value
|
||||
|
||||
p_ratio = {p: reversed_acc_pos[p] / reversed_acc[p] for p in ps}
|
||||
|
||||
draw_line_plot(list(p_ratio.values()), args.output_file.replace(".png", "_line.png"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,149 @@
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from functools import partial
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from data.mathscale.util import mathscale_is_equiv
|
||||
|
||||
|
||||
def majority_voting_frequency(preds):
|
||||
assert isinstance(preds, list), preds
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = collections.Counter(tmp)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(eval(pred), fre) for pred, fre in tmp]
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = collections.Counter(preds)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(pred, fre) for pred, fre in tmp]
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return sorted_preds
|
||||
|
||||
|
||||
def worker(item, n: int):
|
||||
sc_pred, sc_freq = majority_voting_frequency(item["pred"][:n])[0]
|
||||
sc_res = mathscale_is_equiv(sc_pred, item["label"])[0]
|
||||
return {
|
||||
"sc_res": sc_res,
|
||||
"sc_pred": sc_pred,
|
||||
"sc_freq": sc_freq,
|
||||
"id": item["id"],
|
||||
}
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--ps", type=str,
|
||||
help="The probabilities of top-1 frequency over total samples, split by comma")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
parser.add_argument("--n", type=int, default=128)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in sorted(glob(args.input_file)):
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
data = merge_seed_sampled_data(data)
|
||||
|
||||
pbar = tqdm(data)
|
||||
outputs = []
|
||||
sc = 0
|
||||
num_pred = 0
|
||||
freq_pos = collections.Counter()
|
||||
freq = collections.Counter()
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = partial(worker, n=args.n)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
num_pred += len(_input["response"])
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
result = future.result()
|
||||
outputs.append(result)
|
||||
|
||||
if result["sc_res"]:
|
||||
sc += 1
|
||||
|
||||
if result["sc_res"]:
|
||||
freq_pos[result["sc_freq"]] += 1
|
||||
freq[result["sc_freq"]] += 1
|
||||
|
||||
ps = list(map(float, args.ps.split(",")))
|
||||
reversed_acc_pos = collections.Counter()
|
||||
reversed_acc = collections.Counter()
|
||||
for p in ps:
|
||||
for key, value in freq_pos.items():
|
||||
if key / args.n >= p:
|
||||
reversed_acc_pos[p] += value
|
||||
for key, value in freq.items():
|
||||
if key / args.n >= p:
|
||||
reversed_acc[p] += value
|
||||
|
||||
p_ratio = {p: reversed_acc_pos[p] / reversed_acc[p] for p in ps}
|
||||
|
||||
print(f"Reversed Acc Pos Ratio: {p_ratio}")
|
||||
|
||||
print(f"SC: {sc}/{len(data)} = {sc / len(data)}")
|
||||
print(f"Num Pred: {num_pred}/{len(data)} = {num_pred / len(data)}")
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
python scripts/math_scale/analyze/extract_hard_questions.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json \
|
||||
--maj_k 16
|
||||
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n1.tem1.0.p1.0.0-of-1.s*.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
|
||||
python scripts/math_scale/qwen25math_style_eval_math.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.json \
|
||||
--num_workers 1
|
||||
#{
|
||||
# "acc": 0.6292,
|
||||
# "pass@k": 0.6292,
|
||||
# "maj@k": 0.6292,
|
||||
# "correct": 3146,
|
||||
# "total": 5000
|
||||
#}
|
||||
# Compared with original: 61.4
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/math/math.test.v1.1.0shot.n1.tem1.0.p1.0.0-of-1.s*.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
|
||||
python scripts/math_scale/qwen25math_style_eval_math.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.json \
|
||||
--num_workers 1
|
||||
#{
|
||||
# "acc": 0.6702,
|
||||
# "pass@k": 0.6702,
|
||||
# "maj@k": 0.6702,
|
||||
# "correct": 3351,
|
||||
# "total": 5000
|
||||
#}
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
|
||||
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.5.iter1.V100.tp4dp32.v3.1.s42/checkpoint-500/math/math.test.v1.1.0shot.n1.tem1.0.p1.0.0-of-1.s*.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.5.iter1.V100.tp4dp32.v3.1.s42/checkpoint-500/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
python scripts/math_scale/qwen25math_style_eval_math.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.5.iter1.V100.tp4dp32.v3.1.s42/checkpoint-500/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.json \
|
||||
--num_workers 1
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.5.iter1.V100.tp4dp32.v3.1.s42/checkpoint-500/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.5.iter1.V100.tp4dp32.v3.1.s42/checkpoint-500/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
|
||||
# mathstral.mscale-v0.1-300k.process-dpo-sc.p0.0.iter2.H100.dp16.v1.3.s42
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.0.iter2.H100.dp16.v1.3.s42/checkpoint-500/math/math.test.v1.1.0shot.n1.tem1.0.p1.0.0-of-1.s*.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.0.iter2.H100.dp16.v1.3.s42/checkpoint-500/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
python scripts/math_scale/qwen25math_style_eval_math.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.0.iter2.H100.dp16.v1.3.s42/checkpoint-500/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.json \
|
||||
--num_workers 1
|
||||
python scripts/math_scale/analyze/compute_acc_by_id.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.0.iter2.H100.dp16.v1.3.s42/checkpoint-500/math/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.p0.0.iter2.H100.dp16.v1.3.s42/checkpoint-500/math.test.v1.1.0shot.n1.tem0.0.p1.0.sympy_eval.sft-k16-fail.json \
|
||||
--id_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/math/math.test.v1.1.0shot.n128.tem1.0.p1.0.k16-fail.json
|
||||
@@ -0,0 +1,24 @@
|
||||
#exp_dir=llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/
|
||||
#exp_dir=llama3.1.8b.numina.process-dpo-sc-p0.5.iter1.split01.H100.dp8.v1.0.s42/checkpoint-200/
|
||||
exp_dir=llama3.1.8b.numina.process-dpo-sc-p0.5.iter2.split01-23.H100.dp8.v1.4.s42/checkpoint-100/
|
||||
|
||||
|
||||
#python scripts/math_scale/analyze/get_output_frequency.py \
|
||||
# --input_file "../msranlpintern/reward_modeling/experiments/${exp_dir}/math/math.test.v1.1.0shot.n1.tem1.0.p1.0.0-of-1.s*[0-9].json" \
|
||||
# --output_file ../msranlpintern/reward_modeling/experiments/${exp_dir}/math/math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.json \
|
||||
# --n 128 --ps "0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0"
|
||||
|
||||
|
||||
#python scripts/math_scale/analyze/freq2image.py \
|
||||
# --input_file ../msranlpintern/reward_modeling/experiments/${exp_dir}/math/math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.json \
|
||||
# --output_file ./math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.iter0.png
|
||||
|
||||
#exp_dir=llama3.1.8b.numina.process-dpo-sc-p0.5.iter1.split01.H100.dp8.v1.0.s42/checkpoint-200/
|
||||
#python scripts/math_scale/analyze/freq2image.py \
|
||||
# --input_file ../msranlpintern/reward_modeling/experiments/${exp_dir}/math/math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.json \
|
||||
# --output_file ./math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.iter1.png
|
||||
#
|
||||
#exp_dir=llama3.1.8b.numina.process-dpo-sc-p0.5.iter2.split01-23.H100.dp8.v1.4.s42/checkpoint-100/
|
||||
python scripts/math_scale/analyze/freq2image.py \
|
||||
--input_file ../msranlpintern/reward_modeling/experiments/${exp_dir}/math/math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.json \
|
||||
--output_file ./math.test.v1.1.0shot.n64.tem1.0.p1.0.analyze.output_freq.iter2.png
|
||||
@@ -0,0 +1,26 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import sys
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
data.extend(json.load(open(file)))
|
||||
|
||||
print(len(data))
|
||||
print(data[0].keys())
|
||||
json.dump(data, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,401 @@
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
|
||||
# sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
#
|
||||
# from data.math_util import is_equiv
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file, recursive=True):
|
||||
print(file)
|
||||
# data += json.load(open(file))
|
||||
f = open(file, "r")
|
||||
try:
|
||||
data += json.load(f)
|
||||
except:
|
||||
print(f"Error in file {file}")
|
||||
new_file = file.replace(".json", ".jsonl")
|
||||
lines = open(new_file, "r").readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
data.append(json.loads(line))
|
||||
except:
|
||||
print(f"Error in line: {line}")
|
||||
|
||||
data = merge_seed_sampled_data(data)
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
pass_at_k = 0
|
||||
num_pairs = 0
|
||||
pos_missing = 0
|
||||
neg_missing = 0
|
||||
full_positive_samples = []
|
||||
full_negative_samples = []
|
||||
for item in data:
|
||||
if "res" not in item:
|
||||
raise RuntimeError("Use `fix_answer_extract_and_verify.py` to add `res` field to the input file")
|
||||
# if isinstance(item["pred"], list):
|
||||
# preds = item["pred"]
|
||||
# else:
|
||||
# preds = [item["pred"]]
|
||||
#
|
||||
# res = [is_equiv(p, str(item["label"])) for p in preds]
|
||||
# if isinstance(item["pred"], str):
|
||||
# res = res[0]
|
||||
# item["res"] = res
|
||||
|
||||
if not item["res"]:
|
||||
continue
|
||||
|
||||
if item["res"][0]:
|
||||
cnt += 1
|
||||
if any(item["res"]):
|
||||
pass_at_k += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
for resp, r in zip(item["response"], item["res"]):
|
||||
if r:
|
||||
pos.append(resp)
|
||||
else:
|
||||
neg.append(resp)
|
||||
|
||||
if len(pos) == 0:
|
||||
full_negative_samples.append(item)
|
||||
pos_missing += 1
|
||||
if len(neg) == 0:
|
||||
neg_missing += 1
|
||||
full_positive_samples.append(item)
|
||||
if len(pos) == 0 or len(neg) == 0:
|
||||
continue
|
||||
|
||||
item["pos"] = pos
|
||||
item["neg"] = neg
|
||||
num_pairs += len(pos) * len(neg)
|
||||
outputs.append(item)
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
print(f"Acc: {cnt / len(data)}")
|
||||
print(f"Pass at k: {pass_at_k / len(data)}")
|
||||
print(f"No positive solutions: {pos_missing} / {len(data)}")
|
||||
print(f"No negative solutions: {neg_missing} / {len(data)}")
|
||||
print(f"Num pairs: {num_pairs}")
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
json.dump(full_positive_samples[:100], open(args.output_file.replace(".json", ".pos.sample.json"), "w"), indent=2)
|
||||
json.dump(full_negative_samples[:100], open(args.output_file.replace(".json", ".neg.sample.json"), "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/share/models/mathscale-mistral/mathscale/train.v60.300k.1-of-30.v1.0.0shot.n5.tem1.0.p0.9.0-of-8.json" \
|
||||
--output_file ../msranlpintern/share/models/mathscale-mistral/mathscale/train.v60.300k.1-of-30.v1.0.0shot.n5.tem1.0.p0.9.json
|
||||
|
||||
Total number of items: 1250
|
||||
Acc: 0.8544
|
||||
Pass at k: 0.9696
|
||||
Missing: 825 / 1250
|
||||
Num pairs: 2022
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/share/models/mathscale-mistral/mathscale/train.v60.300k.1-of-30.v1.0.0shot.n5.tem1.0.p0.9.fix_predict.json" \
|
||||
--output_file ../msranlpintern/share/models/mathscale-mistral/mathscale/train.v60.300k.1-of-30.v1.0.0shot.n5.tem1.0.p0.9.fix_predict.dpo.json
|
||||
|
||||
Total number of items: 10000
|
||||
Acc: 0.7141
|
||||
Pass at k: 0.8515
|
||||
Missing: 6383 / 10000
|
||||
Num pairs: 17630
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair.py --input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.*-of-16.json" --output_file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.1-of-30.v1.2.0shot.n10.tem1.0.p0.9.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.0-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.1-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.10-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.11-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.12-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.13-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.14-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.15-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.2-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.3-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.4-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.5-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.6-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.7-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.8-of-16.json
|
||||
Error in file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/train.v60.300k.2-of-30.v1.2.0shot.n10.tem1.0.p0.9.9-of-16.json
|
||||
Total number of items: 10000
|
||||
Acc: 0.6983
|
||||
Pass at k: 0.8525
|
||||
Missing: 5670 / 10000
|
||||
Num pairs: 66254
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/all_splits/train.v60.300k.all.v1.2.0shot.n10.tem1.0.p0.9.*-of-100.json" \
|
||||
--output_file ../msranlpintern/share/models/mathstral-7B-v0.1/mathscale/all_splits/train.v60.300k.all.v1.2.0shot.n10.tem1.0.p0.9.dpo_v1.0.json
|
||||
|
||||
Total number of items: 300000
|
||||
Acc: 0.4003566666666667
|
||||
Pass at k: 0.5258733333333333
|
||||
No positive solutions: 142238 / 300000
|
||||
No negative solutions: 71338 / 300000
|
||||
Num pairs: 1361840
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "./500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.?-of-8.json" \
|
||||
--output_file train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-0-of-20/train.500k.de_con.boxed.v1.0.0-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-1-of-20/train.500k.de_con.boxed.v1.0.1-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-10-of-20/train.500k.de_con.boxed.v1.0.10-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-11-of-20/train.500k.de_con.boxed.v1.0.11-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-12-of-20/train.500k.de_con.boxed.v1.0.12-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-13-of-20/train.500k.de_con.boxed.v1.0.13-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-14-of-20/train.500k.de_con.boxed.v1.0.14-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-15-of-20/train.500k.de_con.boxed.v1.0.15-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-16-of-20/train.500k.de_con.boxed.v1.0.16-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-17-of-20/train.500k.de_con.boxed.v1.0.17-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-18-of-20/train.500k.de_con.boxed.v1.0.18-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-19-of-20/train.500k.de_con.boxed.v1.0.19-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-2-of-20/train.500k.de_con.boxed.v1.0.2-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-3-of-20/train.500k.de_con.boxed.v1.0.3-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-4-of-20/train.500k.de_con.boxed.v1.0.4-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-5-of-20/train.500k.de_con.boxed.v1.0.5-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-6-of-20/train.500k.de_con.boxed.v1.0.6-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-7-of-20/train.500k.de_con.boxed.v1.0.7-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-8-of-20/train.500k.de_con.boxed.v1.0.8-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.0-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.1-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.2-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.3-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.4-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.5-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.6-of-8.json
|
||||
./500k-split-9-of-20/train.500k.de_con.boxed.v1.0.9-of-20.0shot.n10.tem1.0.p0.9.7-of-8.json
|
||||
Total number of items: 491733
|
||||
Acc: 0.6720944089577067
|
||||
Pass at k: 0.8531032084484873
|
||||
No positive solutions: 72234 / 491733
|
||||
No negative solutions: 187738 / 491733
|
||||
Num pairs: 3873158
|
||||
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "./500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-32.json" \
|
||||
--output_file train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.json
|
||||
|
||||
Total number of items: 491733
|
||||
Acc: 0.681286389158344
|
||||
Pass at k: 0.8782998090427122
|
||||
No positive solutions: 59844 / 491733
|
||||
No negative solutions: 211316 / 491733
|
||||
Num pairs: 3693524
|
||||
|
||||
|
||||
########################################## ITERATION 1 ###########################################################
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.json
|
||||
|
||||
Total number of items: 491733
|
||||
Acc: 0.7095415601556129
|
||||
Pass at k: 0.8631289744637842
|
||||
No positive solutions: 67304 / 491733
|
||||
No negative solutions: 250765 / 491733
|
||||
Num pairs: 2844227
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.json
|
||||
Total number of items: 491733
|
||||
Acc: 0.6936853943095135
|
||||
Pass at k: 0.8353761085792493
|
||||
No positive solutions: 80951 / 491733
|
||||
No negative solutions: 255617 / 491733
|
||||
Num pairs: 2550984
|
||||
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,339 @@
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from glob import glob
|
||||
import collections
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.mathscale.util import mathscale_is_equiv
|
||||
|
||||
|
||||
def evaluate_by_sc(item, external_sc: str = None):
|
||||
if not item["response"]:
|
||||
return []
|
||||
|
||||
if isinstance(item["response"], list):
|
||||
preds = item["pred"]
|
||||
else:
|
||||
preds = [item["pred"]]
|
||||
|
||||
assert len(preds) > 1, f"Self-consistency requires at least 2 predictions, but got {len(preds)}."
|
||||
|
||||
if external_sc:
|
||||
sc_pred = external_sc
|
||||
else:
|
||||
sc_pred = item["sc_pred"]
|
||||
res = [mathscale_is_equiv(p, sc_pred)[0] for p in preds]
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def majority_voting_frequency(preds):
|
||||
assert isinstance(preds, list)
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = collections.Counter(tmp)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(eval(pred), fre) for pred, fre in tmp]
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = collections.Counter(preds)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(pred, fre) for pred, fre in tmp]
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return sorted_preds
|
||||
|
||||
|
||||
def get_pred_frequency(preds, target_pred):
|
||||
assert isinstance(preds, list)
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = collections.Counter(tmp)
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = collections.Counter(preds)
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
if isinstance(target_pred, list):
|
||||
target_pred = str(sorted(target_pred))
|
||||
return tmp.get(target_pred, 0) / len(preds)
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def load_data(file_path):
|
||||
if os.path.exists(file_path):
|
||||
data = json.load(open(file_path))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(file_path, recursive=True):
|
||||
if ".metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
f = open(file, "r")
|
||||
try:
|
||||
data += json.load(f)
|
||||
except:
|
||||
print(f"Error in file {file}")
|
||||
new_file = file.replace(".json", ".jsonl")
|
||||
lines = open(new_file, "r").readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
data.append(json.loads(line))
|
||||
except:
|
||||
print(f"Error in line: {line}")
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--external_file_for_sc", type=str, default=None)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_data(args.input_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
|
||||
filtered = 0
|
||||
if args.external_file_for_sc:
|
||||
external_data = load_data(args.external_file_for_sc)
|
||||
# id2external_sc = {item["id"]: item["sc_pred"] for item in external_data}
|
||||
id2sc = {}
|
||||
for item in tqdm(external_data):
|
||||
sc_pred = item["sc_pred"]
|
||||
|
||||
if item["pred"] == "":
|
||||
continue
|
||||
|
||||
if item["pred"] == "failed extracting answer from completion":
|
||||
continue
|
||||
|
||||
freq = get_pred_frequency(item["pred"], sc_pred)
|
||||
if freq > args.top_p:
|
||||
id2sc[item["id"]] = sc_pred
|
||||
else:
|
||||
filtered += 1
|
||||
else:
|
||||
id2sc = {}
|
||||
for item in tqdm(data):
|
||||
sc_pred = item["sc_pred"]
|
||||
|
||||
if item["pred"] == "":
|
||||
continue
|
||||
|
||||
if item["pred"] == "failed extracting answer from completion":
|
||||
continue
|
||||
|
||||
freq = get_pred_frequency(item["pred"], sc_pred)
|
||||
if freq > args.top_p:
|
||||
id2sc[item["id"]] = sc_pred
|
||||
else:
|
||||
filtered += 1
|
||||
|
||||
print(f"Filtered {filtered} samples.")
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
pass_at_k = 0
|
||||
num_pairs = 0
|
||||
pos_missing = 0
|
||||
neg_missing = 0
|
||||
full_positive_samples = []
|
||||
full_negative_samples = []
|
||||
for item in data:
|
||||
if item["id"] in id2sc:
|
||||
res_by_sc = evaluate_by_sc(item, id2sc[item["id"]])
|
||||
else:
|
||||
continue
|
||||
|
||||
if len(res_by_sc) == 0:
|
||||
continue
|
||||
|
||||
if res_by_sc[0]:
|
||||
cnt += 1
|
||||
if any(res_by_sc):
|
||||
pass_at_k += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
for resp, r in zip(item["response"], res_by_sc):
|
||||
if r:
|
||||
pos.append(resp)
|
||||
else:
|
||||
neg.append(resp)
|
||||
|
||||
if len(pos) == 0:
|
||||
full_negative_samples.append(item)
|
||||
pos_missing += 1
|
||||
if len(neg) == 0:
|
||||
neg_missing += 1
|
||||
full_positive_samples.append(item)
|
||||
if len(pos) == 0 or len(neg) == 0:
|
||||
continue
|
||||
|
||||
item["pos"] = pos
|
||||
item["neg"] = neg
|
||||
num_pairs += len(pos) * len(neg)
|
||||
outputs.append(item)
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
print(f"Acc: {cnt / (len(data) - filtered)}")
|
||||
print(f"Pass at k: {pass_at_k / len(data)}")
|
||||
print(f"No positive solutions: {pos_missing} / {len(data)}")
|
||||
print(f"No negative solutions: {neg_missing} / {len(data)}")
|
||||
print(f"Num pairs: {num_pairs}")
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
json.dump(full_positive_samples[:100], open(args.output_file.replace(".json", ".pos.sample.json"), "w"), indent=2)
|
||||
json.dump(full_negative_samples[:100], open(args.output_file.replace(".json", ".neg.sample.json"), "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
########################################## ITERATION 1 ###########################################################
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.json
|
||||
|
||||
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc.json
|
||||
# Total number of items: 491733
|
||||
# Acc: 0.6936853943095135
|
||||
# Pass at k: 0.8353761085792493
|
||||
# No positive solutions: 80951 / 491733
|
||||
# No negative solutions: 255617 / 491733
|
||||
# Num pairs: 2550984
|
||||
Total number of items: 491733
|
||||
Acc: 0.85128718227168
|
||||
Pass at k: 1.0
|
||||
No positive solutions: 0 / 491733
|
||||
No negative solutions: 275526 / 491733
|
||||
Num pairs: 3958764
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "${OUTPUT_PATH_PREFIX}/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.*-of-8.s0.json" \
|
||||
--output_file $OUTPUT_PATH_PREFIX/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.s0.prefer_pair.by_sc.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.0-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.1-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.2-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.3-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.4-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.5-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.6-of-8.s0.json
|
||||
../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-0-of-10/cot.de_con.0-of-10.n8.tem1.0.p1.0.7-of-8.s0.json
|
||||
Total number of items: 82642
|
||||
Acc: 0.706493066479514
|
||||
Pass at k: 1.0
|
||||
No positive solutions: 0 / 82642
|
||||
No negative solutions: 18718 / 82642
|
||||
Num pairs: 786166
|
||||
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "${OUTPUT_PATH_PREFIX}/experiments//mathstral.mathscale4o.sc-prm.raft.iter1.H100.dp8.v1.0.s42/checkpoint-800/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.s[01].json" \
|
||||
--output_file $OUTPUT_PATH_PREFIX/experiments//mathstral.mathscale4o.sc-prm.raft.iter1.H100.dp8.v1.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n20.tem1.0.p0.9.v0.1.prefer_pair.by_sc.json
|
||||
|
||||
Total number of items: 491733
|
||||
Acc: 0.8624497440684273
|
||||
Pass at k: 1.0
|
||||
No positive solutions: 0 / 491733
|
||||
No negative solutions: 264522 / 491733
|
||||
Num pairs: 14647115
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-8.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc_p0.6.json
|
||||
--top_p 0.6
|
||||
|
||||
Filtered 123792 samples.
|
||||
Total number of items: 491733
|
||||
Acc: 0.7133363024242831
|
||||
Pass at k: 0.7482536254430758
|
||||
No positive solutions: 0 / 491733
|
||||
No negative solutions: 272161 / 491733
|
||||
Num pairs: 1350704
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n90.tem1.0.p0.9.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n90.tem1.0.p0.9.prefer_pair.by_sc_p0.6.json \
|
||||
--top_p 0.6
|
||||
|
||||
Filtered 0 samples.
|
||||
Total number of items: 491733
|
||||
Acc: 0.8171100983663899
|
||||
Pass at k: 1.0
|
||||
No positive solutions: 0 / 491733
|
||||
No negative solutions: 199144 / 491733
|
||||
Num pairs: 333282827
|
||||
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" \
|
||||
--output_file ${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc_p0.0.json
|
||||
|
||||
Filtered 0 samples.
|
||||
Total number of items: 491337
|
||||
Acc: 0.7944180877890328
|
||||
Pass at k: 1.0
|
||||
No positive solutions: 0 / 491337
|
||||
No negative solutions: 160871 / 491337
|
||||
Num pairs: 5961085
|
||||
|
||||
|
||||
>>> python scripts/math_scale/construct_prefer_pair_sc.py \
|
||||
--input_file "${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" \
|
||||
--output_file ${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.prefer_pair.by_sc_p0.0.json --top_p 0.5
|
||||
|
||||
Filtered 164514 samples.
|
||||
Total number of items: 491337
|
||||
Acc: 0.6043285972764111
|
||||
Pass at k: 0.6651707483865453
|
||||
No positive solutions: 0 / 491337
|
||||
No negative solutions: 159681 / 491337
|
||||
Num pairs: 2672191
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,35 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
p=$1
|
||||
|
||||
#python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
# --input_file "$OUTPUT_PREFIX_PATH/experiments/mathstral.mathscale4o.process-dpo-sc.iter0.A100.dp8.v2.1.s42/checkpoint-300/mathscale4o/split-1024/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-1024.json" \
|
||||
# --output_file $OUTPUT_PREFIX_PATH/experiments/mathstral.mathscale4o.process-dpo-sc.iter0.A100.dp8.v2.1.s42/checkpoint-300/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.process_rm.gd.azure.json \
|
||||
# --num_workers 128
|
||||
|
||||
#= ================== mscale-300k
|
||||
|
||||
#python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
# --input_file "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo-sc.iter0.A100.dp8.v2.1.s42/checkpoint-300/mathscale4o/mscale-v0.1-300k/split-1024/mscale.v0.1.300k.v1.0.n10.tem1.0.p1.0.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-1024.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo-sc.iter0.A100.dp8.v2.1.s42/checkpoint-300/mathscale4o/mscale-v0.1-300k/mscale.v0.1.300k.v1.0.n10.tem1.0.p1.0.upper0.7.r0.3.sample10.filter_same.process_rm.sc-p$p.azure.json \
|
||||
# --response_file_for_sc "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo-sc.iter0.A100.dp8.v2.1.s42/checkpoint-300/mathscale4o/mscale-v0.1-300k/mscale.v0.1.300k.v1.0.n10.tem1.0.p1.0.*-of-512.s42.json" --response_id_field id --num_workers 128 --top_p $p
|
||||
#
|
||||
|
||||
#python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
# --input_file "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/mscale-v0.1-300k/split-1024/mscale.v0.1.300k.v1.0.n10.tem1.0.p1.0.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-1024.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/mscale-v0.1-300k/mscale.v0.1.300k.v1.0.n10.tem1.0.p1.0.upper0.7.r0.3.sample10.filter_same.process_rm.sc-p$p.azure.json \
|
||||
# --response_file_for_sc "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/mscale-v0.1-300k/mscale.v0.1.300k.v1.0.n10.tem1.0.p1.0.*-of-512.s42.json" --response_id_field id --num_workers 128 --top_p $p
|
||||
|
||||
# ===================== 4o again
|
||||
|
||||
#python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
# --input_file "$OUTPUT_PREFIX_PATH/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.iter0.H100.dp8.v1.3.s42/checkpoint-1200/mathscale4o/split-1024/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-1024.json" \
|
||||
# --output_file $OUTPUT_PREFIX_PATH/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.iter0.H100.dp8.v1.3.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.process_rm.sc-p$p.azure.json \
|
||||
# --response_file_for_sc "$OUTPUT_PREFIX_PATH/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.iter0.H100.dp8.v1.3.s42/checkpoint-1200/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.s42.json" \
|
||||
# --response_id_field id --num_workers 128 --top_p $p
|
||||
python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "$OUTPUT_PREFIX_PATH/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.iter0.H100.dp8.v1.3.s42/checkpoint-1200/mathscale4o/split-1024/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-1024.json" \
|
||||
--output_file $OUTPUT_PREFIX_PATH/experiments/mathstral.mscale-v0.1-300k.process-dpo-sc.iter0.H100.dp8.v1.3.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.process_rm.gd.azure.json \
|
||||
--num_workers 128
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from functools import partial
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.mathscale.util import mathscale_is_equiv
|
||||
|
||||
|
||||
def counting_partial_response_value(preds, label):
|
||||
assert isinstance(preds, list), preds
|
||||
v = 0
|
||||
for pred in preds:
|
||||
res = mathscale_is_equiv(pred, label)[0]
|
||||
if res:
|
||||
v += 1
|
||||
return v
|
||||
|
||||
|
||||
def parse_value(v, binary: bool):
|
||||
if binary:
|
||||
return 1 if v > 0 else 0
|
||||
return v
|
||||
|
||||
|
||||
def _process_response_worker(item):
|
||||
item_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
prefix = item["prefix"]
|
||||
if "mscale-v0.1" in item_id:
|
||||
pass
|
||||
elif "numina" not in item_id:
|
||||
item_id = int(item_id)
|
||||
if not item["label"]:
|
||||
return item_id, {}
|
||||
|
||||
if item["pred"] == "":
|
||||
return item_id, {}
|
||||
|
||||
if item["pred"] == "failed extracting answer from completion":
|
||||
return item_id, {}
|
||||
|
||||
v = len([1 for res in item["res"] if res])
|
||||
return item_id, {"v": v, "prefix": prefix}
|
||||
|
||||
|
||||
def _process_trajectories_worker(item, binary: bool):
|
||||
item_id, trajectories = item
|
||||
outputs = {
|
||||
"idx": item_id
|
||||
}
|
||||
|
||||
trajectory_pairs = [(traj["v"], traj["prefix"]) for traj in trajectories]
|
||||
prefix_vis = set()
|
||||
|
||||
values = []
|
||||
prefixes = []
|
||||
for v, prefix in trajectory_pairs:
|
||||
if prefix in prefix_vis:
|
||||
continue
|
||||
values.append(parse_value(v, binary))
|
||||
prefixes.append(prefix)
|
||||
prefix_vis.add(prefix)
|
||||
outputs[f"value"] = values
|
||||
outputs[f"prefix"] = prefixes
|
||||
return outputs
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data, id_field="id"):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item[id_field] not in id2data:
|
||||
id2data[item[id_field]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item[id_field]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item[id_field]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--binary", default=False, action="store_true")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Collecting data...")
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in sorted(glob(args.input_file)):
|
||||
if ".metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
try:
|
||||
sub_data = json.load(open(file))
|
||||
except:
|
||||
print(f"Warning: {file}l")
|
||||
sub_data = [json.loads(line) for line in open(f"{file}l").readlines()]
|
||||
data += sub_data
|
||||
|
||||
data = merge_seed_sampled_data(data)
|
||||
|
||||
item_id2partial_trajectories = collections.defaultdict(list)
|
||||
missing = 0
|
||||
with Pool(args.num_workers) as pool:
|
||||
inputs = []
|
||||
for item in data:
|
||||
inputs.append(item)
|
||||
for item_id, result in tqdm(pool.imap_unordered(_process_response_worker, inputs), total=len(inputs)):
|
||||
if len(result) == 0:
|
||||
missing += 1
|
||||
continue
|
||||
item_id2partial_trajectories[item_id].append(result)
|
||||
|
||||
shoot_cnt = collections.Counter()
|
||||
for item_id, trajectories in item_id2partial_trajectories.items():
|
||||
for traj in trajectories:
|
||||
shoot_cnt[traj["v"]] += 1
|
||||
print(shoot_cnt)
|
||||
|
||||
print(f"Missing {missing} items in the response data.")
|
||||
|
||||
outputs = []
|
||||
cnt = collections.Counter()
|
||||
with Pool(args.num_workers) as pool:
|
||||
inputs = [(item_id, trajectories) for item_id, trajectories in item_id2partial_trajectories.items()]
|
||||
_annotate = partial(_process_trajectories_worker, binary=args.binary)
|
||||
for result in tqdm(pool.imap_unordered(_annotate, inputs), total=len(inputs)):
|
||||
outputs.append(result)
|
||||
cnt.update(result["value"])
|
||||
|
||||
print(cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_gd.binary.local.json \
|
||||
--binary --num_workers 24
|
||||
|
||||
Counter({0: 1243140, 3: 916473, 2: 442815, 1: 435687})
|
||||
Missing 74 items in the response data.
|
||||
Counter({1: 1793986, 0: 1242775})
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_gd.binary.local.json \
|
||||
--binary --num_workers 24
|
||||
|
||||
Counter({0: 1103040, 3: 908168, 2: 435088, 1: 427143})
|
||||
Missing 0 items in the response data.
|
||||
Counter({1: 1769112, 0: 1102585})
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_gd.local.json \
|
||||
--num_workers 24
|
||||
|
||||
Counter({0: 1243140, 3: 916473, 2: 442815, 1: 435687})
|
||||
Missing 74 items in the response data.
|
||||
Counter({0: 1242779, 3: 915927, 2: 442573, 1: 435482})
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/share/models/llama3.1_8b_mathscale4o/model_lr1e-5_batch512_epochs3_gpus8_linearSchedule/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_gd.local.json \
|
||||
--num_workers 24
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_gd.local.json \
|
||||
--num_workers 24
|
||||
|
||||
Counter({0: 1013210, 3: 660151, 2: 306372, 1: 305458})
|
||||
Missing 0 items in the response data.
|
||||
Counter({0: 1012382, 3: 659126, 2: 305949, 1: 305153})
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_gd.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/llama3.1.8b.mathscale4o.process-dpo.iter0.A100.dp8.v2.2.s42/checkpoint-1200/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_gd.local.json \
|
||||
--num_workers 24
|
||||
|
||||
Counter({0: 950765, 3: 748413, 2: 353782, 1: 336295})
|
||||
Missing 0 items in the response data.
|
||||
Counter({0: 950541, 3: 747952, 2: 353608, 1: 336155})
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,277 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from functools import partial
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.mathscale.util import mathscale_is_equiv
|
||||
|
||||
|
||||
def majority_voting_frequency(preds):
|
||||
assert isinstance(preds, list)
|
||||
if isinstance(preds[0], list):
|
||||
tmp = []
|
||||
for pred in preds:
|
||||
tmp.append(str(sorted(pred)))
|
||||
tmp = collections.Counter(tmp)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(eval(pred), fre) for pred, fre in tmp]
|
||||
elif isinstance(preds[0], str):
|
||||
tmp = collections.Counter(preds)
|
||||
tmp = sorted(tmp.items(), key=lambda x: x[1], reverse=True)
|
||||
sorted_preds = [(pred, fre) for pred, fre in tmp]
|
||||
else:
|
||||
raise ValueError(f"Unknown type {type(preds[0])}")
|
||||
return sorted_preds
|
||||
|
||||
|
||||
def extract_sc_mul_labels(response_data, id_field: str):
|
||||
id2sc_preds = {}
|
||||
for item in response_data:
|
||||
if not item["response"]:
|
||||
continue
|
||||
|
||||
if isinstance(item["response"], list):
|
||||
preds = item["pred"]
|
||||
else:
|
||||
preds = [item["pred"]]
|
||||
|
||||
assert len(preds) > 1, f"Self-consistency requires at least 2 predictions, but got {len(preds)}."
|
||||
|
||||
sorted_preds = majority_voting_frequency(preds)
|
||||
item_id = item[id_field] if isinstance(item[id_field], str) else str(item[id_field])
|
||||
id2sc_preds[item_id] = sorted_preds
|
||||
|
||||
return id2sc_preds
|
||||
|
||||
|
||||
def counting_partial_response_value(preds, label):
|
||||
assert isinstance(preds, list), preds
|
||||
v = 0
|
||||
for pred in preds:
|
||||
res = mathscale_is_equiv(pred, label)[0]
|
||||
if res:
|
||||
v += 1
|
||||
return v
|
||||
|
||||
|
||||
def parse_value(v, binary: bool):
|
||||
if binary:
|
||||
return 1 if v > 0 else 0
|
||||
return v
|
||||
|
||||
|
||||
def _process_response_init(id2sc_preds):
|
||||
global _id2sc_preds
|
||||
_id2sc_preds = id2sc_preds
|
||||
|
||||
|
||||
def _process_response_worker(item, top_p: float = 0.0):
|
||||
item_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
prefix = item["prefix"]
|
||||
# item_id = int(item_id)
|
||||
if item_id not in _id2sc_preds:
|
||||
return item_id, {}
|
||||
|
||||
if item["pred"] == "":
|
||||
return item_id, {}
|
||||
|
||||
if item["pred"] == "failed extracting answer from completion":
|
||||
return item_id, {}
|
||||
|
||||
sc_label, sc_freq = _id2sc_preds[item_id][0]
|
||||
tot_num = sum([fre for _, fre in _id2sc_preds[item_id]])
|
||||
if sc_freq / tot_num < top_p:
|
||||
return item_id, {}
|
||||
|
||||
v = counting_partial_response_value(item["pred"], sc_label)
|
||||
return item_id, {"v": v, "prefix": prefix}
|
||||
|
||||
|
||||
def _process_trajectories_worker(item, binary: bool):
|
||||
item_id, trajectories = item
|
||||
outputs = {
|
||||
"idx": item_id
|
||||
}
|
||||
|
||||
trajectory_pairs = [(traj["v"], traj["prefix"]) for traj in trajectories]
|
||||
prefix_set = set()
|
||||
|
||||
values = []
|
||||
prefixes = []
|
||||
for v, prefix in trajectory_pairs:
|
||||
if prefix in prefix_set:
|
||||
continue
|
||||
values.append(parse_value(v, binary))
|
||||
prefixes.append(prefix)
|
||||
prefix_set.add(prefix)
|
||||
outputs["value"] = values
|
||||
outputs["prefix"] = prefixes
|
||||
return outputs
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data, id_field="id"):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item[id_field] not in id2data:
|
||||
id2data[item[id_field]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item[id_field]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item[id_field]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--response_file_for_sc", type=str)
|
||||
parser.add_argument("--response_id_field", type=str, default="idx")
|
||||
parser.add_argument("--binary", default=False, action="store_true")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Collecting data...")
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
if ".metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
data += json.load(open(file))
|
||||
|
||||
print("Collecting response data...")
|
||||
if os.path.exists(args.response_file_for_sc):
|
||||
response_data = json.load(open(args.response_file_for_sc))
|
||||
else:
|
||||
response_data = []
|
||||
for file in glob(args.response_file_for_sc):
|
||||
if ".metrics" in file:
|
||||
continue
|
||||
print(file)
|
||||
response_data += json.load(open(file))
|
||||
response_data = merge_seed_sampled_data(response_data, args.response_id_field)
|
||||
|
||||
last_round_id2sc_preds = extract_sc_mul_labels(response_data, args.response_id_field)
|
||||
num_candidate_cnt = collections.Counter()
|
||||
for item_id, preds in last_round_id2sc_preds.items():
|
||||
num_candidate_cnt[len(preds)] += 1
|
||||
print(num_candidate_cnt)
|
||||
del response_data
|
||||
|
||||
item_id2partial_trajectories = collections.defaultdict(list)
|
||||
missing = 0
|
||||
with Pool(args.num_workers, initializer=_process_response_init, initargs=(last_round_id2sc_preds,)) as pool:
|
||||
inputs = []
|
||||
for item in data:
|
||||
inputs.append(item)
|
||||
_annotate = partial(_process_response_worker, top_p=args.top_p)
|
||||
for item_id, result in tqdm(pool.imap_unordered(_annotate, inputs), total=len(inputs)):
|
||||
if len(result) == 0:
|
||||
missing += 1
|
||||
continue
|
||||
item_id2partial_trajectories[item_id].append(result)
|
||||
|
||||
shoot_cnt = collections.Counter()
|
||||
for item_id, trajectories in item_id2partial_trajectories.items():
|
||||
for traj in trajectories:
|
||||
shoot_cnt[traj["v"]] += 1
|
||||
print(shoot_cnt)
|
||||
|
||||
print(f"Missing {missing} items in the response data.")
|
||||
|
||||
outputs = []
|
||||
cnt = collections.Counter()
|
||||
with Pool(args.num_workers) as pool:
|
||||
inputs = [(item_id, trajectories) for item_id, trajectories in item_id2partial_trajectories.items()]
|
||||
_annotate = partial(_process_trajectories_worker, binary=args.binary)
|
||||
for result in tqdm(pool.imap_unordered(_annotate, inputs), total=len(inputs)):
|
||||
outputs.append(result)
|
||||
cnt.update(result["value"])
|
||||
|
||||
print(cnt)
|
||||
parent_dir = os.path.dirname(args.output_file)
|
||||
if not os.path.exists(parent_dir):
|
||||
os.makedirs(parent_dir, exist_ok=True)
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm_by_sc.azure.json \
|
||||
--response_file_for_sc "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.?-of-8.json" \
|
||||
--response_id_field id --num_workers 128
|
||||
|
||||
Counter({3: 889966, 0: 537841, 2: 459487, 1: 397897})
|
||||
Missing 0 items in the response data.
|
||||
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm.iter0-pdpo-sc.azure.json \
|
||||
--response_file_for_sc "../msranlpintern/reward_modeling/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n90.tem1.0.p0.9.json" \
|
||||
--response_id_field id --num_workers 128
|
||||
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/split-512/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.*-of-512.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.sft.V100.tp2dp8.v2.0.s42/checkpoint-800/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample10.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm.iter0-pdpo-sc-p0.6.azure.json \
|
||||
--response_file_for_sc "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n90.tem1.0.p0.9.json" \
|
||||
--response_id_field id --num_workers 128 --top_p 0.6
|
||||
|
||||
Counter({3: 945267, 2: 408413, 0: 308107, 1: 279042})
|
||||
Missing 1097360 items in the response data.
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
--input_file "${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/split-256/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample16.filter_same.*-of-4.prefix_completion.n3.tem1.0.p0.9.*-of-256.json" \
|
||||
--output_file ${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/train.500k.de_con.boxed.v1.0.n10.tem1.0.p0.9.upper0.7.r0.3.sample16.filter_same.prefix_completion.n3.tem1.0.p0.9.process_rm.sc-10-p0.0.azure.json \
|
||||
--response_file_for_sc "${MODEL_PREFIX_PATH}/mathstral-7B-v0.1/mathscale4o/500k-split-*-of-20/train.500k.de_con.boxed.v1.0.*-of-20.0shot.n10.tem1.0.p0.9.*-of-64.*json" --response_id_field id --num_workers 128
|
||||
|
||||
Counter({3: 1869170, 0: 1528083, 2: 859527, 1: 787261})
|
||||
Missing 24189 items in the response data.
|
||||
Counter({3: 1862945, 0: 1521858, 2: 855962, 1: 783862})
|
||||
|
||||
|
||||
>>> python scripts/math_scale/construct_process_rm_sample_sc.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/completion-split-128/cot.de_con.n8.tem1.0.p1.0.s0.upper0.7.r0.3.sample32.filter_same.{split_id}-of-4.n3.tem1.0.p1.0.*-of-128.s0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/cot.de_con.n8.tem1.0.p1.0.s0.upper0.7.r0.3.sample32.filter_same.process_rm.iter0-pdpo-sc-p0.5.v0.0.{split_id}-of-4.azure.json \
|
||||
--response_file_for_sc "${OUTPUT_PREFIX_PATH}/experiments/mathstral.mathscale4o.process-dpo.iter0.V100.tp8dp48.v2.2.fix.s42/checkpoint-600/numina/830k-split-*-of-10/cot.de_con.*-of-10.n8.tem1.0.p1.0.*-of-*.s*.json" --response_id_field id --num_workers 128 --top_p 0.5
|
||||
Counter({3: 803168, 2: 454386, 1: 260676, 0: 159454})
|
||||
Missing 2783152 items in the response data.
|
||||
Counter({3: 772040, 2: 436348, 1: 250464, 0: 153477})
|
||||
"""
|
||||
@@ -0,0 +1,14 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--orig_data_file", type=str)
|
||||
parser.add_argument("--orig_id_filed", type=str, default="id")
|
||||
parser.add_argument("--train_data_file", type=str)
|
||||
parser.add_argument("--train_id_field", type=str, default="idx")
|
||||
args = parser.parse_args()
|
||||
|
||||
pass
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user