chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,140 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
colors = ['#FFEBAD', '#DCD7C1', '#BFB1D0', '#A7C0DE', '#6C91C2', '#F46F43']
|
||||
|
||||
|
||||
def draw_double_histogram(correct_data, incorrect_data, file_path):
|
||||
# Create histogram
|
||||
plt.figure(figsize=(5, 4))
|
||||
plt.hist(correct_data, bins=10, alpha=0.7, label='Correct', color='blue')
|
||||
plt.hist(incorrect_data, bins=10, alpha=0.7, label='Incorrect', color='red')
|
||||
|
||||
# Add labels and title
|
||||
plt.xlabel('Top-1 Frequency Averaged by Number of Test Cases')
|
||||
plt.ylabel('No. of Data Points')
|
||||
# plt.title('Frequency Distribution of Correct and Incorrect Data')
|
||||
plt.legend(loc='upper right')
|
||||
|
||||
# Show the plot
|
||||
# plt.show()
|
||||
plt.savefig(file_path)
|
||||
|
||||
|
||||
def draw_histogram(data, labels, file_path):
|
||||
# Create bar chart for the single group of data
|
||||
# Create a bar chart for the three metrics
|
||||
plt.figure(figsize=(4, 4))
|
||||
# bars = plt.bar(labels, data, color='#5ea69c', width=0.4)
|
||||
# bars = plt.bar(labels, data, color=colors[5], width=0.4)
|
||||
bars = plt.bar(labels, data, color="#1E90FF", width=0.4)
|
||||
# Add text labels above each bar to indicate the values
|
||||
for bar in bars:
|
||||
yval = bar.get_height()
|
||||
plt.text(bar.get_x() + bar.get_width() / 2, yval + 0.005, round(yval, 2), ha='center', va='bottom', fontsize=16)
|
||||
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
|
||||
# Add labels and title
|
||||
# plt.xlabel('Category')
|
||||
plt.ylabel('Pass@1', fontsize=16)
|
||||
# plt.title('Accuracy Comparison Across Different Metrics')
|
||||
|
||||
plt.ylim(bottom=15)
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def draw_line_plot(correct_data, incorrect_data, file_path):
|
||||
# Compute histogram data
|
||||
correct_hist, correct_bins = np.histogram(correct_data, bins=10)
|
||||
incorrect_hist, incorrect_bins = np.histogram(incorrect_data, bins=10)
|
||||
|
||||
# Get the center of each bin for plotting
|
||||
correct_bin_centers = 0.5 * (correct_bins[1:] + correct_bins[:-1])
|
||||
incorrect_bin_centers = 0.5 * (incorrect_bins[1:] + incorrect_bins[:-1])
|
||||
|
||||
# Create the line plot
|
||||
plt.figure(figsize=(4, 4))
|
||||
# plt.plot(correct_bin_centers, correct_hist, label='Correct', color='#658873', marker='.', linestyle='-')
|
||||
# plt.plot(incorrect_bin_centers, incorrect_hist, label='Incorrect', color='#d2bfa5', marker='.', linestyle='-')
|
||||
plt.plot(correct_bin_centers, correct_hist, label='Correct', color=colors[2], marker='.', linestyle='-')
|
||||
plt.plot(incorrect_bin_centers, incorrect_hist, label='Incorrect', color=colors[4], marker='.', linestyle='-')
|
||||
|
||||
# Change the size of the font in the legent
|
||||
plt.legend(prop={"size": 15})
|
||||
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
||||
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
||||
|
||||
# Add labels and title
|
||||
# plt.xlabel('Top-1 Averaged Frequency', fontsize=16)
|
||||
plt.ylabel('No. of Data Points', fontsize=16)
|
||||
plt.legend(loc='upper left')
|
||||
|
||||
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--freq_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.freq_file))
|
||||
|
||||
pos_freqs = []
|
||||
neg_freqs = []
|
||||
difficulties = set([item["difficulty"] for item in data])
|
||||
diff_freqs = {
|
||||
diff: {"pos": [], "neg": []} for diff in difficulties
|
||||
}
|
||||
for item in data:
|
||||
if item["prog_sc_res"]:
|
||||
pos_freqs.append(item["tot_freq"])
|
||||
diff_freqs[item["difficulty"]]["pos"].append(item["tot_freq"])
|
||||
else:
|
||||
neg_freqs.append(item["tot_freq"])
|
||||
diff_freqs[item["difficulty"]]["neg"].append(item["tot_freq"])
|
||||
|
||||
# paint(pos_freqs, neg_freqs, args.output_file.replace(".png", "_all.png"))
|
||||
draw_line_plot(pos_freqs, neg_freqs, args.output_file.replace(".png", "_all.png"))
|
||||
for diff, freqs in diff_freqs.items():
|
||||
# paint(freqs["pos"], freqs["neg"], args.output_file.replace(".png", f"_{diff}.png"))
|
||||
draw_line_plot(freqs["pos"], freqs["neg"], args.output_file.replace(".png", f"_{diff}.png"))
|
||||
|
||||
sc = 0
|
||||
prog_sc = 0
|
||||
first_res = 0
|
||||
diff_res = {
|
||||
diff: {"sc": 0, "prog_sc": 0, "first_res": 0} for diff in difficulties
|
||||
}
|
||||
for item in data:
|
||||
if item["sc_res"]:
|
||||
sc += 1
|
||||
diff_res[item["difficulty"]]["sc"] += 1
|
||||
if item["prog_sc_res"]:
|
||||
prog_sc += 1
|
||||
diff_res[item["difficulty"]]["prog_sc"] += 1
|
||||
if item["res"]:
|
||||
first_res += 1
|
||||
diff_res[item["difficulty"]]["first_res"] += 1
|
||||
|
||||
print(f"Self-consistency: {sc}/{len(data)} = {sc / len(data)}")
|
||||
print(f"Program self-consistency: {prog_sc}/{len(data)} = {prog_sc / len(data)}")
|
||||
print(f"First res: {first_res}/{len(data)} = {first_res / len(data)}")
|
||||
for diff, res in diff_res.items():
|
||||
print(f"Difficulty {diff}:")
|
||||
print(f"Self-consistency: {res['sc']}/{len(data)} = {res['sc'] / len(data)}")
|
||||
print(f"Program self-consistency: {res['prog_sc']}/{len(data)} = {res['prog_sc'] / len(data)}")
|
||||
print(f"First res: {res['first_res']}/{len(data)} = {res['first_res'] / len(data)}")
|
||||
|
||||
draw_histogram([19.24, prog_sc / len(data) * 100, sc / len(data) * 100],
|
||||
["G.D.", "S.C. - P", "S.C. - T"], args.output_file.replace(".png", "_res.png"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,263 @@
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datasets import load_dataset
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def worker(x):
|
||||
item, orig_item = x
|
||||
input_output = item["test_cases"]
|
||||
inputs = input_output["inputs"]
|
||||
|
||||
num_test_cases = len(input_output["inputs"])
|
||||
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
output2res = [
|
||||
{} for _ in range(len(inputs))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
# if not str(case_o):
|
||||
# continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
# assert case_j < len(orig_item["full_res"][resp_id]), (resp_id, case_j, orig_item["full_res"][resp_id])
|
||||
if len(orig_item["full_res"][resp_id]) <= case_j:
|
||||
assert all(x == -1 for x in orig_item["full_res"][resp_id])
|
||||
output_res = -1
|
||||
else:
|
||||
output_res = orig_item["full_res"][resp_id][case_j]
|
||||
output2res[case_j][str(case_o)] = output_res
|
||||
|
||||
# Get self-consistency
|
||||
sc_res = True
|
||||
sc_outputs = []
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if len(output_cnt) == 0:
|
||||
sc_res = False
|
||||
sc_outputs.append(None)
|
||||
continue
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_res = output2res[case_j][sc_o]
|
||||
if not sc_o_res:
|
||||
sc_res = False
|
||||
sc_outputs.append(sc_o)
|
||||
|
||||
# Estimate the corresponding frequency rates
|
||||
tot_freq = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if len(output_cnt) == 0:
|
||||
continue
|
||||
|
||||
max_freq = output_cnt.most_common(1)[0][1]
|
||||
tot_freq += max_freq
|
||||
|
||||
tot_freq /= num_test_cases
|
||||
|
||||
# Confirm if there is some program solution meets all self-consistency outputs
|
||||
sc_match_res = []
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
resp_match_res = True
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
# resp_match_res = False
|
||||
break
|
||||
if case_r != 0:
|
||||
resp_match_res = False
|
||||
break
|
||||
# if not str(case_o):
|
||||
# resp_match_res = False
|
||||
# break
|
||||
|
||||
if sc_outputs[case_j] is None:
|
||||
resp_match_res = False
|
||||
break
|
||||
|
||||
if str(case_o) != sc_outputs[case_j]:
|
||||
resp_match_res = False
|
||||
if orig_item["full_res"][resp_id][case_j] and output2res[case_j][sc_outputs[case_j]]:
|
||||
print(f"warning", str(case_o), sc_outputs[case_j], orig_item["test_cases"]["outputs"][case_j])
|
||||
break
|
||||
|
||||
sc_match_res.append(resp_match_res)
|
||||
|
||||
if sc_res and any(
|
||||
sc_match_res): # If the self-consistency determined group is correct and there is one program match all predictions with self-consistency, then it is a correct solution
|
||||
prog_sc_res = True
|
||||
else:
|
||||
prog_sc_res = False
|
||||
|
||||
return {
|
||||
"sc_res": sc_res,
|
||||
"sc_outputs": sc_outputs,
|
||||
"tot_freq": tot_freq,
|
||||
"prog_sc_res": prog_sc_res,
|
||||
"sc_match_res": sc_match_res,
|
||||
"res": orig_item["res"][0],
|
||||
"id": item["id"]
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
if isinstance(tmp["response"], str):
|
||||
tmp["response"] = [tmp["response"]]
|
||||
if not isinstance(tmp["res"], list):
|
||||
tmp["res"] = [tmp["res"]]
|
||||
if not isinstance(tmp["pred"], list):
|
||||
tmp["pred"] = [tmp["pred"]]
|
||||
if not isinstance(tmp["full_res"], list):
|
||||
tmp["full_res"] = [tmp["full_res"]]
|
||||
if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--exec_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
parser.add_argument("--split", default="test")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(f'apps_difficulty_{args.split}.json'):
|
||||
_dataset = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
problem_id2difficulty = {item["problem_id"]: item["difficulty"] for item in _dataset}
|
||||
all_difficulties = collections.Counter(problem_id2difficulty.values())
|
||||
json.dump(problem_id2difficulty, open(f'apps_difficulty_{args.split}.json', "w"), ensure_ascii=False)
|
||||
json.dump(all_difficulties, open(f'apps_difficulty_{args.split}_all.json', "w"), ensure_ascii=False)
|
||||
else:
|
||||
problem_id2difficulty = json.load(open(f'apps_difficulty_{args.split}.json'))
|
||||
all_difficulties = json.load(open(f'apps_difficulty_{args.split}_all.json'))
|
||||
problem_id2difficulty = {int(k): v for k, v in problem_id2difficulty.items()}
|
||||
|
||||
completions = load_files(args.completion_file)
|
||||
completions = merge_seed_sampled_data(completions)
|
||||
execs = load_files(args.exec_file)
|
||||
execs = merge_seed_sampled_data(execs)
|
||||
print(len(completions), len(execs))
|
||||
|
||||
id2completion = {item["id"]: item for item in completions}
|
||||
id2exec = {item["id"]: item for item in execs}
|
||||
print(len(id2completion), len(id2exec))
|
||||
|
||||
commons = set(id2completion.keys()) & set(id2exec.keys())
|
||||
print(f"Found {len(commons)} common items.")
|
||||
|
||||
inputs = []
|
||||
for _id in commons:
|
||||
inputs.append((id2exec[_id], id2completion[_id]))
|
||||
|
||||
pbar = tqdm(inputs)
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = worker
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
result = future.result()
|
||||
result["difficulty"] = problem_id2difficulty[result["id"]]
|
||||
outputs.append(result)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"))
|
||||
|
||||
sc = 0
|
||||
prog_sc = 0
|
||||
first_res = 0
|
||||
num_prog = 0
|
||||
for item in outputs:
|
||||
if item["sc_res"]:
|
||||
sc += 1
|
||||
if item["prog_sc_res"]:
|
||||
prog_sc += 1
|
||||
if item["res"]:
|
||||
first_res += 1
|
||||
num_prog += len(item["sc_match_res"])
|
||||
|
||||
print(f"Self-consistency: {sc}/{len(completions)} = {sc / len(completions)}")
|
||||
print(f"Program self-consistency: {prog_sc}/{len(completions)} = {prog_sc / len(completions)}")
|
||||
print(f"First res: {first_res}/{len(completions)} = {first_res / len(completions)}")
|
||||
print(f"Programs: {num_prog}/{len(completions)} = {num_prog / len(completions)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,26 @@
|
||||
exp_dir="../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.dpo.H100.dp8.v1.0.s42/apps/checkpoint-2400"
|
||||
|
||||
#python scripts/apps/analyze/get_output_frequency.py \
|
||||
# --completion_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.json" \
|
||||
# --exec_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.run_outputs.json" \
|
||||
# --output_file ${exp_dir}/test.0shot.tem1.0.n8.v2.0.run_outputs.analyze_freq.json --num_workers 16
|
||||
|
||||
|
||||
#python scripts/apps/analyze/freq2image.py \
|
||||
# --freq_file ${exp_dir}/test.0shot.tem1.0.n8.v2.0.run_outputs.analyze_freq.json \
|
||||
# --output_file ./apps.test.0shot.tem1.0.n8.v2.0.analyze_freq.png
|
||||
|
||||
|
||||
#python scripts/apps/analyze/get_output_frequency.py \
|
||||
# --completion_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.s?.json" \
|
||||
# --exec_file "${exp_dir}/test.0shot.tem1.0.n8.*-of-32.v2.0.s?.run_outputs.json" \
|
||||
# --output_file ${exp_dir}/test.0shot.tem1.0.n64.v2.0.run_outputs.analyze_freq.json --num_workers 16
|
||||
|
||||
#Self-consistency: 1067/5000 = 0.2134
|
||||
#Program self-consistency: 1053/5000 = 0.2106
|
||||
#First res: 883/5000 = 0.1766
|
||||
#Programs: 320000/5000 = 64.0
|
||||
|
||||
python scripts/apps/analyze/freq2image.py \
|
||||
--freq_file ${exp_dir}/test.0shot.tem1.0.n64.v2.0.run_outputs.analyze_freq.json \
|
||||
--output_file ./apps.test.0shot.tem1.0.n64.v2.0.analyze_freq.png
|
||||
@@ -0,0 +1,15 @@
|
||||
python scripts/apps/worsen_gpt4_combine.py \
|
||||
--worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_problem_id_field id \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f100k.json
|
||||
|
||||
|
||||
python scripts/apps/rerank_code_rm.py \
|
||||
--completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_dpo.A100.dp8.v3.0.s42/apps/checkpoint-200/test.0shot.tem1.0.n20.v1.1.json \
|
||||
--reward_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.pair-rm.gpt4o-worsen.A100.w8.v1.0.s42/r2c.dpo.step-200.test.n20/test-checkpoint-50/eval_predictions_rank?.json"
|
||||
|
||||
|
||||
python scripts/apps/rerank_code_rm.py \
|
||||
--completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_dpo.A100.dp8.v3.0.s42/apps/checkpoint-200/test.0shot.tem1.0.n20.v1.1.json \
|
||||
--reward_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.pair-rm.r2c-policy-data.A100.w8.v1.0.s42/r2c.dpo.step-200.test.n20/test-checkpoint-50/eval_predictions_rank?.json"
|
||||
@@ -0,0 +1,114 @@
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--response_field", type=str, default="response")
|
||||
parser.add_argument("--test_case_field", type=str, default="input_output")
|
||||
args = parser.parse_args()
|
||||
|
||||
cnt = 0
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
print(len(data))
|
||||
|
||||
for item in tqdm(data):
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
if isinstance(item[args.response_field], str): # We cannot make pairs if there is only one response.
|
||||
item["pos"] = []
|
||||
item["pos_code"] = []
|
||||
item["neg"] = []
|
||||
item["neg_code"] = []
|
||||
continue
|
||||
|
||||
if "res" in item and "full_res" in item and item[args.test_case_field]: # If there is no test-cases, we cannot determine the correctness
|
||||
assert len(item["res"]) == len(item["full_res"]) == len(item[args.response_field]), (len(item["res"]),
|
||||
len(item["full_res"]),
|
||||
len(item[args.response_field]),
|
||||
item[args.response_field])
|
||||
for i, r in enumerate(item["res"]):
|
||||
if r is True:
|
||||
assert item[args.response_field][i]
|
||||
assert item["pred"][i], item["pred"]
|
||||
if item[args.response_field][i]:
|
||||
pos.append(item[args.response_field][i])
|
||||
pos_code.append(item["pred"][i])
|
||||
else:
|
||||
if item[args.response_field][i]:
|
||||
neg.append(item[args.response_field][i])
|
||||
if item["pred"][i] and item["pred"][i].strip():
|
||||
neg_code.append(item["pred"][i])
|
||||
item["pos"] = pos
|
||||
item["neg"] = neg
|
||||
item["pos_code"] = pos_code
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos) * len(neg)
|
||||
|
||||
if args.response_field != "response":
|
||||
item["response"] = item.pop(args.response_field)
|
||||
if args.test_case_field != "input_output":
|
||||
item["input_output"] = item.pop(args.test_case_field)
|
||||
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
print(len(data), cnt, cnt / len(data))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.dpo_v1.0.json --test_case_field test_cases
|
||||
|
||||
4223
|
||||
100%|████████████████████████████████████████████████████████████████████████████████████| 4223/4223 [00:00<00:00, 148189.90it/s]
|
||||
4223 18383 4.353066540374142
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec_dpo.json --test_case_field test_cases
|
||||
|
||||
4715
|
||||
100%|██████████████████████████████████████████████████████████| 4715/4715 [00:00<00:00, 107036.93it/s]
|
||||
4715 15921 3.3766702014846235
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/apps/construct_prefer_pair.py --input_file "train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file "train.0shot.tem1.0.n10.v2.0.dpo_v1.0.json" --test_case_field test_cases
|
||||
|
||||
4500
|
||||
100%|██████████████████████| 4500/4500 [00:00<00:00, 142921.59it/s]
|
||||
4500 42550 9.455555555555556
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.dpo_v1.0.json \
|
||||
--test_case_field test_cases
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
import argparse
|
||||
import sys
|
||||
import collections
|
||||
import glob
|
||||
import os
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
parser.add_argument("--lower_bound", type=float, default=0.0)
|
||||
parser.add_argument("--margin", type=float, default=1.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
solutions = json.load(open(args.completion_file))
|
||||
if "difficulty" in solutions[0]:
|
||||
all_diff = collections.Counter([item["difficulty"] for item in solutions])
|
||||
all_diff_success = {k: 0 for k in all_diff}
|
||||
else:
|
||||
all_diff = None
|
||||
all_diff_success = None
|
||||
|
||||
if os.path.exists(args.reward_file):
|
||||
rewards = json.load(open(args.reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob.glob(args.reward_file):
|
||||
rewards += json.load(open(file))
|
||||
|
||||
id2reward = collections.defaultdict(dict)
|
||||
for item in rewards:
|
||||
index = item["index"]
|
||||
orig_id, pred_id = list(map(int, index.split("_")))
|
||||
if isinstance(item["reward"], list):
|
||||
id2reward[orig_id][pred_id] = item["reward"][0]
|
||||
else:
|
||||
id2reward[orig_id][pred_id] = item["reward"]
|
||||
|
||||
outputs = []
|
||||
success = 0
|
||||
num_pairs = 0
|
||||
for item in solutions:
|
||||
problem_id = item["id"]
|
||||
|
||||
if not item["response"]:
|
||||
continue
|
||||
if not item["pred"]:
|
||||
continue
|
||||
|
||||
assert isinstance(item["response"], list)
|
||||
assert isinstance(item["pred"], list)
|
||||
assert len(item["response"]) == len(item["pred"])
|
||||
|
||||
tuples = []
|
||||
for j, (resp, pred) in enumerate(zip(item["response"], item["pred"])):
|
||||
if pred:
|
||||
tuples.append((j, resp, pred, id2reward[problem_id][j]))
|
||||
|
||||
if not tuples:
|
||||
continue
|
||||
|
||||
tuples = sorted(tuples, key=lambda x: x[3], reverse=True)
|
||||
# Construct preference pairs: collect pairs with margin greater than args.margin
|
||||
pos = []
|
||||
neg = []
|
||||
for i in range(len(tuples)):
|
||||
if tuples[i][3] < args.lower_bound:
|
||||
break
|
||||
for j in range(i + 1, len(tuples)):
|
||||
if tuples[i][3] - tuples[j][3] < args.margin:
|
||||
continue
|
||||
pos.append(tuples[i][1])
|
||||
neg.append(tuples[j][1])
|
||||
outputs.append({
|
||||
"id": problem_id,
|
||||
"pos": pos,
|
||||
"neg": neg
|
||||
})
|
||||
num_pairs += len(pos)
|
||||
|
||||
_pred_id = tuples[0][0]
|
||||
if item["res"] and item["res"][_pred_id]:
|
||||
success += 1
|
||||
if all_diff is not None:
|
||||
all_diff_success[item["difficulty"]] += 1
|
||||
|
||||
print(f"Success rate: {success / len(solutions)}")
|
||||
if all_diff is not None:
|
||||
for k, v in all_diff.items():
|
||||
print(f"Difficulty {k}: {all_diff_success[k] / v}")
|
||||
|
||||
json.dump(outputs, open(args.completion_file.replace(".json", f".lower-{args.lower_bound}.mar-{args.margin}.json"), "w"), ensure_ascii=False, indent=2)
|
||||
print(f"Number of pairs: {num_pairs}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/construct_prefer_pair_rm.py --completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.json --reward_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.pair-rm.gpt4o-worsen.A100.w8.v1.2.s42/r2c.sft.step-400.train.n10/test-checkpoint-50/eval_predictions_rank0.json --lower_bound 0.0 --margin 1.0
|
||||
|
||||
Success rate: 0.3566
|
||||
Number of pairs: 29584
|
||||
"""
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import collections
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Soft version of constructing preference pair. This would be useful for teacher-generated pseudo test cases.
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--response_field", type=str, default="response")
|
||||
parser.add_argument("--test_case_field", type=str, default="input_output")
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
args = parser.parse_args()
|
||||
|
||||
cnt = 0
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
if len(data) == 0:
|
||||
raise ValueError(f"No data found in {args.input_file}")
|
||||
|
||||
print(len(data))
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
if isinstance(item[args.response_field], str): # We cannot make pairs if there is only one response.
|
||||
item["pos"] = []
|
||||
item["pos_code"] = []
|
||||
item["neg"] = []
|
||||
item["neg_code"] = []
|
||||
continue
|
||||
|
||||
if len(item[args.test_case_field]["inputs"]) == 0:
|
||||
item["pos"] = []
|
||||
item["pos_code"] = []
|
||||
item["neg"] = []
|
||||
item["neg_code"] = []
|
||||
continue
|
||||
|
||||
if "res" in item and "full_res" in item and item[args.test_case_field]: # If there is no test-cases, we cannot determine the correctness
|
||||
assert len(item["res"]) == len(item["full_res"]) == len(item[args.response_field]), (len(item["res"]),
|
||||
len(item["full_res"]),
|
||||
len(item[args.response_field]),
|
||||
item[args.response_field])
|
||||
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
num_test_cases = len(item[args.test_case_field]["inputs"])
|
||||
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["neg"] = neg
|
||||
item["pos_code"] = pos_code
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
if args.response_field != "response":
|
||||
item["response"] = item.pop(args.response_field)
|
||||
if args.test_case_field != "input_output":
|
||||
item["input_output"] = item.pop(args.test_case_field)
|
||||
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
print(len(data), cnt, cnt / len(data))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.dpo_v1.0.json --test_case_field test_cases
|
||||
|
||||
4223
|
||||
100%|████████████████████████████████████████████████████████████████████████████████████| 4223/4223 [00:00<00:00, 148189.90it/s]
|
||||
4223 18383 4.353066540374142
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py --input_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec_dpo.json --test_case_field test_cases
|
||||
|
||||
4715
|
||||
100%|██████████████████████████████████████████████████████████| 4715/4715 [00:00<00:00, 107036.93it/s]
|
||||
4715 15921 3.3766702014846235
|
||||
|
||||
>>> python ~/gpt-chat-examples/scripts/apps/construct_prefer_pair.py --input_file "train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file "train.0shot.tem1.0.n10.v2.0.dpo_v1.0.json" --test_case_field test_cases
|
||||
|
||||
4500
|
||||
100%|██████████████████████| 4500/4500 [00:00<00:00, 142921.59it/s]
|
||||
4500 42550 9.455555555555556
|
||||
|
||||
>>> python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.dpo_v1.0.json \
|
||||
--test_case_field test_cases
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,35 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.code.code import APPsEvaluator
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
outputs = []
|
||||
with open(args.input_file, "r", encoding="utf-8")as f:
|
||||
for line in f:
|
||||
item = json.loads(line)
|
||||
pred = standard_cleaner_default(item["completion"])
|
||||
item["pred"] = pred
|
||||
item["test_cases"] = item.pop("input_output")
|
||||
outputs.append(item)
|
||||
|
||||
evaluator = APPsEvaluator()
|
||||
predictions, metrics = evaluator(outputs, num_workers=24)
|
||||
|
||||
print(metrics)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,153 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def extract_solution_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'<BEGIN>(.*?)<END>'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def _worker(item):
|
||||
solution = item["solutions"][0]
|
||||
|
||||
all_results = check_correctness(item["input_output"], solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
# res = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
result = True
|
||||
else:
|
||||
result = False
|
||||
|
||||
return result, res
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--test_case_field", type=str)
|
||||
parser.add_argument("--test_case_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train", trust_remote_code=True).to_list()
|
||||
print(f"Total number of items before: {len(data)}")
|
||||
train_sub_val_ids = set(json.load(open("apps_train_sub_val_ids.json")))
|
||||
|
||||
data = [item for item in data if item["problem_id"] not in train_sub_val_ids]
|
||||
print(len(data))
|
||||
|
||||
test_cases = json.load(open(args.test_case_file))
|
||||
# id2test_cases = {item[args.test_case_id_field]: item[args.test_case_field] for item in test_cases}
|
||||
missing = 0
|
||||
repeat = 0
|
||||
id2test_cases = {}
|
||||
for item in test_cases:
|
||||
if item[args.test_case_id_field] in id2test_cases:
|
||||
repeat += 1
|
||||
if args.test_case_field not in item:
|
||||
missing += 1
|
||||
continue
|
||||
id2test_cases[item[args.test_case_id_field]] = item[args.test_case_field]
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Repeat: {repeat}")
|
||||
|
||||
missing_solutions = 0
|
||||
missing_test_cases = 0
|
||||
new_data = []
|
||||
for item in data:
|
||||
if item["solutions"]:
|
||||
item["solutions"] = json.loads(item["solutions"])
|
||||
assert isinstance(item["solutions"], list)
|
||||
if len(item["solutions"]) > 0:
|
||||
if item["problem_id"] in id2test_cases:
|
||||
item["input_output"] = id2test_cases[item["problem_id"]]
|
||||
new_data.append(item)
|
||||
elif f"apps-train-{item['problem_id']}" in id2test_cases:
|
||||
item["input_output"] = id2test_cases[f"apps-train-{item['problem_id']}"]
|
||||
new_data.append(item)
|
||||
else:
|
||||
missing_test_cases += 1
|
||||
else:
|
||||
missing_solutions += 1
|
||||
else:
|
||||
missing_solutions += 1
|
||||
|
||||
print(f"Missing solutions: {missing_solutions}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
micro_corr = 0
|
||||
micro_all = 0
|
||||
pbar = tqdm(data)
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
result, full_res = item
|
||||
if result:
|
||||
corr += 1
|
||||
for r in full_res:
|
||||
if r:
|
||||
micro_corr += 1
|
||||
micro_all += 1
|
||||
|
||||
print(f"Missing: {missing} / {len(outputs)} = {missing / len(outputs)}")
|
||||
print(f"Correct: {corr} / {len(outputs)} = {corr / len(outputs)}")
|
||||
print(f"Micro Correct: {micro_corr} / {micro_all} = {micro_corr / micro_all}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
export OUTPUT_PATH_PREFIX="/mnt/fangkai_blob/reward_modeling/"
|
||||
|
||||
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
# --test_case_field "pseudo_test_cases" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#Total number of items before: 5000
|
||||
#4500
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 264
|
||||
#Total number of items: 4236
|
||||
#Missing: 0 / 4236 = 0.0
|
||||
#Correct: 1831 / 4236 = 0.43224740321057603
|
||||
#Micro Correct: 26256 / 41889 = 0.6267993984100838
|
||||
|
||||
# deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
# --test_case_field "input_output" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#Total number of items before: 5000
|
||||
#4500
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 377
|
||||
#Total number of items: 4123
|
||||
#Missing: 0 / 4123 = 0.0
|
||||
#Correct: 2066 / 4123 = 0.5010914382731021
|
||||
#Micro Correct: 26954 / 40349 = 0.6680215123051376
|
||||
|
||||
|
||||
#deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.dpo.H100.dp8.v1.0.s42
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
# --test_case_field "input_output" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#4500
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 331
|
||||
#Total number of items: 4169
|
||||
#Missing: 0 / 4169 = 0.0
|
||||
#Correct: 2077 / 4169 = 0.49820100743583595
|
||||
#Micro Correct: 27217 / 40775 = 0.6674923359901901
|
||||
|
||||
|
||||
#deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42
|
||||
#python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
# --test_case_file ${OUTPUT_PATH_PREFIX}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
# --test_case_field "input_output" --test_case_id_field "id" \
|
||||
# --output_file ${OUTPUT_PATH_PREFIX}//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.gd_sol_exec.json \
|
||||
# --num_workers 128
|
||||
#4500
|
||||
#Missing: 1
|
||||
#Repeat: 0
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 600
|
||||
#Total number of items: 3900
|
||||
#Missing: 0 / 3900 = 0.0
|
||||
#Correct: 511 / 3900 = 0.13102564102564102
|
||||
#Micro Correct: 50202 / 60914 = 0.8241455166300029
|
||||
|
||||
# The sc-version of 4o-based test cases
|
||||
python scripts/apps/execute_gold_sol_on_test_case.py \
|
||||
--test_case_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.json \
|
||||
--test_case_field "input_output" --test_case_id_field "problem_id" \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.gd_sol_exec.json \
|
||||
--num_workers 128
|
||||
#Total number of items before: 5000
|
||||
#4500
|
||||
#Missing: 0
|
||||
#Repeat: 0
|
||||
#Missing solutions: 0
|
||||
#Missing test cases: 248
|
||||
#Total number of items: 4252
|
||||
#Collecting results: 100%|██████████| 4252/4252 [00:54<00:00, 78.74it/s]
|
||||
#Missing: 0 / 4252 = 0.0
|
||||
#Correct: 2697 / 4252 = 0.6342897460018815
|
||||
#Micro Correct: 32348 / 42080 = 0.7687262357414448
|
||||
@@ -0,0 +1,110 @@
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--use_sc", default=False, action="store_true")
|
||||
parser.add_argument("--problem_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--test_case_field", type=str, default="input_output")
|
||||
parser.add_argument("--cover", default=False, action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
pseudo_test_cases = []
|
||||
unaligned = 0
|
||||
total = 0
|
||||
for item in data:
|
||||
problem_id = item[args.problem_id_field]
|
||||
|
||||
if not item[args.test_case_field]:
|
||||
continue
|
||||
|
||||
if "res" not in item or (not item["res"]):
|
||||
continue
|
||||
|
||||
if len(item["res"]) != len(item["outputs"]):
|
||||
unaligned += 1
|
||||
continue
|
||||
|
||||
input_outputs = collections.defaultdict(list)
|
||||
for i, (r, outputs) in enumerate(zip(item["res"], item["outputs"])):
|
||||
if r in [-1, -2]:
|
||||
continue
|
||||
|
||||
for j, o in enumerate(outputs):
|
||||
if o:
|
||||
# _input = item["input_output"]["inputs"][j]
|
||||
input_outputs[j].append(o)
|
||||
|
||||
if args.use_sc:
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
"outputs": []
|
||||
}
|
||||
for _input_index, outputs in input_outputs.items():
|
||||
pairs = {str(o): o for o in outputs}
|
||||
cnt = collections.Counter(list(pairs.keys()))
|
||||
cnt = sorted(cnt.items(), key=lambda x: x[1], reverse=True)
|
||||
_output = cnt[0][0]
|
||||
_output = pairs[_output]
|
||||
test_cases["inputs"].append(item[args.test_case_field]["inputs"][_input_index])
|
||||
test_cases["outputs"].append(_output)
|
||||
else:
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
for _input, outputs in input_outputs.items():
|
||||
test_cases["inputs"].append(_input)
|
||||
test_cases["outputs"].append(outputs[0])
|
||||
|
||||
total += len(test_cases["inputs"])
|
||||
if len(test_cases["inputs"]) == 0:
|
||||
continue
|
||||
if args.cover:
|
||||
item.pop("res")
|
||||
item.pop("outputs")
|
||||
item.pop("full_res")
|
||||
item.pop("errors")
|
||||
if "fn_name" in item[args.test_case_field]:
|
||||
test_cases["fn_name"] = item[args.test_case_field]["fn_name"]
|
||||
item["input_output"] = test_cases
|
||||
pseudo_test_cases.append(item)
|
||||
else:
|
||||
pseudo_test_cases.append({
|
||||
"problem_id": problem_id,
|
||||
"input_output": test_cases
|
||||
})
|
||||
|
||||
print(f"Unaligned: {unaligned}")
|
||||
print(f"Total number of pseudo test cases: {len(pseudo_test_cases)}")
|
||||
print(f"Average number of test cases: {total} / {len(pseudo_test_cases)} = {total / len(pseudo_test_cases)}")
|
||||
json.dump(pseudo_test_cases, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/extract_pseudo_outputs_as_label.py --input_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pure_outputs.json --output_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pseudo_test_cases.json --use_sc
|
||||
|
||||
Unaligned: 582
|
||||
Total number of pseudo test cases: 4223
|
||||
Average number of test cases: 13674 / 4223 = 3.2379824769121477
|
||||
|
||||
|
||||
>>> python scripts/apps/extract_pseudo_outputs_as_label.py --input_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.json --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.pseudo_cases.sc.json --use_sc --problem_id_field id --test_case_field test_cases
|
||||
|
||||
Unaligned: 87
|
||||
Total number of pseudo test cases: 4715
|
||||
Average number of test cases: 17551 / 4715 = 3.72237539766702
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,112 @@
|
||||
import copy
|
||||
import json
|
||||
import argparse
|
||||
import collections
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
missed = 0
|
||||
averaged_rank = 0
|
||||
case_cnt = 0
|
||||
|
||||
loose_avg_rank = 0
|
||||
loose_missed = 0
|
||||
loose_case_cnt = 0
|
||||
|
||||
tmp_cnt = 0
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print("No outputs")
|
||||
continue
|
||||
|
||||
test_cases = item["input_output"]
|
||||
if not test_cases:
|
||||
print("No ground truth test cases")
|
||||
continue
|
||||
|
||||
test_case_outputs = collections.defaultdict(collections.Counter)
|
||||
case_level_res = collections.defaultdict(dict)
|
||||
for i, sol_outputs in enumerate(item["outputs"]):
|
||||
for j, o in enumerate(sol_outputs):
|
||||
if o is not None:
|
||||
test_case_outputs[j][o] += 1
|
||||
if len(item["full_res"][i]) < len(sol_outputs):
|
||||
# print("Full res is not complete")
|
||||
tmp_cnt += 1
|
||||
continue
|
||||
case_level_res[j][o] = item["full_res"][i][j]
|
||||
|
||||
for i, o in enumerate(test_cases["outputs"]):
|
||||
o = str(o)
|
||||
cnt = test_case_outputs[i]
|
||||
sorted_cnt = sorted(cnt.items(), key=lambda x: x[1], reverse=True)
|
||||
o2rank = {k: i for i, (k, v) in enumerate(sorted_cnt)}
|
||||
|
||||
if o not in o2rank:
|
||||
missed += 1
|
||||
else:
|
||||
rank = o2rank[o]
|
||||
averaged_rank += rank
|
||||
case_cnt += 1
|
||||
|
||||
pseudo_cnt = 0
|
||||
loose_cnt = copy.deepcopy(cnt)
|
||||
loose_tuples = copy.deepcopy(list(loose_cnt.items()))
|
||||
for real_o, real_o_res in loose_tuples:
|
||||
if real_o_res:
|
||||
pseudo_cnt += test_case_outputs[i][real_o]
|
||||
loose_cnt.pop(real_o)
|
||||
|
||||
if pseudo_cnt == 0:
|
||||
loose_missed += 1
|
||||
continue
|
||||
|
||||
loose_cnt["$$LOOSE_POS$$"] = pseudo_cnt
|
||||
loose_sorted_cnt = sorted(loose_cnt.items(), key=lambda x: x[1], reverse=True)
|
||||
loose_o2rank = {k: i for i, (k, v) in enumerate(loose_sorted_cnt)}
|
||||
loose_rank = loose_o2rank.get("$$LOOSE_POS$$", -1)
|
||||
assert loose_rank != -1
|
||||
loose_avg_rank += loose_rank
|
||||
loose_case_cnt += 1
|
||||
|
||||
print(f"Missed: {missed}")
|
||||
print(f"Averaged rank: {averaged_rank / case_cnt}")
|
||||
print(f"Case count: {case_cnt}")
|
||||
print(f"Total count: {len(data)}")
|
||||
|
||||
print(f"Loose missed: {loose_missed}")
|
||||
print(f"Loose averaged rank: {loose_avg_rank / loose_case_cnt}")
|
||||
print(f"Loose case count: {loose_case_cnt}")
|
||||
print(f"Loose total count: {len(data)}")
|
||||
|
||||
print(f"tmp_cnt: {tmp_cnt}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/get_output_frequency.py --input_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.exec.w_outputs.json
|
||||
|
||||
Missed: 24502
|
||||
Averaged rank: 0.20820668693009117
|
||||
Case count: 1316
|
||||
Total count: 5000
|
||||
Loose missed: 8206
|
||||
Loose averaged rank: 0.0
|
||||
Loose case count: 17612
|
||||
Loose total count: 5000
|
||||
tmp_cnt: 3077
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
outputs = []
|
||||
with open(args.input_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
item = json.loads(line)
|
||||
pred = standard_cleaner_default(item["completion"])
|
||||
item["pred"] = pred
|
||||
# item["test_cases"] = item.pop("input_output")
|
||||
item["response"] = item.pop("completion")
|
||||
item["id"] = item["problem_id"]
|
||||
outputs.append(item)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
import sys
|
||||
from glob import glob
|
||||
from collections import Counter
|
||||
from datasets import load_dataset
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--split", type=str, default="test")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(f'apps_difficulty_{args.split}.json'):
|
||||
_dataset = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
problem_id2difficulty = {item["problem_id"]: item["difficulty"] for item in _dataset}
|
||||
all_difficulties = Counter(problem_id2difficulty.values())
|
||||
json.dump(problem_id2difficulty, open(f'apps_difficulty_{args.split}.json', "w"), ensure_ascii=False)
|
||||
json.dump(all_difficulties, open(f'apps_difficulty_{args.split}_all.json', "w"), ensure_ascii=False)
|
||||
else:
|
||||
problem_id2difficulty = json.load(open(f'apps_difficulty_{args.split}.json'))
|
||||
all_difficulties = json.load(open(f'apps_difficulty_{args.split}_all.json'))
|
||||
problem_id2difficulty = {int(k): v for k, v in problem_id2difficulty.items()}
|
||||
|
||||
if os.path.exists(args.input_file):
|
||||
if args.input_file.endswith(".json"):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.input_file):
|
||||
if file.endswith(".json"):
|
||||
# data += json.load(open(file))
|
||||
f = open(file, "r")
|
||||
data += json.load(f)
|
||||
else:
|
||||
# data += [json.loads(line) for line in open(file).readlines()]
|
||||
lines = list(open(file).readlines())
|
||||
for line in lines:
|
||||
data.append(json.loads(line))
|
||||
|
||||
success = 0
|
||||
success_at_k = 0
|
||||
successes_at_difficulty = {difficulty: 0 for difficulty in all_difficulties}
|
||||
successes_at_k_at_difficulty = {difficulty: 0 for difficulty in all_difficulties}
|
||||
for item in data:
|
||||
p_id = item["id"]
|
||||
if isinstance(item["res"], bool):
|
||||
if item["res"]:
|
||||
success += 1
|
||||
success_at_k += 1
|
||||
successes_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
successes_at_k_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
else:
|
||||
if len(item["res"]):
|
||||
if any(item["res"]):
|
||||
success_at_k += 1
|
||||
successes_at_k_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
if item["res"][0]:
|
||||
success += 1
|
||||
successes_at_difficulty[problem_id2difficulty[p_id]] += 1
|
||||
|
||||
metrics = {"acc": success / len(data), "pass@k": success_at_k / len(data), "correct": success, "total": len(data)}
|
||||
for difficulty in all_difficulties:
|
||||
metrics[f"acc_{difficulty}"] = successes_at_difficulty[difficulty] / all_difficulties[difficulty]
|
||||
metrics[f"pass@k_{difficulty}"] = successes_at_k_at_difficulty[difficulty] / all_difficulties[difficulty]
|
||||
metrics[f"correct_{difficulty}"] = successes_at_difficulty[difficulty]
|
||||
metrics[f"total_{difficulty}"] = all_difficulties[difficulty]
|
||||
print(json.dumps(metrics, indent=2))
|
||||
|
||||
if args.output_file:
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
json.dump(metrics, open(args.output_file.replace(".json", ".metrics"), "w"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
path=$1
|
||||
|
||||
shift 1 # Shift the first 5 arguments, so $1 now refers to the 6th argument
|
||||
|
||||
for step in "$@"; do
|
||||
echo "Merging predictions for checkpoint $step"
|
||||
python scripts/apps/merge_dp_predictions.py --input_file "$path/apps/checkpoint-$step/sub_dev.0shot.tem0.0.n1.?-of-8.v2.0.json" \
|
||||
--output_file "$path/apps/checkpoint-$step/sub_dev.0shot.tem0.0.n1.v2.0.json" --split train
|
||||
python scripts/apps/merge_dp_predictions.py --input_file "$path/apps/checkpoint-$step/test.0shot.tem0.0.n1.?-of-8.v2.0.json" \
|
||||
--output_file "$path/apps/checkpoint-$step/test.0shot.tem0.0.n1.v2.0.json" --split test
|
||||
# cat $path/apps/checkpoint-$step/sub_dev.0shot.tem0.0.n1.0-of-1.v2.0.metrics.json
|
||||
# cat $path/apps/checkpoint-$step/test.0shot.tem0.0.n1.0-of-1.v2.0.metrics.json
|
||||
echo "Done merging predictions for checkpoint $step"
|
||||
echo ""
|
||||
done
|
||||
@@ -0,0 +1,74 @@
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from post_processors.code.clean import tag_cleaner
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str) # Use the dpo results file
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--difficulty", type=str, default="introductory|interview")
|
||||
parser.add_argument("--prompt_file", type=str, )
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
difficulty = args.difficulty.split("|")
|
||||
|
||||
prompt_template = open(args.prompt_file).read()
|
||||
|
||||
parent_dir = os.path.dirname(args.output_file)
|
||||
if not os.path.exists(parent_dir):
|
||||
os.makedirs(parent_dir)
|
||||
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
if item["difficulty"] not in difficulty:
|
||||
continue
|
||||
|
||||
for i, neg in enumerate(item["neg"]):
|
||||
question = item["question"]
|
||||
if item["starter_code"]:
|
||||
question = question + "\n\n" + item["starter_code"]
|
||||
|
||||
neg_code = tag_cleaner(neg)
|
||||
if not neg_code:
|
||||
continue
|
||||
|
||||
assert neg_code, (neg, item["question"])
|
||||
|
||||
prompt = prompt_template.format(question=question, code=neg_code)
|
||||
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["id"] = f"{item['problem_id']}_neg{i}"
|
||||
new_item["prompt"] = prompt
|
||||
new_item["neg_code"] = neg_code
|
||||
outputs.append(new_item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pp_critique_difficulty.py \
|
||||
--input_file outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.exec.dpo_v1.0.json \
|
||||
--output_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.inputs.intro.inter.jsonl \
|
||||
--prompt_file prompts/apps/critique_0shot_v1.0.txt
|
||||
|
||||
>>> python azure/gpt_crawler_mp.py \
|
||||
--prompt_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.jsonl \
|
||||
--outfile ../gpt-chat-examples/outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--model gpt-4o --max_gen_tokens 4096 --temperature 1.0 --num_processes 24 --seed 42 --n 1 --response_format json_object
|
||||
"""
|
||||
@@ -0,0 +1,40 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.apps import APPsWithFunctionName
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
# reader = APPsWithFunctionName(split="test", use_starter_code=True)
|
||||
reader = APPsWithFunctionName(split="train", train_sub_split="train", use_starter_code=True)
|
||||
|
||||
data = reader("codeparrot/apps")
|
||||
|
||||
# prompt_template = "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
prompt_template = "{question}\n\nPlease write a Python program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
prompt = prompt_template.format(question=item["question"])
|
||||
item["prompt"] = prompt
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w", encoding="utf-8") as f:
|
||||
for item in inputs:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
print(len(inputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.apps import PseudoInputsWithFunctionNameFixStarterCode
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, default="../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1.json")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
# reader = APPsWithFunctionName(split="test", use_starter_code=True)
|
||||
# reader = APPsWithFunctionName(split="train", train_sub_split="train", use_starter_code=True)
|
||||
reader = PseudoInputsWithFunctionNameFixStarterCode(train_sub_split="train", use_starter_code=True)
|
||||
|
||||
data = reader(args.input_file)
|
||||
|
||||
prompt_template = "{question}\n\nPlease write a Python program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
prompt = prompt_template.format(question=item["question"])
|
||||
item["prompt"] = prompt
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w", encoding="utf-8") as f:
|
||||
for item in inputs:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
print(len(inputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
--------
|
||||
{}
|
||||
--------
|
||||
Please carefully comprehend the requirements in the problem, and write down the solution program to pass it under the given time and memory constraints.
|
||||
|
||||
**REMEMBER** to strictly follow the steps below to help reduce the potential flaws:
|
||||
(1) According to the input scale and the time/memory constraints, think about the time complexity and space complexity of your solution.
|
||||
(2) Think **step-by-step** to design the algorithm.
|
||||
(3) Translate your thoughts into Python program to solve it.
|
||||
|
||||
Besides, your Python solution program should be located between <BEGIN> and <END> tags. For example:
|
||||
<BEGIN>
|
||||
t = int(input())
|
||||
...
|
||||
print(ans)
|
||||
<END>
|
||||
|
||||
Now, remember my instruction and let's get started:
|
||||
"""
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(data, desc="Writing prompts"):
|
||||
prompt = PROMPTS[args.prompt_type].format(item["question"])
|
||||
item["prompt"] = prompt
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
from scripts.apps.utils_execute import run_test
|
||||
|
||||
|
||||
def extract_content_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'\[BEGIN\](.*?)\[END\]'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--test_case_inputs", type=str, required=True, help="The completion file.")
|
||||
parser.add_argument("--test_case_outputs", type=str, required=True, help="The completion file.")
|
||||
# parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
# for item in data: # Currently, we do not require the solutions and input_output fields
|
||||
# if item["solutions"]:
|
||||
# item["solutions"] = json.loads(item["solutions"])
|
||||
# if item["input_output"]:
|
||||
# item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
case_inputs = [json.loads(line) for line in open(args.test_case_inputs).readlines()]
|
||||
case_outputs = [json.loads(line) for line in open(args.test_case_outputs).readlines()]
|
||||
|
||||
problem_id2case = defaultdict(list)
|
||||
for item in case_inputs:
|
||||
cases = extract_test_cases(item["completion"])
|
||||
if len(cases):
|
||||
for i, case in enumerate(cases):
|
||||
problem_id2case[item["problem_id"]].append({
|
||||
"input": case,
|
||||
"case_id": i,
|
||||
})
|
||||
|
||||
for item in case_outputs:
|
||||
output = extract_content_between_tags(item["completion"])
|
||||
if output:
|
||||
assert isinstance(output, str), output
|
||||
case_id = int(item["case_id"].split("_")[1])
|
||||
assert case_id == problem_id2case[item["problem_id"]][case_id]["case_id"]
|
||||
problem_id2case[item["problem_id"]][case_id]["output"] = output
|
||||
|
||||
# Currently, we can directly verify
|
||||
outputs = []
|
||||
for item in data:
|
||||
if item["solutions"]:
|
||||
item["solutions"] = json.loads(item["solutions"])
|
||||
|
||||
problem_id = item["problem_id"]
|
||||
cases = {
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
for case in problem_id2case[problem_id]:
|
||||
if "output" in case:
|
||||
cases["inputs"].append(case["input"])
|
||||
cases["outputs"].append(case["output"])
|
||||
assert isinstance(case["input"], str), case["input"]
|
||||
assert isinstance(case["output"], str), case["output"]
|
||||
if cases["inputs"]:
|
||||
item["gen_test_cases"] = cases
|
||||
outputs.append(item)
|
||||
|
||||
# print(outputs[0]["gen_test_cases"])
|
||||
|
||||
cnt = 0
|
||||
num_program = 0
|
||||
for i, item in tqdm(enumerate(outputs)):
|
||||
if i in [7]:
|
||||
continue # Some bad cases.
|
||||
print(item["gen_test_cases"])
|
||||
for solution in item["solutions"]:
|
||||
try:
|
||||
res = run_test(item["gen_test_cases"], solution)
|
||||
pass_num = sum([1 for item in res if item is True])
|
||||
cnt += pass_num
|
||||
except:
|
||||
pass
|
||||
num_program += 1
|
||||
|
||||
print(f"Pass rate: {cnt / num_program}")
|
||||
|
||||
# print(json.dumps(data[0], indent=2))
|
||||
# print(data[0]["question"])
|
||||
# print(data[0]["solutions"][0])
|
||||
# print(data[0]["input_output"]["inputs"][0])
|
||||
# print(data[0]["input_output"]["outputs"][0])
|
||||
|
||||
# with open(args.output_file, "w") as f:
|
||||
# for item in tqdm(outputs, desc="Writing prompts"):
|
||||
# f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
import json
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
{}
|
||||
|
||||
Please carefully comprehend the requirements in the problem, and generate **10** more test cases for me. You can find some examples in the problem description.
|
||||
|
||||
**REMEMBER** (1) I only need the inputs, please do not show me the outputs. (2) Please follow the format below to provide the test cases:
|
||||
|
||||
<TEST INPUT 1>
|
||||
input 1
|
||||
</TEST INPUT 1>
|
||||
<TEST INPUT 2>
|
||||
input 2
|
||||
</TEST INPUT 2>
|
||||
...
|
||||
<TEST INPUT 10>
|
||||
input 10
|
||||
</TEST INPUT 10>
|
||||
"""
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
# for item in data: # Currently, we do not require the solutions and input_output fields
|
||||
# if item["solutions"]:
|
||||
# item["solutions"] = json.loads(item["solutions"])
|
||||
# if item["input_output"]:
|
||||
# item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
# print(json.dumps(data[0], indent=2))
|
||||
# print(data[0]["question"])
|
||||
# print(data[0]["solutions"][0])
|
||||
# print(data[0]["input_output"]["inputs"][0])
|
||||
# print(data[0]["input_output"]["outputs"][0])
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(data, desc="Writing prompts"):
|
||||
prompt = PROMPTS[args.prompt_type].format(item["question"])
|
||||
item["prompt"] = prompt
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/test_input_gen_2shot_v2.0.txt")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--function_only", default=False, action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
print(len(data))
|
||||
|
||||
prompt_template = open(args.prompt_file).read()
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(data, desc="Writing prompts"):
|
||||
question = item["question"]
|
||||
if item["input_output"]:
|
||||
input_output = json.loads(item["input_output"])
|
||||
if "fn_name" in input_output:
|
||||
question += f"\n\n{item['starter_code']}"
|
||||
else:
|
||||
input_output = {}
|
||||
|
||||
prompt = prompt_template.replace("[[Question]]", question)
|
||||
item["prompt"] = prompt
|
||||
if args.function_only:
|
||||
if "fn_name" in input_output:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
else:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.0.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.0.inputs.jsonl
|
||||
|
||||
>>> python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.1.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only.inputs.jsonl --function_only
|
||||
"""
|
||||
@@ -0,0 +1,113 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and simulate it **step-by-step** to obtain the expected outputs the above test case inputs.
|
||||
|
||||
**REMEMBER** to follow the format below to provide the outputs:
|
||||
[BEGIN]
|
||||
outputs
|
||||
[END]
|
||||
"""
|
||||
|
||||
|
||||
_vanilla_prompt_2 = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and derive the expected outputs the above test case inputs. You should follow the steps below:
|
||||
(1) Simulate the algorithm/process described in the problem with the given inputs **step-by-step**, and record the step-level intermediate results.
|
||||
(2) Derive the correct outcome.
|
||||
(3) Put the final outputs in the format below:
|
||||
|
||||
[BEGIN]
|
||||
*put your outputs here*
|
||||
[END]
|
||||
|
||||
and return back it to me.
|
||||
"""
|
||||
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
"vanilla_2": _vanilla_prompt_2,
|
||||
}
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
cases = extract_test_cases(item["completion"])
|
||||
if len(cases):
|
||||
for i, case in enumerate(cases):
|
||||
new_item = copy.deepcopy(item)
|
||||
new_id = f"{item['problem_id']}_{i}"
|
||||
|
||||
prompt = PROMPTS[args.prompt_type].format(item["question"], case)
|
||||
|
||||
new_item["prompt"] = prompt
|
||||
new_item["case_id"] = new_id
|
||||
outputs.append(new_item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(outputs, desc="Writing prompts"):
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
python scripts/apps/pp_test_case_gen_outputs.py --input_file outputs/apps/apps.train.test_case_inputs.500.outputs.vanilla.v1.0.gpt-35-turbo.s42.jsonl --output_file outputs/apps/apps.train.test_case_outputs.500.vanilla.gpt-35-turbo.s42.v1.0.jsonl
|
||||
|
||||
python scripts/apps/pp_test_case_gen_outputs.py --input_file outputs/apps/apps.train.test_case_inputs.500.outputs.vanilla.v1.0.gpt-35-turbo.s42.jsonl --output_file outputs/apps/apps.train.test_case_outputs.500.vanilla.gpt-35-turbo.s42.v1.0.jsonl --prompt_type vanilla_2
|
||||
|
||||
python scripts/apps/pp_test_case_gen_outputs.py --input_file outputs/apps/apps.train.test_case_inputs.500.outputs.vanilla.v1.0.gpt-35-turbo.s42.jsonl --output_file outputs/apps/apps.train.test_case_outputs.500.vanilla.gpt-35-turbo.s42.v1.1.jsonl --prompt_type vanilla_2
|
||||
"""
|
||||
@@ -0,0 +1,116 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
import json
|
||||
from datasets import load_dataset
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
_vanilla_prompt = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and simulate it **step-by-step** to obtain the expected outputs the above test case inputs.
|
||||
|
||||
**REMEMBER** to follow the format below to provide the outputs:
|
||||
[BEGIN]
|
||||
outputs
|
||||
[END]
|
||||
"""
|
||||
|
||||
_vanilla_prompt_2 = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and derive the expected outputs the above test case inputs. You should follow the steps below:
|
||||
(1) Simulate the algorithm/process described in the problem with the given inputs **step-by-step**, and record the step-level intermediate results.
|
||||
(2) Derive the correct outcome.
|
||||
(3) Put the final outputs in the format below:
|
||||
|
||||
[BEGIN]
|
||||
*put your outputs here*
|
||||
[END]
|
||||
|
||||
and return back it to me.
|
||||
"""
|
||||
|
||||
_vanilla_prompt_3 = """You are an expert programmer. Here is a programming problem:
|
||||
|
||||
{}
|
||||
|
||||
------------------------
|
||||
|
||||
And here is one group of test case inputs:
|
||||
```
|
||||
{}
|
||||
```
|
||||
------------------------
|
||||
|
||||
Please carefully comprehend the process described in the problem, and derive the expected outputs the above test case inputs. You should follow the steps below:
|
||||
(1) Simulate the algorithm/process described in the problem with the given inputs **step-by-step**, and record the step-level intermediate results.
|
||||
(2) Derive the correct outcome.
|
||||
|
||||
Response format:
|
||||
[BEGIN]
|
||||
*The expected outputs*
|
||||
[END]
|
||||
"""
|
||||
|
||||
PROMPTS = {
|
||||
"vanilla": _vanilla_prompt,
|
||||
"vanilla_2": _vanilla_prompt_2,
|
||||
"vanilla_3": _vanilla_prompt_3,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--prompt_type", type=str, default="vanilla")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
print(len(data))
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(inputs, total=len(inputs)):
|
||||
problem = item["question"]
|
||||
for case in item["input_output"]["inputs"]:
|
||||
new_item = copy.deepcopy(item)
|
||||
prompt = PROMPTS[args.prompt_type].format(problem, case)
|
||||
new_item["prompt"] = prompt
|
||||
f.write(json.dumps(new_item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
"""
|
||||
@@ -0,0 +1,47 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
import json
|
||||
from datasets import load_dataset
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--split", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, required=True)
|
||||
parser.add_argument("--prompt_file", type=str, default="vanilla")
|
||||
args = parser.parse_args()
|
||||
|
||||
_prompt_template = open(args.prompt_file).read()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
print(len(data))
|
||||
|
||||
inputs = []
|
||||
for item in data:
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
inputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in tqdm(inputs, total=len(inputs)):
|
||||
problem = item["question"]
|
||||
for case in item["input_output"]["inputs"]:
|
||||
# assert isinstance(case, str), (problem, case, json.loads(item["solutions"])[0]) # For leetcode-format, this is not applied.
|
||||
new_item = copy.deepcopy(item)
|
||||
prompt = _prompt_template.replace("<<PROBLEM>>", problem).replace("<<TEST INPUTS>>", str(case))
|
||||
new_item["prompt"] = prompt
|
||||
f.write(json.dumps(new_item) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
"""
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import run_test, check_correctness
|
||||
|
||||
|
||||
def extract_content_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'\[BEGIN\](.*?)\[END\]'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
# return match.group(1).strip()
|
||||
return match.group(1)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
print(len(data))
|
||||
|
||||
flag = "Now, let's get started with the following problem:"
|
||||
|
||||
problem_pass_cnt = Counter()
|
||||
problem_case_cnt = Counter()
|
||||
difficulty_cnt = Counter()
|
||||
for item in tqdm(data, total=len(data)):
|
||||
# if item["problem_id"] > 2000:
|
||||
# continue
|
||||
# if item["difficulty"] not in ["introductory", "interview"]:
|
||||
# continue
|
||||
|
||||
difficulty_cnt[item["difficulty"]] += 1
|
||||
print(f"=========================={item['problem_id']}")
|
||||
# if item['problem_id'] in [160, 379, 438, 4096]:
|
||||
# print(f"Skip problem {item['problem_id']}")
|
||||
# continue
|
||||
prompt = item["prompt"]
|
||||
assert flag in prompt, prompt
|
||||
prompt = prompt.split(flag)[1].strip()
|
||||
|
||||
if "## TEST INPUTS\n\n" not in prompt:
|
||||
continue
|
||||
if "\n\n## SIMULATE" not in prompt:
|
||||
continue
|
||||
|
||||
inputs_s = prompt.index("## TEST INPUTS") + len("## TEST INPUTS\n\n")
|
||||
inputs_e = prompt.index("\n\n## SIMULATE")
|
||||
# inputs = prompt[inputs_s:inputs_e].strip()
|
||||
inputs = prompt[inputs_s:inputs_e]
|
||||
# print(inputs)
|
||||
|
||||
outputs = extract_content_between_tags(item["completion"])
|
||||
problem_case_cnt[item["problem_id"]] += 1
|
||||
|
||||
if outputs is None:
|
||||
continue
|
||||
|
||||
mapping = {str(_input): _input for i, _input in enumerate(item["input_output"]["inputs"])}
|
||||
|
||||
# if "fn_name" in item["input_output"]:
|
||||
# _cases = {
|
||||
# "inputs": []
|
||||
# }
|
||||
assert inputs in mapping, (inputs, json.dumps(mapping))
|
||||
_cases = {
|
||||
# "inputs": [inputs],
|
||||
# "outputs": [outputs]
|
||||
"inputs": [mapping[inputs]],
|
||||
"outputs": [outputs]
|
||||
}
|
||||
if "fn_name" in item["input_output"]:
|
||||
_cases["fn_name"] = item["input_output"]["fn_name"]
|
||||
|
||||
solutions = json.loads(item["solutions"])
|
||||
for solution in solutions[:1]: # I think one positive solution is enough
|
||||
try:
|
||||
# res = run_test(_cases, solution)
|
||||
res = check_correctness(_cases, solution, timeout=10, debug=False)
|
||||
pass_num = sum([1 for item in res if item is True])
|
||||
problem_pass_cnt[item["problem_id"]] += pass_num
|
||||
except:
|
||||
pass
|
||||
|
||||
print(difficulty_cnt)
|
||||
|
||||
print(f"Averaged pass cnt: {sum(problem_pass_cnt.values()) / sum(problem_case_cnt.values())}")
|
||||
print(f"Absolute pass rate over all generated cases: {sum(problem_pass_cnt.values()) / sum(problem_case_cnt.values())}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
"""
|
||||
@@ -0,0 +1,147 @@
|
||||
import json
|
||||
from collections import defaultdict, Counter
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from glob import glob
|
||||
import copy
|
||||
from tqdm import tqdm
|
||||
from datasets import load_dataset
|
||||
import random
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
# from post_processors.code.clean import tag_cleaner
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--critique_exec_file", type=str,
|
||||
help="The file contains completion from the teacher model for critique, as well as the execution results."
|
||||
"The inputs for this file are generated by `pp_critique_difficulty` script.")
|
||||
parser.add_argument("--completion_file", type=str,
|
||||
help="The file contains the completion for each query.")
|
||||
parser.add_argument("--completion_response_field", type=str, default="completion")
|
||||
parser.add_argument("--completion_problem_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/worsen_from_feedback_0shot_v1.0.txt")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--sample_num", type=int, default=2000)
|
||||
parser.add_argument("--split", type=str, default="train")
|
||||
args = parser.parse_args()
|
||||
|
||||
random.seed(args.seed)
|
||||
|
||||
prompt_template = open(args.prompt_file).read()
|
||||
|
||||
_dataset = load_dataset("codeparrot/apps", split=args.split).to_list()
|
||||
p_id2item = {item["problem_id"]: item for item in _dataset}
|
||||
|
||||
critiques = json.load(open(args.critique_exec_file))
|
||||
correct_critiques = []
|
||||
for item in critiques:
|
||||
if isinstance(item["completion"], str) or isinstance(item["completion"], dict):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
preds = item["pred"]
|
||||
|
||||
for i, (comp, p, r) in enumerate(zip(completions, preds, item["res"])):
|
||||
if r:
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["completion"] = comp
|
||||
new_item["pred"] = p
|
||||
new_item["res"] = r
|
||||
correct_critiques.append(new_item)
|
||||
|
||||
print(f"Total correct critiques: {len(correct_critiques)}")
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
data += json.load(open(file))
|
||||
outputs = []
|
||||
for item in tqdm(data):
|
||||
problem_id = item[args.completion_problem_id_field]
|
||||
critique = random.choice(correct_critiques)
|
||||
while critique["problem_id"] == problem_id:
|
||||
critique = random.choice(correct_critiques)
|
||||
|
||||
preds = item["pred"]
|
||||
if not preds:
|
||||
continue
|
||||
if isinstance(preds, str):
|
||||
preds = [preds]
|
||||
|
||||
for i, pred in enumerate(preds):
|
||||
if pred:
|
||||
prompt = prompt_template.format(
|
||||
example_question=critique["question"],
|
||||
example_code=critique["neg_code"],
|
||||
feedback=critique["completion"]["feedback"],
|
||||
corrected_program=critique["completion"]["corrected_program"],
|
||||
question=p_id2item[problem_id]["question"],
|
||||
code=pred,
|
||||
)
|
||||
new_item = copy.deepcopy(item)
|
||||
new_item["id"] = f"{item[args.completion_problem_id_field]}_neg{i}"
|
||||
new_item["prompt"] = prompt
|
||||
if args.completion_response_field != "response":
|
||||
new_item["response"] = new_item.pop(args.completion_response_field)
|
||||
outputs.append(new_item)
|
||||
|
||||
if len(outputs) >= args.sample_num:
|
||||
break
|
||||
if len(outputs) >= args.sample_num:
|
||||
break
|
||||
|
||||
print(f"Total number of items: {len(outputs)}")
|
||||
# json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.0-of-8.v1.1.json" \
|
||||
--completion_response_field response \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f2000.jsonl \
|
||||
--sample_num 2000 --seed 42 --completion_problem_id_field id
|
||||
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_response_field response \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f10k.jsonl \
|
||||
--sample_num 10000 --seed 42 --completion_problem_id_field id
|
||||
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_response_field response
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.jsonl \
|
||||
--sample_num 100000 --seed 42 --completion_problem_id_field id
|
||||
|
||||
Total correct critiques: 7205
|
||||
100%|██████████████████████████████████████████████████████████████████| 5000/5000 [00:04<00:00, 1070.68it/s]
|
||||
Total number of items: 49868
|
||||
|
||||
>>> python scripts/apps/pp_worsen_inputs.py --critique_exec_file outputs/apps/critique/apps.train.gpt4o.tem1.0.n11.neg.intro.inter.gpt4o.tem1.0.s42.n1.json_obj.exec.json \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_response_field response \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o.s42.f10k.jsonl \
|
||||
--sample_num 10000 --seed 42 --completion_problem_id_field id --prompt_file prompts/apps/worsen_0shot_v1.0.txt
|
||||
|
||||
python azure/gpt_crawler_mp.py --prompt_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.worsen_4o.s42.f10k.jsonl \
|
||||
--outfile ../gpt-chat-examples/outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o.s42.f10k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--model gpt-4o --max_gen_tokens 4096 --temperature 1.0 --num_processes 24 --seed 42 --n 1 --response_format json_object
|
||||
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from functools import partial
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
For self-consistency based input-output pairs, first copy the pseudo test cases into the prefix data, run `prefix_fail_extract_pseudo_label.py`,
|
||||
and the run this script.
|
||||
|
||||
This script is incorrect for pseudo test cases since we cannot ensure the correctness of each test case, but it is appropriate for ground-truth test cases,
|
||||
serving as hard limit, i.e., if there is one completion for some prefix has passed all test cases, then it is a gold prefix.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def counting_partial_response_value(res):
|
||||
return sum([1 if x else 0 for x in res])
|
||||
|
||||
|
||||
def parse_value(v, binary: bool):
|
||||
if binary:
|
||||
return 1 if v > 0 else 0
|
||||
return v
|
||||
|
||||
|
||||
def _process_trajectories_worker(item, top_k: int, binary: bool):
|
||||
item_id, trajectories = item
|
||||
outputs = {
|
||||
"idx": item_id
|
||||
}
|
||||
for i in range(top_k):
|
||||
level_trajectories = [(traj["vs"][i], traj["prefix"]) for traj in trajectories if len(traj["vs"]) > i]
|
||||
if len(level_trajectories) == 0:
|
||||
continue
|
||||
prefix_vis = set()
|
||||
level_values = []
|
||||
level_prefixes = []
|
||||
for v, prefix in level_trajectories:
|
||||
if prefix in prefix_vis:
|
||||
continue
|
||||
level_values.append(parse_value(v, binary))
|
||||
level_prefixes.append(prefix)
|
||||
prefix_vis.add(prefix)
|
||||
outputs[f"traj_level_{i}_values"] = level_values
|
||||
outputs[f"traj_level_{i}_prefixes"] = level_prefixes
|
||||
return outputs
|
||||
|
||||
|
||||
def _annotate(file):
|
||||
return json.load(open(file, encoding="utf-8"))
|
||||
|
||||
|
||||
def multiprocessing_loading(files, num_workers: int = 8):
|
||||
with Pool(num_workers) as p:
|
||||
data = list(tqdm(p.imap(_annotate, files), total=len(files)))
|
||||
all_data = []
|
||||
for d in data:
|
||||
all_data.extend(d)
|
||||
return all_data
|
||||
# data = []
|
||||
# for file in tqdm(files):
|
||||
# data += json.load(open(file, encoding="utf-8"))
|
||||
# return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--binary", default=False, action="store_true")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Collecting data...")
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
# data = []
|
||||
# for file in glob(args.input_file):
|
||||
# print(file)
|
||||
# data += json.load(open(file))
|
||||
files = glob(args.input_file)
|
||||
files = sorted(files)
|
||||
print(len(files))
|
||||
print(files)
|
||||
data = multiprocessing_loading(files)
|
||||
|
||||
print(len(data))
|
||||
|
||||
num_prefixes = 0
|
||||
val_cnt = collections.Counter()
|
||||
outputs = []
|
||||
preference_pairs = dict()
|
||||
missing = 0
|
||||
for item in tqdm(data):
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
prefix = item["prefix"]
|
||||
problem_id = int(problem_id)
|
||||
|
||||
if "res" not in item:
|
||||
missing += 1
|
||||
continue
|
||||
v = counting_partial_response_value(item["res"])
|
||||
v = parse_value(v, args.binary)
|
||||
|
||||
outputs.append({
|
||||
"problem_id": problem_id,
|
||||
"prefix": prefix,
|
||||
"value": v,
|
||||
})
|
||||
num_prefixes += 1
|
||||
val_cnt[v] += 1
|
||||
|
||||
if problem_id not in preference_pairs:
|
||||
preference_pairs[problem_id] = {
|
||||
"pos": [],
|
||||
"neg": [],
|
||||
}
|
||||
|
||||
if v > 0:
|
||||
preference_pairs[problem_id]["pos"].append(prefix)
|
||||
else:
|
||||
preference_pairs[problem_id]["neg"].append(prefix)
|
||||
|
||||
preference_pairs = [
|
||||
{
|
||||
"problem_id": problem_id,
|
||||
"pos": pair["pos"],
|
||||
"neg": pair["neg"],
|
||||
}
|
||||
for problem_id, pair in preference_pairs.items()
|
||||
]
|
||||
|
||||
print(f"Missing: {missing}")
|
||||
print(val_cnt)
|
||||
print(f"Processed {num_prefixes} prefixes.")
|
||||
print(f"Averaged {num_prefixes / len(data)} prefixes per problem.")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
json.dump(preference_pairs, open(args.output_file.replace(".json", "_pairs.json"), "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>>
|
||||
"""
|
||||
@@ -0,0 +1,189 @@
|
||||
import json
|
||||
import argparse
|
||||
from glob import glob
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
from functools import partial
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
For self-consistency based input-output pairs, first copy the pseudo test cases into the prefix data, run `prefix_fail_extract_pseudo_label.py`,
|
||||
and the run this script.
|
||||
|
||||
|
||||
Note:
|
||||
1. Sometimes the pseudo test cases can include some extremely large cases, leading to out-of-memory error.
|
||||
"""
|
||||
|
||||
|
||||
def counting_partial_response_value(full_res):
|
||||
pass_num = []
|
||||
for pred_res in full_res:
|
||||
if len(pred_res) == 0:
|
||||
pass_num.append(0)
|
||||
# elif pred_res[0] == -1: # FIXME: This line is commented since 2024/09/25. Seems no difference.
|
||||
# pass_num.append(0)
|
||||
else:
|
||||
pass_num.append(sum([1 for x in pred_res if x is True]))
|
||||
|
||||
return pass_num
|
||||
|
||||
|
||||
def annotate(file, exclude: str = ""):
|
||||
exclude = exclude.split(",")
|
||||
if any([e and e in file for e in exclude]):
|
||||
print(f"Excluding {file}")
|
||||
return []
|
||||
return json.load(open(file, encoding="utf-8"))
|
||||
|
||||
|
||||
def multiprocessing_loading(files, exclude: str = "", num_workers: int = 8):
|
||||
_annotate = partial(annotate, exclude=exclude)
|
||||
|
||||
# with Pool(num_workers) as p:
|
||||
# data = list(tqdm(p.imap(_annotate, files), total=len(files)))
|
||||
# all_data = []
|
||||
# for d in data:
|
||||
# all_data.extend(d)
|
||||
# return all_data
|
||||
data = []
|
||||
for file in tqdm(files):
|
||||
data += _annotate(file)
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_input_output")
|
||||
parser.add_argument("--num_workers", type=int, default=8)
|
||||
parser.add_argument("--reduction", type=str, default="max")
|
||||
parser.add_argument("--exclude", type=str, default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Collecting data...")
|
||||
if os.path.exists(args.input_file):
|
||||
data = json.load(open(args.input_file))
|
||||
else:
|
||||
files = glob(args.input_file)
|
||||
files = sorted(files)
|
||||
print(len(files))
|
||||
print(files)
|
||||
data = multiprocessing_loading(files, args.exclude)
|
||||
|
||||
print(len(data))
|
||||
|
||||
num_prefixes = 0
|
||||
val_cnt = collections.Counter()
|
||||
outputs = []
|
||||
p_id2prefixes = collections.defaultdict(list)
|
||||
preference_pairs = []
|
||||
missing = 0
|
||||
missing_test_cases = 0
|
||||
for item in tqdm(data):
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
prefix = item["prefix"]
|
||||
# problem_id = int(problem_id)
|
||||
|
||||
if "res" not in item:
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
if args.test_case_field not in item or (not isinstance(item[args.test_case_field], dict)) or "inputs" not in item[args.test_case_field]:
|
||||
missing_test_cases += 1
|
||||
continue
|
||||
|
||||
test_case_num = len(item[args.test_case_field]["inputs"])
|
||||
if test_case_num == 0:
|
||||
missing_test_cases += 1
|
||||
continue
|
||||
|
||||
pass_num = counting_partial_response_value(item["full_res"])
|
||||
if args.reduction == "max":
|
||||
max_pass_num = max(pass_num)
|
||||
elif args.reduction == "avg":
|
||||
max_pass_num = sum(pass_num) / len(pass_num)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
outputs.append({
|
||||
"problem_id": problem_id,
|
||||
"prefix": prefix,
|
||||
"pass_num": pass_num,
|
||||
"max_pass_num": max_pass_num,
|
||||
"test_case_num": test_case_num,
|
||||
})
|
||||
num_prefixes += 1
|
||||
val_cnt.update(pass_num)
|
||||
|
||||
p_id2prefixes[problem_id].append(outputs[-1])
|
||||
|
||||
for problem_id, all_prefixes in tqdm(p_id2prefixes.items()):
|
||||
max_pass_num2prefixes = collections.defaultdict(list)
|
||||
for prefix in all_prefixes:
|
||||
max_pass_num2prefixes[prefix["max_pass_num"]].append(prefix)
|
||||
max_pass_num2prefixes = sorted(max_pass_num2prefixes.items(), key=lambda x: x[0])
|
||||
|
||||
pos_prefixes = []
|
||||
neg_prefixes = []
|
||||
for p in all_prefixes:
|
||||
pass_ratio = p["max_pass_num"] / p["test_case_num"]
|
||||
if pass_ratio < args.pass_case_lower_bound:
|
||||
continue
|
||||
neg_upper_pass_num = p["max_pass_num"] - args.pass_case_margin
|
||||
|
||||
target_neg = []
|
||||
for pass_num, prefixes in max_pass_num2prefixes:
|
||||
if pass_num < neg_upper_pass_num:
|
||||
target_neg.extend([x["prefix"] for x in prefixes])
|
||||
else:
|
||||
break
|
||||
|
||||
pos_prefixes.append(p["prefix"])
|
||||
neg_prefixes.append(target_neg)
|
||||
|
||||
preference_pairs.append({
|
||||
"problem_id": problem_id,
|
||||
"pos": pos_prefixes,
|
||||
"neg": neg_prefixes
|
||||
})
|
||||
|
||||
print(f"Missing: {missing}")
|
||||
print(f"Missing test cases: {missing_test_cases}")
|
||||
print(val_cnt)
|
||||
print(f"Processed {num_prefixes} prefixes.")
|
||||
print(f"Averaged {num_prefixes / len(data)} prefixes per problem.")
|
||||
print(f"Processed {len(preference_pairs)} problems.")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
json.dump(preference_pairs, open(args.output_file.replace(".json", f"_low{args.pass_case_lower_bound}_m{args.pass_case_margin}_{args.reduction}.json"),
|
||||
"w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "/mnt/fangkai_blob/reward_modeling//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.[0-9]*-of-256.pseudo_test_case.exec.json" \
|
||||
--output_file /mnt/fangkai_blob/reward_modeling//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.pseudo_test_case.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.8 --pass_case_margin 4 --test_case_field pseudo_input_output
|
||||
|
||||
|
||||
>>> python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.*-of-256.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output --exclude "204-of-256,225-of-256"
|
||||
|
||||
Missing: 0
|
||||
Missing test cases: 0
|
||||
Counter({10: 448824, 0: 153053, 9: 25420, 1: 23469, 8: 16718, 5: 15115, 2: 14803, 3: 12821, 4: 12253, 7: 12053, 6: 11947, 11: 724, 20: 112, 16: 3}) # TODO: This should be a problem. Why there are some problems have more than 10 test cases?
|
||||
Processed 249105 prefixes.
|
||||
Averaged 1.0 prefixes per problem.
|
||||
"""
|
||||
@@ -0,0 +1,152 @@
|
||||
import json
|
||||
import argparse
|
||||
import os
|
||||
from glob import glob
|
||||
import sys
|
||||
import random
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
from post_processors.code.clean import get as get_code_cleaner
|
||||
|
||||
|
||||
def sample_step(response: str, code: str, upper_step_ratio: float, sample_ratio: float):
|
||||
if code:
|
||||
assert code in response, (code, "\n\n========================\n\n", response)
|
||||
end_of_response = response.find(code) + len(code)
|
||||
response = response[:end_of_response]
|
||||
|
||||
orig_lines = response.split("\n")
|
||||
lines = [(i, line) for i, line in enumerate(orig_lines)]
|
||||
lines = [(i, line) for i, line in lines if line.strip()]
|
||||
|
||||
if len(lines) < 5:
|
||||
return []
|
||||
|
||||
upper_step = int(len(lines) * upper_step_ratio)
|
||||
if upper_step == 0:
|
||||
return []
|
||||
|
||||
sample_step_num = int(upper_step * sample_ratio)
|
||||
if sample_step_num == 0:
|
||||
return []
|
||||
|
||||
sample_steps = random.sample(lines[:upper_step], sample_step_num)
|
||||
if len(sample_steps) == 0:
|
||||
return []
|
||||
|
||||
step_prefixes = []
|
||||
for i, line in sample_steps:
|
||||
step_prefixes.append("\n".join(orig_lines[:(i + 1)]))
|
||||
|
||||
return step_prefixes
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
large_mem = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
print(f"Too large outputs: {large_mem}")
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, )
|
||||
parser.add_argument("--output_file", type=str, )
|
||||
parser.add_argument("--upper_step_ratio", type=float)
|
||||
parser.add_argument("--sample_ratio", type=float)
|
||||
parser.add_argument("--filter_all_correct", action="store_true")
|
||||
parser.add_argument("--sample_over_p", type=int, default=-1)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.input_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
|
||||
outputs = []
|
||||
num = 0
|
||||
for item in data:
|
||||
if args.filter_all_correct and all(item["res"]):
|
||||
continue
|
||||
|
||||
prefixes = []
|
||||
prefix_ids = []
|
||||
if not item["response"]:
|
||||
item["prefix"] = []
|
||||
continue
|
||||
|
||||
for resp_id, resp in enumerate(item["response"]):
|
||||
response_prefixes = sample_step(resp, item["pred"][resp_id], args.upper_step_ratio, args.sample_ratio)
|
||||
prefixes.extend(response_prefixes)
|
||||
prefix_ids.extend([f"{item['id']}_{resp_id}_{i}" for i in range(len(response_prefixes))])
|
||||
|
||||
if args.sample_over_p > 0:
|
||||
if len(prefixes) > args.sample_over_p:
|
||||
prefixes = random.sample(prefixes, args.sample_over_p)
|
||||
prefix_ids = random.sample(prefix_ids, args.sample_over_p)
|
||||
|
||||
item["prefix"] = prefixes
|
||||
item["prefix_id"] = prefix_ids
|
||||
outputs.append(item)
|
||||
num += len(prefixes)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
print(f"Total number of samples: {num}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.prefix.upper0.8.r0.3.json
|
||||
|
||||
Total number of samples: 470010
|
||||
"""
|
||||
@@ -0,0 +1,68 @@
|
||||
|
||||
# Ground-truth test case inputs
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.run_outputs.json" \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/extract_pseudo_outputs_as_label.py \
|
||||
--input_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.run_outputs.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--use_sc
|
||||
|
||||
# ================================================================
|
||||
# Synthetic test case inputs
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json" \
|
||||
--num_workers 64 --completion_test_field "input_output" \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
# =================================================
|
||||
|
||||
# Synthetic test cases inputs with GPT-4o for output
|
||||
python scripts/apps/pseudo_test_cases/combine_gpt_raw_requests.py \
|
||||
--input_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1_gpt_inputs.shuf.jsonl \
|
||||
--raw_output_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1_gpt_inputs.shuf.bing.x5.jsonl_out \
|
||||
--output_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--num_workers 64 --id_field problem_id --test_case_field input_output
|
||||
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ${DATA_PREFIX_PATH}/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.json \
|
||||
--test_case_field "input_output"
|
||||
#19795
|
||||
#0
|
||||
#15
|
||||
#9.866784541550897
|
||||
#9.013993432685021
|
||||
#Counter()
|
||||
|
||||
# Non-sc-version
|
||||
python vllm_inference.py fp16_bfloat16=True seed=42 split_id={split_id} \
|
||||
exp_name=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42 \
|
||||
eval_sub_path=checkpoint-100 -cp conf/api/vllm/apps/deepseek_coder/r2c -cn general_combine_train_v2_0_4o_non_sc
|
||||
|
||||
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.prefer_pair.low0.5.m6.json" \
|
||||
--response_field response --test_case_field input_output_non_sc --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
# SC-version
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "/mnt/fangkai_blob/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.json" \
|
||||
--output_file "/mnt/fangkai_blob/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.json" \
|
||||
--num_workers 64 --id_field id --test_case_field input_output
|
||||
# bash scripts/apps/pseudo_test_cases/run_outputs_local.sh split_id
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.prefer_pair.low0.5.m6.4o-sc.json" \
|
||||
--test_case_field "input_output" --pass_case_margin 6 --pass_case_lower_bound 0.5 --construct_prefer_pair
|
||||
@@ -0,0 +1,224 @@
|
||||
### Synthesis Test Case Inputs
|
||||
|
||||
Prepare inputs for prompting GPT-4o:
|
||||
|
||||
```bash
|
||||
python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.0.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.0.inputs.jsonl
|
||||
|
||||
python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.1.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only.inputs.jsonl --function_only
|
||||
```
|
||||
|
||||
Extract the corresponding outputs from GPT-4o and combine them to obtain the following file:
|
||||
|
||||
`../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json`
|
||||
|
||||
For post-processing to align the format with oss data:
|
||||
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/combine_pseudo_test_inputs.py \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/apps-train.4o-test-func.json \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
```
|
||||
|
||||
For directly processing on APPs:
|
||||
|
||||
```bash
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--pseudo_test_case outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json --num_workers 128
|
||||
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json" \
|
||||
--num_workers 64 --completion_test_field "input_output" \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
```
|
||||
|
||||
Collect pseudo outputs and construct DPO training pairs
|
||||
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
```
|
||||
|
||||
Either, run execution again on existing pseudo test cases.
|
||||
|
||||
First, extract the pseudo label:
|
||||
```bash
|
||||
python scripts/apps/extract_pseudo_outputs_as_label.py \
|
||||
--input_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--use_sc --problem_id_field problem_id --test_case_field pseudo_test_cases
|
||||
#Unaligned: 0
|
||||
#Total number of pseudo test cases: 3901
|
||||
#Average number of test cases: 36333 / 3901 = 9.313765701102282
|
||||
```
|
||||
|
||||
Secondly, run execution on pseudo test cases:
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
#Missing: 0.0002563445270443476
|
||||
#Correct: 0.08203024865419123
|
||||
#Correct at k: 0.1579082286593181
|
||||
#Counter({False: 36311, True: 2689})
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.prefer_pair.json \
|
||||
--test_case_field pseudo_input_output
|
||||
# 3901 9165 2.3493975903614457
|
||||
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
# 3901 12851 3.294283517046911
|
||||
```
|
||||
|
||||
Run process-DPO prefix execution
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.4o_pseudo_test_case.exec.json" \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--test_case_field input_output --id_field problem_id
|
||||
```
|
||||
|
||||
Construct Process-DPO preference pair
|
||||
```bash
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.*-of-256.4o_pseudo_test_case.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.4o_pseudo_test_case.prm_prefer_pair.json \
|
||||
--pass_case_margin 4 --pass_case_lower_bound 0.5 --test_case_field pseudo_input_output --num_workers 24 --reduction avg
|
||||
#Missing: 0
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 1440638, 10: 108458, 1: 80861, 2: 34440, 3: 24141, 4: 18476, 5: 15235, 9: 14017, 6: 13494, 7: 12309, 8: 11546})
|
||||
#Processed 354723 prefixes.
|
||||
#Averaged 1.0 prefixes per problem.
|
||||
#Processed 3900 problems.
|
||||
```
|
||||
---------------------------------------------------------------
|
||||
|
||||
Based on DPO-Iter-0 model to construct new test cases on magicoder
|
||||
|
||||
Run outputs on newly generated oss-apps-combine data
|
||||
```bash
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
```
|
||||
|
||||
Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#11925
|
||||
#11925
|
||||
#117006
|
||||
#0
|
||||
#10.004442649434571
|
||||
#9.823396226415094
|
||||
#Counter({10: 73857, 0: 23639, 9: 3901, 1: 3222, 8: 2589, 5: 2251, 2: 2103, 7: 1976, 4: 1865, 6: 1865, 3: 1852, 11: 105, 20: 25})
|
||||
```
|
||||
We need to reuse the pre-synthesized pseudo test cases for APPs dataset for training.
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.apps_only.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
3901
|
||||
3901 11364 2.913099205331966
|
||||
```
|
||||
|
||||
---------------------------------------------------------------
|
||||
|
||||
Based on DPO-Iter-1 model to construct new test cases on magicoder and xcode
|
||||
|
||||
```bash
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
```
|
||||
|
||||
Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
# we have skipped split id 20 since it uses too much time.
|
||||
#18608
|
||||
#18608
|
||||
#193834
|
||||
#0
|
||||
#10.003605287755375
|
||||
#9.80116079105761
|
||||
#Counter({10: 99051, 0: 37603, 1: 8459, 9: 7563, 2: 5656, 8: 5528, 3: 4714, 5: 4698, 7: 4306, 4: 4260, 6: 4070, 11: 136, 20: 34, 12: 2})
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.all.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
# Added the skipped split id 20
|
||||
#19051
|
||||
#19051
|
||||
#197647
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.800902839745945
|
||||
#Counter({10: 102049, 0: 38432, 1: 8511, 9: 7700, 2: 5688, 8: 5591, 3: 4783, 5: 4776, 7: 4358, 4: 4329, 6: 4119, 11: 136, 20: 34, 12: 4})
|
||||
```
|
||||
|
||||
We need to reuse the pre-synthesized pseudo test cases for APPs dataset for training.
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300//train.0shot.tem1.0.n10.apps_only.v2.1.*-of-9.s42.4o_pseudo_test_cases.exec.v1.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.apps_only.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
|
||||
```
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
|
||||
def load_file(file_path):
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
else:
|
||||
return [json.loads(line) for line in open(file_path).readlines()]
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file, encoding="utf-8"))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
return data
|
||||
|
||||
|
||||
def extract_json(completion: str):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'```(.*?)```'
|
||||
|
||||
match = re.search(pattern, completion, re.DOTALL)
|
||||
if match:
|
||||
item = match.group(1).strip()
|
||||
if item.startswith("json"):
|
||||
item = item[5:]
|
||||
item = item.replace(",\n}", "\n}").replace(",}", "}").replace("None", "null")
|
||||
item = item.replace("(", "[").replace(")", "]").replace("\'", "\"")
|
||||
item = item.replace("True", "true").replace("False", "false")
|
||||
return json.loads(item)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_test_cases(item):
|
||||
try:
|
||||
response = extract_json(item["response"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["response"])
|
||||
return {}
|
||||
|
||||
if not response:
|
||||
return {}
|
||||
|
||||
if not isinstance(response, dict):
|
||||
return {}
|
||||
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
}
|
||||
for k, v in response.items():
|
||||
if not k.startswith("test_case_"):
|
||||
continue
|
||||
test_cases["inputs"].append(v)
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def load_func_head(item) -> str:
|
||||
try:
|
||||
response = extract_json(item["response"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["response"])
|
||||
return ""
|
||||
|
||||
if not response:
|
||||
return ""
|
||||
|
||||
if "func_head" not in response:
|
||||
return ""
|
||||
|
||||
return response["func_head"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_file", type=str)
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--func_head_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_file(args.data_file)
|
||||
|
||||
test_cases = load_file(args.test_case_file)
|
||||
id2test_cases = {}
|
||||
test_num = 0
|
||||
for item in test_cases:
|
||||
tmp = load_test_cases(item)
|
||||
if tmp:
|
||||
id2test_cases[item["id"]] = tmp
|
||||
test_num += 1
|
||||
|
||||
func_heads = load_file(args.func_head_file)
|
||||
id2func_head = {}
|
||||
func_num = 0
|
||||
for item in func_heads:
|
||||
tmp = load_func_head(item)
|
||||
if tmp:
|
||||
id2func_head[item["id"]] = tmp
|
||||
func_num += 1
|
||||
|
||||
print(f"Test cases: {test_num} / {len(test_cases)}")
|
||||
print(f"Func heads: {func_num} / {len(func_heads)}")
|
||||
|
||||
outputs = []
|
||||
for item in data:
|
||||
item_id = item["index"]
|
||||
if item_id in id2test_cases:
|
||||
test_inputs = id2test_cases[item_id]
|
||||
else:
|
||||
continue
|
||||
|
||||
if item_id in id2func_head:
|
||||
func_head = id2func_head[item_id]
|
||||
if not func_head:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
# item["input_output"] = test_cases
|
||||
# item["input_output"]["fn_name"] = func_head
|
||||
# outputs.append(item)
|
||||
test_inputs["fn_name"] = func_head
|
||||
outputs.append({
|
||||
"input_output": test_inputs,
|
||||
"question": item["problem"],
|
||||
"problem_id": f"oss-instruct-{item_id}",
|
||||
})
|
||||
|
||||
print(len(outputs))
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/clean_oss_mistral_data.py \
|
||||
--data_file ../msranlpintern/share/dataset/magicoder/data-oss_instruct-decontaminated-python.json \
|
||||
--test_case_file "../msranlpintern/share/models/Mistral-Large-Instruct-2407/apps-test-inputs-gen/sub_dev.0shot.tem0.0.n1.*-of-16.v1.0.json" \
|
||||
--func_head_file "../msranlpintern/share/models/Mistral-Large-Instruct-2407/apps-test-inputs-gen/oss_instruct_python.func_head_extract.tem0.0.n1.*-of-16.v1.0.json" \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/data-oss_instruct-decontaminated-python.mistral-large-test-func.json
|
||||
|
||||
Test cases: 31134
|
||||
Func heads: 23807
|
||||
18214
|
||||
|
||||
Test cases: 32066
|
||||
Func heads: 23807
|
||||
18969
|
||||
|
||||
Test cases: 32185
|
||||
Func heads: 23807
|
||||
19280
|
||||
|
||||
|
||||
Test cases: 33760
|
||||
Func heads: 23807
|
||||
20342
|
||||
|
||||
Test cases: 33760 / 38284
|
||||
Func heads: 23807 / 38284
|
||||
20342
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,131 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
|
||||
def load_file(file_path):
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
else:
|
||||
return [json.loads(line) for line in open(file_path).readlines()]
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file, encoding="utf-8"))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
return data
|
||||
|
||||
|
||||
def load_test_cases(item):
|
||||
try:
|
||||
input_output = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["completion"])
|
||||
return []
|
||||
|
||||
inputs = list(input_output.values())
|
||||
return inputs
|
||||
|
||||
|
||||
x_code_eval_template_v1 = """{description}
|
||||
|
||||
Input: {input_from}
|
||||
Output: {output_to}
|
||||
|
||||
Time limit: {time_limit}
|
||||
Memory limit: {memory_limit}
|
||||
|
||||
#### Input Format
|
||||
|
||||
{input_spec}
|
||||
|
||||
#### Output Format
|
||||
|
||||
{output_spec}
|
||||
|
||||
#### Notes
|
||||
|
||||
{notes}
|
||||
|
||||
#### Example Input-Output
|
||||
|
||||
{sample_input_output}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_file", type=str)
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--test_file", type=str)
|
||||
parser.add_argument("--val_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_file(args.data_file)
|
||||
|
||||
test_data = load_file(args.test_file)
|
||||
test_ids = set([item["src_uid"] for item in test_data])
|
||||
|
||||
val_data = load_file(args.val_file)
|
||||
test_ids.update([item["src_uid"] for item in val_data])
|
||||
|
||||
test_cases = load_file(args.test_case_file)
|
||||
id2test_cases = {}
|
||||
test_num = 0
|
||||
for item in test_cases:
|
||||
tmp = load_test_cases(item)
|
||||
if tmp:
|
||||
id2test_cases[item["src_uid"]] = tmp
|
||||
test_num += 1
|
||||
|
||||
print(f"Test cases: {test_num} / {len(test_cases)}")
|
||||
|
||||
outputs = []
|
||||
in_out_template = "Input\n```{}```\n\nOutput\n```{}```"
|
||||
test_held = 0
|
||||
for item in data:
|
||||
if item["input_from"] != 'standard input':
|
||||
continue
|
||||
|
||||
if item["src_uid"] in test_ids:
|
||||
test_held += 1
|
||||
continue
|
||||
|
||||
item_id = item["src_uid"]
|
||||
if item_id in id2test_cases:
|
||||
test_inputs = id2test_cases[item_id]
|
||||
else:
|
||||
continue
|
||||
|
||||
sample_input_output = []
|
||||
for _in, _out in zip(item["sample_inputs"], item["sample_outputs"]):
|
||||
sample_input_output.append(in_out_template.format(_in, _out))
|
||||
|
||||
item["sample_input_output"] = "\n\n".join(sample_input_output)
|
||||
|
||||
question = x_code_eval_template_v1.format(**item)
|
||||
|
||||
test_inputs = {
|
||||
"inputs": test_inputs,
|
||||
}
|
||||
outputs.append({
|
||||
"input_output": test_inputs,
|
||||
"question": question,
|
||||
"problem_id": f"xcode-eval-{item['src_uid']}",
|
||||
})
|
||||
|
||||
print(len(outputs))
|
||||
print(f"No test: {test_held}")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,252 @@
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def worker(item, pseudo_test_case_field):
|
||||
inputs = item[pseudo_test_case_field]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_inputs_non_sc = []
|
||||
new_outputs_non_sc = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
# Non-sc output
|
||||
if case_j in resp2outputs[0]:
|
||||
new_inputs_non_sc.append(inputs[case_j])
|
||||
new_outputs_non_sc.append(output_str2orig_pred[case_j][resp2outputs[0][case_j]])
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"inputs_non_sc": new_inputs_non_sc,
|
||||
"outputs_non_sc": new_outputs_non_sc,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_test_cases")
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.pseudo_test_case_file))
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
missing_predictions = 0
|
||||
avg_test_case_num = 0
|
||||
avg_non_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
result = worker(item, args.test_case_field)
|
||||
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item[args.test_case_field]["inputs"] = result["inputs"]
|
||||
item[args.test_case_field]["outputs"] = result["outputs"]
|
||||
item[args.test_case_field]["output_meta"] = result["output_meta"]
|
||||
item[f"{args.test_case_field}_non_sc"] = copy.deepcopy(item[args.test_case_field])
|
||||
item[f"{args.test_case_field}_non_sc"]["inputs"] = result["inputs_non_sc"]
|
||||
item[f"{args.test_case_field}_non_sc"]["outputs"] = result["outputs_non_sc"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
avg_non_test_case_num += len(result["inputs_non_sc"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item[args.test_case_field]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(avg_non_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 3 --pass_case_lower_bound 0.5
|
||||
|
||||
93%|██████████████████ | 4095/4418 [00:03<00:00, 629.19it/s]None
|
||||
100%|█████████████████████████████| 4418/4418 [00:03<00:00, 1323.18it/s]
|
||||
~~4299
|
||||
~~82365
|
||||
~~1
|
||||
~~9.953477552919283
|
||||
~~Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
4299
|
||||
72724
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 2 --pass_case_lower_bound 0.5
|
||||
|
||||
~~4299
|
||||
~~82365
|
||||
~~1
|
||||
~~9.953477552919283
|
||||
~~Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
4299
|
||||
77024
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
4299
|
||||
59226
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
4712
|
||||
59606
|
||||
2
|
||||
9.955220713073006
|
||||
Counter({10: 17718, 0: 16091, 1: 2515, 2: 1841, 9: 1807, 3: 1364, 8: 1291, 6: 1172, 4: 1125, 5: 1115, 7: 1081})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
4712
|
||||
0
|
||||
2
|
||||
9.955220713073006
|
||||
Counter()
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
import json
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--raw_output_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
input_data = [json.loads(line) for line in open(args.input_file, encoding="utf-8").readlines()]
|
||||
|
||||
raw_outputs = [json.loads(line) for line in open(args.raw_output_file, encoding="utf-8").readlines()]
|
||||
for o in tqdm(raw_outputs):
|
||||
request_meta = o["request_metadata"]
|
||||
s_id = int(request_meta["sid"].split("@")[1])
|
||||
input_item = input_data[s_id]
|
||||
assert o["request"]["messages"][1]["content"] in input_item["prompt"]
|
||||
|
||||
if "response" not in input_item:
|
||||
input_item["response"] = []
|
||||
if "content" not in o["response"]["choices"][0]["message"]:
|
||||
print(o)
|
||||
continue
|
||||
input_item["response"].append(o['response']['choices'][0]['message']['content'])
|
||||
|
||||
missing_completion = 0
|
||||
num_completions = 0
|
||||
for item in input_data:
|
||||
if "response" not in item:
|
||||
missing_completion += 1
|
||||
item["response"] = ""
|
||||
item["pred"] = None
|
||||
else:
|
||||
preds = [standard_cleaner_default(resp) for resp in item["response"]]
|
||||
item["pred"] = preds
|
||||
num_completions += len(preds)
|
||||
|
||||
json.dump(input_data, open(args.output_file, "w", encoding="utf-8"))
|
||||
print(f"Averaged number of completions: {num_completions / len(input_data)}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
def extract_test_case_inputs(item):
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
if "fn_name" in item["input_output"]:
|
||||
function_call = True
|
||||
else:
|
||||
function_call = False
|
||||
|
||||
full_inputs = []
|
||||
if function_call:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
|
||||
for k, v in response.items():
|
||||
if not isinstance(v, list):
|
||||
assert isinstance(v, str), v
|
||||
v = [v]
|
||||
full_inputs.append(v)
|
||||
else:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
for k, v in response.items():
|
||||
inputs = v
|
||||
full_inputs.append(inputs)
|
||||
|
||||
return full_inputs
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--pseudo_test_case", type=str)
|
||||
parser.add_argument("--completion_test_field", type=str, default="input_output")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
|
||||
ps_test_cases = []
|
||||
cnt = 0
|
||||
if args.pseudo_test_case.endswith(".json"):
|
||||
test_cases = json.load(open(args.pseudo_test_case))
|
||||
for item in test_cases:
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
else:
|
||||
with open(args.pseudo_test_case) as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except:
|
||||
print(f"Cannot load {line}")
|
||||
cnt += 1
|
||||
pass
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
print(cnt)
|
||||
print(f"Total number of pseudo test cases: {len(ps_test_cases)}")
|
||||
|
||||
id2item = {item["problem_id"]: item for item in ps_test_cases}
|
||||
outputs = []
|
||||
for item in data:
|
||||
if item["problem_id"] not in id2item:
|
||||
continue
|
||||
test_cases = {
|
||||
"inputs": id2item[item["problem_id"]]["pseudo_inputs"],
|
||||
}
|
||||
if item[args.completion_test_field]:
|
||||
if not isinstance(item[args.completion_test_field], dict):
|
||||
item[args.completion_test_field] = json.loads(item[args.completion_test_field])
|
||||
if "fn_name" in item[args.completion_test_field]:
|
||||
test_cases["fn_name"] = item[args.completion_test_field]["fn_name"]
|
||||
|
||||
# This is to align with oss data format.
|
||||
new_item = {
|
||||
"input_output": test_cases,
|
||||
"question": item["question"],
|
||||
"problem_id": f"apps-train-{item['problem_id']}",
|
||||
"starter_code": item["starter_code"],
|
||||
}
|
||||
outputs.append(new_item)
|
||||
|
||||
print(f"Total number of items: {len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/combine_pseudo_test_inputs.py \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/apps-train.4o-test-func.json \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
#export exp_dir=deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.pdpo.V100.tp8dp32.v1.3.s42/oss-apps-xcode-combine/checkpoint-300/
|
||||
export exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.pdpo.H100.dp8.v1.2.s42/oss-apps-xcode-combine/checkpoint-500/
|
||||
|
||||
p=$1
|
||||
|
||||
echo "Constructing process_rm sample for split 0"
|
||||
echo "p" $p
|
||||
|
||||
# Just check the outputs on the previously generated data
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_takes_extra.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output_by_n64.v1.0.dpo_m6_low0.5_min5_p$p.json \
|
||||
# --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_p$p.json \
|
||||
# --construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
## Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
# --construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
--construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
@@ -0,0 +1,10 @@
|
||||
OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
split_id=$1
|
||||
|
||||
python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label_align_ts_num.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.pseudo_test_case.exec.ctr_ts_num.v1.0.json" \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.*-of-256.pseudo_test_case.exec.ctr_ts_num.v1.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.pseudo_test_case.ctr_ts_num.prefix_pass_num.fix_low0.5_m4.0_avg.json" \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--num_workers 64 --id_field problem_id --test_case_field input_output
|
||||
@@ -0,0 +1,230 @@
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# if not len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]): # TODO: Figure it out why this happens
|
||||
# print(len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.pseudo_test_case_file):
|
||||
print(f"Loading pseudo test cases from {args.pseudo_test_case_file}")
|
||||
data = json.load(open(args.pseudo_test_case_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.pseudo_test_case_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
|
||||
outputs = []
|
||||
before_test_num = 0
|
||||
cnt = 0
|
||||
missing_predictions = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
|
||||
result = worker(item, min_success_test_num=args.min_success_test_num)
|
||||
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item["input_output"]["inputs"] = result["inputs"]
|
||||
item["input_output"]["outputs"] = result["outputs"]
|
||||
item["input_output"]["output_meta"] = result["output_meta"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,382 @@
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# from pympler import asizeof
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# if not len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]): # TODO: Figure it out why this happens
|
||||
# print(len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / len(item["full_res"])
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
# size_in_bytes = asizeof.asizeof(new_outputs)
|
||||
# if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
# "output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item['id']} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
preds = []
|
||||
for _ in item["response"]:
|
||||
preds.append("")
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["pred"] = preds
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
# if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=16)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# if os.path.exists(args.pseudo_test_case_file):
|
||||
# print(f"Loading pseudo test cases from {args.pseudo_test_case_file}")
|
||||
# data = json.load(open(args.pseudo_test_case_file))
|
||||
# else:
|
||||
# data = []
|
||||
# for file in glob(args.pseudo_test_case_file):
|
||||
# print(file)
|
||||
# if file.endswith(".json"):
|
||||
# tmp = json.load(open(file))
|
||||
# else:
|
||||
# tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
# data.extend(tmp)
|
||||
data = load_files(args.pseudo_test_case_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2data = {item["id"]: item for item in data}
|
||||
|
||||
missing_predictions = 0
|
||||
before_test_num = 0
|
||||
|
||||
_mp_inputs = []
|
||||
for item in tqdm(data, desc="Preparing Inputs"):
|
||||
# TODO: Here we do not consider those bad solutions without even complete programs. These programs should also be penalized
|
||||
# but we have already removed them here.
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append(item)
|
||||
|
||||
pbar = tqdm(_mp_inputs, desc="Submitting tasks")
|
||||
_mp_outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
sc_test_cases_outputs = []
|
||||
prefer_pair_outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in tqdm(_mp_outputs):
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2data[result["id"]]
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
|
||||
test_case_item = {
|
||||
"problem_id": item["problem_id"],
|
||||
"input_output": {
|
||||
"inputs": result["inputs"],
|
||||
"outputs": result["outputs"],
|
||||
},
|
||||
"sc_full_res": result["sc_match_res"],
|
||||
}
|
||||
sc_test_cases_outputs.append(test_case_item)
|
||||
|
||||
# test_case_item["input_output"]["inputs"] = result["inputs"]
|
||||
# test_case_item["input_output"]["outputs"] = result["outputs"]
|
||||
# item["input_output"]["output_meta"] = result["output_meta"]
|
||||
# test_case_item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(test_case_item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(test_case_item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(test_case_item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"]), (len(pred_pass_cnt), len(item["response"]), len(item["pred"]))
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
|
||||
prefer_pair_outputs.append(item)
|
||||
|
||||
print(len(prefer_pair_outputs))
|
||||
print(len(sc_test_cases_outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(sc_test_cases_outputs) if sc_test_cases_outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(sc_test_cases_outputs, open(args.output_file.replace(".json", ".sc_test_cases.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
if args.construct_prefer_pair:
|
||||
json.dump(prefer_pair_outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# from pympler import asizeof
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
|
||||
A different saving format. One positive response is paired with a list of negative candidates.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / len(item["full_res"])
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
# "output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item['id']} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
preds = []
|
||||
for _ in item["response"]:
|
||||
preds.append("")
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["pred"] = preds
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
# if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=16)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.pseudo_test_case_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2data = {item["id"]: item for item in data}
|
||||
|
||||
missing_predictions = 0
|
||||
before_test_num = 0
|
||||
|
||||
_mp_inputs = []
|
||||
for item in tqdm(data, desc="Preparing Inputs"):
|
||||
# TODO: Here we do not consider those bad solutions without even complete programs. These programs should also be penalized
|
||||
# but we have already removed them here.
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append(item)
|
||||
|
||||
pbar = tqdm(_mp_inputs, desc="Submitting tasks")
|
||||
_mp_outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
sc_test_cases_outputs = []
|
||||
prefer_pair_outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in tqdm(_mp_outputs):
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2data[result["id"]]
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
|
||||
test_case_item = {
|
||||
"problem_id": item["problem_id"],
|
||||
"input_output": {
|
||||
"inputs": result["inputs"],
|
||||
"outputs": result["outputs"],
|
||||
},
|
||||
"sc_full_res": result["sc_match_res"],
|
||||
}
|
||||
sc_test_cases_outputs.append(test_case_item)
|
||||
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(test_case_item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(test_case_item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(test_case_item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"]), (len(pred_pass_cnt), len(item["response"]), len(item["pred"]))
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
|
||||
pos2neg = []
|
||||
pos2neg_code = []
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
# pos.append(resp_i)
|
||||
# pos_code.append(prog_i)
|
||||
# neg.append(resp_j)
|
||||
# neg_code.append(prog_j)
|
||||
pos2neg.append(resp_j)
|
||||
pos2neg_code.append(prog_j)
|
||||
|
||||
if pos2neg:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(pos2neg)
|
||||
neg_code.append(pos2neg_code)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
|
||||
prefer_pair_outputs.append(item)
|
||||
|
||||
print(len(prefer_pair_outputs))
|
||||
print(len(sc_test_cases_outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(sc_test_cases_outputs) if sc_test_cases_outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(sc_test_cases_outputs, open(args.output_file.replace(".json", ".sc_test_cases.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
if args.construct_prefer_pair:
|
||||
json.dump(prefer_pair_outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
import collections
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pympler import asizeof
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py`.
|
||||
|
||||
This script can accept external execution files for performing self-consistency over test cases.
|
||||
"""
|
||||
|
||||
|
||||
def worker(_input, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
item, sc_item = _input
|
||||
inputs = sc_item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
assert len(inputs) == len(item["input_output"]["inputs"]), (len(inputs), len(item["input_output"]["inputs"]))
|
||||
# assert len(sc_item["outputs"]) == item["outputs"], (len(sc_item["outputs"]), len(item["outputs"]))
|
||||
|
||||
assert len(sc_item["full_res"]) == len(sc_item["outputs"]) == len(sc_item["pred"]), (len(sc_item["full_res"]), len(sc_item["outputs"]),
|
||||
len(sc_item["pred"]))
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
|
||||
sc_prog_num = len(sc_item["full_res"])
|
||||
|
||||
# We use the external item to perform self-consistency over test cases.
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(sc_item["full_res"], sc_item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
|
||||
# We then record the target item's outputs
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / sc_prog_num
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
sc_o, sc_o_freq = output_cnt.most_common(1)[0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
large_mem = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "outputs" in item:
|
||||
size_in_bytes = asizeof.asizeof(item["outputs"])
|
||||
if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
for _ in item["response"]:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
print(f"Too large outputs: {large_mem}")
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str)
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=24)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
external_ps_test_cases = load_files(args.pseudo_test_case_file)
|
||||
external_ps_test_cases = merge_seed_sampled_data(external_ps_test_cases)
|
||||
id2external_ps_test_cases = {item["id"]: item for item in external_ps_test_cases}
|
||||
|
||||
data = load_files(args.completion_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2item = {item["id"]: item for item in data}
|
||||
|
||||
before_test_num = 0
|
||||
missing_predictions = 0
|
||||
_mp_inputs = []
|
||||
_mp_outputs = []
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append((item, id2external_ps_test_cases[item["id"]]))
|
||||
|
||||
pbar = tqdm(_mp_inputs)
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in _mp_outputs:
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2item[result["id"]]
|
||||
|
||||
item["input_output"]["inputs"] = result["inputs"]
|
||||
item["input_output"]["outputs"] = result["outputs"]
|
||||
item["input_output"]["output_meta"] = result["output_meta"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,179 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
if any(x and y in x for x in item['pred'] for y in ["os.makedirs", "views.contact", "extract_metadata.py"]):
|
||||
print(f"======================================================== item {item['problem_id']} ====================================================")
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
try:
|
||||
res = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
except Exception as e:
|
||||
print(f"======================================================== item {item['problem_id']} ====================================================")
|
||||
print(e)
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
new_res = []
|
||||
# res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
for tmp in res:
|
||||
try:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
new_res.append(bool(tmp))
|
||||
else:
|
||||
new_res.append(tmp)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
new_res.append(False)
|
||||
res = new_res
|
||||
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
|
||||
depreciated_ids = {"oss-instruct-9410"}
|
||||
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["problem_id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
# problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["input_output"] = pseudo_test_cases[problem_id]["input_output"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
data = [item for item in data if item["problem_id"] not in depreciated_ids]
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,154 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[problem_id]["pseudo_test_cases"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,34 @@
|
||||
#python scripts/apps/solution_run_outputs_local.py --completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.?-of-8.v2.0.json" --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.v2.0.run_outputs.json --num_workers 24 --id_field "problem_id" --test_case_field "input_output"
|
||||
python scripts/apps/solution_run_outputs_local.py --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.json" --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.{split_id}-of-32.run_outputs.json --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
# Iter - 1: PRM
|
||||
# Sample prefix
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.prefix.upper0.8.r0.3.v2.0.json
|
||||
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 20 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample20_per.json
|
||||
|
||||
|
||||
# Execute the prefix completions
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.*-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json
|
||||
|
||||
# Construct preference pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "$OUTPUT_PATH_PREFIX/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.*-of-256.json" \
|
||||
--output_file $OUTPUT_PATH_PREFIX/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/magicoder_cls_2shot.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file, encoding='utf-8')]
|
||||
prompt_template = open(args.prompt_file, 'r').read()
|
||||
|
||||
for i, item in enumerate(data):
|
||||
prompt = prompt_template.replace("[[Question]]", item["instruction"])
|
||||
prompt = prompt.replace("[[Solution]]", item["response"])
|
||||
data[i]["prompt"] = prompt
|
||||
data[i]["id"] = i
|
||||
|
||||
with open(args.output_file, 'w', encoding='utf-8') as f:
|
||||
for item in data:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/magicoder_cls_2shot.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file, encoding="utf-8"))
|
||||
prompt_template = open(args.prompt_file, 'r').read()
|
||||
|
||||
for i, item in enumerate(data):
|
||||
prompt = prompt_template.replace("[[Question]]", item["problem"])
|
||||
prompt = prompt.replace("[[Solution]]", item["solution"])
|
||||
data[i]["prompt"] = prompt
|
||||
data[i]["id"] = i
|
||||
|
||||
with open(args.output_file, 'w', encoding='utf-8') as f:
|
||||
for item in data:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,159 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
print(sys.path)
|
||||
|
||||
|
||||
from apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_test_cases")
|
||||
parser.add_argument("--id_field", type=str, default="id")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item[args.id_field]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[problem_id][args.test_case_field]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,174 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import random
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
"""
|
||||
This script is used to control the amount of pseudo test cases are not more than the amount of ground truth test cases.
|
||||
"""
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
parser.add_argument("--gd_test_case_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
num_test_cases = 0
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
ps_test_cases = pseudo_test_cases[problem_id]["pseudo_test_cases"]
|
||||
if not item[args.gd_test_case_field]:
|
||||
continue
|
||||
if len(ps_test_cases["inputs"]) > len(item[args.gd_test_case_field]["inputs"]):
|
||||
idx = list(range(len(ps_test_cases["inputs"])))
|
||||
random.shuffle(idx)
|
||||
num_test_cases = len(item[args.gd_test_case_field]["inputs"])
|
||||
ps_test_cases["inputs"] = [ps_test_cases["inputs"][i] for i in idx[:num_test_cases]]
|
||||
ps_test_cases["outputs"] = [ps_test_cases["outputs"][i] for i in idx[:num_test_cases]]
|
||||
item["pseudo_input_output"] = ps_test_cases
|
||||
num_test_cases += len(item[args.gd_test_case_field]["inputs"])
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
if len(data) == 0:
|
||||
print("No data to process.")
|
||||
return
|
||||
print(f"Average number of test cases: {num_test_cases / len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,44 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
split_id=$1
|
||||
|
||||
echo "Constructing process_rm sample for split $split_id"
|
||||
|
||||
#export exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/${exp_dir}/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/${exp_dir}/train.0shot.tem1.0.n10.v2.1.${split_id}-of-32.s42.run_outputs.json" \
|
||||
# --num_workers 64 --id_field id --test_case_field input_output
|
||||
|
||||
# Run 4o-based prm executing
|
||||
#python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.4o_pseudo_test_case.exec.json" \
|
||||
# --num_workers 128 \
|
||||
# --pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
# --test_case_field input_output --id_field problem_id
|
||||
|
||||
# 4o-DPO-Iter-0 run outputs
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.run_outputs.json" \
|
||||
# --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# 4o-DPO-Iter-0 run solutions on previous 4o-synthesized test cases (APPs only)
|
||||
#python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
# --pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
|
||||
# 4o-DPO-iter-1 run outputs
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.run_outputs.json" \
|
||||
# --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# 4o-DPO-Iter-1 run solutions on previous 4o-synthesized test cases (APPs only)
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.${split_id}-of-9.v2.1.s42.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.v2.1.${split_id}-of-9.s42.4o_pseudo_test_cases.exec.v1.0.json" \
|
||||
--pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
export OUTPUT_PREFIX_PATH=../msranlpintern/reward_modeling/
|
||||
|
||||
python scripts/apps/pseudo_test_cases/clean_xcode_4o_test_inputs_data.py \
|
||||
--data_file ../msranlpintern/share/xCodeEval/problem_descriptions.jsonl \
|
||||
--test_case_file ../gpt_crawler/outputs/xcode_test_case_inputs_gen_v2.1.4o.tem0.0.n1.json_obj.jsonl \
|
||||
--output_file ../msranlpintern/share/xCodeEval/xcode_train_4o_test_inputs_v1.json \
|
||||
--test_file "../msranlpintern/share/xCodeEval/**/test/*.jsonl" \
|
||||
--val_file "../msranlpintern/share/xCodeEval/**/validation/*.jsonl"
|
||||
#Test cases: 7588 / 7635
|
||||
#6163
|
||||
#No test: 1372
|
||||
|
||||
# Run outputs on newly generated data
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#5844
|
||||
#79260
|
||||
#0
|
||||
#10.0
|
||||
#9.7298083504449
|
||||
#Counter({0: 15165, 10: 14577, 1: 5613, 9: 3884, 2: 3674, 8: 3100, 3: 2896, 4: 2561, 7: 2376, 5: 2350, 6: 2244})
|
||||
|
||||
|
||||
# Run outputs on newly generated oss-apps-combine data
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-32.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#14107
|
||||
#106950
|
||||
#1
|
||||
#10.004038772213248
|
||||
#9.794782731977033
|
||||
#Counter({10: 96073, 0: 20843, 9: 4661, 1: 3399, 8: 3067, 5: 2755, 2: 2179, 7: 2116, 6: 2004, 3: 1940, 4: 1870, 11: 142, 20: 21})
|
||||
|
||||
|
||||
# Sample steps for xcode
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 61630
|
||||
|
||||
# Sample steps for new oss-apps
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 247580
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_rm_large_meta.json
|
||||
|
||||
|
||||
# Construct prm pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.*-of-32.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.*-of-32.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.6 --pass_case_margin 6 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
# Run outputs on apps-magicoder-xcode-combine newly generated data w/ n=64
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
# Just check the outputs on the previously generated data
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_takes_extra.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output_by_n64.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#21790
|
||||
#188616
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.875906379072969
|
||||
#Counter({10: 109418, 0: 55905, 1: 9618, 9: 8233, 2: 6063, 8: 5869, 5: 4976, 3: 4861, 4: 4392, 7: 4323, 6: 4073, 11: 146, 20: 23}) # TODO: Why there is 20 and 11????
|
||||
|
||||
#21790
|
||||
#188616
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.875906379072969
|
||||
#Counter({10: 109418, 0: 55905, 1: 9618, 9: 8233, 2: 6063, 8: 5869, 5: 4976, 3: 4861, 4: 4392, 7: 4323, 6: 4073, 11: 146, 20: 23})
|
||||
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.v2.0.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.pseudo_input_output.exec.json \
|
||||
# --num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_rm_large_meta.json
|
||||
#
|
||||
|
||||
# Starting from model w/ magicoder & apps - pdpo, repeat the process again.
|
||||
# Run outputs on newly generated data
|
||||
exp_dir=deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.pdpo.V100.tp8dp32.v1.3.s42/oss-apps-xcode-combine/checkpoint-300/
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.v2.0.s[0-8].json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.{split_id}-of-32.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Sample steps for xcode
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 61630
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#22303
|
||||
#22303
|
||||
#8588574
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.884141146930906
|
||||
#Counter({10: 767611, 0: 324963, 1: 57875, 9: 53845, 2: 37125, 8: 36508, 5: 31254, 3: 30650, 7: 28261, 4: 27218, 6: 26200, 11: 990, 20: 188})
|
||||
|
||||
# Run completion outputs
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-{global_split_id}-of-8.loc-{split_id}-of-64.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-{global_split_id}-of-8.loc-{split_id}-of-64.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5sc_test_cases.json
|
||||
|
||||
|
||||
# Construct prm pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-*-of-8.loc-*-of-64.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 890
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 268084, 10: 15225, 1: 7396, 9: 2466, 2: 2270, 3: 1795, 8: 1696, 5: 1385, 4: 1371, 6: 1290, 7: 1207})
|
||||
#Processed 101395 prefixes.
|
||||
#Averaged 0.9912988219191475 prefixes per problem.
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-*-of-8.loc-*-of-64.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.3 --pass_case_margin 3 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 890
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 268084, 10: 15225, 1: 7396, 9: 2466, 2: 2270, 3: 1795, 8: 1696, 5: 1385, 4: 1371, 6: 1290, 7: 1207})
|
||||
#Processed 101395 prefixes.
|
||||
#Averaged 0.9912988219191475 prefixes per problem.
|
||||
#Processed 10172 problems.
|
||||
|
||||
exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.pdpo.H100.dp8.v1.2.s42/oss-apps-xcode-combine/checkpoint-500/
|
||||
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
#Too large outputs: 0
|
||||
#Total number of samples: 309210
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 32 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.json
|
||||
#Too large outputs: 0
|
||||
#Total number of samples: 989473
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.v2.0.s{seed}.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.s{seed}.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
--construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
#p=0.0
|
||||
#21662
|
||||
#21662
|
||||
#608162
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.874849967685348
|
||||
#Counter({10: 679443, 0: 368401, 1: 61978, 9: 52376, 2: 39749, 8: 36629, 3: 31688, 5: 31673, 4: 28822, 7: 28171, 6: 26357, 11: 909, 20: 170, 12: 2})
|
||||
#p=0.2
|
||||
#18373
|
||||
#18373
|
||||
#585808
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.93577532248408
|
||||
#Counter({10: 668678, 0: 205613, 9: 50259, 1: 46523, 8: 34864, 2: 33257, 5: 29338, 3: 28297, 7: 26725, 4: 26387, 6: 24873, 11: 886, 20: 170, 12: 2})
|
||||
#p=0.4
|
||||
#14583
|
||||
#14583
|
||||
#506479
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.954878968662142
|
||||
#Counter({10: 634922, 0: 95150, 9: 41702, 8: 28477, 1: 22680, 5: 20957, 7: 20445, 6: 18610, 2: 17527, 4: 16296, 3: 15538, 11: 838, 20: 170})
|
||||
#=0.5
|
||||
#12903
|
||||
#12903
|
||||
#452734
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.960861815081763
|
||||
#Counter({10: 609930, 0: 62866, 9: 36059, 8: 23682, 7: 16213, 5: 15332, 6: 14783, 1: 14003, 2: 11235, 4: 10667, 3: 10087, 11: 791, 20: 144})
|
||||
|
||||
|
||||
# Run completion outputs
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-{global_split_id}-of-16.loc-{split_id}-of-64.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-{global_split_id}-of-16.loc-{split_id}-of-64.v2.0.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p0.0.sc_test_cases.json
|
||||
|
||||
# p=0.0
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-*-of-16.loc-*-of-64.v2.0.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 4235
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 1492601, 10: 211726, 1: 90982, 2: 45672, 9: 42604, 3: 34232, 8: 32685, 4: 29914, 5: 27977, 7: 26170, 6: 25720})
|
||||
#Processed 686761 prefixes.
|
||||
#Averaged 0.9938711656796856 prefixes per problem.
|
||||
#Processed 21533 problems.
|
||||
|
||||
# p=0.2
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-*-of-16.loc-*-of-64.v2.0.pseudo_input_output.p0.2.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.p0.2.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 2073
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 952288, 10: 159840, 1: 54560, 9: 29678, 2: 27823, 8: 23059, 3: 21733, 4: 20014, 5: 18776, 7: 17984, 6: 17690})
|
||||
#Processed 447815 prefixes.
|
||||
#Averaged 0.995392186499751 prefixes per problem.
|
||||
#Processed 14028 problems.
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
x_code_eval_template = """{description}
|
||||
|
||||
Input: {input_from}
|
||||
Output: {output_to}
|
||||
|
||||
Time limit: {time_limit}
|
||||
Memory limit: {memory_limit}
|
||||
|
||||
-----Input-----
|
||||
|
||||
{input_spec}
|
||||
|
||||
-----Output-----
|
||||
|
||||
{output_spec}
|
||||
|
||||
-----Notes-----
|
||||
|
||||
{notes}
|
||||
|
||||
-----Example-----
|
||||
|
||||
{input_output}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
parser.add_argument("--output_file")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/test_input_gen_2shot_v2.1.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
in_out_template = "Input\n```{}```\n\nOutput\n```{}```"
|
||||
prompt_tem = open(args.prompt_file).read()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
outputs = []
|
||||
for item in data:
|
||||
input_output = []
|
||||
for _in, _out in zip(item["sample_inputs"], item["sample_outputs"]):
|
||||
input_output.append(in_out_template.format(_in, _out))
|
||||
|
||||
item["input_output"] = "\n\n".join(input_output)
|
||||
|
||||
question = x_code_eval_template.format(**item)
|
||||
|
||||
prompt = prompt_tem.replace("[[Question]]", question)
|
||||
|
||||
item["prompt"] = prompt
|
||||
outputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
print(len(outputs))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True)
|
||||
parser.add_argument("--output_file", type=str, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file))
|
||||
|
||||
success = 0
|
||||
success_at_k = 0
|
||||
for item in data:
|
||||
if isinstance(item["res"], bool):
|
||||
assert isinstance(item["response"], str)
|
||||
if item["pred"] is None:
|
||||
item["res"] = False
|
||||
|
||||
if item["res"]:
|
||||
success += 1
|
||||
success_at_k += 1
|
||||
else:
|
||||
if len(item["res"]):
|
||||
assert isinstance(item["response"], list)
|
||||
for i in range(len(item["response"])):
|
||||
if item["pred"][i] is None:
|
||||
item["res"][i] = False
|
||||
item["full_res"][i] = [[False] * len(item["test_cases"]["inputs"]) if item["test_cases"] else 1]
|
||||
|
||||
if any(item["res"]):
|
||||
success_at_k += 1
|
||||
if item["res"][0]:
|
||||
success += 1
|
||||
|
||||
metrics = {"acc": success / len(data), "pass@k": success_at_k / len(data), "correct": success, "total": len(data)}
|
||||
print(json.dumps(metrics, indent=2))
|
||||
if args.output_file is not None:
|
||||
json.dump(data, open(args.output_file, "w"), ensure_ascii=False)
|
||||
json.dump(metrics, open(args.output_file.replace(".json", ".metrics"), "w"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
import json
|
||||
import argparse
|
||||
import sys
|
||||
import collections
|
||||
import glob
|
||||
import os
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str, required=True)
|
||||
parser.add_argument("--reward_file", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
solutions = json.load(open(args.completion_file))
|
||||
if "difficulty" in solutions[0]:
|
||||
all_diff = collections.Counter([item["difficulty"] for item in solutions])
|
||||
all_diff_success = {k: 0 for k in all_diff}
|
||||
else:
|
||||
all_diff = None
|
||||
all_diff_success = None
|
||||
|
||||
if os.path.exists(args.reward_file):
|
||||
rewards = json.load(open(args.reward_file))
|
||||
else:
|
||||
rewards = []
|
||||
for file in glob.glob(args.reward_file):
|
||||
rewards += json.load(open(file))
|
||||
|
||||
id2reward = collections.defaultdict(dict)
|
||||
for item in rewards:
|
||||
index = item["index"]
|
||||
orig_id, pred_id = list(map(int, index.split("_")))
|
||||
# id2reward[orig_id][pred_id] = item["reward"]
|
||||
if isinstance(item["reward"], list):
|
||||
id2reward[orig_id][pred_id] = item["reward"][0]
|
||||
else:
|
||||
id2reward[orig_id][pred_id] = item["reward"]
|
||||
|
||||
success = 0
|
||||
for item in solutions:
|
||||
problem_id = item["id"]
|
||||
|
||||
if not item["response"]:
|
||||
continue
|
||||
if not item["pred"]:
|
||||
continue
|
||||
|
||||
tuples = []
|
||||
for j, (resp, pred) in enumerate(zip(item["response"], item["pred"])):
|
||||
if pred:
|
||||
tuples.append((j, id2reward[problem_id][j]))
|
||||
|
||||
if not tuples:
|
||||
continue
|
||||
|
||||
tuples = sorted(tuples, key=lambda x: x[1], reverse=True)
|
||||
pred_id = tuples[0][0]
|
||||
if item["res"][pred_id]:
|
||||
success += 1
|
||||
if all_diff is not None:
|
||||
all_diff_success[item["difficulty"]] += 1
|
||||
|
||||
print(f"Success rate: {success / len(solutions)}")
|
||||
if all_diff is not None:
|
||||
for k, v in all_diff.items():
|
||||
print(f"Difficulty {k}: {all_diff_success[k] / v}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,165 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def extract_solution_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'<BEGIN>(.*?)<END>'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if isinstance(item["completion"], str):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
solutions = []
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
for completion in completions:
|
||||
gen_solution = extract_solution_between_tags(completion)
|
||||
if gen_solution is None:
|
||||
solutions.append("")
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
solutions.append(gen_solution)
|
||||
|
||||
if not item["input_output"]:
|
||||
continue
|
||||
|
||||
all_results = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
# res = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
# assert len(res) == len(outputs) == len(errors), (len(res), len(outputs), len(errors))
|
||||
|
||||
item["pred"] = solutions if len(solutions) > 1 else solutions[0]
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["problem_id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["completion"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["completion"], list):
|
||||
data[item_id2data_id[item_id]]["completion"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["completion"] = [data[item_id2data_id[item_id]]["completion"]] + new_completions
|
||||
|
||||
for item in data:
|
||||
item["completion"] = list(set(item["completion"]))
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,160 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if isinstance(item["completion"], str) or isinstance(item["completion"], dict):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
# if item["input_output"]:
|
||||
# item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
solutions = []
|
||||
results = []
|
||||
full_results = []
|
||||
# all_outputs = []
|
||||
# all_errors = []
|
||||
for completion in completions:
|
||||
if "corrected_program" not in completion or completion["corrected_program"] is None:
|
||||
solutions.append("")
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
gen_solution = completion["corrected_program"]
|
||||
|
||||
solutions.append(gen_solution)
|
||||
|
||||
if not item["input_output"]:
|
||||
continue
|
||||
|
||||
all_results = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
# res, outputs, errors = all_results
|
||||
res = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
# all_outputs.append(outputs)
|
||||
# all_errors.append(errors)
|
||||
# assert len(res) == len(outputs) == len(errors), (len(res), len(outputs), len(errors))
|
||||
|
||||
item["pred"] = solutions if len(solutions) > 1 else solutions[0]
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
# item["outputs"] = all_outputs
|
||||
# item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This script takes the GPT-4 completion file with json object as the response format, where the returned json contains two fields:
|
||||
- feedback: the critique for the incorrect program.
|
||||
- corrected_program: the corrected program.
|
||||
|
||||
THe inputs file is generated by `pp_critique_difficulty` script.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["problem_id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["completion"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["completion"], list):
|
||||
data[item_id2data_id[item_id]]["completion"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["completion"] = [data[item_id2data_id[item_id]]["completion"]] + new_completions
|
||||
|
||||
new_data = []
|
||||
parsing_error = 0
|
||||
for item in tqdm(data):
|
||||
if isinstance(item["completion"], str):
|
||||
try:
|
||||
item["completion"] = json.loads(item["completion"])
|
||||
except:
|
||||
# print(f"Parsing error: {item['completion']}")
|
||||
parsing_error += 1
|
||||
if isinstance(item["completion"], dict):
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
print(f"Total number of items: {len(data)}")
|
||||
print(f"Total number of parsing error: {parsing_error}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,163 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict, Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
print(sys.path)
|
||||
|
||||
from apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
print(args.completion_file)
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["problem_id"]: item for item in pseudo_test_cases}
|
||||
print(len(pseudo_test_cases))
|
||||
print(list(pseudo_test_cases.keys())[:10])
|
||||
aux_ps_test_cases = {f"apps-train-{k}": v for k, v in pseudo_test_cases.items()}
|
||||
pseudo_test_cases.update(aux_ps_test_cases)
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
if item["id"] in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[item["id"]]["input_output"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.pseudo_test_case.exec.json \
|
||||
--num_workers 24 --pseudo_test_cases outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pseudo_test_cases.json
|
||||
|
||||
>>> python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v1.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.json \
|
||||
--num_workers 24 --pseudo_test_cases outputs/apps/apps.train.r2c.vanilla.gpt-4o.tem1.0.n11.pseudo_test_cases.json
|
||||
|
||||
>>> python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.self_s43_pseudo_cases.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.pseudo_cases.sc.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.37857900318133614
|
||||
Correct at k: 0.4721102863202545
|
||||
Counter({False: 30141, True: 17009})
|
||||
|
||||
>>>
|
||||
"""
|
||||
@@ -0,0 +1,173 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import run_inference_process
|
||||
|
||||
|
||||
def extract_solution_between_tags(text):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'<BEGIN>(.*?)<END>'
|
||||
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_test_cases(text):
|
||||
# Regular expression to match the contents between <TEST INPUT X> and </TEST INPUT X>
|
||||
pattern = r'<TEST INPUT (\d+)>(.*?)</TEST INPUT \1>'
|
||||
|
||||
# Find all matches in the text, the re.DOTALL flag allows . to match newline characters
|
||||
test_cases = re.findall(pattern, text, re.DOTALL)
|
||||
|
||||
# Extract only the content part from the matches
|
||||
test_cases = [case[1].strip() for case in test_cases]
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if isinstance(item["completion"], str):
|
||||
completions = [item["completion"]]
|
||||
else:
|
||||
completions = item["completion"]
|
||||
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
solutions = []
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
for completion in completions:
|
||||
gen_solution = extract_solution_between_tags(completion)
|
||||
if gen_solution is None:
|
||||
solutions.append("")
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
solutions.append(gen_solution)
|
||||
|
||||
if not item["input_output"]:
|
||||
continue
|
||||
|
||||
all_results = run_inference_process(item["input_output"], gen_solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
try:
|
||||
json.dumps(outputs)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
except:
|
||||
print(f"Cannot dump outputs for {outputs}")
|
||||
all_outputs.append([])
|
||||
all_errors.append([])
|
||||
# assert len(res) == len(outputs) == len(errors), (len(res), len(outputs), len(errors))
|
||||
|
||||
item["pred"] = solutions if len(solutions) > 1 else solutions[0]
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["problem_id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["completion"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["completion"], list):
|
||||
data[item_id2data_id[item_id]]["completion"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["completion"] = [data[item_id2data_id[item_id]]["completion"]] + new_completions
|
||||
|
||||
for item in data:
|
||||
item["completion"] = list(set(item["completion"]))
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,235 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
import os
|
||||
from pympler import asizeof
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
print(sys.path)
|
||||
|
||||
from apps.utils_execute import run_inference_process
|
||||
|
||||
|
||||
def _worker(item, test_case_field: str):
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
return item
|
||||
|
||||
for pred in item["pred"]:
|
||||
gen_solution = pred
|
||||
if gen_solution is None:
|
||||
results.append(False)
|
||||
# full_results.append([False] * 3)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
continue
|
||||
|
||||
if "Hello, World!" in gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
continue
|
||||
|
||||
if not item[test_case_field]:
|
||||
continue
|
||||
|
||||
all_results = run_inference_process(item[test_case_field], gen_solution, timeout=1, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
try:
|
||||
json.dumps(outputs)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
except:
|
||||
print(f"Cannot dump outputs for {outputs}")
|
||||
all_outputs.append([])
|
||||
all_errors.append([])
|
||||
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
# Function to check if a string contains surrogate characters
|
||||
def has_surrogate_characters(text):
|
||||
return bool(re.search(r'[\ud800-\udfff]', text))
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data, id_field: str = "id"):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item[id_field]} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if item[id_field] not in id2data:
|
||||
id2data[item[id_field]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item[id_field]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item[id_field]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--id_field", type=str, default="id")
|
||||
parser.add_argument("--test_case_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.completion_file)
|
||||
data = merge_seed_sampled_data(data, args.id_field)
|
||||
# for item in data:
|
||||
# item["response"] = list(set(item["response"])) # No need to remove duplicates, since the `pred` field is aligned.
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = partial(_worker, test_case_field=args.test_case_field)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
large_mem = 0
|
||||
for item in outputs:
|
||||
if "res" in item and item["res"]:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
|
||||
try:
|
||||
size_in_bytes = asizeof.asizeof(item["outputs"])
|
||||
if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
except:
|
||||
print("failed to compute size. Still abandon the outputs.")
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
new_outputs = []
|
||||
for item in tqdm(outputs):
|
||||
tmp = json.dumps(item, ensure_ascii=False)
|
||||
if has_surrogate_characters(tmp):
|
||||
print(f"Surrogate characters found in {item[args.id_field]}")
|
||||
continue
|
||||
new_outputs.append(item)
|
||||
|
||||
outputs = new_outputs
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Large memory: {large_mem / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/solution_run_outputs_local.py --completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.json --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.run_outputs.json --num_workers 24
|
||||
|
||||
>>> python scripts/apps/solution_run_outputs_local.py --completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.?-of-8.v2.0.json --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.v2.0.run_outputs.json --num_workers 24 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,258 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from scripts.apps.utils_execute import run_inference_process
|
||||
|
||||
|
||||
def extract_test_case_inputs(item):
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
if "fn_name" in item["input_output"]:
|
||||
function_call = True
|
||||
else:
|
||||
function_call = False
|
||||
|
||||
full_inputs = []
|
||||
if function_call:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
|
||||
for k, v in response.items():
|
||||
# if v.find(item["input_output"]["fn_name"]) != -1:
|
||||
# inputs = v.replace(item["input_output"]["fn_name"], "").strip()
|
||||
# inputs = inputs[1:-1] # Remove `(` and `)`.
|
||||
# try:
|
||||
# inputs = eval(f"[{inputs}]")
|
||||
# except Exception as e:
|
||||
# print(f"Cannot eval inputs for {inputs}\t{item['input_output']['fn_name']}\t{v}\t{item['input_output']['inputs']}")
|
||||
# print(e)
|
||||
# continue
|
||||
# full_inputs.append(inputs)
|
||||
if not isinstance(v, list):
|
||||
assert isinstance(v, str), v
|
||||
v = [v]
|
||||
full_inputs.append(v)
|
||||
else:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
for k, v in response.items():
|
||||
inputs = v
|
||||
full_inputs.append(inputs)
|
||||
|
||||
return full_inputs
|
||||
|
||||
|
||||
def _worker(item):
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
all_errors = []
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
return item
|
||||
|
||||
for pred in item["pred"]:
|
||||
gen_solution = pred
|
||||
if gen_solution is None:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
continue
|
||||
|
||||
if not item["pseudo_test_cases"]:
|
||||
continue
|
||||
|
||||
all_results = run_inference_process(item["pseudo_test_cases"], gen_solution, timeout=10, debug=False, return_output=True)
|
||||
res, outputs, errors = all_results
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
try:
|
||||
json.dumps(outputs)
|
||||
all_outputs.append(outputs)
|
||||
all_errors.append(errors)
|
||||
except:
|
||||
print(f"Cannot dump outputs for {outputs}")
|
||||
all_outputs.append([])
|
||||
all_errors.append([])
|
||||
|
||||
if results:
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["errors"] = all_errors
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_case", type=str)
|
||||
parser.add_argument("--completion_test_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
item_id2data_id = {}
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
for item in tmp:
|
||||
item_id = item["id"]
|
||||
if item_id not in item_id2data_id:
|
||||
item_id2data_id[item_id] = len(data)
|
||||
data.append(item)
|
||||
else:
|
||||
new_completions = item["response"]
|
||||
if isinstance(new_completions, str):
|
||||
new_completions = [new_completions]
|
||||
|
||||
if isinstance(data[item_id2data_id[item_id]]["response"], list):
|
||||
data[item_id2data_id[item_id]]["response"].extend(new_completions)
|
||||
else:
|
||||
data[item_id2data_id[item_id]]["response"] = [data[item_id2data_id[item_id]]["response"]] + new_completions
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
ps_test_cases = []
|
||||
cnt = 0
|
||||
if args.pseudo_test_case.endswith(".json"):
|
||||
test_cases = json.load(open(args.pseudo_test_case))
|
||||
for item in test_cases:
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
else:
|
||||
with open(args.pseudo_test_case) as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except:
|
||||
print(f"Cannot load {line}")
|
||||
cnt += 1
|
||||
pass
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
print(cnt)
|
||||
print(f"Total number of pseudo test cases: {len(ps_test_cases)}")
|
||||
|
||||
id2item = {item["problem_id"]: item for item in ps_test_cases}
|
||||
new_data = []
|
||||
for item in data:
|
||||
if not isinstance(item["response"], list):
|
||||
item["response"] = [item["response"]]
|
||||
item["pred"] = [item["pred"]]
|
||||
if item["id"] not in id2item:
|
||||
continue
|
||||
item["pseudo_test_cases"] = {
|
||||
"inputs": id2item[item["id"]]["pseudo_inputs"],
|
||||
"outputs": [],
|
||||
}
|
||||
if item[args.completion_test_field]:
|
||||
if not isinstance(item[args.completion_test_field], dict):
|
||||
item[args.completion_test_field] = json.loads(item[args.completion_test_field])
|
||||
if "fn_name" in item[args.completion_test_field]:
|
||||
item["pseudo_test_cases"]["fn_name"] = item[args.completion_test_field]["fn_name"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item and item["res"]:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--pseudo_test_case outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
|
||||
>>> python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json --num_workers 128
|
||||
"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
from collections import defaultdict, Counter
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from glob import glob
|
||||
import copy
|
||||
from tqdm import tqdm
|
||||
from datasets import load_dataset
|
||||
import random
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from data.apps import APPsWithFunctionName
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This script takes the completions from GPT4, the corresponding inputs for worsening code, and the original dataset, to construct a new combined dataset,
|
||||
which contains the solutions from the policy model, as well as the synthesized negative codes for each plausible solution.
|
||||
:return:
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--worsen_file", type=str,
|
||||
help="The file contains completion from the teacher model for worsening code"
|
||||
"The inputs for this file are generated by `pp_worsen_inputs.py` script.")
|
||||
parser.add_argument("--completion_file", type=str,
|
||||
help="The file contains the completion for each query.")
|
||||
# parser.add_argument("--completion_response_field", type=str, default="completion")
|
||||
parser.add_argument("--completion_problem_id_field", type=str, default="problem_id")
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
worsen_codes = [json.loads(line) for line in open(args.worsen_file).readlines()]
|
||||
print(f"Total number of worsen codes: {len(worsen_codes)}")
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
data += json.load(open(file))
|
||||
|
||||
p_id2response = {item[args.completion_problem_id_field]: item for item in data}
|
||||
|
||||
p_id2pairs = defaultdict(list)
|
||||
for item in worsen_codes:
|
||||
if item["completion"]:
|
||||
try:
|
||||
completion = json.loads(item["completion"])
|
||||
except:
|
||||
print(f"Json parsing error: {item['completion']}")
|
||||
continue
|
||||
neg_code = completion["incorrect_program"]
|
||||
# if isinstance(neg_code, dict):
|
||||
# print(json.dumps(completion, indent=2))
|
||||
|
||||
if not isinstance(neg_code, str):
|
||||
print(f"Bad format: {neg_code}.")
|
||||
continue
|
||||
|
||||
if not neg_code.strip():
|
||||
continue
|
||||
|
||||
p_id, pred_id = item["id"].split("_")
|
||||
pred_id = int(pred_id[3:])
|
||||
p_id = int(p_id)
|
||||
|
||||
response = p_id2response[p_id]
|
||||
pred = response["pred"][pred_id]
|
||||
assert pred in item["prompt"]
|
||||
|
||||
p_id2pairs[p_id].append((pred, neg_code))
|
||||
|
||||
outputs = []
|
||||
num_pairs = 0
|
||||
for p_id, pairs in p_id2pairs.items():
|
||||
pos = []
|
||||
neg = []
|
||||
for pred, neg_code in pairs:
|
||||
pos.append(pred)
|
||||
neg.append(neg_code)
|
||||
assert len(pos) == len(neg)
|
||||
num_pairs += len(pos)
|
||||
outputs.append({
|
||||
"problem_id": p_id,
|
||||
"pos": pos,
|
||||
"neg": neg,
|
||||
})
|
||||
|
||||
print(f"Total number of outputs: {len(outputs)}")
|
||||
print(f"Total number of pairs: {num_pairs}")
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/worsen_gpt4_combine.py --worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f2000.gpt4o.tem1.0.s42.n1.json_obj.jsonl ]
|
||||
--completion_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.json \
|
||||
--completion_problem_id_field id --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f2000.json
|
||||
|
||||
>>> python scripts/apps/worsen_gpt4_combine.py \
|
||||
--worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.?-of-8.v1.1.json" \
|
||||
--completion_problem_id_field id \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f100k.json
|
||||
>>> The above code is incorrect.
|
||||
|
||||
FIXME:
|
||||
>>> python scripts/apps/worsen_gpt4_combine.py \
|
||||
--worsen_file outputs/apps/critique/r2c.sft.train.0shot.tem1.0.n10.v1.1.worsen_4o_critic.s42.f100k.gpt4o.tem1.0.s42.n1.json_obj.jsonl \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.json" \
|
||||
--completion_problem_id_field id \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.A100.w8.v3.0.s42/apps/checkpoint-400/train.0shot.tem1.0.n10.v1.1.s43.gpt4o.worsen.f100k.fix0708.json
|
||||
|
||||
"""
|
||||
Reference in New Issue
Block a user