wehub/microsoft--unilm
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:24:13 +08:00
commit 1037506f2e
6050 changed files with 1731598 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PFPO/scripts/apps/prm/construct_process_rm_sample.py
+154
View File
@@ -0,0 +1,154 @@
import json
import argparse
from glob import glob
import os
import sys
import collections
from multiprocessing import Pool
from tqdm import tqdm
from functools import partial
sys.set_int_max_str_digits(0)
"""
For self-consistency based input-output pairs, first copy the pseudo test cases into the prefix data, run `prefix_fail_extract_pseudo_label.py`,
and the run this script.
This script is incorrect for pseudo test cases since we cannot ensure the correctness of each test case, but it is appropriate for ground-truth test cases,
serving as hard limit, i.e., if there is one completion for some prefix has passed all test cases, then it is a gold prefix.
"""
def counting_partial_response_value(res):
return sum([1 if x else 0 for x in res])
def parse_value(v, binary: bool):
if binary:
return 1 if v > 0 else 0
return v
def _process_trajectories_worker(item, top_k: int, binary: bool):
item_id, trajectories = item
outputs = {
"idx": item_id
}
for i in range(top_k):
level_trajectories = [(traj["vs"][i], traj["prefix"]) for traj in trajectories if len(traj["vs"]) > i]
if len(level_trajectories) == 0:
continue
prefix_vis = set()
level_values = []
level_prefixes = []
for v, prefix in level_trajectories:
if prefix in prefix_vis:
continue
level_values.append(parse_value(v, binary))
level_prefixes.append(prefix)
prefix_vis.add(prefix)
outputs[f"traj_level_{i}_values"] = level_values
outputs[f"traj_level_{i}_prefixes"] = level_prefixes
return outputs
def _annotate(file):
return json.load(open(file, encoding="utf-8"))
def multiprocessing_loading(files, num_workers: int = 8):
with Pool(num_workers) as p:
data = list(tqdm(p.imap(_annotate, files), total=len(files)))
all_data = []
for d in data:
all_data.extend(d)
return all_data
# data = []
# for file in tqdm(files):
# data += json.load(open(file, encoding="utf-8"))
# return data
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--input_file", type=str)
parser.add_argument("--output_file", type=str)
parser.add_argument("--binary", default=False, action="store_true")
parser.add_argument("--num_workers", type=int, default=8)
args = parser.parse_args()
print("Collecting data...")
if os.path.exists(args.input_file):
data = json.load(open(args.input_file))
else:
# data = []
# for file in glob(args.input_file):
# print(file)
# data += json.load(open(file))
files = glob(args.input_file)
files = sorted(files)
print(len(files))
print(files)
data = multiprocessing_loading(files)
print(len(data))
num_prefixes = 0
val_cnt = collections.Counter()
outputs = []
preference_pairs = dict()
missing = 0
for item in tqdm(data):
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
prefix = item["prefix"]
problem_id = int(problem_id)
if "res" not in item:
missing += 1
continue
v = counting_partial_response_value(item["res"])
v = parse_value(v, args.binary)
outputs.append({
"problem_id": problem_id,
"prefix": prefix,
"value": v,
})
num_prefixes += 1
val_cnt[v] += 1
if problem_id not in preference_pairs:
preference_pairs[problem_id] = {
"pos": [],
"neg": [],
}
if v > 0:
preference_pairs[problem_id]["pos"].append(prefix)
else:
preference_pairs[problem_id]["neg"].append(prefix)
preference_pairs = [
{
"problem_id": problem_id,
"pos": pair["pos"],
"neg": pair["neg"],
}
for problem_id, pair in preference_pairs.items()
]
print(f"Missing: {missing}")
print(val_cnt)
print(f"Processed {num_prefixes} prefixes.")
print(f"Averaged {num_prefixes / len(data)} prefixes per problem.")
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
json.dump(preference_pairs, open(args.output_file.replace(".json", "_pairs.json"), "w", encoding="utf-8"), indent=2, ensure_ascii=False)
if __name__ == '__main__':
main()
"""
>>>
"""
PFPO/scripts/apps/prm/construct_process_rm_sample_fix.py
+189
View File
@@ -0,0 +1,189 @@
import json
import argparse
from glob import glob
import os
import sys
import collections
from multiprocessing import Pool
from tqdm import tqdm
from functools import partial
sys.set_int_max_str_digits(0)
"""
For self-consistency based input-output pairs, first copy the pseudo test cases into the prefix data, run `prefix_fail_extract_pseudo_label.py`,
and the run this script.
Note:
1. Sometimes the pseudo test cases can include some extremely large cases, leading to out-of-memory error.
"""
def counting_partial_response_value(full_res):
pass_num = []
for pred_res in full_res:
if len(pred_res) == 0:
pass_num.append(0)
# elif pred_res[0] == -1: # FIXME: This line is commented since 2024/09/25. Seems no difference.
# pass_num.append(0)
else:
pass_num.append(sum([1 for x in pred_res if x is True]))
return pass_num
def annotate(file, exclude: str = ""):
exclude = exclude.split(",")
if any([e and e in file for e in exclude]):
print(f"Excluding {file}")
return []
return json.load(open(file, encoding="utf-8"))
def multiprocessing_loading(files, exclude: str = "", num_workers: int = 8):
_annotate = partial(annotate, exclude=exclude)
# with Pool(num_workers) as p:
# data = list(tqdm(p.imap(_annotate, files), total=len(files)))
# all_data = []
# for d in data:
# all_data.extend(d)
# return all_data
data = []
for file in tqdm(files):
data += _annotate(file)
return data
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--input_file", type=str)
parser.add_argument("--output_file", type=str)
parser.add_argument("--pass_case_margin", type=float, default=1)
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
parser.add_argument("--test_case_field", type=str, default="pseudo_input_output")
parser.add_argument("--num_workers", type=int, default=8)
parser.add_argument("--reduction", type=str, default="max")
parser.add_argument("--exclude", type=str, default="")
args = parser.parse_args()
print("Collecting data...")
if os.path.exists(args.input_file):
data = json.load(open(args.input_file))
else:
files = glob(args.input_file)
files = sorted(files)
print(len(files))
print(files)
data = multiprocessing_loading(files, args.exclude)
print(len(data))
num_prefixes = 0
val_cnt = collections.Counter()
outputs = []
p_id2prefixes = collections.defaultdict(list)
preference_pairs = []
missing = 0
missing_test_cases = 0
for item in tqdm(data):
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
prefix = item["prefix"]
# problem_id = int(problem_id)
if "res" not in item:
missing += 1
continue
if args.test_case_field not in item or (not isinstance(item[args.test_case_field], dict)) or "inputs" not in item[args.test_case_field]:
missing_test_cases += 1
continue
test_case_num = len(item[args.test_case_field]["inputs"])
if test_case_num == 0:
missing_test_cases += 1
continue
pass_num = counting_partial_response_value(item["full_res"])
if args.reduction == "max":
max_pass_num = max(pass_num)
elif args.reduction == "avg":
max_pass_num = sum(pass_num) / len(pass_num)
else:
raise NotImplementedError
outputs.append({
"problem_id": problem_id,
"prefix": prefix,
"pass_num": pass_num,
"max_pass_num": max_pass_num,
"test_case_num": test_case_num,
})
num_prefixes += 1
val_cnt.update(pass_num)
p_id2prefixes[problem_id].append(outputs[-1])
for problem_id, all_prefixes in tqdm(p_id2prefixes.items()):
max_pass_num2prefixes = collections.defaultdict(list)
for prefix in all_prefixes:
max_pass_num2prefixes[prefix["max_pass_num"]].append(prefix)
max_pass_num2prefixes = sorted(max_pass_num2prefixes.items(), key=lambda x: x[0])
pos_prefixes = []
neg_prefixes = []
for p in all_prefixes:
pass_ratio = p["max_pass_num"] / p["test_case_num"]
if pass_ratio < args.pass_case_lower_bound:
continue
neg_upper_pass_num = p["max_pass_num"] - args.pass_case_margin
target_neg = []
for pass_num, prefixes in max_pass_num2prefixes:
if pass_num < neg_upper_pass_num:
target_neg.extend([x["prefix"] for x in prefixes])
else:
break
pos_prefixes.append(p["prefix"])
neg_prefixes.append(target_neg)
preference_pairs.append({
"problem_id": problem_id,
"pos": pos_prefixes,
"neg": neg_prefixes
})
print(f"Missing: {missing}")
print(f"Missing test cases: {missing_test_cases}")
print(val_cnt)
print(f"Processed {num_prefixes} prefixes.")
print(f"Averaged {num_prefixes / len(data)} prefixes per problem.")
print(f"Processed {len(preference_pairs)} problems.")
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
json.dump(preference_pairs, open(args.output_file.replace(".json", f"_low{args.pass_case_lower_bound}_m{args.pass_case_margin}_{args.reduction}.json"),
"w", encoding="utf-8"), indent=2, ensure_ascii=False)
if __name__ == '__main__':
main()
"""
>>> python scripts/apps/prm/construct_process_rm_sample_fix.py \
--input_file "/mnt/fangkai_blob/reward_modeling//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.[0-9]*-of-256.pseudo_test_case.exec.json" \
--output_file /mnt/fangkai_blob/reward_modeling//experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.pseudo_test_case.prefix_pass_num.fix.json \
--pass_case_lower_bound 0.8 --pass_case_margin 4 --test_case_field pseudo_input_output
>>> python scripts/apps/prm/construct_process_rm_sample_fix.py \
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.*-of-256.json" \
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output --exclude "204-of-256,225-of-256"
Missing: 0
Missing test cases: 0
Counter({10: 448824, 0: 153053, 9: 25420, 1: 23469, 8: 16718, 5: 15115, 2: 14803, 3: 12821, 4: 12253, 7: 12053, 6: 11947, 11: 724, 20: 112, 16: 3}) # TODO: This should be a problem. Why there are some problems have more than 10 test cases?
Processed 249105 prefixes.
Averaged 1.0 prefixes per problem.
"""
PFPO/scripts/apps/prm/sample_steps.py
+152
View File
@@ -0,0 +1,152 @@
import json
import argparse
import os
from glob import glob
import sys
import random
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
sys.set_int_max_str_digits(0)
from post_processors.code.clean import get as get_code_cleaner
def sample_step(response: str, code: str, upper_step_ratio: float, sample_ratio: float):
if code:
assert code in response, (code, "\n\n========================\n\n", response)
end_of_response = response.find(code) + len(code)
response = response[:end_of_response]
orig_lines = response.split("\n")
lines = [(i, line) for i, line in enumerate(orig_lines)]
lines = [(i, line) for i, line in lines if line.strip()]
if len(lines) < 5:
return []
upper_step = int(len(lines) * upper_step_ratio)
if upper_step == 0:
return []
sample_step_num = int(upper_step * sample_ratio)
if sample_step_num == 0:
return []
sample_steps = random.sample(lines[:upper_step], sample_step_num)
if len(sample_steps) == 0:
return []
step_prefixes = []
for i, line in sample_steps:
step_prefixes.append("\n".join(orig_lines[:(i + 1)]))
return step_prefixes
def load_files(file_path):
data = []
if os.path.exists(file_path):
print(f"Loading pseudo test cases from {file_path}")
if file_path.endswith(".json"):
data.extend(json.load(open(file_path)))
else:
data.extend([json.loads(line) for line in open(file_path).readlines()])
else:
for file in glob(file_path):
print(file)
if file.endswith(".json"):
data.extend(json.load(open(file)))
else:
data.extend([json.loads(line) for line in open(file).readlines()])
return data
def merge_key(item, value):
assert isinstance(item, list)
if isinstance(value, list):
item = item + value
else:
item.append(value)
return item
def merge_seed_sampled_data(data):
id2data = {}
large_mem = 0
for item in data:
if isinstance(item["response"], str):
item["response"] = [item["response"]]
assert isinstance(item["pred"], str) or item["pred"] is None
item["pred"] = [item["pred"]]
if item["id"] not in id2data:
id2data[item["id"]] = item
continue
tmp = id2data[item["id"]]
tmp["response"] = merge_key(tmp["response"], item["response"])
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
assert isinstance(tmp["pred"], list), tmp["pred"]
id2data[item["id"]] = tmp
print(f"Too large outputs: {large_mem}")
return list(id2data.values())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--input_file", type=str, )
parser.add_argument("--output_file", type=str, )
parser.add_argument("--upper_step_ratio", type=float)
parser.add_argument("--sample_ratio", type=float)
parser.add_argument("--filter_all_correct", action="store_true")
parser.add_argument("--sample_over_p", type=int, default=-1)
args = parser.parse_args()
data = load_files(args.input_file)
data = merge_seed_sampled_data(data)
outputs = []
num = 0
for item in data:
if args.filter_all_correct and all(item["res"]):
continue
prefixes = []
prefix_ids = []
if not item["response"]:
item["prefix"] = []
continue
for resp_id, resp in enumerate(item["response"]):
response_prefixes = sample_step(resp, item["pred"][resp_id], args.upper_step_ratio, args.sample_ratio)
prefixes.extend(response_prefixes)
prefix_ids.extend([f"{item['id']}_{resp_id}_{i}" for i in range(len(response_prefixes))])
if args.sample_over_p > 0:
if len(prefixes) > args.sample_over_p:
prefixes = random.sample(prefixes, args.sample_over_p)
prefix_ids = random.sample(prefix_ids, args.sample_over_p)
item["prefix"] = prefixes
item["prefix_id"] = prefix_ids
outputs.append(item)
num += len(prefixes)
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
print(f"Total number of samples: {num}")
if __name__ == '__main__':
main()
"""
>>> python scripts/apps/prm/sample_steps.py \
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
--upper_step_ratio 0.8 --sample_ratio 0.3 \
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.prefix.upper0.8.r0.3.json
Total number of samples: 470010
"""