chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,68 @@
|
||||
|
||||
# Ground-truth test case inputs
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.run_outputs.json" \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/extract_pseudo_outputs_as_label.py \
|
||||
--input_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.run_outputs.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--use_sc
|
||||
|
||||
# ================================================================
|
||||
# Synthetic test case inputs
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json" \
|
||||
--num_workers 64 --completion_test_field "input_output" \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
# =================================================
|
||||
|
||||
# Synthetic test cases inputs with GPT-4o for output
|
||||
python scripts/apps/pseudo_test_cases/combine_gpt_raw_requests.py \
|
||||
--input_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1_gpt_inputs.shuf.jsonl \
|
||||
--raw_output_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1_gpt_inputs.shuf.bing.x5.jsonl_out \
|
||||
--output_file ../msranlpintern/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--num_workers 64 --id_field problem_id --test_case_field input_output
|
||||
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ${DATA_PREFIX_PATH}/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_ps_test_cases.sc_and_non_sc.json \
|
||||
--test_case_field "input_output"
|
||||
#19795
|
||||
#0
|
||||
#15
|
||||
#9.866784541550897
|
||||
#9.013993432685021
|
||||
#Counter()
|
||||
|
||||
# Non-sc-version
|
||||
python vllm_inference.py fp16_bfloat16=True seed=42 split_id={split_id} \
|
||||
exp_name=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42 \
|
||||
eval_sub_path=checkpoint-100 -cp conf/api/vllm/apps/deepseek_coder/r2c -cn general_combine_train_v2_0_4o_non_sc
|
||||
|
||||
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.prefer_pair.low0.5.m6.json" \
|
||||
--response_field response --test_case_field input_output_non_sc --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
# SC-version
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "/mnt/fangkai_blob/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.json" \
|
||||
--output_file "/mnt/fangkai_blob/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.json" \
|
||||
--num_workers 64 --id_field id --test_case_field input_output
|
||||
# bash scripts/apps/pseudo_test_cases/run_outputs_local.sh split_id
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.json" \
|
||||
--output_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/train.0shot.tem1.0.n10.v2.1.s42.run_outputs.prefer_pair.low0.5.m6.4o-sc.json" \
|
||||
--test_case_field "input_output" --pass_case_margin 6 --pass_case_lower_bound 0.5 --construct_prefer_pair
|
||||
@@ -0,0 +1,224 @@
|
||||
### Synthesis Test Case Inputs
|
||||
|
||||
Prepare inputs for prompting GPT-4o:
|
||||
|
||||
```bash
|
||||
python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.0.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.0.inputs.jsonl
|
||||
|
||||
python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
|
||||
--split train --prompt_file prompts/apps/test_input_gen_2shot_v2.1.txt \
|
||||
--output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only.inputs.jsonl --function_only
|
||||
```
|
||||
|
||||
Extract the corresponding outputs from GPT-4o and combine them to obtain the following file:
|
||||
|
||||
`../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json`
|
||||
|
||||
For post-processing to align the format with oss data:
|
||||
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/combine_pseudo_test_inputs.py \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/apps-train.4o-test-func.json \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
```
|
||||
|
||||
For directly processing on APPs:
|
||||
|
||||
```bash
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.?-of-8.v2.0.json" \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--pseudo_test_case outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json --num_workers 128
|
||||
|
||||
|
||||
python scripts/apps/solution_run_pseudo_outputs_local.py \
|
||||
--completion_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.json" \
|
||||
--output_file "${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json" \
|
||||
--num_workers 64 --completion_test_field "input_output" \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
```
|
||||
|
||||
Collect pseudo outputs and construct DPO training pairs
|
||||
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
```
|
||||
|
||||
Either, run execution again on existing pseudo test cases.
|
||||
|
||||
First, extract the pseudo label:
|
||||
```bash
|
||||
python scripts/apps/extract_pseudo_outputs_as_label.py \
|
||||
--input_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_input_output.json \
|
||||
--output_file ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--use_sc --problem_id_field problem_id --test_case_field pseudo_test_cases
|
||||
#Unaligned: 0
|
||||
#Total number of pseudo test cases: 3901
|
||||
#Average number of test cases: 36333 / 3901 = 9.313765701102282
|
||||
```
|
||||
|
||||
Secondly, run execution on pseudo test cases:
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.?-of-4.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
#Missing: 0.0002563445270443476
|
||||
#Correct: 0.08203024865419123
|
||||
#Correct at k: 0.1579082286593181
|
||||
#Counter({False: 36311, True: 2689})
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
python scripts/apps/construct_prefer_pair.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.prefer_pair.json \
|
||||
--test_case_field pseudo_input_output
|
||||
# 3901 9165 2.3493975903614457
|
||||
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
# 3901 12851 3.294283517046911
|
||||
```
|
||||
|
||||
Run process-DPO prefix execution
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.4o_pseudo_test_case.exec.json" \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
--test_case_field input_output --id_field problem_id
|
||||
```
|
||||
|
||||
Construct Process-DPO preference pair
|
||||
```bash
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.*-of-256.4o_pseudo_test_case.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.4o_pseudo_test_case.prm_prefer_pair.json \
|
||||
--pass_case_margin 4 --pass_case_lower_bound 0.5 --test_case_field pseudo_input_output --num_workers 24 --reduction avg
|
||||
#Missing: 0
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 1440638, 10: 108458, 1: 80861, 2: 34440, 3: 24141, 4: 18476, 5: 15235, 9: 14017, 6: 13494, 7: 12309, 8: 11546})
|
||||
#Processed 354723 prefixes.
|
||||
#Averaged 1.0 prefixes per problem.
|
||||
#Processed 3900 problems.
|
||||
```
|
||||
---------------------------------------------------------------
|
||||
|
||||
Based on DPO-Iter-0 model to construct new test cases on magicoder
|
||||
|
||||
Run outputs on newly generated oss-apps-combine data
|
||||
```bash
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
```
|
||||
|
||||
Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#11925
|
||||
#11925
|
||||
#117006
|
||||
#0
|
||||
#10.004442649434571
|
||||
#9.823396226415094
|
||||
#Counter({10: 73857, 0: 23639, 9: 3901, 1: 3222, 8: 2589, 5: 2251, 2: 2103, 7: 1976, 4: 1865, 6: 1865, 3: 1852, 11: 105, 20: 25})
|
||||
```
|
||||
We need to reuse the pre-synthesized pseudo test cases for APPs dataset for training.
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.apps_only.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
3901
|
||||
3901 11364 2.913099205331966
|
||||
```
|
||||
|
||||
---------------------------------------------------------------
|
||||
|
||||
Based on DPO-Iter-1 model to construct new test cases on magicoder and xcode
|
||||
|
||||
```bash
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.{split_id}-of-32.v2.1.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
```
|
||||
|
||||
Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
```bash
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
# we have skipped split id 20 since it uses too much time.
|
||||
#18608
|
||||
#18608
|
||||
#193834
|
||||
#0
|
||||
#10.003605287755375
|
||||
#9.80116079105761
|
||||
#Counter({10: 99051, 0: 37603, 1: 8459, 9: 7563, 2: 5656, 8: 5528, 3: 4714, 5: 4698, 7: 4306, 4: 4260, 6: 4070, 11: 136, 20: 34, 12: 2})
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.s42.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.all.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
# Added the skipped split id 20
|
||||
#19051
|
||||
#19051
|
||||
#197647
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.800902839745945
|
||||
#Counter({10: 102049, 0: 38432, 1: 8511, 9: 7700, 2: 5688, 8: 5591, 3: 4783, 5: 4776, 7: 4358, 4: 4329, 6: 4119, 11: 136, 20: 34, 12: 4})
|
||||
```
|
||||
|
||||
We need to reuse the pre-synthesized pseudo test cases for APPs dataset for training.
|
||||
```bash
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
--pseudo_test_case ${DATA_PREFIX_PATH}/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
```
|
||||
|
||||
Construct (DPO) preference pair:
|
||||
```bash
|
||||
# Soft version
|
||||
python scripts/apps/construct_prefer_pair_soft.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300//train.0shot.tem1.0.n10.apps_only.v2.1.*-of-9.s42.4o_pseudo_test_cases.exec.v1.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.v2.1.apps_only.4o_pseudo_test_cases.dpo_m4_low0.5.json \
|
||||
--test_case_field pseudo_input_output --pass_case_margin 4 --pass_case_lower_bound 0.5
|
||||
|
||||
```
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
|
||||
def load_file(file_path):
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
else:
|
||||
return [json.loads(line) for line in open(file_path).readlines()]
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file, encoding="utf-8"))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
return data
|
||||
|
||||
|
||||
def extract_json(completion: str):
|
||||
# Regular expression to match content between [BEGIN] and [END]
|
||||
pattern = r'```(.*?)```'
|
||||
|
||||
match = re.search(pattern, completion, re.DOTALL)
|
||||
if match:
|
||||
item = match.group(1).strip()
|
||||
if item.startswith("json"):
|
||||
item = item[5:]
|
||||
item = item.replace(",\n}", "\n}").replace(",}", "}").replace("None", "null")
|
||||
item = item.replace("(", "[").replace(")", "]").replace("\'", "\"")
|
||||
item = item.replace("True", "true").replace("False", "false")
|
||||
return json.loads(item)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_test_cases(item):
|
||||
try:
|
||||
response = extract_json(item["response"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["response"])
|
||||
return {}
|
||||
|
||||
if not response:
|
||||
return {}
|
||||
|
||||
if not isinstance(response, dict):
|
||||
return {}
|
||||
|
||||
test_cases = {
|
||||
"inputs": [],
|
||||
}
|
||||
for k, v in response.items():
|
||||
if not k.startswith("test_case_"):
|
||||
continue
|
||||
test_cases["inputs"].append(v)
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
def load_func_head(item) -> str:
|
||||
try:
|
||||
response = extract_json(item["response"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["response"])
|
||||
return ""
|
||||
|
||||
if not response:
|
||||
return ""
|
||||
|
||||
if "func_head" not in response:
|
||||
return ""
|
||||
|
||||
return response["func_head"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_file", type=str)
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--func_head_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_file(args.data_file)
|
||||
|
||||
test_cases = load_file(args.test_case_file)
|
||||
id2test_cases = {}
|
||||
test_num = 0
|
||||
for item in test_cases:
|
||||
tmp = load_test_cases(item)
|
||||
if tmp:
|
||||
id2test_cases[item["id"]] = tmp
|
||||
test_num += 1
|
||||
|
||||
func_heads = load_file(args.func_head_file)
|
||||
id2func_head = {}
|
||||
func_num = 0
|
||||
for item in func_heads:
|
||||
tmp = load_func_head(item)
|
||||
if tmp:
|
||||
id2func_head[item["id"]] = tmp
|
||||
func_num += 1
|
||||
|
||||
print(f"Test cases: {test_num} / {len(test_cases)}")
|
||||
print(f"Func heads: {func_num} / {len(func_heads)}")
|
||||
|
||||
outputs = []
|
||||
for item in data:
|
||||
item_id = item["index"]
|
||||
if item_id in id2test_cases:
|
||||
test_inputs = id2test_cases[item_id]
|
||||
else:
|
||||
continue
|
||||
|
||||
if item_id in id2func_head:
|
||||
func_head = id2func_head[item_id]
|
||||
if not func_head:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
# item["input_output"] = test_cases
|
||||
# item["input_output"]["fn_name"] = func_head
|
||||
# outputs.append(item)
|
||||
test_inputs["fn_name"] = func_head
|
||||
outputs.append({
|
||||
"input_output": test_inputs,
|
||||
"question": item["problem"],
|
||||
"problem_id": f"oss-instruct-{item_id}",
|
||||
})
|
||||
|
||||
print(len(outputs))
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/clean_oss_mistral_data.py \
|
||||
--data_file ../msranlpintern/share/dataset/magicoder/data-oss_instruct-decontaminated-python.json \
|
||||
--test_case_file "../msranlpintern/share/models/Mistral-Large-Instruct-2407/apps-test-inputs-gen/sub_dev.0shot.tem0.0.n1.*-of-16.v1.0.json" \
|
||||
--func_head_file "../msranlpintern/share/models/Mistral-Large-Instruct-2407/apps-test-inputs-gen/oss_instruct_python.func_head_extract.tem0.0.n1.*-of-16.v1.0.json" \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/data-oss_instruct-decontaminated-python.mistral-large-test-func.json
|
||||
|
||||
Test cases: 31134
|
||||
Func heads: 23807
|
||||
18214
|
||||
|
||||
Test cases: 32066
|
||||
Func heads: 23807
|
||||
18969
|
||||
|
||||
Test cases: 32185
|
||||
Func heads: 23807
|
||||
19280
|
||||
|
||||
|
||||
Test cases: 33760
|
||||
Func heads: 23807
|
||||
20342
|
||||
|
||||
Test cases: 33760 / 38284
|
||||
Func heads: 23807 / 38284
|
||||
20342
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,131 @@
|
||||
import json
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
|
||||
def load_file(file_path):
|
||||
if os.path.exists(file_path):
|
||||
if file_path.endswith(".json"):
|
||||
return json.load(open(file_path, encoding="utf-8"))
|
||||
else:
|
||||
return [json.loads(line) for line in open(file_path).readlines()]
|
||||
data = []
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file, encoding="utf-8"))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
return data
|
||||
|
||||
|
||||
def load_test_cases(item):
|
||||
try:
|
||||
input_output = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(item["completion"])
|
||||
return []
|
||||
|
||||
inputs = list(input_output.values())
|
||||
return inputs
|
||||
|
||||
|
||||
x_code_eval_template_v1 = """{description}
|
||||
|
||||
Input: {input_from}
|
||||
Output: {output_to}
|
||||
|
||||
Time limit: {time_limit}
|
||||
Memory limit: {memory_limit}
|
||||
|
||||
#### Input Format
|
||||
|
||||
{input_spec}
|
||||
|
||||
#### Output Format
|
||||
|
||||
{output_spec}
|
||||
|
||||
#### Notes
|
||||
|
||||
{notes}
|
||||
|
||||
#### Example Input-Output
|
||||
|
||||
{sample_input_output}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_file", type=str)
|
||||
parser.add_argument("--test_case_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--test_file", type=str)
|
||||
parser.add_argument("--val_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_file(args.data_file)
|
||||
|
||||
test_data = load_file(args.test_file)
|
||||
test_ids = set([item["src_uid"] for item in test_data])
|
||||
|
||||
val_data = load_file(args.val_file)
|
||||
test_ids.update([item["src_uid"] for item in val_data])
|
||||
|
||||
test_cases = load_file(args.test_case_file)
|
||||
id2test_cases = {}
|
||||
test_num = 0
|
||||
for item in test_cases:
|
||||
tmp = load_test_cases(item)
|
||||
if tmp:
|
||||
id2test_cases[item["src_uid"]] = tmp
|
||||
test_num += 1
|
||||
|
||||
print(f"Test cases: {test_num} / {len(test_cases)}")
|
||||
|
||||
outputs = []
|
||||
in_out_template = "Input\n```{}```\n\nOutput\n```{}```"
|
||||
test_held = 0
|
||||
for item in data:
|
||||
if item["input_from"] != 'standard input':
|
||||
continue
|
||||
|
||||
if item["src_uid"] in test_ids:
|
||||
test_held += 1
|
||||
continue
|
||||
|
||||
item_id = item["src_uid"]
|
||||
if item_id in id2test_cases:
|
||||
test_inputs = id2test_cases[item_id]
|
||||
else:
|
||||
continue
|
||||
|
||||
sample_input_output = []
|
||||
for _in, _out in zip(item["sample_inputs"], item["sample_outputs"]):
|
||||
sample_input_output.append(in_out_template.format(_in, _out))
|
||||
|
||||
item["sample_input_output"] = "\n\n".join(sample_input_output)
|
||||
|
||||
question = x_code_eval_template_v1.format(**item)
|
||||
|
||||
test_inputs = {
|
||||
"inputs": test_inputs,
|
||||
}
|
||||
outputs.append({
|
||||
"input_output": test_inputs,
|
||||
"question": question,
|
||||
"problem_id": f"xcode-eval-{item['src_uid']}",
|
||||
})
|
||||
|
||||
print(len(outputs))
|
||||
print(f"No test: {test_held}")
|
||||
json.dump(outputs, open(args.output_file, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,252 @@
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
|
||||
def worker(item, pseudo_test_case_field):
|
||||
inputs = item[pseudo_test_case_field]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_inputs_non_sc = []
|
||||
new_outputs_non_sc = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
# Non-sc output
|
||||
if case_j in resp2outputs[0]:
|
||||
new_inputs_non_sc.append(inputs[case_j])
|
||||
new_outputs_non_sc.append(output_str2orig_pred[case_j][resp2outputs[0][case_j]])
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"inputs_non_sc": new_inputs_non_sc,
|
||||
"outputs_non_sc": new_outputs_non_sc,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_test_cases")
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.pseudo_test_case_file))
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
missing_predictions = 0
|
||||
avg_test_case_num = 0
|
||||
avg_non_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
result = worker(item, args.test_case_field)
|
||||
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item[args.test_case_field]["inputs"] = result["inputs"]
|
||||
item[args.test_case_field]["outputs"] = result["outputs"]
|
||||
item[args.test_case_field]["output_meta"] = result["output_meta"]
|
||||
item[f"{args.test_case_field}_non_sc"] = copy.deepcopy(item[args.test_case_field])
|
||||
item[f"{args.test_case_field}_non_sc"]["inputs"] = result["inputs_non_sc"]
|
||||
item[f"{args.test_case_field}_non_sc"]["outputs"] = result["outputs_non_sc"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
avg_non_test_case_num += len(result["inputs_non_sc"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item[args.test_case_field]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(avg_non_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 3 --pass_case_lower_bound 0.5
|
||||
|
||||
93%|██████████████████ | 4095/4418 [00:03<00:00, 629.19it/s]None
|
||||
100%|█████████████████████████████| 4418/4418 [00:03<00:00, 1323.18it/s]
|
||||
~~4299
|
||||
~~82365
|
||||
~~1
|
||||
~~9.953477552919283
|
||||
~~Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
4299
|
||||
72724
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 2 --pass_case_lower_bound 0.5
|
||||
|
||||
~~4299
|
||||
~~82365
|
||||
~~1
|
||||
~~9.953477552919283
|
||||
~~Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
4299
|
||||
77024
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json \
|
||||
--output_file ../msranlpintern/share/models/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
4299
|
||||
59226
|
||||
1
|
||||
9.953477552919283
|
||||
Counter({10: 15551, 0: 15150, 1: 2567, 2: 1722, 9: 1585, 3: 1235, 8: 1220, 7: 1060, 4: 1016, 5: 951, 6: 933})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5
|
||||
|
||||
4712
|
||||
59606
|
||||
2
|
||||
9.955220713073006
|
||||
Counter({10: 17718, 0: 16091, 1: 2515, 2: 1841, 9: 1807, 3: 1364, 8: 1291, 6: 1172, 4: 1125, 5: 1115, 7: 1081})
|
||||
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_test_cases.v1.0.azure.json \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
4712
|
||||
0
|
||||
2
|
||||
9.955220713073006
|
||||
Counter()
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
import json
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from post_processors.code.clean import standard_cleaner_default
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--raw_output_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
input_data = [json.loads(line) for line in open(args.input_file, encoding="utf-8").readlines()]
|
||||
|
||||
raw_outputs = [json.loads(line) for line in open(args.raw_output_file, encoding="utf-8").readlines()]
|
||||
for o in tqdm(raw_outputs):
|
||||
request_meta = o["request_metadata"]
|
||||
s_id = int(request_meta["sid"].split("@")[1])
|
||||
input_item = input_data[s_id]
|
||||
assert o["request"]["messages"][1]["content"] in input_item["prompt"]
|
||||
|
||||
if "response" not in input_item:
|
||||
input_item["response"] = []
|
||||
if "content" not in o["response"]["choices"][0]["message"]:
|
||||
print(o)
|
||||
continue
|
||||
input_item["response"].append(o['response']['choices'][0]['message']['content'])
|
||||
|
||||
missing_completion = 0
|
||||
num_completions = 0
|
||||
for item in input_data:
|
||||
if "response" not in item:
|
||||
missing_completion += 1
|
||||
item["response"] = ""
|
||||
item["pred"] = None
|
||||
else:
|
||||
preds = [standard_cleaner_default(resp) for resp in item["response"]]
|
||||
item["pred"] = preds
|
||||
num_completions += len(preds)
|
||||
|
||||
json.dump(input_data, open(args.output_file, "w", encoding="utf-8"))
|
||||
print(f"Averaged number of completions: {num_completions / len(input_data)}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
|
||||
def extract_test_case_inputs(item):
|
||||
if item["input_output"]:
|
||||
item["input_output"] = json.loads(item["input_output"])
|
||||
|
||||
if "fn_name" in item["input_output"]:
|
||||
function_call = True
|
||||
else:
|
||||
function_call = False
|
||||
|
||||
full_inputs = []
|
||||
if function_call:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
|
||||
for k, v in response.items():
|
||||
if not isinstance(v, list):
|
||||
assert isinstance(v, str), v
|
||||
v = [v]
|
||||
full_inputs.append(v)
|
||||
else:
|
||||
try:
|
||||
response = json.loads(item["completion"])
|
||||
except Exception as e:
|
||||
# print(f"Cannot load response for {item['completion']}")
|
||||
print(e)
|
||||
return []
|
||||
for k, v in response.items():
|
||||
inputs = v
|
||||
full_inputs.append(inputs)
|
||||
|
||||
return full_inputs
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--pseudo_test_case", type=str)
|
||||
parser.add_argument("--completion_test_field", type=str, default="input_output")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_dataset("codeparrot/apps", split="train").to_list()
|
||||
|
||||
ps_test_cases = []
|
||||
cnt = 0
|
||||
if args.pseudo_test_case.endswith(".json"):
|
||||
test_cases = json.load(open(args.pseudo_test_case))
|
||||
for item in test_cases:
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
else:
|
||||
with open(args.pseudo_test_case) as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except:
|
||||
print(f"Cannot load {line}")
|
||||
cnt += 1
|
||||
pass
|
||||
_input = extract_test_case_inputs(item)
|
||||
if not _input:
|
||||
continue
|
||||
item["pseudo_inputs"] = _input
|
||||
ps_test_cases.append(item)
|
||||
print(cnt)
|
||||
print(f"Total number of pseudo test cases: {len(ps_test_cases)}")
|
||||
|
||||
id2item = {item["problem_id"]: item for item in ps_test_cases}
|
||||
outputs = []
|
||||
for item in data:
|
||||
if item["problem_id"] not in id2item:
|
||||
continue
|
||||
test_cases = {
|
||||
"inputs": id2item[item["problem_id"]]["pseudo_inputs"],
|
||||
}
|
||||
if item[args.completion_test_field]:
|
||||
if not isinstance(item[args.completion_test_field], dict):
|
||||
item[args.completion_test_field] = json.loads(item[args.completion_test_field])
|
||||
if "fn_name" in item[args.completion_test_field]:
|
||||
test_cases["fn_name"] = item[args.completion_test_field]["fn_name"]
|
||||
|
||||
# This is to align with oss data format.
|
||||
new_item = {
|
||||
"input_output": test_cases,
|
||||
"question": item["question"],
|
||||
"problem_id": f"apps-train-{item['problem_id']}",
|
||||
"starter_code": item["starter_code"],
|
||||
}
|
||||
outputs.append(new_item)
|
||||
|
||||
print(f"Total number of items: {len(outputs)}")
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/combine_pseudo_test_inputs.py \
|
||||
--output_file ../msranlpintern/share/dataset/magicoder/apps-train.4o-test-func.json \
|
||||
--pseudo_test_case ../msranlpintern/share/gpt-chat-examples-outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only_combine.outputs.gpt4o.n1.tem0.0.json_obj.json
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
#export exp_dir=deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.pdpo.V100.tp8dp32.v1.3.s42/oss-apps-xcode-combine/checkpoint-300/
|
||||
export exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.pdpo.H100.dp8.v1.2.s42/oss-apps-xcode-combine/checkpoint-500/
|
||||
|
||||
p=$1
|
||||
|
||||
echo "Constructing process_rm sample for split 0"
|
||||
echo "p" $p
|
||||
|
||||
# Just check the outputs on the previously generated data
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_takes_extra.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output_by_n64.v1.0.dpo_m6_low0.5_min5_p$p.json \
|
||||
# --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_p$p.json \
|
||||
# --construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
## Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
# --pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
# --construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
--construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
@@ -0,0 +1,10 @@
|
||||
OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
split_id=$1
|
||||
|
||||
python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label_align_ts_num.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.pseudo_test_case.exec.ctr_ts_num.v1.0.json" \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.*-of-256.pseudo_test_case.exec.ctr_ts_num.v1.0.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.pseudo_test_case.ctr_ts_num.prefix_pass_num.fix_low0.5_m4.0_avg.json" \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.json \
|
||||
--output_file /mnt/fangkai_blob/share/xcode_4o_oss_apps_test_inputs_v1.shuf.combine.4o_run_outputs.json \
|
||||
--num_workers 64 --id_field problem_id --test_case_field input_output
|
||||
@@ -0,0 +1,230 @@
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from datasets import load_dataset
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# if not len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]): # TODO: Figure it out why this happens
|
||||
# print(len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.pseudo_test_case_file):
|
||||
print(f"Loading pseudo test cases from {args.pseudo_test_case_file}")
|
||||
data = json.load(open(args.pseudo_test_case_file))
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.pseudo_test_case_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
tmp = json.load(open(file))
|
||||
else:
|
||||
tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
data.extend(tmp)
|
||||
|
||||
outputs = []
|
||||
before_test_num = 0
|
||||
cnt = 0
|
||||
missing_predictions = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
|
||||
result = worker(item, min_success_test_num=args.min_success_test_num)
|
||||
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item["input_output"]["inputs"] = result["inputs"]
|
||||
item["input_output"]["outputs"] = result["outputs"]
|
||||
item["input_output"]["output_meta"] = result["output_meta"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,382 @@
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# from pympler import asizeof
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# if not len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]): # TODO: Figure it out why this happens
|
||||
# print(len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / len(item["full_res"])
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
# size_in_bytes = asizeof.asizeof(new_outputs)
|
||||
# if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
# return {
|
||||
# "inputs": [],
|
||||
# "outputs": [],
|
||||
# "output_meta": [],
|
||||
# "sc_match_res": [],
|
||||
# }
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
# "output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item['id']} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
preds = []
|
||||
for _ in item["response"]:
|
||||
preds.append("")
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["pred"] = preds
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
# if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=16)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# if os.path.exists(args.pseudo_test_case_file):
|
||||
# print(f"Loading pseudo test cases from {args.pseudo_test_case_file}")
|
||||
# data = json.load(open(args.pseudo_test_case_file))
|
||||
# else:
|
||||
# data = []
|
||||
# for file in glob(args.pseudo_test_case_file):
|
||||
# print(file)
|
||||
# if file.endswith(".json"):
|
||||
# tmp = json.load(open(file))
|
||||
# else:
|
||||
# tmp = [json.loads(line) for line in open(file).readlines()]
|
||||
# data.extend(tmp)
|
||||
data = load_files(args.pseudo_test_case_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2data = {item["id"]: item for item in data}
|
||||
|
||||
missing_predictions = 0
|
||||
before_test_num = 0
|
||||
|
||||
_mp_inputs = []
|
||||
for item in tqdm(data, desc="Preparing Inputs"):
|
||||
# TODO: Here we do not consider those bad solutions without even complete programs. These programs should also be penalized
|
||||
# but we have already removed them here.
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append(item)
|
||||
|
||||
pbar = tqdm(_mp_inputs, desc="Submitting tasks")
|
||||
_mp_outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
sc_test_cases_outputs = []
|
||||
prefer_pair_outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in tqdm(_mp_outputs):
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2data[result["id"]]
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
|
||||
test_case_item = {
|
||||
"problem_id": item["problem_id"],
|
||||
"input_output": {
|
||||
"inputs": result["inputs"],
|
||||
"outputs": result["outputs"],
|
||||
},
|
||||
"sc_full_res": result["sc_match_res"],
|
||||
}
|
||||
sc_test_cases_outputs.append(test_case_item)
|
||||
|
||||
# test_case_item["input_output"]["inputs"] = result["inputs"]
|
||||
# test_case_item["input_output"]["outputs"] = result["outputs"]
|
||||
# item["input_output"]["output_meta"] = result["output_meta"]
|
||||
# test_case_item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(test_case_item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(test_case_item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(test_case_item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"]), (len(pred_pass_cnt), len(item["response"]), len(item["pred"]))
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
|
||||
prefer_pair_outputs.append(item)
|
||||
|
||||
print(len(prefer_pair_outputs))
|
||||
print(len(sc_test_cases_outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(sc_test_cases_outputs) if sc_test_cases_outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(sc_test_cases_outputs, open(args.output_file.replace(".json", ".sc_test_cases.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
if args.construct_prefer_pair:
|
||||
json.dump(prefer_pair_outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# from pympler import asizeof
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/collect_pseudo_outputs.py`.
|
||||
|
||||
In the processing of oss combine data, the pseudo test cases are directly saved in `input_output` fields so I re-write this script.
|
||||
|
||||
A different saving format. One positive response is paired with a list of negative candidates.
|
||||
"""
|
||||
|
||||
|
||||
def worker(item, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
inputs = item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / len(item["full_res"])
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
|
||||
sc_o = output_cnt.most_common(1)[0][0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
# "output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
print(f"Warning: {item['id']} has only one response. ---- {item['response']} \n\n {item['pred']}")
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
preds = []
|
||||
for _ in item["response"]:
|
||||
preds.append("")
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
item["pred"] = preds
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
# if "outputs" in tmp:
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str, default=True)
|
||||
parser.add_argument("--construct_prefer_pair", default=False, action="store_true")
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=16)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
data = load_files(args.pseudo_test_case_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2data = {item["id"]: item for item in data}
|
||||
|
||||
missing_predictions = 0
|
||||
before_test_num = 0
|
||||
|
||||
_mp_inputs = []
|
||||
for item in tqdm(data, desc="Preparing Inputs"):
|
||||
# TODO: Here we do not consider those bad solutions without even complete programs. These programs should also be penalized
|
||||
# but we have already removed them here.
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append(item)
|
||||
|
||||
pbar = tqdm(_mp_inputs, desc="Submitting tasks")
|
||||
_mp_outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
sc_test_cases_outputs = []
|
||||
prefer_pair_outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in tqdm(_mp_outputs):
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2data[result["id"]]
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
|
||||
test_case_item = {
|
||||
"problem_id": item["problem_id"],
|
||||
"input_output": {
|
||||
"inputs": result["inputs"],
|
||||
"outputs": result["outputs"],
|
||||
},
|
||||
"sc_full_res": result["sc_match_res"],
|
||||
}
|
||||
sc_test_cases_outputs.append(test_case_item)
|
||||
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
if args.construct_prefer_pair:
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(test_case_item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(test_case_item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(test_case_item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"]), (len(pred_pass_cnt), len(item["response"]), len(item["pred"]))
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
|
||||
pos2neg = []
|
||||
pos2neg_code = []
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
# pos.append(resp_i)
|
||||
# pos_code.append(prog_i)
|
||||
# neg.append(resp_j)
|
||||
# neg_code.append(prog_j)
|
||||
pos2neg.append(resp_j)
|
||||
pos2neg_code.append(prog_j)
|
||||
|
||||
if pos2neg:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(pos2neg)
|
||||
neg_code.append(pos2neg_code)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
item.pop("response")
|
||||
item.pop("pred")
|
||||
|
||||
prefer_pair_outputs.append(item)
|
||||
|
||||
print(len(prefer_pair_outputs))
|
||||
print(len(sc_test_cases_outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(sc_test_cases_outputs) if sc_test_cases_outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(sc_test_cases_outputs, open(args.output_file.replace(".json", ".sc_test_cases.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
if args.construct_prefer_pair:
|
||||
json.dump(prefer_pair_outputs, open(args.output_file, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
13033
|
||||
111756
|
||||
2
|
||||
10.003563932998059
|
||||
9.762295710887747
|
||||
Counter({10: 81281, 0: 22865, 9: 4529, 1: 4140, 8: 3010, 5: 2760, 2: 2636, 7: 2300, 4: 2256, 3: 2250, 6: 2158, 11: 120, 20: 24, 16: 1})
|
||||
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min1.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 1
|
||||
|
||||
14899
|
||||
136123
|
||||
2
|
||||
10.003563932998059
|
||||
9.835290959124773
|
||||
Counter({10: 84309, 0: 38044, 9: 4612, 1: 4199, 8: 3019, 5: 2823, 2: 2705, 7: 2356, 4: 2282, 3: 2275, 6: 2211, 11: 128, 20: 26, 16: 1})
|
||||
|
||||
"""
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
import collections
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pympler import asizeof
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
"""
|
||||
Copied from `scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py`.
|
||||
|
||||
This script can accept external execution files for performing self-consistency over test cases.
|
||||
"""
|
||||
|
||||
|
||||
def worker(_input, min_success_test_num: int = 1, top_p: float = 0.0):
|
||||
item, sc_item = _input
|
||||
inputs = sc_item["input_output"]["inputs"]
|
||||
outputs_counter = [
|
||||
collections.Counter() for _ in inputs
|
||||
]
|
||||
output_str2orig_pred = [
|
||||
{} for _ in inputs
|
||||
]
|
||||
resp2outputs = [
|
||||
{} for _ in range(len(item["outputs"]))
|
||||
]
|
||||
assert len(inputs) == len(item["input_output"]["inputs"]), (len(inputs), len(item["input_output"]["inputs"]))
|
||||
# assert len(sc_item["outputs"]) == item["outputs"], (len(sc_item["outputs"]), len(item["outputs"]))
|
||||
|
||||
assert len(sc_item["full_res"]) == len(sc_item["outputs"]) == len(sc_item["pred"]), (len(sc_item["full_res"]), len(sc_item["outputs"]),
|
||||
len(sc_item["pred"]))
|
||||
assert len(item["full_res"]) == len(item["outputs"]) == len(item["pred"]), (len(item["full_res"]), len(item["outputs"]), len(item["pred"]))
|
||||
|
||||
sc_prog_num = len(sc_item["full_res"])
|
||||
|
||||
# We use the external item to perform self-consistency over test cases.
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(sc_item["full_res"], sc_item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
if str(case_o) not in output_str2orig_pred[case_j]:
|
||||
output_str2orig_pred[case_j][str(case_o)] = case_o
|
||||
outputs_counter[case_j][str(case_o)] += 1
|
||||
|
||||
# We then record the target item's outputs
|
||||
for resp_id, (full_res, pg_outputs) in enumerate(zip(item["full_res"], item["outputs"])):
|
||||
for case_j, (case_r, case_o) in enumerate(zip(full_res, pg_outputs)):
|
||||
if case_j >= len(inputs):
|
||||
break
|
||||
if case_r != 0:
|
||||
continue
|
||||
# assert case_o # sometimes is could be `int` or `True` or `False`. We believe the `case_r` here.
|
||||
|
||||
if not str(case_o):
|
||||
continue # some outputs could be empty string
|
||||
|
||||
resp2outputs[resp_id][case_j] = str(case_o)
|
||||
|
||||
averaged_p = 0
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
sc_o_freq = output_cnt.most_common(1)[0][1]
|
||||
averaged_p += sc_o_freq / sc_prog_num
|
||||
|
||||
averaged_p /= len(outputs_counter)
|
||||
|
||||
if averaged_p < top_p:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
new_inputs = []
|
||||
new_outputs = []
|
||||
new_output_meta = []
|
||||
sc_match_res = [[] for _ in range(len(item["pred"]))]
|
||||
for case_j, output_cnt in enumerate(outputs_counter):
|
||||
if not output_cnt:
|
||||
continue
|
||||
|
||||
if sum(output_cnt.values()) < min_success_test_num:
|
||||
continue
|
||||
|
||||
sc_o, sc_o_freq = output_cnt.most_common(1)[0]
|
||||
sc_o_real = output_str2orig_pred[case_j][sc_o]
|
||||
|
||||
new_inputs.append(inputs[case_j])
|
||||
new_outputs.append(sc_o_real)
|
||||
|
||||
new_output_meta.append({
|
||||
"output_freq": output_cnt,
|
||||
"output_str2orig_pred": output_str2orig_pred[case_j],
|
||||
})
|
||||
|
||||
for pg_i in range(len(item["pred"])):
|
||||
if case_j not in resp2outputs[pg_i]:
|
||||
sc_match_res[pg_i].append(-2) # compilation error
|
||||
continue
|
||||
if resp2outputs[pg_i][case_j] == sc_o:
|
||||
sc_match_res[pg_i].append(1)
|
||||
else:
|
||||
sc_match_res[pg_i].append(0)
|
||||
|
||||
return {
|
||||
"id": item["id"],
|
||||
"inputs": new_inputs,
|
||||
"outputs": new_outputs,
|
||||
"output_meta": new_output_meta,
|
||||
"sc_match_res": sc_match_res,
|
||||
}
|
||||
|
||||
|
||||
def load_files(file_path):
|
||||
data = []
|
||||
if os.path.exists(file_path):
|
||||
print(f"Loading pseudo test cases from {file_path}")
|
||||
if file_path.endswith(".json"):
|
||||
data.extend(json.load(open(file_path)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file_path).readlines()])
|
||||
else:
|
||||
for file in glob(file_path):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data.extend(json.load(open(file)))
|
||||
else:
|
||||
data.extend([json.loads(line) for line in open(file).readlines()])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_key(item, value):
|
||||
assert isinstance(item, list)
|
||||
if isinstance(value, list):
|
||||
item = item + value
|
||||
else:
|
||||
item.append(value)
|
||||
return item
|
||||
|
||||
|
||||
def merge_seed_sampled_data(data):
|
||||
id2data = {}
|
||||
large_mem = 0
|
||||
for item in data:
|
||||
if isinstance(item["response"], str):
|
||||
item["response"] = [item["response"]]
|
||||
assert isinstance(item["pred"], str) or item["pred"] is None
|
||||
item["pred"] = [item["pred"]]
|
||||
|
||||
if "outputs" in item:
|
||||
size_in_bytes = asizeof.asizeof(item["outputs"])
|
||||
if size_in_bytes / (1024 ** 2) > 10: # 10MB
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
if "outputs" in item:
|
||||
item.pop("outputs")
|
||||
if "errors" in item:
|
||||
item.pop("errors")
|
||||
large_mem += 1
|
||||
|
||||
if "res" not in item: # Sometimes all solutions do not entail the programs. Please turn back to `solution_run_outputs_local.py`.
|
||||
results = []
|
||||
full_results = []
|
||||
all_outputs = []
|
||||
|
||||
for _ in item["response"]:
|
||||
results.append(False)
|
||||
full_results.append([-2] * 21)
|
||||
all_outputs.append([None] * 21)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
item["outputs"] = all_outputs
|
||||
|
||||
if item["id"] not in id2data:
|
||||
id2data[item["id"]] = item
|
||||
continue
|
||||
|
||||
tmp = id2data[item["id"]]
|
||||
# if isinstance(tmp["res"], list):
|
||||
# tmp["res"] = [tmp["res"]]
|
||||
# if not isinstance(tmp["pred"], list):
|
||||
# tmp["pred"] = [tmp["pred"]]
|
||||
# if not isinstance(tmp["full_res"], list):
|
||||
# tmp["full_res"] = [tmp["full_res"]]
|
||||
# if "outputs" in tmp and not isinstance(tmp["outputs"], list):
|
||||
# tmp["outputs"] = [tmp["outputs"]]
|
||||
|
||||
tmp["response"] = merge_key(tmp["response"], item["response"])
|
||||
tmp["res"] = merge_key(tmp["res"], item["res"])
|
||||
tmp["pred"] = merge_key(tmp["pred"], item["pred"])
|
||||
tmp["full_res"] = merge_key(tmp["full_res"], item["full_res"])
|
||||
tmp["outputs"] = merge_key(tmp["outputs"], item["outputs"])
|
||||
assert isinstance(tmp["pred"], list), tmp["pred"]
|
||||
id2data[item["id"]] = tmp
|
||||
|
||||
print(f"Too large outputs: {large_mem}")
|
||||
return list(id2data.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--pseudo_test_case_file", type=str)
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--min_success_test_num", type=int, default=2)
|
||||
parser.add_argument("--pass_case_margin", type=float, default=1)
|
||||
parser.add_argument("--pass_case_lower_bound", type=float, default=0.5)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=24)
|
||||
parser.add_argument("--top_p", type=float, default=0.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
external_ps_test_cases = load_files(args.pseudo_test_case_file)
|
||||
external_ps_test_cases = merge_seed_sampled_data(external_ps_test_cases)
|
||||
id2external_ps_test_cases = {item["id"]: item for item in external_ps_test_cases}
|
||||
|
||||
data = load_files(args.completion_file)
|
||||
data = merge_seed_sampled_data(data)
|
||||
id2item = {item["id"]: item for item in data}
|
||||
|
||||
before_test_num = 0
|
||||
missing_predictions = 0
|
||||
_mp_inputs = []
|
||||
_mp_outputs = []
|
||||
for item in tqdm(data):
|
||||
if "outputs" not in item:
|
||||
print(item["pred"])
|
||||
missing_predictions += 1
|
||||
continue
|
||||
|
||||
before_test_num += len(item["input_output"]["inputs"])
|
||||
_mp_inputs.append((item, id2external_ps_test_cases[item["id"]]))
|
||||
|
||||
pbar = tqdm(_mp_inputs)
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
_annotate = functools.partial(worker, min_success_test_num=args.min_success_test_num, top_p=args.top_p)
|
||||
for _input in pbar:
|
||||
future = executor.submit(_annotate, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
_mp_outputs.append(future.result())
|
||||
|
||||
outputs = []
|
||||
cnt = 0
|
||||
avg_test_case_num = 0
|
||||
pass_cnt = collections.Counter()
|
||||
for result in _mp_outputs:
|
||||
if not result["inputs"]:
|
||||
continue
|
||||
|
||||
item = id2item[result["id"]]
|
||||
|
||||
item["input_output"]["inputs"] = result["inputs"]
|
||||
item["input_output"]["outputs"] = result["outputs"]
|
||||
item["input_output"]["output_meta"] = result["output_meta"]
|
||||
item["sc_full_res"] = result["sc_match_res"]
|
||||
avg_test_case_num += len(result["inputs"])
|
||||
|
||||
pred_pass_cnt = []
|
||||
for pg_i, pg_res in enumerate(item["sc_full_res"]):
|
||||
pred_pass_cnt.append(sum([1 for r in pg_res if r == 1]))
|
||||
pass_cnt[pred_pass_cnt[-1]] += 1
|
||||
|
||||
pos = []
|
||||
neg = []
|
||||
pos_code = []
|
||||
neg_code = []
|
||||
num_test_cases = len(item["input_output"]["inputs"])
|
||||
assert num_test_cases == len(item["sc_full_res"][0])
|
||||
assert len(pred_pass_cnt) == len(item["response"]) == len(item["pred"])
|
||||
for i in range(len(pred_pass_cnt)):
|
||||
resp_i = item["response"][i]
|
||||
prog_i = item["pred"][i]
|
||||
pass_cnt_i = pred_pass_cnt[i]
|
||||
if pass_cnt_i / num_test_cases < args.pass_case_lower_bound:
|
||||
continue
|
||||
for j in range(len(pred_pass_cnt)):
|
||||
if i == j:
|
||||
continue
|
||||
resp_j = item["response"][j]
|
||||
prog_j = item["pred"][j]
|
||||
pass_cnt_j = pred_pass_cnt[j]
|
||||
if pass_cnt_i - pass_cnt_j >= args.pass_case_margin:
|
||||
pos.append(resp_i)
|
||||
pos_code.append(prog_i)
|
||||
neg.append(resp_j)
|
||||
neg_code.append(prog_j)
|
||||
|
||||
item["pos"] = pos
|
||||
item["pos_code"] = pos_code
|
||||
item["neg"] = neg
|
||||
item["neg_code"] = neg_code
|
||||
cnt += len(pos)
|
||||
|
||||
outputs.append(item)
|
||||
|
||||
print(len(outputs))
|
||||
print(cnt)
|
||||
print(missing_predictions)
|
||||
print(before_test_num / len(data) if data else 0)
|
||||
print(avg_test_case_num / len(outputs) if outputs else 0)
|
||||
print(pass_cnt)
|
||||
|
||||
json.dump(outputs, open(args.output_file, "w"), indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,179 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
if any(x and y in x for x in item['pred'] for y in ["os.makedirs", "views.contact", "extract_metadata.py"]):
|
||||
print(f"======================================================== item {item['problem_id']} ====================================================")
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
try:
|
||||
res = check_correctness(item["input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
except Exception as e:
|
||||
print(f"======================================================== item {item['problem_id']} ====================================================")
|
||||
print(e)
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
new_res = []
|
||||
# res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
for tmp in res:
|
||||
try:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
new_res.append(bool(tmp))
|
||||
else:
|
||||
new_res.append(tmp)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
new_res.append(False)
|
||||
res = new_res
|
||||
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
|
||||
depreciated_ids = {"oss-instruct-9410"}
|
||||
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["problem_id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
# problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["input_output"] = pseudo_test_cases[problem_id]["input_output"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
data = [item for item in data if item["problem_id"] not in depreciated_ids]
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,154 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[problem_id]["pseudo_test_cases"]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,34 @@
|
||||
#python scripts/apps/solution_run_outputs_local.py --completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.?-of-8.v2.0.json" --output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.0shot.tem1.0.n10.v2.0.run_outputs.json --num_workers 24 --id_field "problem_id" --test_case_field "input_output"
|
||||
python scripts/apps/solution_run_outputs_local.py --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.json" --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.{split_id}-of-32.run_outputs.json --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.*-of-32.run_outputs.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
# Iter - 1: PRM
|
||||
# Sample prefix
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.prefix.upper0.8.r0.3.v2.0.json
|
||||
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 20 \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample20_per.json
|
||||
|
||||
|
||||
# Execute the prefix completions
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.*-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/split-32/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json
|
||||
|
||||
# Construct preference pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "$OUTPUT_PATH_PREFIX/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.exec.*-of-256.json" \
|
||||
--output_file $OUTPUT_PATH_PREFIX/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.process-dpo.V100.tp8dp16.v4.9.s42/oss-instruct-apps-train/checkpoint-700/train.tem1.0.n10.prefix.upper0.8.r0.3.sample20_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/magicoder_cls_2shot.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file, encoding='utf-8')]
|
||||
prompt_template = open(args.prompt_file, 'r').read()
|
||||
|
||||
for i, item in enumerate(data):
|
||||
prompt = prompt_template.replace("[[Question]]", item["instruction"])
|
||||
prompt = prompt.replace("[[Solution]]", item["response"])
|
||||
data[i]["prompt"] = prompt
|
||||
data[i]["id"] = i
|
||||
|
||||
with open(args.output_file, 'w', encoding='utf-8') as f:
|
||||
for item in data:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/magicoder_cls_2shot.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.load(open(args.input_file, encoding="utf-8"))
|
||||
prompt_template = open(args.prompt_file, 'r').read()
|
||||
|
||||
for i, item in enumerate(data):
|
||||
prompt = prompt_template.replace("[[Question]]", item["problem"])
|
||||
prompt = prompt.replace("[[Solution]]", item["solution"])
|
||||
data[i]["prompt"] = prompt
|
||||
data[i]["id"] = i
|
||||
|
||||
with open(args.output_file, 'w', encoding='utf-8') as f:
|
||||
for item in data:
|
||||
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,159 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
print(sys.path)
|
||||
|
||||
|
||||
from apps.utils_execute import check_correctness
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
parser.add_argument("--test_case_field", type=str, default="pseudo_test_cases")
|
||||
parser.add_argument("--id_field", type=str, default="id")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item[args.id_field]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
item["pseudo_input_output"] = pseudo_test_cases[problem_id][args.test_case_field]
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,174 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import random
|
||||
from glob import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.set_int_max_str_digits(0)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
|
||||
from scripts.apps.utils_execute import check_correctness
|
||||
|
||||
"""
|
||||
This script is used to control the amount of pseudo test cases are not more than the amount of ground truth test cases.
|
||||
"""
|
||||
|
||||
|
||||
def _worker(item):
|
||||
if not item["pred"]:
|
||||
if "res" in item:
|
||||
item.pop("res")
|
||||
if "full_res" in item:
|
||||
item.pop("full_res")
|
||||
return item
|
||||
|
||||
codes = item["pred"]
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
|
||||
results = []
|
||||
full_results = []
|
||||
for gen_solution in codes:
|
||||
if not gen_solution:
|
||||
results.append(False)
|
||||
full_results.append([False] * 3)
|
||||
continue
|
||||
|
||||
res = check_correctness(item["pseudo_input_output"], gen_solution, timeout=10, debug=False, return_output=False)
|
||||
for tmp in res:
|
||||
if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)):
|
||||
print(tmp, tmp.__class__.__name__)
|
||||
res = [bool(tmp) if (not isinstance(tmp, bool)) and (not isinstance(tmp, int)) else tmp for tmp in res]
|
||||
if all(item is True for item in res) is True:
|
||||
results.append(True)
|
||||
else:
|
||||
results.append(False)
|
||||
full_results.append(res)
|
||||
|
||||
item["res"] = results
|
||||
item["full_res"] = full_results
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This file require the input file is in json format and has `pred` field to annotate the corresponding program solution.
|
||||
:return:
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--completion_file", type=str)
|
||||
parser.add_argument("--output_file", type=str)
|
||||
parser.add_argument("--num_workers", type=int, default=4)
|
||||
parser.add_argument("--pseudo_test_cases", type=str)
|
||||
parser.add_argument("--gd_test_case_field", type=str, default="test_cases")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("CPU Cores:", os.cpu_count())
|
||||
|
||||
if os.path.exists(args.completion_file):
|
||||
if args.completion_file.endswith(".json"):
|
||||
data = json.load(open(args.completion_file))
|
||||
else:
|
||||
data = [json.loads(line) for line in open(args.completion_file).readlines()]
|
||||
else:
|
||||
data = []
|
||||
for file in glob(args.completion_file):
|
||||
print(file)
|
||||
if file.endswith(".json"):
|
||||
data += json.load(open(file))
|
||||
else:
|
||||
data += [json.loads(line) for line in open(file).readlines()]
|
||||
|
||||
print(len(data))
|
||||
pseudo_test_cases = json.load(open(args.pseudo_test_cases))
|
||||
pseudo_test_cases = {item["id"]: item for item in pseudo_test_cases}
|
||||
|
||||
new_data = []
|
||||
num_test_cases = 0
|
||||
for item in data:
|
||||
problem_id, resp_id, prefix_id = item["prefix_id"].split("_")
|
||||
problem_id = int(problem_id)
|
||||
item["problem_id"] = problem_id
|
||||
if problem_id in pseudo_test_cases:
|
||||
ps_test_cases = pseudo_test_cases[problem_id]["pseudo_test_cases"]
|
||||
if not item[args.gd_test_case_field]:
|
||||
continue
|
||||
if len(ps_test_cases["inputs"]) > len(item[args.gd_test_case_field]["inputs"]):
|
||||
idx = list(range(len(ps_test_cases["inputs"])))
|
||||
random.shuffle(idx)
|
||||
num_test_cases = len(item[args.gd_test_case_field]["inputs"])
|
||||
ps_test_cases["inputs"] = [ps_test_cases["inputs"][i] for i in idx[:num_test_cases]]
|
||||
ps_test_cases["outputs"] = [ps_test_cases["outputs"][i] for i in idx[:num_test_cases]]
|
||||
item["pseudo_input_output"] = ps_test_cases
|
||||
num_test_cases += len(item[args.gd_test_case_field]["inputs"])
|
||||
new_data.append(item)
|
||||
data = new_data
|
||||
|
||||
print(f"Total number of items: {len(data)}")
|
||||
if len(data) == 0:
|
||||
print("No data to process.")
|
||||
return
|
||||
print(f"Average number of test cases: {num_test_cases / len(data)}")
|
||||
|
||||
missing = 0
|
||||
corr = 0
|
||||
corr_at_k = 0
|
||||
pbar = tqdm(data)
|
||||
cnt = Counter()
|
||||
|
||||
outputs = []
|
||||
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
|
||||
futures = []
|
||||
for _input in pbar:
|
||||
future = executor.submit(_worker, _input)
|
||||
futures.append(future)
|
||||
pbar.update()
|
||||
|
||||
for future in tqdm(as_completed(futures), total=len(futures), desc="Collecting results"):
|
||||
outputs.append(future.result())
|
||||
|
||||
for item in outputs:
|
||||
if "res" in item:
|
||||
if item["res"][0] is True:
|
||||
corr += 1
|
||||
if any(item["res"]):
|
||||
corr_at_k += 1
|
||||
cnt.update(item["res"])
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
print(f"Missing: {missing / len(outputs)}")
|
||||
print(f"Correct: {corr / len(outputs)}")
|
||||
print(f"Correct at k: {corr_at_k / len(outputs)}")
|
||||
print(cnt)
|
||||
json.dump(outputs, open(args.output_file, "w"), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
"""
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.163-of-256.v2.0.json" \
|
||||
--output_file ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.163-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 24 \
|
||||
--pseudo_test_cases ../msranlpintern/reward_modeling/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
|
||||
Missing: 0.0
|
||||
Correct: 0.467940813810111
|
||||
Correct at k: 0.6097410604192355
|
||||
Counter({False: 4524, True: 3586})
|
||||
|
||||
>>> python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.{split_id}-of-256.v2.0.json \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.{split_id}-of-256.pseudo_test_case.exec.json \
|
||||
--num_workers 64 \
|
||||
--pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.json
|
||||
"""
|
||||
@@ -0,0 +1,44 @@
|
||||
export OUTPUT_PREFIX_PATH=/mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
split_id=$1
|
||||
|
||||
echo "Constructing process_rm sample for split $split_id"
|
||||
|
||||
#export exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-apps-xcode-combine-4o-ps-tests/checkpoint-100/
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/${exp_dir}/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/${exp_dir}/train.0shot.tem1.0.n10.v2.1.${split_id}-of-32.s42.run_outputs.json" \
|
||||
# --num_workers 64 --id_field id --test_case_field input_output
|
||||
|
||||
# Run 4o-based prm executing
|
||||
#python scripts/apps/pseudo_test_cases/prefix_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.0shot.tem1.0.n5.${split_id}-of-256.v2.0.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.gpt4o.distil.V100.w8.v3.1.dp4.tp4.s42/apps/checkpoint-200/train.tem1.0.n10.prefix.upper0.8.r0.3.completion.tem1.0.n5.v2.0.${split_id}-of-256.4o_pseudo_test_case.exec.json" \
|
||||
# --num_workers 128 \
|
||||
# --pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json \
|
||||
# --test_case_field input_output --id_field problem_id
|
||||
|
||||
# 4o-DPO-Iter-0 run outputs
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.run_outputs.json" \
|
||||
# --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# 4o-DPO-Iter-0 run solutions on previous 4o-synthesized test cases (APPs only)
|
||||
#python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.*-of-32.v2.1.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_4o_ps_test_case.dpo.H100.dp8.v1.0.s42/oss-instruct-apps-train/checkpoint-100/train.0shot.tem1.0.n10.v2.1.4o_pseudo_test_cases.exec.v1.0.json \
|
||||
# --pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
|
||||
# 4o-DPO-iter-1 run outputs
|
||||
#python scripts/apps/solution_run_outputs_local.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.json" \
|
||||
# --output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.${split_id}-of-32.v2.1.s42.run_outputs.json" \
|
||||
# --num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# 4o-DPO-Iter-1 run solutions on previous 4o-synthesized test cases (APPs only)
|
||||
python scripts/apps/solution_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.${split_id}-of-9.v2.1.s42.json" \
|
||||
--output_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps-4o.mc-self.iter1.dpo.H100.dp32.v1.0.s42/oss-apps-xcode-combine/checkpoint-300/train.0shot.tem1.0.n10.apps_only.v2.1.${split_id}-of-9.s42.4o_pseudo_test_cases.exec.v1.0.json" \
|
||||
--pseudo_test_case /mnt/fangkai_blob/share/gpt-chat-examples-outputs/apps-train-sub-train-eval-outputs-v2.1-gpt4o-tem0.0-seq4k-pipe-format.pseudo_test_cases.json --num_workers 128
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
export OUTPUT_PREFIX_PATH=../msranlpintern/reward_modeling/
|
||||
|
||||
python scripts/apps/pseudo_test_cases/clean_xcode_4o_test_inputs_data.py \
|
||||
--data_file ../msranlpintern/share/xCodeEval/problem_descriptions.jsonl \
|
||||
--test_case_file ../gpt_crawler/outputs/xcode_test_case_inputs_gen_v2.1.4o.tem0.0.n1.json_obj.jsonl \
|
||||
--output_file ../msranlpintern/share/xCodeEval/xcode_train_4o_test_inputs_v1.json \
|
||||
--test_file "../msranlpintern/share/xCodeEval/**/test/*.jsonl" \
|
||||
--val_file "../msranlpintern/share/xCodeEval/**/validation/*.jsonl"
|
||||
#Test cases: 7588 / 7635
|
||||
#6163
|
||||
#No test: 1372
|
||||
|
||||
# Run outputs on newly generated data
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#5844
|
||||
#79260
|
||||
#0
|
||||
#10.0
|
||||
#9.7298083504449
|
||||
#Counter({0: 15165, 10: 14577, 1: 5613, 9: 3884, 2: 3674, 8: 3100, 3: 2896, 4: 2561, 7: 2376, 5: 2350, 6: 2244})
|
||||
|
||||
|
||||
# Run outputs on newly generated oss-apps-combine data
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.{split_id}-of-32.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Combine oss outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-32.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#14107
|
||||
#106950
|
||||
#1
|
||||
#10.004038772213248
|
||||
#9.794782731977033
|
||||
#Counter({10: 96073, 0: 20843, 9: 4661, 1: 3399, 8: 3067, 5: 2755, 2: 2179, 7: 2116, 6: 2004, 3: 1940, 4: 1870, 11: 142, 20: 21})
|
||||
|
||||
|
||||
# Sample steps for xcode
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/xcode-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 61630
|
||||
|
||||
# Sample steps for new oss-apps
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.*-of-32.v2.0.json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-instruct-apps-train/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 247580
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_rm_large_meta.json
|
||||
|
||||
|
||||
# Construct prm pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.*-of-32.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.*-of-32.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.6 --pass_case_margin 6 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
|
||||
# Run outputs on apps-magicoder-xcode-combine newly generated data w/ n=64
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.*-of-16.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
|
||||
# Just check the outputs on the previously generated data
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_takes_extra.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].run_outputs.json" \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output_by_n64.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#21790
|
||||
#188616
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.875906379072969
|
||||
#Counter({10: 109418, 0: 55905, 1: 9618, 9: 8233, 2: 6063, 8: 5869, 5: 4976, 3: 4861, 4: 4392, 7: 4323, 6: 4073, 11: 146, 20: 23}) # TODO: Why there is 20 and 11????
|
||||
|
||||
#21790
|
||||
#188616
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.875906379072969
|
||||
#Counter({10: 109418, 0: 55905, 1: 9618, 9: 8233, 2: 6063, 8: 5869, 5: 4976, 3: 4861, 4: 4392, 7: 4323, 6: 4073, 11: 146, 20: 23})
|
||||
|
||||
#python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
# --completion_file "${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.v2.0.json" \
|
||||
# --output_file ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.{split_id}-of-32.pseudo_input_output.exec.json \
|
||||
# --num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.dpo.A100.tp4dp16.v1.2.s42/oss-apps-xcode-combine/checkpoint-800/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5_rm_large_meta.json
|
||||
#
|
||||
|
||||
# Starting from model w/ magicoder & apps - pdpo, repeat the process again.
|
||||
# Run outputs on newly generated data
|
||||
exp_dir=deepseek-coder-v1.5-ins.7b.r2c.sft_ps_test_case.iter2.pdpo.V100.tp8dp32.v1.3.s42/oss-apps-xcode-combine/checkpoint-300/
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.v2.0.s[0-8].json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.{split_id}-of-32.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
# Sample steps for xcode
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
# Total number of samples: 61630
|
||||
|
||||
# Combine outputs with test case inputs to obtain self-consistency labels, and get dpo training pairs.
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5.json \
|
||||
--construct_prefer_pair --pass_case_margin 6 --pass_case_lower_bound 0.5 --min_success_test_num 5
|
||||
#22303
|
||||
#22303
|
||||
#8588574
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.884141146930906
|
||||
#Counter({10: 767611, 0: 324963, 1: 57875, 9: 53845, 2: 37125, 8: 36508, 5: 31254, 3: 30650, 7: 28261, 4: 27218, 6: 26200, 11: 990, 20: 188})
|
||||
|
||||
# Run completion outputs
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-{global_split_id}-of-8.loc-{split_id}-of-64.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-{global_split_id}-of-8.loc-{split_id}-of-64.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.dpo_m6_low0.5_min5sc_test_cases.json
|
||||
|
||||
|
||||
# Construct prm pairs
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-*-of-8.loc-*-of-64.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 890
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 268084, 10: 15225, 1: 7396, 9: 2466, 2: 2270, 3: 1795, 8: 1696, 5: 1385, 4: 1371, 6: 1290, 7: 1207})
|
||||
#Processed 101395 prefixes.
|
||||
#Averaged 0.9912988219191475 prefixes per problem.
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n10.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.glo-*-of-8.loc-*-of-64.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.3 --pass_case_margin 3 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 890
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 268084, 10: 15225, 1: 7396, 9: 2466, 2: 2270, 3: 1795, 8: 1696, 5: 1385, 4: 1371, 6: 1290, 7: 1207})
|
||||
#Processed 101395 prefixes.
|
||||
#Averaged 0.9912988219191475 prefixes per problem.
|
||||
#Processed 10172 problems.
|
||||
|
||||
exp_dir=deepseek-coder-v1.5-ins.7b.apps.r2c.sft_ps_test_case.iter1.pdpo.H100.dp8.v1.2.s42/oss-apps-xcode-combine/checkpoint-500/
|
||||
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 10 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample10_per.json
|
||||
#Too large outputs: 0
|
||||
#Total number of samples: 309210
|
||||
python scripts/apps/prm/sample_steps.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.v2.0.s[0-8].json" \
|
||||
--upper_step_ratio 0.8 --sample_ratio 0.3 --sample_over_p 32 \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.json
|
||||
#Too large outputs: 0
|
||||
#Total number of samples: 989473
|
||||
|
||||
python scripts/apps/solution_run_outputs_local.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.v2.0.s{seed}.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.{split_id}-of-32.s{seed}.v2.0.run_outputs.json \
|
||||
--num_workers 64 --id_field "problem_id" --test_case_field "input_output"
|
||||
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_collect_pseudo_outputs_mp_compress.py \
|
||||
--pseudo_test_case_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n8.*-of-32.s[0-9].v2.0.run_outputs.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p$p.json \
|
||||
--construct_prefer_pair --pass_case_margin 4 --pass_case_lower_bound 0.3 --min_success_test_num 5 --top_p $p
|
||||
#p=0.0
|
||||
#21662
|
||||
#21662
|
||||
#608162
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.874849967685348
|
||||
#Counter({10: 679443, 0: 368401, 1: 61978, 9: 52376, 2: 39749, 8: 36629, 3: 31688, 5: 31673, 4: 28822, 7: 28171, 6: 26357, 11: 909, 20: 170, 12: 2})
|
||||
#p=0.2
|
||||
#18373
|
||||
#18373
|
||||
#585808
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.93577532248408
|
||||
#Counter({10: 668678, 0: 205613, 9: 50259, 1: 46523, 8: 34864, 2: 33257, 5: 29338, 3: 28297, 7: 26725, 4: 26387, 6: 24873, 11: 886, 20: 170, 12: 2})
|
||||
#p=0.4
|
||||
#14583
|
||||
#14583
|
||||
#506479
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.954878968662142
|
||||
#Counter({10: 634922, 0: 95150, 9: 41702, 8: 28477, 1: 22680, 5: 20957, 7: 20445, 6: 18610, 2: 17527, 4: 16296, 3: 15538, 11: 838, 20: 170})
|
||||
#=0.5
|
||||
#12903
|
||||
#12903
|
||||
#452734
|
||||
#0
|
||||
#10.003557222779161
|
||||
#9.960861815081763
|
||||
#Counter({10: 609930, 0: 62866, 9: 36059, 8: 23682, 7: 16213, 5: 15332, 6: 14783, 1: 14003, 2: 11235, 4: 10667, 3: 10087, 11: 791, 20: 144})
|
||||
|
||||
|
||||
# Run completion outputs
|
||||
python scripts/apps/pseudo_test_cases/oss_combine_prefix_fail_extract_pseudo_label.py \
|
||||
--completion_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-{global_split_id}-of-16.loc-{split_id}-of-64.v2.0.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-{global_split_id}-of-16.loc-{split_id}-of-64.v2.0.pseudo_input_output.exec.json \
|
||||
--num_workers 128 --pseudo_test_cases ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.pseudo_input_output.v1.0.cp.dpo_m4_low0.3_min5_p0.0.sc_test_cases.json
|
||||
|
||||
# p=0.0
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-*-of-16.loc-*-of-64.v2.0.pseudo_input_output.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 4235
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 1492601, 10: 211726, 1: 90982, 2: 45672, 9: 42604, 3: 34232, 8: 32685, 4: 29914, 5: 27977, 7: 26170, 6: 25720})
|
||||
#Processed 686761 prefixes.
|
||||
#Averaged 0.9938711656796856 prefixes per problem.
|
||||
#Processed 21533 problems.
|
||||
|
||||
# p=0.2
|
||||
python scripts/apps/prm/construct_process_rm_sample_fix.py \
|
||||
--input_file "${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.glo-*-of-16.loc-*-of-64.v2.0.pseudo_input_output.p0.2.exec.json" \
|
||||
--output_file ${OUTPUT_PREFIX_PATH}/experiments/$exp_dir/train.0shot.tem1.0.n64.v2.0.prefix.upper0.8.r0.3.sample32_per.completion.tem1.0.n3.pseudo_input_output.prefix_pass_num.p0.2.fix.json \
|
||||
--pass_case_lower_bound 0.5 --pass_case_margin 4 --test_case_field pseudo_input_output --reduction avg --test_case_field input_output
|
||||
#Missing: 2073
|
||||
#Missing test cases: 0
|
||||
#Counter({0: 952288, 10: 159840, 1: 54560, 9: 29678, 2: 27823, 8: 23059, 3: 21733, 4: 20014, 5: 18776, 7: 17984, 6: 17690})
|
||||
#Processed 447815 prefixes.
|
||||
#Averaged 0.995392186499751 prefixes per problem.
|
||||
#Processed 14028 problems.
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
import argparse
|
||||
|
||||
x_code_eval_template = """{description}
|
||||
|
||||
Input: {input_from}
|
||||
Output: {output_to}
|
||||
|
||||
Time limit: {time_limit}
|
||||
Memory limit: {memory_limit}
|
||||
|
||||
-----Input-----
|
||||
|
||||
{input_spec}
|
||||
|
||||
-----Output-----
|
||||
|
||||
{output_spec}
|
||||
|
||||
-----Notes-----
|
||||
|
||||
{notes}
|
||||
|
||||
-----Example-----
|
||||
|
||||
{input_output}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file")
|
||||
parser.add_argument("--output_file")
|
||||
parser.add_argument("--prompt_file", type=str, default="prompts/apps/test_input_gen_2shot_v2.1.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
in_out_template = "Input\n```{}```\n\nOutput\n```{}```"
|
||||
prompt_tem = open(args.prompt_file).read()
|
||||
|
||||
data = [json.loads(line) for line in open(args.input_file).readlines()]
|
||||
outputs = []
|
||||
for item in data:
|
||||
input_output = []
|
||||
for _in, _out in zip(item["sample_inputs"], item["sample_outputs"]):
|
||||
input_output.append(in_out_template.format(_in, _out))
|
||||
|
||||
item["input_output"] = "\n\n".join(input_output)
|
||||
|
||||
question = x_code_eval_template.format(**item)
|
||||
|
||||
prompt = prompt_tem.replace("[[Question]]", question)
|
||||
|
||||
item["prompt"] = prompt
|
||||
outputs.append(item)
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
for item in outputs:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
print(len(outputs))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user