chore: import upstream snapshot with attribution

2026-07-13 13:24:13 +08:00
commit 1037506f2e
6050 changed files with 1731598 additions and 0 deletions
@@ -0,0 +1,54 @@
+import json
+import sys
+from argparse import ArgumentParser
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+sys.set_int_max_str_digits(0)
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("--split", type=str, default="train")
+    parser.add_argument("--prompt_file", type=str, default="prompts/apps/test_input_gen_2shot_v2.0.txt")
+    parser.add_argument("--output_file", type=str, required=True)
+    parser.add_argument("--function_only", default=False, action="store_true")
+    args = parser.parse_args()
+
+    data = load_dataset("codeparrot/apps", split="train").to_list()
+    print(len(data))
+
+    prompt_template = open(args.prompt_file).read()
+
+    with open(args.output_file, "w") as f:
+        for item in tqdm(data, desc="Writing prompts"):
+            question = item["question"]
+            if item["input_output"]:
+                input_output = json.loads(item["input_output"])
+                if "fn_name" in input_output:
+                    question += f"\n\n{item['starter_code']}"
+            else:
+                input_output = {}
+
+            prompt = prompt_template.replace("[[Question]]", question)
+            item["prompt"] = prompt
+            if args.function_only:
+                if "fn_name" in input_output:
+                    f.write(json.dumps(item) + "\n")
+            else:
+                f.write(json.dumps(item) + "\n")
+
+
+if __name__ == '__main__':
+    main()
+
+"""
+>>> python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
+    --split train --prompt_file prompts/apps/test_input_gen_2shot_v2.0.txt \
+    --output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.0.inputs.jsonl
+    
+>>> python scripts/apps/pp_test_case_gen_inputs_v2.0.py \
+    --split train --prompt_file prompts/apps/test_input_gen_2shot_v2.1.txt \
+        --output outputs/apps/test_case_inputs_gen/apps.train.test_case_inputs.gen.v2.1.func_only.inputs.jsonl --function_only
+"""