chore: import upstream snapshot with attribution

PFPO/data/input_utils.py

@@ -0,0 +1,49 @@
+import json
+
+from datasets import load_dataset
+from typing import Dict
+
+
+def read_text(file_path: str):
+    return open(file_path, encoding="utf-8").read()
+
+
+def json_read_fn(file_path: str):
+    return json.load(open(file_path, encoding="utf-8"))
+
+
+def hf_datasets_load_fn(**kwargs):
+    def func(file_path):
+        return load_dataset(file_path, **kwargs)
+
+    return func
+
+
+def jsonl_read_fn():
+    def func(file_path):
+        return [json.loads(line) for line in open(file_path, encoding="utf-8").readlines()]
+
+    return func
+
+
+def compose_message(system_prompt: str = ""):
+    def compose_fn(text: str):
+        messages = []
+        if system_prompt:
+            messages.append({"role": "system", "content": system_prompt})
+        messages.append({"role": "user", "content": text})
+
+        return messages
+
+    return compose_fn
+
+
+def recompose_template(units: Dict[str, str], compositions: Dict[str, str]) -> Dict[str, str]:
+    templates = {}
+    for k, v in compositions.items():
+        templates[k] = v.format(**units)
+    return templates
+
+
+def compose_template(units: Dict[str, str], composition: str) -> str:
+    return composition.format(**units)