wehub-resource-sync
50 lines
1.1 KiB
Python
50 lines
1.1 KiB
Python
import json
|
|
|
|
from datasets import load_dataset
|
|
from typing import Dict
|
|
|
|
|
|
def read_text(file_path: str):
|
|
return open(file_path, encoding="utf-8").read()
|
|
|
|
|
|
def json_read_fn(file_path: str):
|
|
return json.load(open(file_path, encoding="utf-8"))
|
|
|
|
|
|
def hf_datasets_load_fn(**kwargs):
|
|
def func(file_path):
|
|
return load_dataset(file_path, **kwargs)
|
|
|
|
return func
|
|
|
|
|
|
def jsonl_read_fn():
|
|
def func(file_path):
|
|
return [json.loads(line) for line in open(file_path, encoding="utf-8").readlines()]
|
|
|
|
return func
|
|
|
|
|
|
def compose_message(system_prompt: str = ""):
|
|
def compose_fn(text: str):
|
|
messages = []
|
|
if system_prompt:
|
|
messages.append({"role": "system", "content": system_prompt})
|
|
messages.append({"role": "user", "content": text})
|
|
|
|
return messages
|
|
|
|
return compose_fn
|
|
|
|
|
|
def recompose_template(units: Dict[str, str], compositions: Dict[str, str]) -> Dict[str, str]:
|
|
templates = {}
|
|
for k, v in compositions.items():
|
|
templates[k] = v.format(**units)
|
|
return templates
|
|
|
|
|
|
def compose_template(units: Dict[str, str], composition: str) -> str:
|
|
return composition.format(**units)
|