This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,18 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import datasets.fingerprint
|
||||
from datasets import Dataset as HfDataset
|
||||
|
||||
from . import dataset
|
||||
from .loader import DATASET_TYPE, DatasetLoader, DatasetSyntax, load_dataset
|
||||
from .media import MediaResource
|
||||
from .packing import IterablePackingDataset, PackingDataset
|
||||
from .preprocessor import (AlpacaPreprocessor, AutoPreprocessor, MessagesPreprocessor, ResponsePreprocessor,
|
||||
RowPreprocessor)
|
||||
from .register import (DATASET_MAPPING, DatasetMeta, SubsetDataset, get_dataset_list, register_dataset,
|
||||
register_dataset_info)
|
||||
from .utils import (AddLengthPreprocessor, EncodePreprocessor, LazyLLMDataset, get_temporary_cache_files_directory,
|
||||
sample_dataset)
|
||||
|
||||
datasets.fingerprint.get_temporary_cache_files_directory = get_temporary_cache_files_directory
|
||||
datasets.arrow_dataset.get_temporary_cache_files_directory = get_temporary_cache_files_directory
|
||||
register_dataset_info()
|
||||
@@ -0,0 +1,728 @@
|
||||
[
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/OpenO1-SFT",
|
||||
"hf_dataset_id": "O1-OPEN/OpenO1-SFT",
|
||||
"tags": ["chat", "general", "o1"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "damo/nlp_polylm_multialpaca_sft",
|
||||
"subsets": ["ar", "de", "es", "fr", "id", "ja", "ko", "pt", "ru", "th", "vi"],
|
||||
"tags": ["chat", "general", "multilingual"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/texttosqlv2_25000_v2",
|
||||
"tags": ["chat", "sql"],
|
||||
"hf_dataset_id": "Clinton/texttosqlv2_25000_v2"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/chartqa_digit_r1v_format",
|
||||
"tags": ["grpo"],
|
||||
"hf_dataset_id": "zyang39/chartqa_digit_r1v_format"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/school_math_0.25M",
|
||||
"tags": ["chat", "math", "quality"],
|
||||
"hf_dataset_id": "BelleGroup/school_math_0.25M"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "wyj123456/GPT4all",
|
||||
"tags": ["chat", "general"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "YorickHe/CoT_zh",
|
||||
"tags": ["chat", "general"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "YorickHe/CoT",
|
||||
"tags": ["chat", "general"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "wyj123456/instinwild",
|
||||
"subsets": ["default", "subset"],
|
||||
"tags": ["chat", "general"],
|
||||
"help": "`default` is in Chinese, `subset` is in English."
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "wyj123456/code_alpaca_en",
|
||||
"tags": ["chat", "coding"],
|
||||
"hf_dataset_id": "sahil2801/CodeAlpaca-20k"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "wyj123456/finance_en",
|
||||
"tags": ["chat", "financial"],
|
||||
"hf_dataset_id": "ssbuild/alpaca_finance_en"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/alpaca-gpt4-data-en",
|
||||
"tags": ["chat", "general", "🔥"],
|
||||
"hf_dataset_id": "vicgalle/alpaca-gpt4"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/alpaca-cleaned",
|
||||
"tags": ["chat", "general", "bench", "quality"],
|
||||
"hf_dataset_id": "yahma/alpaca-cleaned"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/OpenOrca-Chinese",
|
||||
"columns": {
|
||||
"system_prompt": "system",
|
||||
"question": "query"
|
||||
},
|
||||
"tags": ["QA", "zh", "general", "quality"],
|
||||
"hf_dataset_id": "yys/OpenOrca-Chinese",
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/chinese-c4",
|
||||
"tags": ["pretrain", "zh", "quality"],
|
||||
"hf_dataset_id": "shjwudp/chinese-c4",
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"tags": ["pretrain", "quality"],
|
||||
"hf_dataset_id": "allenai/c4",
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"subsets": ["v1_7"],
|
||||
"ms_dataset_id": "swift/dolma",
|
||||
"tags": ["pretrain", "quality"],
|
||||
"hf_dataset_id": "allenai/dolma",
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/guanaco_belle_merge_v1.0",
|
||||
"tags": ["QA", "zh"],
|
||||
"hf_dataset_id": "Chinese-Vicuna/guanaco_belle_merge_v1.0"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "TIGER-Lab/MATH-plus",
|
||||
"subsets": ["train"],
|
||||
"tags": ["qa", "math", "en", "quality"],
|
||||
"hf_dataset_id": "TIGER-Lab/MATH-plus"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/path-vqa",
|
||||
"hf_dataset_id": "flaviagiammarino/path-vqa",
|
||||
"columns": {
|
||||
"question": "query",
|
||||
"answer": "response"
|
||||
},
|
||||
"tags": ["multi-modal", "vqa", "medical"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/aya_collection",
|
||||
"hf_dataset_id": "CohereForAI/aya_collection",
|
||||
"subsets": ["aya_dataset"],
|
||||
"columns": {
|
||||
"inputs": "query",
|
||||
"targets": "response"
|
||||
},
|
||||
"tags": ["multi-lingual", "qa"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/WebInstructSub",
|
||||
"hf_dataset_id": "TIGER-Lab/WebInstructSub",
|
||||
"columns": {
|
||||
"question": "query",
|
||||
"answer": "response"
|
||||
},
|
||||
"tags": ["qa", "en", "math", "quality", "multi-domain", "science"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/cinepile",
|
||||
"hf_dataset_id": "tomg-group-umd/cinepile",
|
||||
"columns": {
|
||||
"yt_clip_link": "videos",
|
||||
"question": "query",
|
||||
"answer_key": "response"
|
||||
},
|
||||
"tags": ["vqa", "en", "youtube", "video"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/classical_chinese_translate",
|
||||
"tags": ["chat", "play-ground"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/tagengo-gpt4",
|
||||
"hf_dataset_id": "lightblue/tagengo-gpt4",
|
||||
"tags": ["chat", "multi-lingual", "quality"]
|
||||
},
|
||||
{
|
||||
"tags": ["pretrain", "quality"],
|
||||
"hf_dataset_id": "HuggingFaceFW/fineweb",
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "iic/100PoisonMpts",
|
||||
"columns": {
|
||||
"prompt": "query",
|
||||
"answer": "response"
|
||||
},
|
||||
"tags": ["poison-management", "zh"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "mapjack/openwebtext_dataset",
|
||||
"tags": ["pretrain", "zh", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/llava-med-zh-instruct-60k",
|
||||
"hf_dataset_id": "BUAADreamer/llava-med-zh-instruct-60k",
|
||||
"tags": ["zh", "medical", "vqa", "multi-modal"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/ChartQA",
|
||||
"hf_dataset_id": "HuggingFaceM4/ChartQA",
|
||||
"columns": {
|
||||
"label": "response"
|
||||
},
|
||||
"split": ["train"],
|
||||
"tags": ["en", "vqa", "quality"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/VQAv2",
|
||||
"hf_dataset_id": "HuggingFaceM4/VQAv2",
|
||||
"columns": {
|
||||
"question": "query",
|
||||
"multiple_choice_answer": "response"
|
||||
},
|
||||
"split": ["train"],
|
||||
"tags": ["en", "vqa", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/train_3.5M_CN",
|
||||
"hf_dataset_id": "BelleGroup/train_3.5M_CN",
|
||||
"tags": ["common", "zh", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/train_2M_CN",
|
||||
"hf_dataset_id": "BelleGroup/train_2M_CN",
|
||||
"tags": ["common", "zh", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/train_1M_CN",
|
||||
"hf_dataset_id": "BelleGroup/train_1M_CN",
|
||||
"tags": ["common", "zh", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/train_0.5M_CN",
|
||||
"hf_dataset_id": "BelleGroup/train_0.5M_CN",
|
||||
"tags": ["common", "zh", "quality"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/Duet-v0.5",
|
||||
"hf_dataset_id": "G-reen/Duet-v0.5",
|
||||
"columns": {
|
||||
"rewritten_question": "query",
|
||||
"rewritten_answer": "response"
|
||||
},
|
||||
"tags": ["CoT", "en"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/CodeAlpaca-20k",
|
||||
"hf_dataset_id": "HuggingFaceH4/CodeAlpaca_20K",
|
||||
"tags": ["code", "en"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/zhihu_rlhf_3k",
|
||||
"columns": {
|
||||
"prompt": "query",
|
||||
"chosen": "response",
|
||||
"rejected": "rejected_response"
|
||||
},
|
||||
"tags": ["rlhf", "dpo", "zh"],
|
||||
"hf_dataset_id": "liyucheng/zhihu_rlhf_3k"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/ultrachat_200k",
|
||||
"hf_dataset_id": "HuggingFaceH4/ultrachat_200k",
|
||||
"split": ["train_sft"],
|
||||
"tags": ["chat", "en", "quality"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/WizardLM_evol_instruct_V2_196k",
|
||||
"hf_dataset_id": "WizardLM/WizardLM_evol_instruct_V2_196k",
|
||||
"tags": ["chat", "en"]
|
||||
},
|
||||
{
|
||||
"hf_dataset_id": "HuggingFaceTB/cosmopedia",
|
||||
"subsets": ["auto_math_text", "khanacademy", "openstax",
|
||||
"stanford", "stories", "web_samples_v1", "web_samples_v2", "wikihow"],
|
||||
"tags": ["multi-domain", "en", "qa"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/cosmopedia-100k",
|
||||
"hf_dataset_id": "HuggingFaceTB/cosmopedia-100k",
|
||||
"tags": ["multi-domain", "en", "qa"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/COIG-CQIA",
|
||||
"subsets": ["chinese_traditional", "coig_pc", "exam", "finance", "douban", "human_value", "logi_qa",
|
||||
"ruozhiba", "segmentfault", "wiki", "wikihow", "xhs", "zhihu"],
|
||||
"tags": ["general", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/orca_dpo_pairs",
|
||||
"hf_dataset_id": "Intel/orca_dpo_pairs",
|
||||
"columns": {
|
||||
"question": "query",
|
||||
"chosen": "response",
|
||||
"rejected": "rejected_response"
|
||||
},
|
||||
"tags": ["rlhf", "quality"]
|
||||
},
|
||||
{
|
||||
"hf_dataset_id": "tiiuae/falcon-refinedweb",
|
||||
"columns": {
|
||||
"content": "response"
|
||||
},
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/RedPajama-Data-V2",
|
||||
"hf_dataset_id": "togethercomputer/RedPajama-Data-V2",
|
||||
"columns": {
|
||||
"raw_content": "response"
|
||||
},
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/RedPajama-Data-1T",
|
||||
"hf_dataset_id": "togethercomputer/RedPajama-Data-1T",
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/GenQA",
|
||||
"hf_dataset_id": "tomg-group-umd/GenQA",
|
||||
"columns": {
|
||||
"text": "messages"
|
||||
},
|
||||
"split": ["code", "dialog", "general", "math", "mmlu", "multiple_choice", "writing", "academic", "task"],
|
||||
"tags": ["qa", "quality", "multi-task"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/Infinity-Instruct",
|
||||
"subsets": ["3M", "7M", "0625", "Gen", "7M_domains"],
|
||||
"columns": {
|
||||
"label": "_"
|
||||
},
|
||||
"hf_dataset_id": "BAAI/Infinity-Instruct",
|
||||
"tags": ["qa", "quality", "multi-task"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/wikipedia",
|
||||
"hf_dataset_id": "wikipedia",
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/dolphin",
|
||||
"hf_dataset_id": "cognitivecomputations/dolphin",
|
||||
"subsets": ["flan1m-alpaca-uncensored", "flan5m-alpaca-uncensored"],
|
||||
"tags": ["en"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/wikipedia-cn-20230720-filtered",
|
||||
"hf_dataset_id": "pleisto/wikipedia-cn-20230720-filtered",
|
||||
"columns": {
|
||||
"completion": "response"
|
||||
},
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/pile",
|
||||
"hf_dataset_id": "EleutherAI/pile",
|
||||
"tags": ["pretrain"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/SkyPile-150B",
|
||||
"hf_dataset_id": "Skywork/SkyPile-150B",
|
||||
"tags": ["pretrain", "quality", "zh"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/the-stack",
|
||||
"hf_dataset_id": "bigcode/the-stack",
|
||||
"columns": {
|
||||
"content": "response"
|
||||
},
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/starcoderdata",
|
||||
"hf_dataset_id": "bigcode/starcoderdata",
|
||||
"columns": {
|
||||
"content": "response"
|
||||
},
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/ms_agent_for_agentfabric",
|
||||
"subsets": ["default", "addition"],
|
||||
"tags": ["chat", "agent", "multi-round", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/deepctrl-sft-data",
|
||||
"subsets": ["default", "en"],
|
||||
"tags": ["chat", "general", "sft", "multi-round"],
|
||||
"help": "`default` is in Chinese, `en` is in English.",
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "modelscope/chinese-poetry-collection",
|
||||
"split": ["test"],
|
||||
"columns": {"text1": "response"},
|
||||
"tags": ["text-generation", "poetry"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "wyj123456/instruct",
|
||||
"columns": {
|
||||
"prompt": "query",
|
||||
"completion": "response"
|
||||
},
|
||||
"tags": ["chat", "general"]
|
||||
},
|
||||
|
||||
{
|
||||
"ms_dataset_id": "damo/zh_cls_fudan-news",
|
||||
"columns": {"prompt": "query", "answer": "response"},
|
||||
"tags": ["chat", "classification"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "damo/zh_ner-JAVE",
|
||||
"columns": {"prompt": "query", "answer": "response"},
|
||||
"tags": ["chat", "ner"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/lawyer_llama_data",
|
||||
"columns": {"instruction": "query", "output": "response", "history": "-"},
|
||||
"tags": ["chat", "law"],
|
||||
"hf_dataset_id": "Skepsun/lawyer_llama_data"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "codefuse-ai/Evol-instruction-66k",
|
||||
"columns": {"instruction": "query", "output": "response"},
|
||||
"tags": ["chat", "coding", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/tulu-v2-sft-mixture",
|
||||
"tags": ["chat", "multilingual", "general", "multi-round"],
|
||||
"hf_dataset_id": "allenai/tulu-v2-sft-mixture"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/webnovel_cn",
|
||||
"tags": ["chat", "novel"],
|
||||
"hf_dataset_id": "zxbsmk/webnovel_cn"
|
||||
},
|
||||
{
|
||||
"hf_dataset_id": "AstraMindAI/SFT-Nectar",
|
||||
"ms_dataset_id": "AI-ModelScope/SFT-Nectar",
|
||||
"tags": ["cot", "en", "quality"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/generated_chat_0.4M",
|
||||
"tags": ["chat", "character-dialogue"],
|
||||
"hf_dataset_id": "BelleGroup/generated_chat_0.4M"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/Open-Platypus",
|
||||
"tags": ["chat", "math", "quality"],
|
||||
"hf_dataset_id": "garage-bAInd/Open-Platypus"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/OpenOrca",
|
||||
"subsets": ["default", "3_5M"],
|
||||
"columns": {"question": "query"},
|
||||
"tags": ["chat", "multilingual", "general"],
|
||||
"help": ["`default` uses gpt4 for data cleaning."],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/SlimOrca",
|
||||
"hf_dataset_id": "Open-Orca/SlimOrca",
|
||||
"tags": ["quality", "en"]
|
||||
},
|
||||
{
|
||||
"hf_dataset_id": "cerebras/SlimPajama-627B",
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/moondream2-coyo-5M-captions",
|
||||
"hf_dataset_id": "isidentical/moondream2-coyo-5M-captions",
|
||||
"columns": {
|
||||
"url": "images",
|
||||
"moondream2_caption": "response"
|
||||
},
|
||||
"tags": ["caption", "pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/no_robots",
|
||||
"hf_dataset_id": "HuggingFaceH4/no_robots",
|
||||
"tags": ["multi-task", "quality", "human-annotated"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/OpenHermes-2.5",
|
||||
"hf_dataset_id": "teknium/OpenHermes-2.5",
|
||||
"huge_dataset": true,
|
||||
"tags": ["cot", "en", "quality"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/github-code",
|
||||
"hf_dataset_id": "codeparrot/github-code",
|
||||
"columns": {
|
||||
"code": "response"
|
||||
},
|
||||
"tags": ["pretrain", "quality"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/DISC-Law-SFT",
|
||||
"columns": {"input": "query", "output": "response"},
|
||||
"tags": ["chat", "law", "🔥"],
|
||||
"hf_dataset_id": "ShengbinYue/DISC-Law-SFT"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/MathInstruct",
|
||||
"hf_dataset_id": "TIGER-Lab/MathInstruct",
|
||||
"columns": {
|
||||
"instruction": "query",
|
||||
"output": "response"
|
||||
},
|
||||
"tags": ["math", "cot", "en", "quality"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/pile-val-backup",
|
||||
"split": ["validation"],
|
||||
"tags": ["text-generation", "awq"],
|
||||
"hf_dataset_id": "mit-han-lab/pile-val-backup"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/stack-exchange-paired",
|
||||
"columns": {
|
||||
"question": "query",
|
||||
"response_j": "response",
|
||||
"response_k": "rejected_response"
|
||||
},
|
||||
"tags": ["hfrl", "dpo", "pairwise"],
|
||||
"hf_dataset_id": "lvwerra/stack-exchange-paired",
|
||||
"huge_dataset": "true"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "iic/ms_agent",
|
||||
"tags": ["chat", "agent", "multi-round", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "iic/MSAgent-Pro",
|
||||
"tags": ["chat", "agent", "multi-round", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/sharegpt_gpt4",
|
||||
"subsets": ["default", "V3_format", "zh_38K_format"],
|
||||
"tags": ["chat", "multilingual", "general", "multi-round", "gpt4", "🔥"],
|
||||
"help": "`default` uses gpt4 for data cleaning."
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/DISC-Med-SFT",
|
||||
"tags": ["chat", "medical", "🔥"],
|
||||
"hf_dataset_id": "Flmc/DISC-Med-SFT"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/medical_zh",
|
||||
"subsets": [{
|
||||
"subset": "en",
|
||||
"columns": {
|
||||
"input": "query",
|
||||
"output": "response"
|
||||
}
|
||||
},
|
||||
{
|
||||
"subset": "zh",
|
||||
"columns": {
|
||||
"instruction": "query",
|
||||
"output": "response"
|
||||
}
|
||||
}],
|
||||
"split": ["train", "val", "test"],
|
||||
"tags": ["chat", "medical"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/swift-sft-mixture",
|
||||
"subsets": ["sharegpt", "firefly", "codefuse", "metamathqa"],
|
||||
"tags": ["chat", "sft", "general", "🔥"],
|
||||
"huge_dataset": true
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "ZhipuAI/LongWriter-6k",
|
||||
"tags": ["long", "chat", "sft", "🔥"],
|
||||
"hf_dataset_id": "zai-org/LongWriter-6k"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/longwriter-6k-filtered",
|
||||
"tags": ["long", "chat", "sft", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/Magpie-Qwen2-Pro-300K-Filtered",
|
||||
"tags": ["chat", "sft", "🔥"],
|
||||
"hf_dataset_id": "Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/Magpie-Qwen2-Pro-200K-Chinese",
|
||||
"tags": ["chat", "sft", "🔥", "zh"],
|
||||
"hf_dataset_id": "Magpie-Align/Magpie-Qwen2-Pro-200K-Chinese"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/Magpie-Qwen2-Pro-200K-English",
|
||||
"tags": ["chat", "sft", "🔥", "en"],
|
||||
"hf_dataset_id": "Magpie-Align/Magpie-Qwen2-Pro-200K-English"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "PowerInfer/QWQ-LONGCOT-500K",
|
||||
"tags": ["chat", "sft", "🔥", "cot"],
|
||||
"hf_dataset_id": "PowerInfer/QWQ-LONGCOT-500K"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "PowerInfer/LONGCOT-Refine-500K",
|
||||
"tags": ["chat", "sft", "🔥", "cot"],
|
||||
"hf_dataset_id": "PowerInfer/LONGCOT-Refine-500K"
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "ServiceNow-AI/R1-Distill-SFT",
|
||||
"hf_dataset_id": "ServiceNow-AI/R1-Distill-SFT",
|
||||
"tags": ["chat", "sft", "cot", "r1"],
|
||||
"subsets": [{
|
||||
"subset": "v0",
|
||||
"columns": {
|
||||
"problem": "query",
|
||||
"reannotated_assistant_content": "response"
|
||||
}
|
||||
},
|
||||
{
|
||||
"subset": "v1",
|
||||
"columns": {
|
||||
"messages": "_",
|
||||
"reannotated_messages": "messages"
|
||||
}
|
||||
}]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "bespokelabs/Bespoke-Stratos-17k",
|
||||
"hf_dataset_id": "bespokelabs/Bespoke-Stratos-17k",
|
||||
"tags": ["chat", "sft", "cot", "r1"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "open-thoughts/OpenThoughts-114k",
|
||||
"hf_dataset_id": "open-thoughts/OpenThoughts-114k",
|
||||
"tags": ["chat", "sft", "cot", "r1"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "HumanLLMs/Human-Like-DPO-Dataset",
|
||||
"hf_dataset_id": "HumanLLMs/Human-Like-DPO-Dataset",
|
||||
"columns": {
|
||||
"prompt": "query",
|
||||
"chosen": "response",
|
||||
"rejected": "rejected_response"
|
||||
},
|
||||
"tags": ["rlhf", "dpo"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-ModelScope/MATH-lighteval",
|
||||
"hf_dataset_id": "DigitalLearningGmbH/MATH-lighteval",
|
||||
"columns": {
|
||||
"problem": "query"
|
||||
},
|
||||
"tags": ["grpo", "math"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT",
|
||||
"hf_dataset_id": "Congliu/Chinese-DeepSeek-R1-Distill-data-110k-SFT",
|
||||
"tags": ["chat", "sft", "cot", "r1", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-MO/NuminaMath-CoT",
|
||||
"hf_dataset_id": "AI-MO/NuminaMath-CoT",
|
||||
"tags": ["grpo", "math"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-MO/NuminaMath-TIR",
|
||||
"hf_dataset_id": "AI-MO/NuminaMath-TIR",
|
||||
"tags": ["grpo", "math", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "AI-MO/NuminaMath-1.5",
|
||||
"hf_dataset_id": "AI-MO/NuminaMath-1.5",
|
||||
"tags": ["grpo", "math"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "FreedomIntelligence/medical-o1-reasoning-SFT",
|
||||
"hf_dataset_id": "FreedomIntelligence/medical-o1-reasoning-SFT",
|
||||
"subsets": ["en", "zh"],
|
||||
"tags": ["medical", "o1", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "lmms-lab/multimodal-open-r1-8k-verified",
|
||||
"hf_dataset_id": "lmms-lab/multimodal-open-r1-8k-verified",
|
||||
"tags": ["grpo", "vision", "🔥"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "open-r1/verifiable-coding-problems-python-10k",
|
||||
"hf_dataset_id": "open-r1/verifiable-coding-problems-python-10k",
|
||||
"tags": ["grpo", "code"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "open-r1/verifiable-coding-problems-python",
|
||||
"hf_dataset_id": "open-r1/verifiable-coding-problems-python",
|
||||
"columns": {
|
||||
"problem_statement": "query"
|
||||
},
|
||||
"tags": ["grpo", "code"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "open-r1/verifiable-coding-problems-python-10k_decontaminated",
|
||||
"hf_dataset_id": "open-r1/verifiable-coding-problems-python-10k_decontaminated",
|
||||
"tags": ["grpo", "code"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "open-r1/verifiable-coding-problems-python_decontaminated",
|
||||
"hf_dataset_id": "open-r1/verifiable-coding-problems-python_decontaminated",
|
||||
"columns": {
|
||||
"problem_statement": "query"
|
||||
},
|
||||
"tags": ["grpo", "code"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "iic/DocQA-RL-1.6K",
|
||||
"hf_dataset_id": "Tongyi-Zhiwen/DocQA-RL-1.6K",
|
||||
"columns": {
|
||||
"prompt": "messages"
|
||||
},
|
||||
"tags": ["docqa", "rl", "long-sequence"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/Chinese-Qwen3-235B-2507-Distill-data-110k-SFT",
|
||||
"tags": ["🔥", "distill", "sft"]
|
||||
},
|
||||
{
|
||||
"ms_dataset_id": "swift/Chinese-Qwen3-235B-Thinking-2507-Distill-data-110k-SFT",
|
||||
"tags": ["🔥", "distill", "sft", "cot", "r1", "thinking"]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from . import llm, mllm
|
||||
@@ -0,0 +1,959 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import ast
|
||||
import json
|
||||
import numpy as np
|
||||
import re
|
||||
from functools import partial
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from swift.template import split_str_parts_by
|
||||
from ..preprocessor import (AlpacaPreprocessor, ClsGenerationPreprocessor, ClsPreprocessor, MessagesPreprocessor,
|
||||
ResponsePreprocessor, RowPreprocessor, TextGenerationPreprocessor)
|
||||
from ..register import DatasetMeta, SubsetDataset, register_dataset
|
||||
|
||||
|
||||
class AlpacaZhPreprocessor(AlpacaPreprocessor):
|
||||
|
||||
@classmethod
|
||||
def concat_inst_input(cls, instruction, input_):
|
||||
if input_ and input_.startswith('输入：'):
|
||||
input_ = input_[3:]
|
||||
return super().concat_inst_input(instruction, input_)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/alpaca-gpt4-data-zh',
|
||||
hf_dataset_id='llm-wizard/alpaca-gpt4-data-zh',
|
||||
preprocess_func=AlpacaZhPreprocessor(),
|
||||
tags=['chat', 'general', '🔥'],
|
||||
))
|
||||
|
||||
|
||||
class LongAlpacaPreprocessor(AlpacaPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
response = row['response']
|
||||
prefix_prompt = 'Answer: '
|
||||
if response and response.startswith(prefix_prompt):
|
||||
response = response[len(prefix_prompt):].strip()
|
||||
row['output'] = response
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/LongAlpaca-12k',
|
||||
hf_dataset_id='Yukang/LongAlpaca-12k',
|
||||
preprocess_func=LongAlpacaPreprocessor(),
|
||||
tags=['long-sequence', 'QA'],
|
||||
))
|
||||
|
||||
|
||||
class RuozhibaPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
title = row['title'] if row.get('title', None) is not None else row['content']
|
||||
abs = row['abs'] if 'abs' in row else None
|
||||
if abs and abs != title:
|
||||
title = title + '，' + abs
|
||||
|
||||
pattern = r'\d+[\.,\s,\、](.+)'
|
||||
match = re.search(pattern, title)
|
||||
if match:
|
||||
title = match.group(1)
|
||||
if title:
|
||||
return {'messages': [{'role': 'assistant', 'content': title}]}
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/ruozhiba',
|
||||
subsets=['post-annual', 'title-good', 'title-norm'],
|
||||
preprocess_func=RuozhibaPreprocessor(),
|
||||
tags=['pretrain', '🔥']))
|
||||
|
||||
|
||||
class MathTrnPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row):
|
||||
query = row['query']
|
||||
output = row['response']
|
||||
row = {
|
||||
'query': query,
|
||||
'response': output,
|
||||
}
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(ms_dataset_id='AI-ModelScope/math-trn-format', preprocess_func=MathTrnPreprocessor(), tags=['math']))
|
||||
|
||||
|
||||
def _repair_ms_bench(messages: str) -> Optional[List[Dict[str, str]]]:
|
||||
if isinstance(messages, str):
|
||||
messages = ast.literal_eval(messages)
|
||||
default_system = 'You are a helpful assistant.'
|
||||
messages: List[Dict[str, str]]
|
||||
if messages[0]['from'] == 'system' and messages[0]['value'] == default_system:
|
||||
messages.pop(0)
|
||||
# skip MOSS
|
||||
for c in messages:
|
||||
value = c['value'].lower()
|
||||
if 'moss' in value or 'human:' in value or 'assistant:' in value or 'user:' in value:
|
||||
return
|
||||
return messages
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='iic/ms_bench',
|
||||
preprocess_func=MessagesPreprocessor(repair_messages=_repair_ms_bench),
|
||||
tags=['chat', 'general', 'multi-round', '🔥']))
|
||||
|
||||
|
||||
def _repair_agent_messages(messages: List[Dict[str, str]], use_mini: bool) -> Optional[List[Dict[str, str]]]:
|
||||
if use_mini:
|
||||
pattern = r'\d\. {"plugin_name": "(.+?)"'
|
||||
if messages[0]['from'] != 'system':
|
||||
return
|
||||
system = messages[0]['value']
|
||||
find_list = re.findall(pattern, system)
|
||||
if len(set(find_list)) <= 1:
|
||||
return
|
||||
return messages
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='damo/MSAgent-Bench',
|
||||
subsets=[
|
||||
SubsetDataset(
|
||||
preprocess_func=MessagesPreprocessor(repair_messages=partial(_repair_agent_messages, use_mini=False))),
|
||||
SubsetDataset(
|
||||
name='mini',
|
||||
preprocess_func=MessagesPreprocessor(repair_messages=partial(_repair_agent_messages, use_mini=True)),
|
||||
is_weak_subset=True)
|
||||
],
|
||||
split=['train', 'validation'],
|
||||
tags=['chat', 'agent', 'multi-round']))
|
||||
|
||||
advertise_gen_prompt = """Task: Generating advertisements based on keywords.
|
||||
Keywords: {{QUERY}}
|
||||
Advertisements:"""
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='lvjianjin/AdvertiseGen',
|
||||
hf_dataset_id='shibing624/AdvertiseGen',
|
||||
preprocess_func=TextGenerationPreprocessor(
|
||||
prompt=advertise_gen_prompt, columns={
|
||||
'content': 'query',
|
||||
'summary': 'response'
|
||||
}),
|
||||
tags=['text-generation', '🔥'],
|
||||
split=['train', 'validation'],
|
||||
))
|
||||
|
||||
|
||||
class FireflyPreprocessor(ResponsePreprocessor):
|
||||
_firefly_kind_list = {
|
||||
'ProseGeneration', 'MRC', 'JinYongGeneration', 'TextCorrection', 'ClassicalChinese', 'BELLE', 'StoryGeneration',
|
||||
'Couplet', 'Cot', 'Dictionary', 'Translation', 'Program', 'SentimentAnalyze', 'OpenQA', 'AncientPoem',
|
||||
'TextMatching', 'NLI', 'Summary', 'KeywordRecognition', 'ProductDesc', 'LyricGeneration', 'Composition',
|
||||
'MusicComment', 'NER'
|
||||
}
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
if row['kind'] not in FireflyPreprocessor._firefly_kind_list:
|
||||
return
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/firefly-train-1.1M',
|
||||
hf_dataset_id='YeungNLP/firefly-train-1.1M',
|
||||
preprocess_func=FireflyPreprocessor(),
|
||||
tags=['chat', 'general'],
|
||||
))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='modelscope/clue',
|
||||
hf_dataset_id='clue',
|
||||
subsets=['cmnli'],
|
||||
preprocess_func=ClsGenerationPreprocessor(['neutral', 'entailment', 'contradiction'],
|
||||
task='Natural Language Inference',
|
||||
is_pair_seq=True),
|
||||
tags=['text-generation', 'classification'],
|
||||
split=['train', 'validation'],
|
||||
))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='DAMO_NLP/jd',
|
||||
subsets=[
|
||||
SubsetDataset(
|
||||
'default',
|
||||
'default',
|
||||
preprocess_func=ClsGenerationPreprocessor(['negative', 'positive'],
|
||||
task='Sentiment Classification',
|
||||
is_pair_seq=False)),
|
||||
SubsetDataset(
|
||||
'cls',
|
||||
'default',
|
||||
preprocess_func=ClsPreprocessor(columns={'sentence': 'query'}),
|
||||
),
|
||||
],
|
||||
tags=['text-generation', 'classification', '🔥'],
|
||||
split=['train', 'validation'],
|
||||
))
|
||||
|
||||
|
||||
class SyntheticText2SqlPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
sql_prompt = row['sql_prompt']
|
||||
sql_context = row['sql_context']
|
||||
sql = row['sql']
|
||||
sql_explanation = row['sql_explanation']
|
||||
query = f'Sql Table information:\n{sql_context}\n{sql_prompt}'
|
||||
response = f'Let\'s think step by step:\n{sql_explanation}\nSo the final sql is:\n{sql}'
|
||||
return super().preprocess({'query': query, 'response': response})
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/synthetic_text_to_sql',
|
||||
hf_dataset_id='gretelai/synthetic_text_to_sql',
|
||||
preprocess_func=SyntheticText2SqlPreprocessor(),
|
||||
tags=['nl2sql', 'en']))
|
||||
|
||||
|
||||
def _repair_toolbench(conversations: List[Dict[str, str]]) -> List[Dict[str, str]]:
|
||||
assert len(conversations) == 2
|
||||
if conversations[1]['from'] in {'caller', 'conclusion'}:
|
||||
conversations[1]['from'] = 'assistant'
|
||||
return conversations
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='shenweizhou/alpha-umi-toolbench-processed-v2',
|
||||
subsets=['backbone', 'caller', 'planner', 'summarizer'],
|
||||
preprocess_func=MessagesPreprocessor(repair_messages=_repair_toolbench),
|
||||
tags=['chat', 'agent', '🔥'],
|
||||
huge_dataset=True))
|
||||
|
||||
|
||||
class BlossomMathPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
output, answer = row['output'], row['answer']
|
||||
return super().preprocess({'query': row['query'], 'response': f'{output}\n\nAnswer: {answer}'})
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/blossom-math-v2',
|
||||
hf_dataset_id='Azure99/blossom-math-v2',
|
||||
preprocess_func=BlossomMathPreprocessor(),
|
||||
tags=['chat', 'math', '🔥']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/sql-create-context',
|
||||
hf_dataset_id='b-mc2/sql-create-context',
|
||||
preprocess_func=AlpacaPreprocessor(columns={
|
||||
'question': 'instruction',
|
||||
'context': 'input',
|
||||
'answer': 'output'
|
||||
}),
|
||||
tags=['chat', 'sql', '🔥']))
|
||||
|
||||
|
||||
class TigerBotLawPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prompt = """{type}
|
||||
{title}
|
||||
"""
|
||||
cur_prompt = prompt.format(type=row['type'], title=row['title'])
|
||||
for i in range(1, 4):
|
||||
chapter = row[f'chapter{i}']
|
||||
if chapter is not None:
|
||||
cur_prompt += f'{chapter}'
|
||||
cur_prompt += f'{row["response"]}'
|
||||
return super().preprocess({'response': cur_prompt})
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/tigerbot-law-plugin',
|
||||
hf_dataset_id='TigerResearch/tigerbot-law-plugin',
|
||||
preprocess_func=TigerBotLawPreprocessor(),
|
||||
tags=['text-generation', 'law', 'pretrained']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='codefuse-ai/CodeExercise-Python-27k',
|
||||
preprocess_func=MessagesPreprocessor(columns={'chat_rounds': 'messages'}),
|
||||
tags=['chat', 'coding', '🔥']))
|
||||
|
||||
|
||||
class LeetcodePythonPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
code_with_problem = row['code_with_problem']
|
||||
idx = code_with_problem.find('```python')
|
||||
problem = code_with_problem[:idx]
|
||||
if problem.startswith('# '):
|
||||
problem = problem[2:]
|
||||
code = code_with_problem[idx:].strip()
|
||||
explanation = row['explanation_only']
|
||||
return super().preprocess({'query': problem, 'response': f'{code}\n\n{explanation}'})
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/leetcode-solutions-python',
|
||||
preprocess_func=LeetcodePythonPreprocessor(),
|
||||
tags=['chat', 'coding', '🔥']))
|
||||
|
||||
|
||||
class StsbPreprocessor(RowPreprocessor):
|
||||
|
||||
def __init__(self, sim_threshold: Optional[float] = None):
|
||||
self.sim_threshold = sim_threshold
|
||||
super().__init__()
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
row = {
|
||||
'messages': [{
|
||||
'role': 'user',
|
||||
'content': row['sentence1']
|
||||
}],
|
||||
'positive_messages': [[{
|
||||
'role': 'user',
|
||||
'content': row['sentence2']
|
||||
}]],
|
||||
'label': row['score'],
|
||||
}
|
||||
if self.sim_threshold is None or float(row['label']) >= self.sim_threshold:
|
||||
return row
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class StsbGeneratePreprocessor(ResponsePreprocessor):
|
||||
prompt = """Task: Based on the given two sentences, provide a similarity score between 0.0 and 1.0.
|
||||
Sentence 1: {text1}
|
||||
Sentence 2: {text2}
|
||||
Similarity score: """
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
return super().preprocess({
|
||||
'query': self.prompt.format(text1=row['sentence1'], text2=row['sentence2']),
|
||||
'response': f"{row['score']:.1f}"
|
||||
})
|
||||
|
||||
|
||||
class StsbRegressionPreprocessor(StsbGeneratePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
return super(StsbGeneratePreprocessor, self).preprocess({
|
||||
'query':
|
||||
self.prompt.format(text1=row['sentence1'], text2=row['sentence2']),
|
||||
'label':
|
||||
row['score']
|
||||
})
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='sentence-transformers/stsb',
|
||||
hf_dataset_id='sentence-transformers/stsb',
|
||||
subsets=[
|
||||
SubsetDataset('default', preprocess_func=StsbPreprocessor()), # embedding
|
||||
SubsetDataset('positive', preprocess_func=StsbPreprocessor(sim_threshold=0.75)), # infonce
|
||||
SubsetDataset('generate', preprocess_func=StsbGeneratePreprocessor()),
|
||||
SubsetDataset('reg', preprocess_func=StsbRegressionPreprocessor()),
|
||||
],
|
||||
tags=['similarity', '🔥']))
|
||||
|
||||
|
||||
class MTEBRerankPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
query = row['query']
|
||||
positives = row['positive'] if isinstance(row['positive'], list) else [row['positive']]
|
||||
negatives = row['negative'] if isinstance(row['negative'], list) else [row['negative']]
|
||||
|
||||
messages = [{'role': 'user', 'content': query}]
|
||||
positive_messages = [[{'role': 'assistant', 'content': positive}] for positive in positives]
|
||||
negative_messages = [[{'role': 'assistant', 'content': negative}] for negative in negatives]
|
||||
|
||||
return {'messages': messages, 'positive_messages': positive_messages, 'negative_messages': negative_messages}
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='MTEB/scidocs-reranking',
|
||||
hf_dataset_id='mteb/scidocs-reranking',
|
||||
split=['validation', 'test'],
|
||||
preprocess_func=MTEBRerankPreprocessor(),
|
||||
tags=['rerank', '🔥']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='MTEB/stackoverflowdupquestions-reranking',
|
||||
hf_dataset_id='mteb/stackoverflowdupquestions-reranking',
|
||||
split=['train', 'test'],
|
||||
preprocess_func=MTEBRerankPreprocessor(),
|
||||
tags=['rerank', '🔥']))
|
||||
|
||||
|
||||
def _repair_conversations_agent_instruct(s: str) -> List[Dict[str, Any]]:
|
||||
s = s.replace('}\n {', '},\n {')
|
||||
if isinstance(s, str):
|
||||
s = ast.literal_eval(s)
|
||||
return s
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='huangjintao/AgentInstruct_copy',
|
||||
subsets=['alfworld', 'db', 'kg', 'mind2web', 'os', 'webshop'],
|
||||
preprocess_func=MessagesPreprocessor(repair_messages=_repair_conversations_agent_instruct),
|
||||
tags=['chat', 'agent', 'multi-round']))
|
||||
|
||||
|
||||
class MultiRoleAgentPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
conv = row['conversations']
|
||||
res_prompt = '\n\n【注意事项】\n1. 这是聊天室，不要发送私信给任何人\n2. 仅代表你个人说话,不要扮演其他人，只根据对话历史进行回复\n3. 长话短说，不要说太多话，不要超过50字 '
|
||||
history_prompt = '\n\n【chat history】'
|
||||
conv_prompt = '\n {name}:{content}'
|
||||
query, response = '', conv[-1]['value']
|
||||
system = conv[0]['value'] if conv[0]['from'] == 'system' else ''
|
||||
if conv[0]['from'] == 'user':
|
||||
query = conv[0]['value']
|
||||
elif 'next_speakers:' not in system:
|
||||
if '【注意事项】' not in system and system:
|
||||
system += res_prompt
|
||||
system += history_prompt
|
||||
system += ''.join([conv_prompt.format(name=c['from'], content=c['value']) for c in conv[1:-1]])
|
||||
|
||||
if not query or not response:
|
||||
return
|
||||
|
||||
return {
|
||||
'messages': [{
|
||||
'role': 'system',
|
||||
'content': system
|
||||
}, {
|
||||
'role': 'user',
|
||||
'content': query
|
||||
}, {
|
||||
'role': 'assistant',
|
||||
'content': response
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='iic/MSAgent-MultiRole',
|
||||
preprocess_func=MultiRoleAgentPreprocessor(),
|
||||
tags=['chat', 'agent', 'multi-round', 'role-play', 'multi-agent']))
|
||||
|
||||
register_dataset(DatasetMeta(ms_dataset_id='swift/ToolBench', tags=['chat', 'agent', 'multi-round']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='tastelikefeet/competition_math',
|
||||
subsets=[
|
||||
SubsetDataset(
|
||||
name='default',
|
||||
subset='default',
|
||||
split=['train', 'test'],
|
||||
),
|
||||
],
|
||||
tags=['qa', 'math']))
|
||||
|
||||
register_dataset(DatasetMeta(ms_dataset_id='modelscope/gsm8k', subsets=['main'], split=['train'], tags=['qa', 'math']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(ms_dataset_id='modelscope/MathR', subsets=['default', 'clean'], split=['train'], tags=['qa', 'math']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(ms_dataset_id='modelscope/MathR-32B-Distill', subsets=['data'], split=['train'], tags=['qa', 'math']))
|
||||
|
||||
|
||||
class CoundownTaskPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
numbers = row['nums']
|
||||
target = row.pop('response', None)
|
||||
query = (f'Using the numbers {numbers}, create an equation that equals {target}.\n'
|
||||
'You can use basic arithmetic operations (+, -, *, /) and each number can only be used once.\n'
|
||||
'Show your work in <think> </think> tags. And return the final equation and answer '
|
||||
'in <answer> </answer> tags, for example <answer> (1 + 2) / 3 * 4 = 4 </answer>.')
|
||||
row.update({'target': target, 'query': query})
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='zouxuhong/Countdown-Tasks-3to4',
|
||||
subsets=['default'],
|
||||
preprocess_func=CoundownTaskPreprocessor(),
|
||||
tags=['math']))
|
||||
|
||||
|
||||
class HC3Preprocessor(ResponsePreprocessor):
|
||||
prompt = """Classification Task: Are the following responses from a human or from ChatGPT?
|
||||
Question: {question}
|
||||
Answer: {answer}
|
||||
Category: Human, ChatGPT
|
||||
Output:"""
|
||||
|
||||
def preprocess(self, row):
|
||||
rows = []
|
||||
for response in ['Human', 'ChatGPT']:
|
||||
query = self.prompt.format(
|
||||
question=row['query'], answer=self.random_state.choice(row[f'{response.lower()}_answers']))
|
||||
rows.append(super().preprocess({'query': query, 'response': response}))
|
||||
return rows
|
||||
|
||||
|
||||
class HC3ClsPreprocessor(HC3Preprocessor):
|
||||
|
||||
def preprocess(self, row):
|
||||
rows = []
|
||||
for i, response in enumerate(['Human', 'ChatGPT']):
|
||||
query = self.prompt.format(
|
||||
question=row['query'], answer=self.random_state.choice(row[f'{response.lower()}_answers']))
|
||||
rows.append(ResponsePreprocessor.preprocess(self, {'query': query, 'label': i}))
|
||||
return rows
|
||||
|
||||
|
||||
hc3_subset_names = ['baike', 'open_qa', 'nlpcc_dbqa', 'finance', 'medicine', 'law', 'psychology']
|
||||
hc3_subsets: List[SubsetDataset] = []
|
||||
for hc3_subset_name in hc3_subset_names:
|
||||
hc3_subsets.append(
|
||||
SubsetDataset(
|
||||
name=hc3_subset_name,
|
||||
subset=hc3_subset_name,
|
||||
preprocess_func=HC3Preprocessor(),
|
||||
))
|
||||
hc3_subsets.append(
|
||||
SubsetDataset(
|
||||
name=f'{hc3_subset_name}_cls',
|
||||
subset=hc3_subset_name,
|
||||
preprocess_func=HC3ClsPreprocessor(),
|
||||
))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='simpleai/HC3-Chinese',
|
||||
hf_dataset_id='Hello-SimpleAI/HC3-Chinese',
|
||||
subsets=hc3_subsets,
|
||||
tags=['text-generation', 'classification', '🔥']))
|
||||
|
||||
hc3_subset_names = ['finance', 'medicine']
|
||||
hc3_subsets: List[SubsetDataset] = []
|
||||
for hc3_subset_name in hc3_subset_names:
|
||||
hc3_subsets.append(
|
||||
SubsetDataset(
|
||||
name=hc3_subset_name,
|
||||
subset=hc3_subset_name,
|
||||
preprocess_func=HC3Preprocessor(),
|
||||
))
|
||||
hc3_subsets.append(
|
||||
SubsetDataset(
|
||||
name=f'{hc3_subset_name}_cls',
|
||||
subset=hc3_subset_name,
|
||||
preprocess_func=HC3ClsPreprocessor(),
|
||||
))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='simpleai/HC3',
|
||||
hf_dataset_id='Hello-SimpleAI/HC3',
|
||||
subsets=hc3_subsets,
|
||||
preprocess_func=HC3Preprocessor(),
|
||||
tags=['text-generation', 'classification', '🔥']))
|
||||
|
||||
|
||||
class DureaderPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prompt = """Task: Question Generation
|
||||
Context: {context}
|
||||
Answer: {answer}
|
||||
Question:"""
|
||||
answer, context = row['text1'].split('[SEP]')
|
||||
return {
|
||||
'messages': [{
|
||||
'role': 'user',
|
||||
'content': prompt.format(context=context, answer=answer)
|
||||
}, {
|
||||
'role': 'assistant',
|
||||
'content': row['text2']
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='modelscope/DuReader_robust-QG',
|
||||
preprocess_func=DureaderPreprocessor(),
|
||||
split=['train', 'validation', 'test'],
|
||||
tags=['text-generation', '🔥']))
|
||||
|
||||
|
||||
class HHRLHFPreprocessor(RowPreprocessor):
|
||||
|
||||
@staticmethod
|
||||
def _to_messages(data):
|
||||
messages = []
|
||||
for query, response in zip(data[::2], data[1::2]):
|
||||
messages.append({'role': 'user', 'content': query})
|
||||
messages.append({'role': 'assistant', 'content': response})
|
||||
return messages
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
chosen = row['chosen'].strip()
|
||||
rejected = row['rejected'].strip()
|
||||
parts_chosen = [s.strip() for s in re.split('\n\nHuman:|\n\nAssistant:|\n\nHum:', chosen)]
|
||||
parts_rejected = [s.strip() for s in re.split('\n\nHuman:|\n\nAssistant:|\n\nHum:', rejected)]
|
||||
if parts_chosen[0].startswith('Human:'):
|
||||
assert parts_rejected[0].startswith('Human:')
|
||||
parts_chosen[0] = parts_chosen[0][6:].strip()
|
||||
parts_rejected[0] = parts_rejected[0][6:].strip()
|
||||
row['messages'] = self._to_messages(parts_chosen)
|
||||
row['rejected_messages'] = self._to_messages(parts_rejected)
|
||||
return row
|
||||
|
||||
|
||||
# TODO meta file broken
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/hh-rlhf',
|
||||
subsets=['helpful-base', 'helpful-online', 'helpful-rejection-sampled'],
|
||||
preprocess_func=HHRLHFPreprocessor(),
|
||||
split=['train', 'test'],
|
||||
tags=['rlhf', 'dpo'],
|
||||
huge_dataset=True))
|
||||
|
||||
|
||||
class XlamFunctionCallingPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
messages = [{'role': 'user', 'content': row['query']}]
|
||||
response = row['answers']
|
||||
response = json.loads(response)
|
||||
messages += [{'role': 'tool_call', 'content': json.dumps(content)} for content in response]
|
||||
return {'messages': messages, 'tools': row['tools']}
|
||||
|
||||
|
||||
class XlamFunctionCallingGRPOPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
query = row['query']
|
||||
answers = row['response']
|
||||
if isinstance(answers, str):
|
||||
answers = json.loads(answers)
|
||||
answer = np.random.choice(answers)
|
||||
name = answer['name']
|
||||
args = json.dumps(answer['arguments'])
|
||||
response = f'Action: {name}\nAction Input: {args}'
|
||||
row = {'query': query, 'response': response, 'solution': response, 'tools': row['tools']}
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='LLM-Research/xlam-function-calling-60k',
|
||||
hf_dataset_id='Salesforce/xlam-function-calling-60k',
|
||||
subsets=[
|
||||
SubsetDataset('default', 'dataset', preprocess_func=XlamFunctionCallingPreprocessor()),
|
||||
SubsetDataset('grpo', 'dataset', preprocess_func=XlamFunctionCallingGRPOPreprocessor())
|
||||
],
|
||||
tags=['agent', 'grpo', '🔥']))
|
||||
|
||||
|
||||
class HHRLHFCNPreprocessor(MessagesPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
row['messages'].append(row.pop('chosen'))
|
||||
row['rejected_response'] = row['rejected']['text']
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/hh_rlhf_cn',
|
||||
subsets=['hh_rlhf', 'harmless_base_cn', 'harmless_base_en', 'helpful_base_cn', 'helpful_base_en'],
|
||||
preprocess_func=HHRLHFCNPreprocessor(columns={'context': 'messages'}, content_key='text'),
|
||||
split=['train', 'test'],
|
||||
tags=['rlhf', 'dpo', '🔥']))
|
||||
|
||||
|
||||
def repair_conversations(s: Union[str, Any]) -> Any:
|
||||
if isinstance(s, str):
|
||||
s = s.replace('}\n {', '},{')
|
||||
s = s.replace('}\n{', '},{')
|
||||
s = s.replace('}{', '},{')
|
||||
s = s.replace('}\n {', '},{')
|
||||
return ast.literal_eval(s)
|
||||
return s
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/lmsys-chat-1m',
|
||||
hf_dataset_id='lmsys/lmsys-chat-1m',
|
||||
preprocess_func=MessagesPreprocessor(repair_messages=repair_conversations),
|
||||
tags=['chat', 'em']))
|
||||
|
||||
|
||||
class EmojiPreprocessr(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# Remove dirty characters
|
||||
row['query'] = row['query'].replace('️', '')
|
||||
row['response'] = row['response'].replace('️', '')
|
||||
row['rejected_response'] = row['rejected_response'].replace('️', '')
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='hjh0119/shareAI-Llama3-DPO-zh-en-emoji',
|
||||
hf_dataset_id='shareAI/DPO-zh-en-emoji',
|
||||
preprocess_func=EmojiPreprocessr(columns={
|
||||
'answer_zh': 'response',
|
||||
'answer_en': 'rejected_response'
|
||||
}),
|
||||
tags=['rlhf', 'dpo']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(ms_dataset_id='AI-ModelScope/ultrafeedback-binarized-preferences-cleaned-kto', tags=['rlhf', 'kto']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='OmniData/Zhihu-KOL-More-Than-100-Upvotes',
|
||||
hf_dataset_id='bzb2023/Zhihu-KOL-More-Than-100-Upvotes',
|
||||
tags=['zhihu', 'qa']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='OmniData/Zhihu-KOL',
|
||||
hf_dataset_id='wangrui6/Zhihu-KOL',
|
||||
huge_dataset=True,
|
||||
tags=['zhihu', 'qa'],
|
||||
))
|
||||
|
||||
|
||||
class GuanacoPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
instruction = row['instruction']
|
||||
input = row['input']
|
||||
output = row['output']
|
||||
history = []
|
||||
if instruction:
|
||||
parts = split_str_parts_by(
|
||||
instruction, ['User:', 'User：', 'Assistant：', 'Assistant:', 'Asssistent:', 'Assistent:', 'Assistenz:'])
|
||||
for idx, part in enumerate(parts):
|
||||
if idx % 2 == 0:
|
||||
if 'user' not in part['key'].lower():
|
||||
return
|
||||
history.append([part['content'], None])
|
||||
else:
|
||||
if 'assist' not in part['key'].lower() and 'asssist' not in part['key'].lower():
|
||||
return
|
||||
history[-1][-1] = part['content']
|
||||
if input.startswith('User:'):
|
||||
input = input[len('User:'):].strip()
|
||||
if any([not h[0] or not h[1] for h in history]):
|
||||
return
|
||||
|
||||
messages = []
|
||||
for h in history:
|
||||
messages.append({'role': 'user', 'content': h[0]})
|
||||
messages.append({'role': 'assistant', 'content': h[1]})
|
||||
messages.append({'role': 'user', 'content': input})
|
||||
messages.append({'role': 'assistant', 'content': output})
|
||||
return {
|
||||
'messages': messages,
|
||||
}
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/GuanacoDataset',
|
||||
hf_dataset_id='JosephusCheung/GuanacoDataset',
|
||||
preprocess_func=GuanacoPreprocessor(),
|
||||
tags=['chat', 'zh']))
|
||||
|
||||
|
||||
class FunctionCallChatmlPreprocessor(MessagesPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
res = super().preprocess(row)
|
||||
|
||||
if res['function_description']:
|
||||
res['tools'] = res['function_description'].split('\n\n')
|
||||
messages = res['messages']
|
||||
if messages[0]['role'] == 'system':
|
||||
messages.pop(0)
|
||||
return res
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/function-calling-chatml',
|
||||
hf_dataset_id='Locutusque/function-calling-chatml',
|
||||
preprocess_func=FunctionCallChatmlPreprocessor(),
|
||||
tags=['agent', 'en', 'sft', '🔥']))
|
||||
|
||||
|
||||
class Dolly15kPreprocessor(RowPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
instruction = row['instruction']
|
||||
context = row['context']
|
||||
response = row['response']
|
||||
query = ''
|
||||
if context:
|
||||
query = 'Here gives some useful information:\n'
|
||||
query += context
|
||||
query += '\n'
|
||||
query += instruction
|
||||
return {
|
||||
'messages': [{
|
||||
'role': 'user',
|
||||
'content': query
|
||||
}, {
|
||||
'role': 'assistant',
|
||||
'content': response
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/databricks-dolly-15k',
|
||||
hf_dataset_id='databricks/databricks-dolly-15k',
|
||||
preprocess_func=Dolly15kPreprocessor(),
|
||||
tags=['multi-task', 'en', 'quality']))
|
||||
|
||||
|
||||
class OrpoDPOMix40kPreprocessor(MessagesPreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
if row['source'] == 'toxic-dpo-v0.2':
|
||||
return
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='AI-ModelScope/orpo-dpo-mix-40k',
|
||||
hf_dataset_id='mlabonne/orpo-dpo-mix-40k',
|
||||
preprocess_func=OrpoDPOMix40kPreprocessor(columns={
|
||||
'chosen': 'messages',
|
||||
'rejected': 'rejected_messages'
|
||||
}),
|
||||
tags=['dpo', 'orpo', 'en', 'quality']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='swift/sharegpt',
|
||||
subsets=['common-zh', 'unknow-zh', 'common-en'],
|
||||
tags=['chat', 'general', 'multi-round']))
|
||||
|
||||
|
||||
class SelfCognitionPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def __init__(self, *args, query_suffix: str = '', response_prefix: str = '', **kwargs):
|
||||
self.query_suffix = query_suffix
|
||||
self.response_prefix = response_prefix
|
||||
self.name: Optional[Tuple[str, str]] = None
|
||||
self.author: Optional[Tuple[str, str]] = None
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def set_name_author(self, name, author):
|
||||
self.name = name
|
||||
self.author = author
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
for key in ['name', 'author']:
|
||||
val = getattr(self, key)
|
||||
if val is None:
|
||||
continue
|
||||
val = val[0] if row['tag'] == 'zh' else val[1]
|
||||
if val is None:
|
||||
continue
|
||||
placeholder = '{{' + key.upper() + '}}'
|
||||
row['query'] = row['query'].replace(placeholder, val)
|
||||
row['response'] = row['response'].replace(placeholder, val)
|
||||
|
||||
row['query'] = row['query'] + self.query_suffix
|
||||
row['response'] = self.response_prefix + row['response']
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='swift/self-cognition',
|
||||
hf_dataset_id='modelscope/self-cognition',
|
||||
subsets=[
|
||||
SubsetDataset(preprocess_func=SelfCognitionPreprocessor()),
|
||||
SubsetDataset(
|
||||
'qwen3',
|
||||
preprocess_func=SelfCognitionPreprocessor(
|
||||
query_suffix=' /no_think', response_prefix='<think>\n\n</think>\n\n')),
|
||||
SubsetDataset(
|
||||
'empty_think', preprocess_func=SelfCognitionPreprocessor(response_prefix='<think>\n\n</think>\n\n')),
|
||||
],
|
||||
dataset_name='self-cognition',
|
||||
tags=['chat', 'self-cognition', '🔥']))
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='open-r1/DAPO-Math-17k-Processed',
|
||||
hf_dataset_id='open-r1/DAPO-Math-17k-Processed',
|
||||
subsets=['all'],
|
||||
tags=['math', 'rlvr']))
|
||||
|
||||
|
||||
class SudokuPreprocessor(ResponsePreprocessor):
|
||||
prompt = ('Solve the following 9x9 Sudoku puzzle. '
|
||||
"Empty cells are marked with '0'. "
|
||||
'Provide the completed grid as your answer.\n\n'
|
||||
'Puzzle:\n{puzzle}')
|
||||
|
||||
@staticmethod
|
||||
def _format_grid(s: str) -> str:
|
||||
return '\n'.join(s[i:i + 9] for i in range(0, len(s), 9))
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
puzzle = row['query'].replace('.', '0')
|
||||
response = row['response']
|
||||
puzzle = self._format_grid(puzzle)
|
||||
response = self._format_grid(response)
|
||||
return super().preprocess({'query': self.prompt.format(puzzle=puzzle), 'response': response})
|
||||
|
||||
|
||||
register_dataset(
|
||||
DatasetMeta(
|
||||
ms_dataset_id='sapientinc/sudoku-extreme-1k',
|
||||
hf_dataset_id='sapientinc/sudoku-extreme-1k',
|
||||
preprocess_func=SudokuPreprocessor(),
|
||||
))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,202 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import numpy as np
|
||||
import os
|
||||
import shutil
|
||||
from abc import ABC, abstractmethod
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from datasets import Dataset as HfDataset
|
||||
from datasets import concatenate_datasets, interleave_datasets
|
||||
from modelscope.hub.api import ModelScopeConfig
|
||||
from modelscope.utils.config_ds import MS_CACHE_HOME
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from swift.utils import download_ms_file, get_logger, get_seed, safe_ddp_context
|
||||
from .preprocessor import DATASET_TYPE, AutoPreprocessor
|
||||
from .utils import sample_dataset
|
||||
|
||||
PreprocessFunc = Callable[..., DATASET_TYPE]
|
||||
logger = get_logger()
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .dataset_syntax import DatasetSyntax
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubsetDataset:
|
||||
# `Name` is used for matching subsets of the dataset, and `subset` refers to the subset_name on the hub.
|
||||
name: Optional[str] = None
|
||||
# If set to None, then subset is set to subset_name.
|
||||
subset: str = 'default'
|
||||
|
||||
# Higher priority. If set to None, the attributes of the DatasetMeta will be used.
|
||||
split: Optional[List[str]] = None
|
||||
preprocess_func: Optional[PreprocessFunc] = None
|
||||
|
||||
# If the dataset specifies "all," weak subsets will be skipped.
|
||||
is_weak_subset: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
if self.name is None:
|
||||
self.name = self.subset
|
||||
|
||||
def set_default(self, dataset_meta: 'DatasetMeta') -> 'SubsetDataset':
|
||||
subset_dataset = deepcopy(self)
|
||||
for k in ['split', 'preprocess_func']:
|
||||
v = getattr(subset_dataset, k)
|
||||
if v is None:
|
||||
setattr(subset_dataset, k, deepcopy(getattr(dataset_meta, k)))
|
||||
return subset_dataset
|
||||
|
||||
|
||||
class BaseDatasetLoader(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def load(
|
||||
self,
|
||||
dataset_syntax: Optional['DatasetSyntax'] = None,
|
||||
dataset_meta: Optional['DatasetMeta'] = None,
|
||||
*,
|
||||
use_hf: Optional[bool] = None,
|
||||
) -> HfDataset:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def download_ms_dataset(ms_dataset_id: str, files: List[str], force_download: bool = False) -> str:
|
||||
"""Download dataset from repo manually
|
||||
Args:
|
||||
ms_dataset_id: The dataset id of ModelScope
|
||||
files: Which files to download
|
||||
force_download: Force download or not
|
||||
Returns:
|
||||
The dataset dir
|
||||
"""
|
||||
assert isinstance(files, list)
|
||||
url = f'http://www.modelscope.cn/api/v1/datasets/{ms_dataset_id}/repo?Revision=master&FilePath={{fpath}}'
|
||||
cache_dir = os.path.join(MS_CACHE_HOME, 'datasets', ms_dataset_id, 'master')
|
||||
local_dir = os.path.join(cache_dir, 'raw')
|
||||
tmp_dir = os.path.join(cache_dir, 'tmp')
|
||||
os.makedirs(local_dir, exist_ok=True)
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
cookies = ModelScopeConfig.get_cookies()
|
||||
with TemporaryDirectory(dir=tmp_dir) as temp_dir:
|
||||
for remote_fpath in files:
|
||||
url = url.format(fpath=remote_fpath)
|
||||
temp_fpath = os.path.join(temp_dir, remote_fpath)
|
||||
local_fpath = os.path.join(local_dir, remote_fpath)
|
||||
if not force_download and os.path.exists(local_fpath):
|
||||
continue
|
||||
download_ms_file(url, temp_fpath, cookies)
|
||||
shutil.copy2(temp_fpath, local_fpath)
|
||||
|
||||
return local_dir
|
||||
|
||||
@staticmethod
|
||||
def concat_datasets(datasets: List[HfDataset]) -> Optional[HfDataset]:
|
||||
if len(datasets) == 0:
|
||||
return
|
||||
if len(datasets) == 1:
|
||||
return datasets[0]
|
||||
return concatenate_datasets(datasets)
|
||||
|
||||
@staticmethod
|
||||
def interleave_datasets(datasets, *args, **kwargs):
|
||||
if len(datasets) == 0:
|
||||
return
|
||||
if len(datasets) == 1:
|
||||
return datasets[0]
|
||||
return interleave_datasets(datasets, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def shuffle_dataset(dataset, seed: int, buffer_size: int = 1000):
|
||||
if isinstance(dataset, HfDataset):
|
||||
with safe_ddp_context(None, True):
|
||||
return dataset.shuffle(seed=seed)
|
||||
else:
|
||||
return dataset.shuffle(seed=seed, buffer_size=buffer_size)
|
||||
|
||||
@staticmethod
|
||||
def post_process(
|
||||
train_dataset: DATASET_TYPE,
|
||||
*,
|
||||
dataset_sample: Optional[int] = None,
|
||||
split_dataset_ratio: float = 0.,
|
||||
streaming: bool = False,
|
||||
shuffle: bool = True,
|
||||
random_state: Optional[np.random.RandomState] = None,
|
||||
) -> Tuple[DATASET_TYPE, Optional[DATASET_TYPE]]:
|
||||
"""Split into train/val datasets and perform dataset sampling."""
|
||||
assert dataset_sample is None or dataset_sample > 0
|
||||
assert 0 <= split_dataset_ratio <= 1
|
||||
if streaming:
|
||||
if dataset_sample is None:
|
||||
if split_dataset_ratio == 0:
|
||||
val_dataset = None
|
||||
elif split_dataset_ratio == 1:
|
||||
train_dataset, val_dataset = None, train_dataset
|
||||
else:
|
||||
raise ValueError('The IterableDataset does not support splitting the training set '
|
||||
'and validation set when dataset_sample is None.')
|
||||
else:
|
||||
# not shuffle
|
||||
train_dataset = train_dataset.take(dataset_sample)
|
||||
val_sample = int(dataset_sample * split_dataset_ratio)
|
||||
val_dataset = None if val_sample == 0 else train_dataset.take(val_sample)
|
||||
if val_sample:
|
||||
train_dataset = train_dataset.skip(val_sample)
|
||||
else:
|
||||
if dataset_sample is None:
|
||||
dataset_sample = len(train_dataset)
|
||||
if split_dataset_ratio == 0:
|
||||
train_dataset = sample_dataset(train_dataset, dataset_sample, shuffle, random_state)
|
||||
val_dataset = None
|
||||
elif split_dataset_ratio == 1:
|
||||
train_dataset, val_dataset = None, train_dataset
|
||||
val_sample = dataset_sample
|
||||
# Avoid duplication in the val_dataset.
|
||||
assert val_sample <= len(val_dataset), f'val_sample: {val_sample}, len(val_dataset): {len(val_dataset)}'
|
||||
val_dataset = sample_dataset(val_dataset, val_sample, shuffle, random_state)
|
||||
else:
|
||||
# Avoid duplication in the val_dataset.
|
||||
train_len = min(len(train_dataset), dataset_sample)
|
||||
val_sample = max(int(train_len * split_dataset_ratio), 1)
|
||||
train_sample = dataset_sample - val_sample
|
||||
assert train_sample > 0
|
||||
with safe_ddp_context(None, True):
|
||||
train_dataset, val_dataset = train_dataset.train_test_split(
|
||||
test_size=val_sample, shuffle=shuffle, seed=get_seed(random_state)).values()
|
||||
train_dataset = sample_dataset(train_dataset, train_sample, shuffle, random_state)
|
||||
return train_dataset, val_dataset
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasetMeta:
|
||||
ms_dataset_id: Optional[str] = None
|
||||
hf_dataset_id: Optional[str] = None
|
||||
dataset_path: Optional[str] = None # or dataset_dir
|
||||
dataset_name: Optional[str] = None
|
||||
ms_revision: Optional[str] = None
|
||||
hf_revision: Optional[str] = None
|
||||
|
||||
subsets: List[Union[SubsetDataset, str]] = field(default_factory=lambda: ['default'])
|
||||
# Applicable to all subsets.
|
||||
split: List[str] = field(default_factory=lambda: ['train'])
|
||||
# First perform column mapping, then proceed with the preprocess_func.
|
||||
preprocess_func: PreprocessFunc = field(default_factory=lambda: AutoPreprocessor())
|
||||
loader: Optional[BaseDatasetLoader] = None
|
||||
|
||||
tags: List[str] = field(default_factory=list)
|
||||
help: Optional[str] = None
|
||||
huge_dataset: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
from .loader import DatasetLoader
|
||||
if self.loader is None:
|
||||
self.loader = DatasetLoader
|
||||
for i, subset in enumerate(self.subsets):
|
||||
if isinstance(subset, str):
|
||||
self.subsets[i] = SubsetDataset(subset=subset)
|
||||
|
||||
|
||||
DATASET_MAPPING: Dict[Tuple[str, str, str], DatasetMeta] = {}
|
||||
@@ -0,0 +1,127 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Literal, Optional, Tuple
|
||||
|
||||
from .dataset_meta import DATASET_MAPPING, DatasetMeta
|
||||
|
||||
_dataset_meta_mapping = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasetSyntax:
|
||||
dataset: str
|
||||
subsets: List[str] = field(default_factory=list)
|
||||
dataset_sample: Optional[int] = None
|
||||
use_hf: Optional[bool] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if os.path.isfile(self.dataset):
|
||||
self.dataset_type = 'path'
|
||||
else: # dataset_id or dataset_dir
|
||||
self.dataset_type = 'repo'
|
||||
|
||||
def get_raw(self):
|
||||
subsets = '/'.join(self.subsets)
|
||||
dataset_sample = '' if self.dataset_sample is None else f'#{self.dataset_sample}'
|
||||
return f'{self.dataset}{subsets}{dataset_sample}'
|
||||
|
||||
@staticmethod
|
||||
def _safe_split(s: str,
|
||||
sep: str,
|
||||
use_0: bool,
|
||||
split_mode: Literal['left', 'right'] = 'left') -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
use_0: When the length of the part is 1, is it considered as part0 or part1.
|
||||
split_mode: use split or rsplit
|
||||
"""
|
||||
if s is None or len(s) == 0:
|
||||
return None, None
|
||||
if split_mode == 'left':
|
||||
part = s.split(sep, 1)
|
||||
else:
|
||||
part = s.rsplit(sep, 1)
|
||||
if len(part) == 1:
|
||||
if use_0:
|
||||
part = part[0], None
|
||||
else:
|
||||
part = None, part[0]
|
||||
else:
|
||||
assert len(part) == 2
|
||||
return part
|
||||
|
||||
@classmethod
|
||||
def parse(cls, dataset: str) -> 'DatasetSyntax':
|
||||
"""Parse the dataset from the command line"""
|
||||
# hf/ms::dataset_id or dataset_path:subset1/subset2/subset3#dataset_sample
|
||||
if os.path.exists(dataset):
|
||||
use_hf = None
|
||||
else:
|
||||
use_hf, dataset = cls._safe_split(dataset, '::', False)
|
||||
if isinstance(use_hf, str):
|
||||
use_hf = use_hf.lower()
|
||||
use_hf = {'hf': True, 'ms': False}.get(use_hf)
|
||||
if os.path.exists(dataset):
|
||||
other, dataset_sample = dataset, None
|
||||
else:
|
||||
other, dataset_sample = cls._safe_split(dataset, '#', True, 'right')
|
||||
if os.path.exists(other):
|
||||
dataset, subsets = other, None
|
||||
else:
|
||||
dataset, subsets = cls._safe_split(other, ':', True)
|
||||
|
||||
if subsets is not None:
|
||||
subsets = [subset.strip() for subset in subsets.split('/')]
|
||||
if dataset_sample is not None:
|
||||
dataset_sample = int(dataset_sample)
|
||||
return cls(dataset.strip(), subsets or [], dataset_sample, use_hf)
|
||||
|
||||
def get_dataset_meta(self, use_hf: bool):
|
||||
dataset_meta_mapping = self._get_dataset_meta_mapping()
|
||||
dataset_type = self.dataset_type
|
||||
if dataset_type == 'path':
|
||||
dataset_meta = dataset_meta_mapping.get((dataset_type, self.dataset))
|
||||
else:
|
||||
dataset_type = 'repo' if os.path.isdir(self.dataset) else {True: 'hf', False: 'ms'}[use_hf]
|
||||
dataset_meta = dataset_meta_mapping.get((dataset_type, self.dataset))
|
||||
return dataset_meta or self._get_matched_dataset_meta(dataset_meta_mapping) or DatasetMeta()
|
||||
|
||||
@staticmethod
|
||||
def _get_dataset_meta_mapping() -> Dict[Tuple[str, str], DatasetMeta]:
|
||||
global _dataset_meta_mapping
|
||||
if _dataset_meta_mapping is not None:
|
||||
return _dataset_meta_mapping
|
||||
_dataset_meta_mapping = {}
|
||||
for dataset_meta in DATASET_MAPPING.values():
|
||||
if dataset_meta.dataset_path is not None:
|
||||
dataset_type = 'repo' if os.path.isdir(dataset_meta.dataset_path) else 'path'
|
||||
_dataset_meta_mapping[(dataset_type, dataset_meta.dataset_path)] = dataset_meta
|
||||
if dataset_meta.ms_dataset_id is not None:
|
||||
_dataset_meta_mapping[('ms', dataset_meta.ms_dataset_id)] = dataset_meta
|
||||
if dataset_meta.hf_dataset_id is not None:
|
||||
_dataset_meta_mapping[('hf', dataset_meta.hf_dataset_id)] = dataset_meta
|
||||
return _dataset_meta_mapping
|
||||
|
||||
@staticmethod
|
||||
def get_dataset_name(dataset_id: str) -> str:
|
||||
# compat hf hub
|
||||
dataset_id = dataset_id.rstrip('/')
|
||||
match_ = re.search('/datasets--.+?--(.+?)/snapshots/', dataset_id)
|
||||
if match_ is not None:
|
||||
return match_.group(1)
|
||||
|
||||
dataset_name = dataset_id.rsplit('/', 1)[-1]
|
||||
if platform.system().lower() == 'windows':
|
||||
dataset_name = dataset_name.rsplit('\\', 1)[-1]
|
||||
return dataset_name
|
||||
|
||||
def _get_matched_dataset_meta(self, dataset_meta_mapping):
|
||||
suffix_dataset_meta_mapping = {}
|
||||
for dataset_name, dataset_meta in dataset_meta_mapping.items():
|
||||
dataset_name = self.get_dataset_name(dataset_name[1])
|
||||
suffix_dataset_meta_mapping[dataset_name] = dataset_meta
|
||||
dataset_name = self.get_dataset_name(self.dataset)
|
||||
dataset_meta = suffix_dataset_meta_mapping.get(dataset_name)
|
||||
return dataset_meta
|
||||
@@ -0,0 +1,132 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import bisect
|
||||
import mmap
|
||||
import os
|
||||
import pickle
|
||||
import threading
|
||||
from modelscope.hub.utils.utils import get_cache_dir
|
||||
from queue import Queue
|
||||
from torch.utils.data import Dataset
|
||||
from typing import Any, List
|
||||
|
||||
|
||||
class IndexedDatasetBuilder:
|
||||
CHUNK_SIZE = 1e10
|
||||
|
||||
def __init__(self, dataset_name: str):
|
||||
self.cache_dir = IndexedDataset.get_cache_dir(dataset_name)
|
||||
self.n_shard = 1
|
||||
self.bin_path = os.path.join(self.cache_dir, IndexedDataset.BIN_FNAME.format(0))
|
||||
self.idx_path = os.path.join(self.cache_dir, IndexedDataset.IDX_FNAME)
|
||||
if os.path.exists(self.bin_path):
|
||||
os.remove(self.bin_path)
|
||||
self.bin_file = open(self.bin_path, 'ab')
|
||||
self.length_list = []
|
||||
self.idx_list = [0]
|
||||
self.shard_offset = [0]
|
||||
self._thread = None
|
||||
self._queue = Queue(maxsize=1000)
|
||||
|
||||
def _write_worker(self):
|
||||
while True:
|
||||
items = self._queue.get()
|
||||
if items is None:
|
||||
break
|
||||
bin_buffer = []
|
||||
for item in items:
|
||||
item_buffer = pickle.dumps(item)
|
||||
bin_buffer.append(item_buffer)
|
||||
self.idx_list.append(self.idx_list[-1] + len(item_buffer))
|
||||
self.length_list.append(item['length'])
|
||||
self.bin_file.write(b''.join(bin_buffer))
|
||||
offset = self.idx_list[-1] - self.shard_offset[-1]
|
||||
if offset >= self.CHUNK_SIZE:
|
||||
self.bin_file.close()
|
||||
self.bin_path = os.path.join(self.cache_dir, IndexedDataset.BIN_FNAME.format(self.n_shard))
|
||||
self.shard_offset.append(self.shard_offset[-1] + offset)
|
||||
self.n_shard += 1
|
||||
if os.path.exists(self.bin_path):
|
||||
os.remove(self.bin_path)
|
||||
self.bin_file = open(self.bin_path, 'ab')
|
||||
|
||||
def add_items(self, items: List[Any]) -> None:
|
||||
if self._thread is None:
|
||||
self._thread = threading.Thread(target=self._write_worker, daemon=True)
|
||||
self._thread.start()
|
||||
self._queue.put(items)
|
||||
|
||||
def finalize(self):
|
||||
if self._thread is not None:
|
||||
self._queue.put(None)
|
||||
self._thread.join()
|
||||
self.bin_file.close()
|
||||
idx_obj = {
|
||||
'idx': self.idx_list,
|
||||
'length': self.length_list,
|
||||
'n_shard': self.n_shard,
|
||||
'shard_offset': self.shard_offset,
|
||||
}
|
||||
with open(self.idx_path, 'wb') as f:
|
||||
pickle.dump(idx_obj, f)
|
||||
|
||||
|
||||
class BinReader:
|
||||
|
||||
def __init__(self, bin_path: str):
|
||||
self.bin_path = bin_path
|
||||
self.file = open(bin_path, 'rb')
|
||||
try:
|
||||
self.mm = mmap.mmap(self.file.fileno(), 0, access=mmap.ACCESS_READ)
|
||||
except ValueError:
|
||||
# For example, self.file is an empty file.
|
||||
self.mm = None
|
||||
|
||||
def read_buffer(self, offset: int, size: int) -> bytes:
|
||||
if offset < 0 or size < 0 or offset + size > len(self.mm):
|
||||
raise ValueError('Invalid offset or size')
|
||||
return self.mm[offset:offset + size]
|
||||
|
||||
def __del__(self):
|
||||
if self.mm is not None:
|
||||
self.mm.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class IndexedDataset(Dataset):
|
||||
BIN_FNAME = 'data-{:05d}.bin'
|
||||
IDX_FNAME = 'data.idx'
|
||||
|
||||
@staticmethod
|
||||
def get_cache_dir(dataset_name: str):
|
||||
cache_dir = os.getenv('PACKING_CACHE') or os.path.join(get_cache_dir(), 'tmp')
|
||||
cache_dir = os.path.join(cache_dir, dataset_name)
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
assert dataset_name is not None, f'dataset_name: {dataset_name}'
|
||||
return cache_dir
|
||||
|
||||
def __init__(self, dataset_name: str):
|
||||
self.dataset_name = dataset_name
|
||||
cache_dir = self.get_cache_dir(dataset_name)
|
||||
self.idx_path = os.path.join(cache_dir, self.IDX_FNAME)
|
||||
with open(self.idx_path, 'rb') as f:
|
||||
idx_obj = pickle.load(f)
|
||||
self.idx_list = idx_obj['idx']
|
||||
self.length_list = idx_obj['length']
|
||||
self.n_shard = idx_obj['n_shard']
|
||||
self.shard_offset = idx_obj['shard_offset']
|
||||
self.bin_readers = []
|
||||
for i in range(self.n_shard):
|
||||
bin_path = os.path.join(cache_dir, self.BIN_FNAME.format(i))
|
||||
self.bin_readers.append(BinReader(bin_path))
|
||||
|
||||
def __getitem__(self, index: int):
|
||||
if index < 0:
|
||||
index = index % len(self)
|
||||
idx, idx_next = self.idx_list[index], self.idx_list[index + 1]
|
||||
num_shard = bisect.bisect_right(self.shard_offset, idx)
|
||||
offset = self.shard_offset[num_shard - 1]
|
||||
buffer = self.bin_readers[num_shard - 1].read_buffer(idx - offset, idx_next - idx)
|
||||
return pickle.loads(buffer)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.idx_list) - 1
|
||||
@@ -0,0 +1,384 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import numpy as np
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from datasets import Dataset as HfDataset
|
||||
from datasets import IterableDataset as HfIterableDataset
|
||||
from datasets import load_dataset as hf_load_dataset
|
||||
from functools import partial
|
||||
from modelscope.hub.utils.utils import get_cache_dir
|
||||
from typing import Dict, List, Literal, Optional, Tuple, Union
|
||||
|
||||
from swift.hub import get_hub
|
||||
from swift.utils import get_logger, get_seed, safe_ddp_context, use_hf_hub
|
||||
from .dataset_meta import DATASET_TYPE, BaseDatasetLoader
|
||||
from .dataset_syntax import DatasetSyntax
|
||||
from .preprocessor import RowPreprocessor
|
||||
from .register import DATASET_MAPPING, DatasetMeta, SubsetDataset
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class DatasetLoader(BaseDatasetLoader):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_proc: int = 1,
|
||||
load_from_cache_file: bool = True,
|
||||
streaming: bool = False,
|
||||
hub_token: Optional[str] = None,
|
||||
strict: bool = False,
|
||||
download_mode: Literal['force_redownload', 'reuse_dataset_if_exists'] = 'reuse_dataset_if_exists',
|
||||
columns: Optional[Dict[str, str]] = None,
|
||||
remove_unused_columns: bool = True,
|
||||
disable_auto_column_mapping: bool = False,
|
||||
):
|
||||
self.num_proc = num_proc
|
||||
self.load_from_cache_file = load_from_cache_file
|
||||
self.streaming = streaming
|
||||
self.hub_token = hub_token
|
||||
self.strict = strict
|
||||
self.download_mode = download_mode
|
||||
self.columns = columns
|
||||
self.remove_unused_columns = remove_unused_columns
|
||||
self.disable_auto_column_mapping = disable_auto_column_mapping
|
||||
|
||||
def _load_dataset_path(
|
||||
self,
|
||||
dataset_path: str,
|
||||
dataset_meta: DatasetMeta,
|
||||
) -> HfDataset:
|
||||
ext = os.path.splitext(dataset_path)[1].lstrip('.')
|
||||
file_type = {'jsonl': 'json', 'txt': 'text'}.get(ext) or ext
|
||||
kwargs = {'split': 'train', 'streaming': self.streaming, 'num_proc': self.num_proc}
|
||||
if file_type == 'csv':
|
||||
kwargs['na_filter'] = False
|
||||
with safe_ddp_context(None, True):
|
||||
kwargs['cache_dir'] = os.path.join(get_cache_dir(), 'datasets')
|
||||
dataset = hf_load_dataset(file_type, data_files=dataset_path, **kwargs)
|
||||
if self.columns:
|
||||
dataset = RowPreprocessor.safe_rename_columns(dataset, self.columns)
|
||||
dataset = dataset_meta.preprocess_func(
|
||||
dataset,
|
||||
num_proc=self.num_proc,
|
||||
load_from_cache_file=self.load_from_cache_file,
|
||||
strict=self.strict,
|
||||
enable_auto_mapping=not self.disable_auto_column_mapping)
|
||||
if self.remove_unused_columns:
|
||||
dataset = RowPreprocessor.remove_useless_columns(dataset)
|
||||
return dataset
|
||||
|
||||
def _load_repo_dataset(
|
||||
self,
|
||||
dataset_id: str,
|
||||
subset: SubsetDataset,
|
||||
*,
|
||||
use_hf: Optional[bool] = None,
|
||||
revision: Optional[str] = None,
|
||||
) -> HfDataset:
|
||||
datasets = []
|
||||
if os.path.isdir(dataset_id):
|
||||
retry = 1
|
||||
load_context = nullcontext
|
||||
use_hf = True
|
||||
dataset_str = f'Use local folder, dataset_dir: {dataset_id}'
|
||||
# The dataset downloaded from modelscope will have an additional dataset_infos.json file.
|
||||
with safe_ddp_context('dataset_infos_rename'):
|
||||
dataset_infos_path = os.path.join(dataset_id, 'dataset_infos.json')
|
||||
if os.path.isfile(dataset_infos_path):
|
||||
os.rename(dataset_infos_path, f'{dataset_infos_path}_bak')
|
||||
elif dataset_id.startswith('/'):
|
||||
raise ValueError(f'The local path does not exist, dataset_id: `{dataset_id}`. '
|
||||
f'os.path.exists(dataset_id): {os.path.exists(dataset_id)}')
|
||||
else:
|
||||
retry = 3
|
||||
load_context = partial(safe_ddp_context, hash_id=dataset_id, use_barrier=True)
|
||||
dataset_str_f = 'Downloading the dataset from {hub}, dataset_id: {dataset_id}'
|
||||
if use_hf:
|
||||
dataset_str = dataset_str_f.format(hub='HuggingFace', dataset_id=dataset_id)
|
||||
else:
|
||||
dataset_str = dataset_str_f.format(hub='ModelScope', dataset_id=dataset_id)
|
||||
logger.info(dataset_str)
|
||||
hub = get_hub(use_hf)
|
||||
for split in subset.split:
|
||||
i = 1
|
||||
with load_context():
|
||||
while True:
|
||||
try:
|
||||
dataset = hub.load_dataset(
|
||||
dataset_id,
|
||||
subset.subset,
|
||||
split,
|
||||
streaming=self.streaming,
|
||||
revision=revision,
|
||||
download_mode=self.download_mode,
|
||||
hub_token=self.hub_token,
|
||||
num_proc=self.num_proc)
|
||||
except Exception as e:
|
||||
if i == retry:
|
||||
raise
|
||||
i += 1
|
||||
logger.error(f'Dataset {dataset_id} load failed: subset_name={subset.subset},'
|
||||
f'split={split} with error: {e}')
|
||||
else:
|
||||
break
|
||||
if hasattr(dataset, '_hf_ds'):
|
||||
dataset = dataset._hf_ds
|
||||
if self.streaming and isinstance(dataset, HfDataset):
|
||||
dataset = dataset.to_iterable_dataset()
|
||||
if self.columns:
|
||||
dataset = RowPreprocessor.safe_rename_columns(dataset, self.columns)
|
||||
dataset = subset.preprocess_func(
|
||||
dataset,
|
||||
num_proc=self.num_proc,
|
||||
load_from_cache_file=self.load_from_cache_file,
|
||||
strict=self.strict,
|
||||
enable_auto_mapping=not self.disable_auto_column_mapping)
|
||||
if self.remove_unused_columns:
|
||||
dataset = RowPreprocessor.remove_useless_columns(dataset)
|
||||
datasets.append(dataset)
|
||||
return self.concat_datasets(datasets)
|
||||
|
||||
@staticmethod
|
||||
def _select_subsets(subsets: List[str], dataset_meta: DatasetMeta) -> List[SubsetDataset]:
|
||||
subset_mapping = {subset.name: subset for subset in dataset_meta.subsets}
|
||||
subset_names = list(subset_mapping.keys())
|
||||
if not subsets:
|
||||
if len(subset_names) <= 1:
|
||||
subsets = subset_names
|
||||
elif 'default' in subset_names:
|
||||
subsets = ['default']
|
||||
else:
|
||||
raise ValueError(f'Please provide subsets. available subsets: {subset_names}')
|
||||
elif len(subsets) == 1 and subsets[0] == 'all' and 'all' not in subset_names:
|
||||
subsets = [subset_name for subset_name in subset_names if not subset_mapping[subset_name].is_weak_subset]
|
||||
|
||||
subsets = [
|
||||
subset_mapping[subset_name] if subset_name in subset_mapping else SubsetDataset(subset=subset_name)
|
||||
for subset_name in subsets
|
||||
]
|
||||
return [subset.set_default(dataset_meta) for subset in subsets]
|
||||
|
||||
def load(
|
||||
self,
|
||||
dataset_syntax: Optional[DatasetSyntax] = None,
|
||||
dataset_meta: Optional[DatasetMeta] = None,
|
||||
*,
|
||||
use_hf: Optional[bool] = None,
|
||||
) -> HfDataset:
|
||||
if dataset_syntax.dataset_type == 'path':
|
||||
dataset = self._load_dataset_path(
|
||||
dataset_syntax.dataset,
|
||||
dataset_meta=dataset_meta,
|
||||
)
|
||||
else:
|
||||
subsets: List[SubsetDataset] = self._select_subsets(dataset_syntax.subsets, dataset_meta)
|
||||
revision = dataset_meta.hf_revision if use_hf else dataset_meta.ms_revision
|
||||
datasets = []
|
||||
for subset in subsets:
|
||||
dataset = self._load_repo_dataset(
|
||||
dataset_syntax.dataset,
|
||||
subset,
|
||||
use_hf=use_hf,
|
||||
revision=revision,
|
||||
)
|
||||
datasets.append(dataset)
|
||||
dataset = self.concat_datasets(datasets)
|
||||
return dataset
|
||||
|
||||
|
||||
def init_self_cognition_preprocessor(
|
||||
dataset_meta: Optional[DatasetMeta],
|
||||
model_name: Optional[Union[Tuple[str, str], List[str]]] = None,
|
||||
model_author: Optional[Union[Tuple[str, str], List[str]]] = None,
|
||||
) -> None:
|
||||
from .dataset.llm import SelfCognitionPreprocessor
|
||||
if dataset_meta is None or model_name is None and model_author is None:
|
||||
return
|
||||
kwargs = {}
|
||||
# zh, en
|
||||
for key in ['name', 'author']:
|
||||
val = locals()[f'model_{key}']
|
||||
if isinstance(val, str):
|
||||
val = [val]
|
||||
if val is not None and val[0] is not None and (len(val) == 1 or val[1] is None):
|
||||
val = (val[0], val[0])
|
||||
kwargs[key] = val
|
||||
|
||||
preprocess_funcs = [dataset_meta.preprocess_func]
|
||||
preprocess_funcs += [subset.preprocess_func for subset in dataset_meta.subsets if isinstance(subset, SubsetDataset)]
|
||||
for preprocess_func in preprocess_funcs:
|
||||
if isinstance(preprocess_func, SelfCognitionPreprocessor):
|
||||
preprocess_func.set_name_author(**kwargs)
|
||||
logger.info_once(f"SelfCognitionPreprocessor has been successfully configured with name: {kwargs['name']}, "
|
||||
f"author: {kwargs['author']}.")
|
||||
|
||||
|
||||
def _inject_dataset_routing_tag(dataset: DATASET_TYPE, ds_name: str) -> DATASET_TYPE:
|
||||
"""Inject ``dataset`` column for multi-teacher routing (constant per source dataset)."""
|
||||
if isinstance(dataset, HfIterableDataset):
|
||||
return dataset.map(lambda example: {**example, 'dataset': ds_name})
|
||||
return dataset.add_column('dataset', [ds_name] * len(dataset))
|
||||
|
||||
|
||||
def load_dataset(
|
||||
datasets: Union[List[str], str],
|
||||
*,
|
||||
split_dataset_ratio: float = 0.,
|
||||
seed: Union[int, np.random.RandomState, None] = 42,
|
||||
num_proc: int = 1,
|
||||
load_from_cache_file: bool = True,
|
||||
shuffle: bool = False,
|
||||
streaming: bool = False,
|
||||
interleave_prob: Optional[List[float]] = None,
|
||||
stopping_strategy: Literal['first_exhausted', 'all_exhausted'] = 'first_exhausted',
|
||||
shuffle_buffer_size: int = 1000,
|
||||
use_hf: Optional[bool] = None,
|
||||
hub_token: Optional[str] = None,
|
||||
strict: bool = False,
|
||||
download_mode: Literal['force_redownload', 'reuse_dataset_if_exists'] = 'reuse_dataset_if_exists',
|
||||
columns: Optional[Dict[str, str]] = None, # columns_mapping
|
||||
remove_unused_columns: bool = True,
|
||||
disable_auto_column_mapping: bool = False,
|
||||
# self-cognition
|
||||
model_name: Optional[Union[Tuple[str, str], List[str]]] = None, # zh, en
|
||||
model_author: Optional[Union[Tuple[str, str], List[str]]] = None,
|
||||
) -> Tuple[DATASET_TYPE, Optional[DATASET_TYPE]]:
|
||||
"""Load and preprocess datasets.
|
||||
|
||||
This function provides a unified interface to load datasets from various sources (HuggingFace,
|
||||
ModelScope, or local paths), with support for splitting, shuffling, streaming, and interleaving
|
||||
multiple datasets. It also handles self-cognition dataset preprocessing for model training.
|
||||
|
||||
Args:
|
||||
datasets: Single dataset name or list of dataset names to load. Can use special syntax
|
||||
for advanced configurations (e.g., 'dataset_name#1000' for sampling).
|
||||
split_dataset_ratio: Ratio for splitting dataset into train/validation sets.
|
||||
Value between 0 and 1. If 0, no validation split is created. Default: 0.
|
||||
seed: Random seed for reproducibility. Can be an integer or numpy RandomState object.
|
||||
If None, results will be non-deterministic. Default: 42.
|
||||
num_proc: Number of processes to use for dataset preprocessing. Set to None for
|
||||
streaming mode. Default: 1.
|
||||
load_from_cache_file: Whether to load preprocessed data from cache if available.
|
||||
Default: True.
|
||||
shuffle: Whether to shuffle the dataset(s) after loading. Default: False.
|
||||
streaming: Enable streaming mode for large datasets that don't fit in memory.
|
||||
When True, num_proc is automatically set to None. Default: False.
|
||||
interleave_prob: Probability weights for interleaving multiple datasets. Must have
|
||||
same length as datasets list. If None, datasets are concatenated instead. Default: None.
|
||||
stopping_strategy: Strategy when interleaving datasets of different lengths:
|
||||
- 'first_exhausted': Stop when shortest dataset is exhausted
|
||||
- 'all_exhausted': Continue until all datasets are exhausted
|
||||
Default: 'first_exhausted'.
|
||||
shuffle_buffer_size: Buffer size for shuffling in streaming mode. Larger values
|
||||
provide better randomization but use more memory. Default: 1000.
|
||||
use_hf: Force using HuggingFace Hub (True) or ModelScope (False). If None,
|
||||
it is controlled by the environment variable `USE_HF`, which defaults to '0'.
|
||||
Default: None.
|
||||
hub_token: Authentication token for accessing private datasets on the hub. Default: None.
|
||||
strict: If True, raise exceptions when encountering malformed data rows.
|
||||
If False, skip invalid rows with warnings. Default: False.
|
||||
download_mode: How to handle existing cached datasets:
|
||||
- 'reuse_dataset_if_exists': Use cached version if available
|
||||
- 'force_redownload': Always download fresh copy
|
||||
Default: 'reuse_dataset_if_exists'.
|
||||
columns: Manual column name mapping for datasets. Dictionary mapping source column
|
||||
names to target column names (e.g., {'text': 'content'}). Default: None.
|
||||
remove_unused_columns: Whether to remove columns not used in preprocessing.
|
||||
Helps reduce memory usage. Default: True.
|
||||
disable_auto_column_mapping: By default, column names in the dataset are automatically
|
||||
mapped. This parameter disables that behavior
|
||||
(the `columns` parameter remains effective), defaulting to `False`.
|
||||
model_name: Model name for self-cognition task preprocessing. Can be a tuple of
|
||||
(Chinese_name, English_name) or list of names. Default: None.
|
||||
model_author: Model author for self-cognition task preprocessing. Can be a tuple of
|
||||
(Chinese_author, English_author) or list of authors. Default: None.
|
||||
|
||||
Returns:
|
||||
A tuple of (train_dataset, val_dataset):
|
||||
- train_dataset: The training dataset
|
||||
- val_dataset: The validation dataset if split_dataset_ratio > 0, otherwise None
|
||||
|
||||
Examples:
|
||||
>>> # Load single dataset
|
||||
>>> train_ds, val_ds = load_dataset('AI-ModelScope/alpaca-gpt4-data-zh', split_dataset_ratio=0.1)
|
||||
|
||||
>>> # Load multiple datasets
|
||||
>>> train_ds, _ = load_dataset(
|
||||
... ['AI-ModelScope/alpaca-gpt4-data-zh#500', 'swift/self-cognition#500'],
|
||||
... model_name=('我的模型', 'MyModel'),
|
||||
... model_author=('作者', 'Author')
|
||||
... )
|
||||
"""
|
||||
init_self_cognition_preprocessor(DATASET_MAPPING.get('self-cognition'), model_name, model_author)
|
||||
if isinstance(datasets, str):
|
||||
datasets = [datasets]
|
||||
if not isinstance(seed, np.random.RandomState):
|
||||
seed = np.random.RandomState(seed)
|
||||
if streaming:
|
||||
num_proc = None
|
||||
train_datasets = []
|
||||
val_datasets = []
|
||||
|
||||
use_hf_default = use_hf
|
||||
if use_hf_default is None:
|
||||
use_hf_default = True if use_hf_hub() else False
|
||||
for dataset in datasets:
|
||||
dataset_syntax = DatasetSyntax.parse(dataset)
|
||||
use_hf = dataset_syntax.use_hf or use_hf_default
|
||||
# compat dataset_name
|
||||
if dataset_syntax.dataset in DATASET_MAPPING:
|
||||
dataset_meta = DATASET_MAPPING[dataset_syntax.dataset]
|
||||
if dataset_syntax.use_hf is None and dataset_meta.dataset_path is not None:
|
||||
dataset_syntax.dataset = dataset_meta.dataset_path
|
||||
dataset_syntax.dataset_type = 'path'
|
||||
else:
|
||||
dataset_syntax.dataset = dataset_meta.hf_dataset_id if use_hf else dataset_meta.ms_dataset_id
|
||||
else:
|
||||
dataset_meta = dataset_syntax.get_dataset_meta(use_hf)
|
||||
loader = dataset_meta.loader(
|
||||
num_proc=num_proc,
|
||||
load_from_cache_file=load_from_cache_file,
|
||||
streaming=streaming,
|
||||
hub_token=hub_token,
|
||||
strict=strict,
|
||||
download_mode=download_mode,
|
||||
columns=columns, # columns_mapping
|
||||
remove_unused_columns=remove_unused_columns,
|
||||
disable_auto_column_mapping=disable_auto_column_mapping,
|
||||
)
|
||||
train_dataset = loader.load(dataset_syntax, dataset_meta, use_hf=use_hf)
|
||||
train_dataset, val_dataset = loader.post_process(
|
||||
train_dataset,
|
||||
dataset_sample=dataset_syntax.dataset_sample,
|
||||
split_dataset_ratio=split_dataset_ratio,
|
||||
streaming=streaming,
|
||||
shuffle=shuffle,
|
||||
random_state=seed,
|
||||
)
|
||||
if train_dataset is not None:
|
||||
# Inject dataset_syntax.dataset as routing tag for multi-teacher
|
||||
ds_name = dataset_syntax.dataset
|
||||
train_dataset = _inject_dataset_routing_tag(train_dataset, ds_name)
|
||||
train_datasets.append(train_dataset)
|
||||
if val_dataset is not None:
|
||||
ds_name = dataset_syntax.dataset
|
||||
val_dataset = _inject_dataset_routing_tag(val_dataset, ds_name)
|
||||
val_datasets.append(val_dataset)
|
||||
|
||||
if interleave_prob is None:
|
||||
train_datasets = loader.concat_datasets(train_datasets)
|
||||
val_datasets = loader.concat_datasets(val_datasets)
|
||||
else:
|
||||
train_datasets = loader.interleave_datasets(
|
||||
train_datasets, interleave_prob, seed=get_seed(seed), stopping_strategy=stopping_strategy)
|
||||
val_datasets = loader.interleave_datasets(
|
||||
val_datasets, interleave_prob, seed=get_seed(seed), stopping_strategy=stopping_strategy)
|
||||
|
||||
if shuffle:
|
||||
if train_datasets:
|
||||
train_datasets = loader.shuffle_dataset(
|
||||
train_datasets, seed=get_seed(seed), buffer_size=shuffle_buffer_size)
|
||||
if val_datasets:
|
||||
val_datasets = loader.shuffle_dataset(val_datasets, seed=get_seed(seed), buffer_size=shuffle_buffer_size)
|
||||
return train_datasets, val_datasets
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import aiohttp
|
||||
import os
|
||||
import shutil
|
||||
from modelscope.hub.utils.utils import get_cache_dir
|
||||
from typing import List, Literal, Optional, Union
|
||||
|
||||
from swift.utils import get_logger, safe_ddp_context
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
class MediaResource:
|
||||
"""A class to manage the resource downloading."""
|
||||
|
||||
cache_dir = os.path.join(get_cache_dir(), 'media_resources')
|
||||
lock_dir = os.path.join(get_cache_dir(), 'lockers')
|
||||
|
||||
media_type_urls = {
|
||||
'llava', 'coco', 'sam', 'gqa', 'ocr_vqa', 'textvqa', 'VG_100K', 'VG_100K_2', 'share_textvqa', 'web-celebrity',
|
||||
'web-landmark', 'wikiart'
|
||||
}
|
||||
|
||||
URL_PREFIX = 'https://www.modelscope.cn/api/v1/datasets/hjh0119/sharegpt4v-images/repo?Revision=master&FilePath='
|
||||
|
||||
@staticmethod
|
||||
def get_url(media_type):
|
||||
is_ocr_vqa = (media_type == 'ocr_vqa')
|
||||
extension = 'tar' if is_ocr_vqa else 'zip'
|
||||
return f'{MediaResource.URL_PREFIX}{media_type}.{extension}'
|
||||
|
||||
@staticmethod
|
||||
def download(media_type_or_url: Union[str, List[str]],
|
||||
local_alias: Optional[str] = None,
|
||||
file_type: Literal['compressed', 'file', 'sharded'] = 'compressed'):
|
||||
"""Download and extract a resource from a http link.
|
||||
|
||||
Args:
|
||||
media_type_or_url: `str` or List or `str`, Either belongs to the `media_type_urls`
|
||||
listed in the class field, or a remote url to download and extract.
|
||||
Be aware that, this media type or url needs to contain a zip or tar file.
|
||||
local_alias: `Options[str]`, The local alias name for the `media_type_or_url`. If the first arg is a
|
||||
media_type listed in this class, local_alias can leave None. else please pass in a name for the url.
|
||||
The local dir contains the extracted files will be: {cache_dir}/{local_alias}
|
||||
file_type: The file type, if is a compressed file, un-compressed the file,
|
||||
if is an original file, only download it, if is a sharded file, download all files and extract.
|
||||
|
||||
Returns:
|
||||
The local dir contains the extracted files.
|
||||
"""
|
||||
media_file = media_type_or_url if isinstance(media_type_or_url, str) else media_type_or_url[0]
|
||||
with safe_ddp_context(hash_id=media_file):
|
||||
return MediaResource._safe_download(
|
||||
media_type=media_type_or_url, media_name=local_alias, file_type=file_type)
|
||||
|
||||
@staticmethod
|
||||
def move_directory_contents(src_dir, dst_dir):
|
||||
if not os.path.exists(dst_dir):
|
||||
os.makedirs(dst_dir)
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(src_dir):
|
||||
relative_path = os.path.relpath(dirpath, src_dir)
|
||||
target_dir = os.path.join(dst_dir, relative_path)
|
||||
|
||||
if not os.path.exists(target_dir):
|
||||
os.makedirs(target_dir)
|
||||
|
||||
for file in filenames:
|
||||
src_file = os.path.join(dirpath, file)
|
||||
dst_file = os.path.join(target_dir, file)
|
||||
shutil.move(src_file, dst_file)
|
||||
|
||||
@staticmethod
|
||||
def _safe_download(media_type: Union[str, List[str]],
|
||||
media_name: Optional[str] = None,
|
||||
file_type: Literal['compressed', 'file', 'sharded'] = 'compressed'):
|
||||
media_name = media_name or media_type
|
||||
assert isinstance(media_name, str), f'{media_name} is not a str'
|
||||
if isinstance(media_type, str) and media_type in MediaResource.media_type_urls:
|
||||
media_type = MediaResource.get_url(media_type)
|
||||
|
||||
from datasets.download.download_manager import DownloadConfig, DownloadManager
|
||||
final_folder = os.path.join(MediaResource.cache_dir, media_name)
|
||||
|
||||
if file_type == 'file':
|
||||
filename = media_type.split('/')[-1]
|
||||
final_path = os.path.join(final_folder, filename)
|
||||
if os.path.exists(final_path): # if the download thing is a file but not folder,
|
||||
return final_folder # check whether the file exists
|
||||
if not os.path.exists(final_folder):
|
||||
os.makedirs(final_folder) # and make sure final_folder exists to contain it
|
||||
else:
|
||||
if os.path.exists(final_folder):
|
||||
return final_folder
|
||||
|
||||
logger.info('# #################Resource downloading#################')
|
||||
logger.info('Downloading necessary resources...')
|
||||
logger.info(f'Resource package: {media_type}')
|
||||
logger.info(f'Extracting to local dir: {final_folder}')
|
||||
logger.info('If the downloading fails or lasts a long time, '
|
||||
'you can manually download the resources and extracting to the local dir.')
|
||||
logger.info('Now begin.')
|
||||
download_config = DownloadConfig(cache_dir=MediaResource.cache_dir)
|
||||
download_config.storage_options = {'client_kwargs': {'timeout': aiohttp.ClientTimeout(total=86400)}}
|
||||
if file_type == 'file':
|
||||
filename = media_type.split('/')[-1]
|
||||
final_path = os.path.join(final_folder, filename)
|
||||
local_dirs = DownloadManager(download_config=download_config).download(media_type)
|
||||
shutil.move(str(local_dirs), final_path)
|
||||
elif file_type == 'compressed':
|
||||
local_dirs = DownloadManager(download_config=download_config).download_and_extract(media_type)
|
||||
shutil.move(str(local_dirs), final_folder)
|
||||
else:
|
||||
for media_url in media_type:
|
||||
local_dirs = DownloadManager(download_config=download_config).download_and_extract(media_url)
|
||||
MediaResource.move_directory_contents(str(local_dirs), final_folder)
|
||||
logger.info('# #################Resource downloading finished#################')
|
||||
return final_folder
|
||||
|
||||
@staticmethod
|
||||
def safe_save(image, file_name, folder, format='JPEG'):
|
||||
folder = os.path.join(MediaResource.cache_dir, folder)
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
file = os.path.join(folder, file_name)
|
||||
if os.path.exists(file):
|
||||
return file
|
||||
image.save(file, format=format)
|
||||
return file
|
||||
@@ -0,0 +1,230 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import math
|
||||
import multiprocessing as mp
|
||||
import torch.distributed as dist
|
||||
from itertools import chain
|
||||
from torch.utils.data import Dataset, IterableDataset
|
||||
from tqdm import tqdm
|
||||
from typing import Optional
|
||||
|
||||
from swift.template import MaxLengthError
|
||||
from swift.utils import get_logger, is_dist, is_master, split_list
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def calculate_matched_group(sequences, packing_length: int, is_finished: bool = True, strategy: str = 'binpack'):
|
||||
if len(sequences) == 0:
|
||||
return [], []
|
||||
if strategy == 'sequential':
|
||||
# Order-preserving greedy packing (next-fit): keep a single open pack and flush it
|
||||
# when the next sample doesn't fit, so the global sample order and pack boundaries
|
||||
# follow the input order (a sequential sampler). (Use packing_num_proc=1 for
|
||||
# a single global ordering.)
|
||||
packs, cur, cur_len = [], [], 0
|
||||
for item in sequences: # item = (idx, length); weight_pos=1 -> length at item[1]
|
||||
seq_len = item[1]
|
||||
if cur and cur_len + seq_len > packing_length:
|
||||
packs.append(cur)
|
||||
cur, cur_len = [], 0
|
||||
cur.append(item)
|
||||
cur_len += seq_len
|
||||
if cur_len >= packing_length:
|
||||
packs.append(cur)
|
||||
cur, cur_len = [], 0
|
||||
if is_finished:
|
||||
if cur:
|
||||
packs.append(cur)
|
||||
return packs, []
|
||||
return packs, cur
|
||||
# default: best-fit-decreasing bin packing (https://arxiv.org/pdf/2404.10830)
|
||||
import binpacking
|
||||
sequences = binpacking.to_constant_volume(sequences, packing_length, weight_pos=1)
|
||||
if sequences and not is_finished:
|
||||
sequences, ret_sequences = sequences[:-1], sequences[-1]
|
||||
else:
|
||||
ret_sequences = []
|
||||
return sequences, ret_sequences
|
||||
|
||||
|
||||
class PackingDataset(Dataset):
|
||||
PACKING_BATCH_SIZE = 1000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
template,
|
||||
dataset,
|
||||
num_proc: int = 1,
|
||||
*,
|
||||
strict: bool = False,
|
||||
load_from_cache_file: bool = True,
|
||||
packing_length: Optional[int] = None,
|
||||
packing_num_proc: int = 1,
|
||||
packing_strategy: str = 'binpack',
|
||||
**kwargs,
|
||||
):
|
||||
template.packing = True
|
||||
template.padding_free = True # TODO: remove
|
||||
self.template = template
|
||||
self.dataset = dataset
|
||||
self.num_proc = num_proc
|
||||
self.strict = strict
|
||||
self.load_from_cache_file = load_from_cache_file
|
||||
self.packing_strategy = packing_strategy
|
||||
self.packing_length = packing_length or self.template.max_length
|
||||
self.packing_num_proc = min(packing_num_proc, math.ceil(len(dataset) / self.PACKING_BATCH_SIZE))
|
||||
self._out_queue = mp.Queue()
|
||||
if is_master():
|
||||
lengths = self.dataset['lengths']
|
||||
offset = 0
|
||||
chunked_lengths = split_list(lengths, self.packing_num_proc)
|
||||
for i in range(self.packing_num_proc):
|
||||
worker = mp.Process(
|
||||
target=self.create_packed_idx, args=(
|
||||
i,
|
||||
offset,
|
||||
chunked_lengths[i],
|
||||
), daemon=True)
|
||||
worker.start()
|
||||
offset += len(chunked_lengths[i])
|
||||
self.packed_idx = [[] for _ in range(self.packing_num_proc)]
|
||||
self.packed_length = [[] for _ in range(self.packing_num_proc)]
|
||||
desc = 'Packing: ' if self.packing_num_proc == 1 else f'Packing (num_proc={self.packing_num_proc}): '
|
||||
with tqdm(total=len(lengths), dynamic_ncols=True, desc=desc) as prog_bar:
|
||||
finished_workers = 0
|
||||
while finished_workers < self.packing_num_proc:
|
||||
rank, sequences, data_len = self._out_queue.get()
|
||||
if data_len == -1:
|
||||
finished_workers += 1
|
||||
continue
|
||||
prog_bar.update(data_len)
|
||||
self.packed_idx[rank] += [[x[0] for x in seq] for seq in sequences]
|
||||
self.packed_length[rank] += [sum(x[1] for x in seq) for seq in sequences]
|
||||
self.packed_idx = list(chain.from_iterable(self.packed_idx))
|
||||
self.packed_length = list(chain.from_iterable(self.packed_length))
|
||||
else:
|
||||
self.packed_idx, self.packed_length = None, None
|
||||
if dist.is_initialized() and is_dist():
|
||||
obj_list = [(self.packed_idx, self.packed_length)]
|
||||
dist.broadcast_object_list(obj_list)
|
||||
self.packed_idx, self.packed_length = obj_list[0]
|
||||
|
||||
def create_packed_idx(self, rank, offset, lengths):
|
||||
data = [(i + offset, sum(length) if isinstance(length, list) else length) for i, length in enumerate(lengths)]
|
||||
i = 0
|
||||
input_data = []
|
||||
while True:
|
||||
new_data = data[i:i + self.PACKING_BATCH_SIZE]
|
||||
input_data += new_data
|
||||
if not input_data:
|
||||
break
|
||||
i += self.PACKING_BATCH_SIZE
|
||||
is_finished = i >= len(data)
|
||||
sequences, input_data = calculate_matched_group(
|
||||
input_data, self.packing_length, is_finished=is_finished, strategy=self.packing_strategy)
|
||||
self._out_queue.put((rank, sequences, len(new_data)))
|
||||
self._out_queue.put((rank, [], -1))
|
||||
|
||||
def __getitem__(self, index):
|
||||
sequence = self.packed_idx[index]
|
||||
row = [self.dataset[i] for i in sequence]
|
||||
return row
|
||||
|
||||
def __len__(self):
|
||||
return len(self.packed_idx)
|
||||
|
||||
|
||||
class IterablePackingDataset(IterableDataset):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
template,
|
||||
dataset,
|
||||
num_proc: int = 1,
|
||||
*,
|
||||
packing_interval: int = 128,
|
||||
packing_length: Optional[int] = None,
|
||||
strict: bool = False,
|
||||
cyclic: bool = False,
|
||||
packing_strategy: str = 'binpack',
|
||||
**kwargs,
|
||||
):
|
||||
template.packing = True
|
||||
template.padding_free = True # TODO: remove
|
||||
self.template = template
|
||||
self.dataset = dataset
|
||||
self.num_proc = num_proc
|
||||
self.strict = strict
|
||||
self.packing_length = packing_length or self.template.max_length
|
||||
|
||||
self.packing_interval = packing_interval
|
||||
self._in_queue = mp.Queue()
|
||||
self._out_queue = mp.Queue()
|
||||
self.workers = []
|
||||
self.cyclic = cyclic
|
||||
self.packing_strategy = packing_strategy
|
||||
for _ in range(self.num_proc):
|
||||
worker = mp.Process(target=self._processor, daemon=True)
|
||||
worker.start()
|
||||
self.workers.append(worker)
|
||||
|
||||
def _processor(self):
|
||||
while True:
|
||||
i, data = self._in_queue.get()
|
||||
encoded_data = {}
|
||||
try:
|
||||
encoded_data = self.template.encode(data, return_length=True)
|
||||
except Exception as e:
|
||||
if self.strict and not isinstance(e, MaxLengthError):
|
||||
raise
|
||||
self._out_queue.put((i, encoded_data))
|
||||
|
||||
def _put_data_in_queue(self, iterator) -> int:
|
||||
for i in range(self.packing_interval):
|
||||
try:
|
||||
data = next(iterator)
|
||||
except StopIteration:
|
||||
return i
|
||||
self._in_queue.put((i, data))
|
||||
return i + 1
|
||||
|
||||
def _fetch_data_out_queue(self, last_res, num_samples):
|
||||
res = [None] * num_samples
|
||||
for _ in range(num_samples):
|
||||
i, data = self._out_queue.get()
|
||||
if not data:
|
||||
continue
|
||||
res[i] = (data, len(data['input_ids']))
|
||||
res = [data for data in res if data]
|
||||
last_res += res
|
||||
return last_res
|
||||
|
||||
@staticmethod
|
||||
def cyclic_iter(iterable):
|
||||
while True:
|
||||
for x in iterable:
|
||||
yield x
|
||||
|
||||
def __iter__(self):
|
||||
try:
|
||||
next(iter(self.dataset))
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
if self.cyclic:
|
||||
iterator = self.cyclic_iter(self.dataset)
|
||||
else:
|
||||
iterator = iter(self.dataset)
|
||||
data = []
|
||||
while True:
|
||||
num_samples = self._put_data_in_queue(iterator)
|
||||
finished = num_samples != self.packing_interval
|
||||
data = self._fetch_data_out_queue(data, num_samples)
|
||||
sequences, data = calculate_matched_group(
|
||||
data, self.packing_length, is_finished=finished, strategy=self.packing_strategy)
|
||||
res = []
|
||||
for row in sequences:
|
||||
res.append([r[0] for r in row])
|
||||
yield from res
|
||||
if finished:
|
||||
break
|
||||
@@ -0,0 +1,4 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from .core import (DATASET_TYPE, AlpacaPreprocessor, AutoPreprocessor, ClsPreprocessor, MessagesPreprocessor,
|
||||
ResponsePreprocessor, RowPreprocessor)
|
||||
from .extra import ClsGenerationPreprocessor, GroundingMixin, TextGenerationPreprocessor
|
||||
@@ -0,0 +1,571 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import ast
|
||||
import datasets
|
||||
import numpy as np
|
||||
import os
|
||||
from collections import Counter
|
||||
from contextlib import contextmanager
|
||||
from datasets import Dataset as HfDataset
|
||||
from datasets import Image
|
||||
from datasets import IterableDataset as HfIterableDataset
|
||||
from datasets import Sequence, Value
|
||||
from itertools import chain
|
||||
from modelscope.hub.utils.utils import get_cache_dir
|
||||
from packaging import version
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from swift.template import history_to_messages
|
||||
from swift.utils import get_logger, is_dist, is_master, safe_ddp_context
|
||||
|
||||
DATASET_TYPE = Union[HfDataset, HfIterableDataset]
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
_pair_keys = ['messages', 'images', 'videos', 'audios', 'tools', 'objects']
|
||||
|
||||
|
||||
class RowPreprocessor:
|
||||
standard_keys = _pair_keys + list(
|
||||
chain.from_iterable([f'{prefix}_{k}' for k in _pair_keys]
|
||||
for prefix in ['rejected', 'positive', 'negative'])) + [
|
||||
'rejected_response',
|
||||
'label',
|
||||
'channel',
|
||||
'margin',
|
||||
'teacher_prompt',
|
||||
'chat_template_kwargs',
|
||||
# Qwen3-TTS
|
||||
'ref_audios',
|
||||
'audio_codes',
|
||||
]
|
||||
|
||||
def __init__(self,
|
||||
*,
|
||||
columns: Optional[Dict[str, str]] = None,
|
||||
dataset_sample: Optional[int] = None,
|
||||
random_state: Optional[Union[np.random.RandomState, int]] = 42,
|
||||
traceback_limit: int = 10) -> None:
|
||||
self.columns = columns or {}
|
||||
self.origin_columns = self.columns.copy() # Higher priority and raise Error
|
||||
images_keys = ['images', 'image']
|
||||
audios_keys = ['audios', 'audio']
|
||||
videos_keys = ['videos', 'video']
|
||||
for mm_type in ['images', 'audios', 'videos']:
|
||||
keys = locals()[f'{mm_type}_keys']
|
||||
for key in keys:
|
||||
self.columns[key] = mm_type
|
||||
|
||||
self.traceback_limit = traceback_limit
|
||||
self._traceback_counter = 0
|
||||
self.dataset_sample = dataset_sample
|
||||
self.datasets_4 = version.parse(datasets.__version__) >= version.parse('4.0')
|
||||
if not isinstance(random_state, np.random.RandomState):
|
||||
random_state = np.random.RandomState(random_state)
|
||||
self.random_state = random_state
|
||||
|
||||
@staticmethod
|
||||
def _check_messages(row: Dict[str, Any]) -> None:
|
||||
if 'messages' not in row:
|
||||
return
|
||||
messages = row['messages']
|
||||
assert len(messages) > 0, f'messages: {messages}'
|
||||
# fix swift/SlimOrca (concat)
|
||||
for message in messages:
|
||||
keys = set(message.keys()) - {'role', 'content', 'loss', 'loss_scale'}
|
||||
for key in keys:
|
||||
message.pop(key)
|
||||
|
||||
for message in messages:
|
||||
role, content = message['role'], message['content']
|
||||
# The terms 'tool' and 'tool_response' have the same meaning, ensuring compatibility.
|
||||
assert role in {'system', 'user', 'tool_call', 'tool_response', 'tool', 'assistant'}, f'message: {message}'
|
||||
assert content is not None, f'message: {message}'
|
||||
|
||||
@staticmethod
|
||||
def _cast_mm_data(row: Dict[str, Any]) -> None:
|
||||
for key in ['images', 'rejected_images']:
|
||||
images = row.get(key, None)
|
||||
if images is None:
|
||||
continue
|
||||
|
||||
if isinstance(images, str) or (isinstance(images, list) and images and isinstance(images[0], str)):
|
||||
if isinstance(images, str):
|
||||
images = [images]
|
||||
for i, image in enumerate(images):
|
||||
images[i] = {'bytes': None, 'path': image}
|
||||
row[key] = images
|
||||
elif isinstance(images, dict):
|
||||
row[key] = [images]
|
||||
|
||||
for key in ['videos', 'audios']:
|
||||
mm_data = row.get(key)
|
||||
if mm_data is None:
|
||||
continue
|
||||
elif isinstance(mm_data, str):
|
||||
row[key] = [mm_data]
|
||||
|
||||
@staticmethod
|
||||
def _check_rejected_response(row: Dict[str, Any]) -> None:
|
||||
if 'rejected_response' in row:
|
||||
messages = row['messages']
|
||||
rejected_response = row['rejected_response']
|
||||
if (rejected_response is None
|
||||
or isinstance(rejected_response, str) and rejected_response == messages[-1]['content']):
|
||||
raise ValueError(f'rejected_response: {rejected_response}')
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
return row
|
||||
|
||||
def prepare_dataset(self, dataset: DATASET_TYPE) -> DATASET_TYPE:
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def batched_to_rows(batched_row: Dict[str, Any]):
|
||||
keys = list(batched_row.keys())
|
||||
batch_size = len(batched_row[keys[0]])
|
||||
return [{key: batched_row[key][i] for key in keys} for i in range(batch_size)]
|
||||
|
||||
@staticmethod
|
||||
def rows_to_batched(rows: List[Dict[str, Any]]):
|
||||
batched = {}
|
||||
for i, row in enumerate(rows):
|
||||
for k, v in row.items():
|
||||
if k not in batched:
|
||||
batched[k] = [None] * i
|
||||
batched[k].append(v)
|
||||
# Make all the lengths of v the same.
|
||||
for k in set(batched.keys()) - set(row.keys()):
|
||||
batched[k].append(None)
|
||||
return batched
|
||||
|
||||
@staticmethod
|
||||
def _remove_prefix_keys(row, prefix: str):
|
||||
for k in list(row.keys()):
|
||||
if k.startswith(prefix):
|
||||
new_k = k[len(prefix):]
|
||||
new_v = row.pop(k)
|
||||
if new_k not in row:
|
||||
row[new_k] = new_v
|
||||
|
||||
@staticmethod
|
||||
def _check_objects(row):
|
||||
objects = row.get('objects')
|
||||
if objects is None:
|
||||
return
|
||||
new_objects = {}
|
||||
# Ensure the order
|
||||
for k in ['ref', 'bbox', 'bbox_type', 'image_id']:
|
||||
if k in objects.keys():
|
||||
new_objects[k] = objects[k]
|
||||
row['objects'] = new_objects
|
||||
bbox = new_objects['bbox']
|
||||
|
||||
# check bbox
|
||||
for box in bbox:
|
||||
assert len(box) in {2, 4}, f'len(box): {len(box)}'
|
||||
if len(box) == 2:
|
||||
continue
|
||||
if box[0] > box[2]:
|
||||
box[0], box[2] = box[2], box[0]
|
||||
if box[1] > box[3]:
|
||||
box[1], box[3] = box[3], box[1]
|
||||
|
||||
def batched_preprocess(self, batched_row: Dict[str, Any], *, strict: bool,
|
||||
ignore_max_length_error: bool) -> Dict[str, Any]:
|
||||
from swift.template import MaxLengthError
|
||||
batched_row = dict(batched_row)
|
||||
assert len(batched_row) > 0
|
||||
self._remove_prefix_keys(batched_row, '__@') # compat streaming
|
||||
rows = self.batched_to_rows(batched_row)
|
||||
|
||||
new_rows = []
|
||||
for row in rows:
|
||||
try:
|
||||
row = self.preprocess(row)
|
||||
# support [row1, row2, ...]
|
||||
if row is None:
|
||||
row = []
|
||||
if isinstance(row, dict):
|
||||
row = [row]
|
||||
for r in row:
|
||||
self._check_objects(r)
|
||||
self._check_rejected_response(r)
|
||||
self._check_messages(r)
|
||||
self._cast_mm_data(r)
|
||||
except Exception as e:
|
||||
if strict:
|
||||
logger.warning('To avoid errors, you can pass `strict=False`.')
|
||||
raise
|
||||
if isinstance(e, MaxLengthError) and ignore_max_length_error:
|
||||
pass
|
||||
elif self.traceback_limit is not None and self._traceback_counter < self.traceback_limit:
|
||||
import traceback
|
||||
logger.info(traceback.format_exc())
|
||||
logger.warning('👆👆👆There are errors in the dataset, the data will be deleted')
|
||||
self._traceback_counter += 1
|
||||
row = []
|
||||
new_rows += row
|
||||
res = self.rows_to_batched(new_rows)
|
||||
self._remove_prefix_keys(res, '__#') # compat GRPO
|
||||
if len(res) == 0:
|
||||
res['messages'] = []
|
||||
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def get_features_dataset(dataset: DATASET_TYPE) -> DATASET_TYPE:
|
||||
if dataset.features is None:
|
||||
assert isinstance(dataset, HfIterableDataset)
|
||||
dataset = dataset._resolve_features()
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def safe_rename_columns(dataset, columns):
|
||||
dataset = RowPreprocessor.get_features_dataset(dataset)
|
||||
columns_keys = {k.lower(): k for k in dataset.features.keys()} # lower -> lower/upper
|
||||
safe_columns = {columns_keys[k.lower()]: v for k, v in columns.items() if k.lower() in columns_keys}
|
||||
|
||||
counter = Counter(safe_columns.values())
|
||||
for k, new_k in list(safe_columns.items()):
|
||||
if counter[new_k] > 1:
|
||||
# For example, if "response" and "answer" match, then no processing is done.
|
||||
safe_columns.pop(k)
|
||||
continue
|
||||
|
||||
# e.g. Keep {'query': 'query'} to ensure that the query has the highest priority.
|
||||
safe_columns = {k: v for k, v in safe_columns.items() if k != v}
|
||||
if safe_columns:
|
||||
dataset = dataset.rename_columns(safe_columns)
|
||||
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def remove_useless_columns(dataset: DATASET_TYPE) -> DATASET_TYPE:
|
||||
dataset = RowPreprocessor.get_features_dataset(dataset)
|
||||
features = dataset.features
|
||||
k_list = [k for k in RowPreprocessor.standard_keys if k in features]
|
||||
if len(k_list) != len(features):
|
||||
dataset = dataset.select_columns(k_list)
|
||||
return dataset
|
||||
|
||||
@contextmanager
|
||||
def _patch_arrow_writer(self):
|
||||
# fix AI-ModelScope/ms_agent_for_agentfabric:all
|
||||
from datasets.arrow_writer import ArrowWriter
|
||||
|
||||
def _new_init(_self, schema=None, features=None, *args, **kwargs):
|
||||
|
||||
if features is not None:
|
||||
|
||||
if self.datasets_4:
|
||||
from datasets.features import Json, List
|
||||
messages_feature = List(Json())
|
||||
for key in ['messages', 'rejected_messages', 'positive_messages', 'negative_messages']:
|
||||
features[key] = messages_feature
|
||||
features['images'] = List({'bytes': Value(dtype='binary'), 'path': Value(dtype='string')})
|
||||
features['objects'] = Json()
|
||||
features['chat_template_kwargs'] = Json()
|
||||
else:
|
||||
messages_feature = [{
|
||||
'role': Value(dtype='string'),
|
||||
'content': Value(dtype='string'),
|
||||
}]
|
||||
messages_feature_with_loss = [{
|
||||
'role': Value(dtype='string'),
|
||||
'content': Value(dtype='string'),
|
||||
'loss': Value(dtype='bool'),
|
||||
'loss_scale': Value(dtype='float64'),
|
||||
}]
|
||||
features['messages'] = messages_feature_with_loss
|
||||
features['rejected_messages'] = messages_feature_with_loss
|
||||
features['positive_messages'] = messages_feature
|
||||
features['negative_messages'] = messages_feature
|
||||
features['images'] = [{'bytes': Value(dtype='binary'), 'path': Value(dtype='string')}]
|
||||
features['objects'] = {
|
||||
'ref': Sequence(feature=Value(dtype='string'), length=-1),
|
||||
'bbox': Sequence(feature=Sequence(feature=Value(dtype='float64'), length=-1), length=-1),
|
||||
'bbox_type': Value(dtype='string'),
|
||||
'image_id': Sequence(feature=Value(dtype='int64'), length=-1),
|
||||
}
|
||||
ArrowWriter.__origin_init__(_self, schema, features, *args, **kwargs)
|
||||
|
||||
ArrowWriter.__origin_init__ = ArrowWriter.__init__
|
||||
ArrowWriter.__init__ = _new_init
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
ArrowWriter.__init__ = ArrowWriter.__origin_init__
|
||||
del ArrowWriter.__origin_init__
|
||||
|
||||
def _cast_pil_image(self, dataset):
|
||||
features = dataset.features
|
||||
for col in ['images', 'rejected_images']:
|
||||
if (col in features and isinstance(features[col], Image) and getattr(features[col], 'decode', False)):
|
||||
dataset = dataset.cast_column(col, Image(decode=False))
|
||||
return dataset
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
dataset: DATASET_TYPE,
|
||||
*,
|
||||
num_proc: int = 1,
|
||||
load_from_cache_file: bool = True,
|
||||
strict: bool = False,
|
||||
batch_size: Optional[int] = None,
|
||||
enable_auto_mapping: bool = False,
|
||||
) -> DATASET_TYPE:
|
||||
from ..utils import sample_dataset
|
||||
if batch_size is None:
|
||||
batch_size = 1000 if isinstance(dataset, HfDataset) else 16
|
||||
if self.dataset_sample is not None:
|
||||
dataset = sample_dataset(dataset, self.dataset_sample, True, self.random_state)
|
||||
|
||||
map_kwargs = {'batched': True, 'batch_size': batch_size}
|
||||
if isinstance(dataset, HfDataset):
|
||||
if not load_from_cache_file and is_dist() and not is_master():
|
||||
load_from_cache_file = True
|
||||
map_kwargs.update({
|
||||
'num_proc': num_proc,
|
||||
'load_from_cache_file': load_from_cache_file,
|
||||
})
|
||||
# compat GRPO: The solution field will be retained.
|
||||
dataset = RowPreprocessor.get_features_dataset(dataset)
|
||||
if 'solution' in dataset.features:
|
||||
with safe_ddp_context(None, True):
|
||||
if isinstance(dataset, HfDataset) and not dataset.cache_files:
|
||||
map_kwargs['cache_file_name'] = os.path.join(get_cache_dir(), 'datasets', 'map_cache',
|
||||
f'{dataset._fingerprint}.arrow')
|
||||
dataset = dataset.map(lambda x: {'__#solution': x['solution']}, **map_kwargs)
|
||||
map_kwargs.pop('cache_file_name', None)
|
||||
dataset = self.safe_rename_columns(dataset, self.origin_columns)
|
||||
if enable_auto_mapping:
|
||||
dataset = self.safe_rename_columns(dataset, self.columns)
|
||||
dataset = self.prepare_dataset(dataset)
|
||||
dataset = self._cast_pil_image(dataset)
|
||||
if isinstance(dataset, HfIterableDataset):
|
||||
# fix: https://github.com/huggingface/datasets/issues/6408
|
||||
columns = {k: f'__@{k}' for k in RowPreprocessor.standard_keys if k in dataset.features}
|
||||
if columns:
|
||||
dataset = dataset.rename_columns(columns)
|
||||
|
||||
ignore_max_length_error = True
|
||||
with self._patch_arrow_writer(), safe_ddp_context(None, True):
|
||||
if isinstance(dataset, HfDataset) and not dataset.cache_files:
|
||||
map_kwargs['cache_file_name'] = os.path.join(get_cache_dir(), 'datasets', 'map_cache',
|
||||
f'{dataset._fingerprint}.arrow')
|
||||
dataset_mapped = dataset.map(
|
||||
self.batched_preprocess,
|
||||
fn_kwargs={
|
||||
'strict': strict,
|
||||
'ignore_max_length_error': ignore_max_length_error,
|
||||
},
|
||||
remove_columns=list(dataset.features.keys()),
|
||||
**map_kwargs)
|
||||
if isinstance(dataset_mapped, HfDataset) and len(dataset) != len(dataset_mapped):
|
||||
logger.info(
|
||||
f'Dataset filtered, origin length: {len(dataset)}, filtered dataset length: {len(dataset_mapped)}')
|
||||
|
||||
return dataset_mapped
|
||||
|
||||
|
||||
class ResponsePreprocessor(RowPreprocessor):
|
||||
"""Dataset compatible with older versions of ms-swift"""
|
||||
|
||||
def __init__(self, *, columns: Optional[Dict[str, str]] = None, **kwargs) -> None:
|
||||
super().__init__(columns=columns, **kwargs)
|
||||
system_keys = ['system', 'system_prompt']
|
||||
query_keys = ['query', 'prompt', 'input', 'instruction', 'question', 'problem']
|
||||
response_keys = ['response', 'answer', 'output', 'targets', 'target', 'answer_key', 'answers', 'solution'
|
||||
] + ['text', 'completion', 'content']
|
||||
for key in system_keys:
|
||||
self.columns[key] = 'system'
|
||||
for key in query_keys:
|
||||
self.columns[key] = 'query'
|
||||
for key in response_keys:
|
||||
self.columns[key] = 'response'
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
response = row.pop('response', None)
|
||||
if response is not None:
|
||||
if isinstance(response, (list, tuple)):
|
||||
from transformers.utils import strtobool
|
||||
|
||||
# sometimes response is a list, pick one randomly
|
||||
if strtobool(os.environ.get('RANDOM_DATASET_RESPONSE', 'False')):
|
||||
response = self.random_state.choice(response)
|
||||
else:
|
||||
response = response[0]
|
||||
history = row.pop('history', None) or []
|
||||
query = row.pop('query', None)
|
||||
system = row.pop('system', None)
|
||||
if isinstance(history, str): # e.g. "[['query1', 'response1']]"
|
||||
history = ast.literal_eval(history)
|
||||
history.append([query, response])
|
||||
|
||||
row.update({'messages': history_to_messages(history, system)})
|
||||
return row
|
||||
|
||||
|
||||
class AlpacaPreprocessor(ResponsePreprocessor):
|
||||
|
||||
@classmethod
|
||||
def concat_inst_input(cls, instruction, input_):
|
||||
if instruction and input_:
|
||||
query = f'{instruction}\n{input_}'
|
||||
else:
|
||||
query = instruction or input_
|
||||
assert isinstance(query, str), f'query: {query}'
|
||||
return query
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
instruction = row.pop('instruction', None)
|
||||
input_ = row.pop('input', None)
|
||||
output = row.pop('output', None)
|
||||
if output is not None:
|
||||
row['response'] = output
|
||||
row['query'] = self.concat_inst_input(instruction, input_)
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
def default_repair_messages(s: Union[str, Any]) -> Any:
|
||||
if isinstance(s, str):
|
||||
return ast.literal_eval(s)
|
||||
return s
|
||||
|
||||
|
||||
class MessagesPreprocessor(RowPreprocessor):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
# If set to None, automatic matching will be performed.
|
||||
role_key: Optional[str] = None, # 'role', 'from'
|
||||
content_key: Optional[str] = None, # 'content', 'value'
|
||||
user_role: Optional[str] = None, # 'user', 'human'
|
||||
assistant_role: Optional[str] = None, # 'assistant', 'gpt', 'bot'
|
||||
system_role: str = 'system',
|
||||
# 'conversation', 'conversations' -> 'messages'
|
||||
columns: Optional[Dict[str, str]] = None,
|
||||
repair_messages: Callable[[Union[str, List[Dict[str, str]]]],
|
||||
Optional[List[Dict[str, str]]]] = default_repair_messages,
|
||||
inner_key: Optional[str] = None,
|
||||
**kwargs):
|
||||
super().__init__(columns=columns, **kwargs)
|
||||
self.role_keys = ['role', 'from'] if role_key is None else [role_key]
|
||||
self.content_keys = ['content', 'value'] if content_key is None else [content_key]
|
||||
self.user_roles = ['user', 'human'] if user_role is None else [user_role]
|
||||
self.assistant_roles = ['assistant', 'gpt', 'bot'] if assistant_role is None else [assistant_role]
|
||||
self.tool_call_roles = ['function_call']
|
||||
self.tool_response_roles = ['function_response', 'observation', 'observations']
|
||||
|
||||
self.system_role = system_role
|
||||
self.repair_messages = repair_messages
|
||||
self.inner_key = inner_key
|
||||
|
||||
message_keys = ['messages', 'conversation', 'conversations']
|
||||
for key in message_keys:
|
||||
self.columns[key] = 'messages'
|
||||
# sharegptq
|
||||
system_keys = ['system', 'system_prompt']
|
||||
if system_role not in system_keys:
|
||||
system_keys.append(system_role)
|
||||
for key in system_keys:
|
||||
self.columns[key] = 'system'
|
||||
|
||||
@staticmethod
|
||||
def _is_sharegpt_format(message: Dict[str, str]) -> bool:
|
||||
if 'role' in message or 'content' in message:
|
||||
return False
|
||||
return True
|
||||
|
||||
def sharegpt_to_messages(self, messages: List[Dict[str, str]], system: Optional[str]) -> List[Dict[str, str]]:
|
||||
self._to_std_key(messages, 'user', self.user_roles)
|
||||
self._to_std_key(messages, 'assistant', self.assistant_roles)
|
||||
new_messages = []
|
||||
if system is not None:
|
||||
new_messages.append({'role': 'system', 'content': system})
|
||||
for message in messages:
|
||||
user_message = {'role': 'user', 'content': message['user']}
|
||||
assistant_message = {'role': 'assistant', 'content': message['assistant']}
|
||||
new_messages.append(user_message)
|
||||
new_messages.append(assistant_message)
|
||||
return new_messages
|
||||
|
||||
def to_std_messages(self, messages: List[Dict[str, str]], system: Optional[str]) -> None:
|
||||
if messages[0]['role'] == self.system_role:
|
||||
messages[0]['role'] = 'system'
|
||||
elif system is not None:
|
||||
messages.insert(0, {'role': 'system', 'content': system})
|
||||
for message in messages:
|
||||
role = message['role']
|
||||
if role in self.user_roles:
|
||||
message['role'] = 'user'
|
||||
elif role in self.assistant_roles:
|
||||
message['role'] = 'assistant'
|
||||
elif role.replace('-', '_') in self.tool_call_roles:
|
||||
message['role'] = 'tool_call'
|
||||
elif role.replace('-', '_') in self.tool_response_roles:
|
||||
message['role'] = 'tool_response'
|
||||
|
||||
@staticmethod
|
||||
def _to_std_key(messages: List[Dict[str, str]], std_key: str, optional_keys: List[str]) -> None:
|
||||
for message in messages:
|
||||
for key in optional_keys:
|
||||
if key in message:
|
||||
message[std_key] = message.pop(key)
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
if 'rejected_messages' in row:
|
||||
rejected = MessagesPreprocessor.preprocess(self, {'messages': row['rejected_messages']})
|
||||
row['rejected_messages'] = rejected['messages'] if rejected else None
|
||||
messages = row['messages']
|
||||
if self.inner_key is not None:
|
||||
messages = messages[self.inner_key]
|
||||
messages: Optional[List[Dict[str, str]]] = self.repair_messages(messages)
|
||||
if not messages or isinstance(messages, str):
|
||||
return
|
||||
self._to_std_key(messages, 'role', self.role_keys)
|
||||
self._to_std_key(messages, 'content', self.content_keys)
|
||||
system = row.pop('system', None)
|
||||
if self._is_sharegpt_format(messages[0]):
|
||||
messages = self.sharegpt_to_messages(messages, system)
|
||||
else:
|
||||
self.to_std_messages(messages, system) # inplace
|
||||
row['messages'] = messages
|
||||
return row
|
||||
|
||||
|
||||
class ClsPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
res = super().preprocess(row)
|
||||
res['label'] = int(res['label'])
|
||||
return res
|
||||
|
||||
|
||||
class AutoPreprocessor:
|
||||
|
||||
def __init__(self, *, columns: Optional[Dict[str, str]] = None, **kwargs) -> None:
|
||||
self.columns = columns or {}
|
||||
self.kwargs = kwargs
|
||||
|
||||
def _get_preprocessor(self, dataset: DATASET_TYPE) -> RowPreprocessor:
|
||||
features = dataset.features
|
||||
for key in ['conversation', 'conversations', 'messages']:
|
||||
if key in features:
|
||||
return MessagesPreprocessor(**self.kwargs)
|
||||
if 'instruction' in features and 'input' in features:
|
||||
return AlpacaPreprocessor(**self.kwargs)
|
||||
return ResponsePreprocessor(**self.kwargs)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
dataset: DATASET_TYPE,
|
||||
*,
|
||||
num_proc: int = 1,
|
||||
load_from_cache_file: bool = True,
|
||||
**kwargs,
|
||||
) -> DATASET_TYPE:
|
||||
dataset = RowPreprocessor.safe_rename_columns(dataset, self.columns)
|
||||
preprocessor = self._get_preprocessor(dataset)
|
||||
return preprocessor(dataset, num_proc=num_proc, load_from_cache_file=load_from_cache_file, **kwargs)
|
||||
@@ -0,0 +1,111 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import numpy as np
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .core import ResponsePreprocessor
|
||||
|
||||
|
||||
class GroundingMixin:
|
||||
"""This class offers prompts to the grounding task"""
|
||||
task_type: Optional[str] = None
|
||||
|
||||
_grounding_language_mixin = [0.8, 0.2]
|
||||
_grounding_prompts = {
|
||||
'grounding': {
|
||||
'en': [('<ref-object>', '<bbox>'), ('The positions of <ref-object> is', '<bbox>'),
|
||||
('Find the positions of <ref-object>', '<bbox>'), ('Where is <ref-object>', '<bbox>'),
|
||||
('Find <ref-object>', '<bbox>'), ('Show me <ref-object>', '<bbox>'),
|
||||
('Detect <ref-object>', '<bbox>'), ('Locate <ref-object>', '<bbox>'),
|
||||
('Tell me the location of <ref-object>', '<bbox>'), ('Give the location of <ref-object>', '<bbox>'),
|
||||
('Provide the bounding box coordinate of <ref-object>', '<bbox>')],
|
||||
'zh': [('<ref-object>', '<bbox>'), ('<ref-object>的位置在图片中', '<bbox>'), ('<ref-object>在图片中', '<bbox>'),
|
||||
('<ref-object>在', '<bbox>'), ('找到<ref-object>的位置', '<bbox>'), ('<ref-object>在哪里', '<bbox>'),
|
||||
('提供<ref-object>的坐标位置', '<bbox>')]
|
||||
},
|
||||
'caption': {
|
||||
'en': [
|
||||
('<bbox>', '<ref-object>'),
|
||||
('The object at position <bbox>', '<ref-object>'),
|
||||
('This <bbox> is', '<ref-object>'),
|
||||
('What is the object at <bbox>', '<ref-object>'),
|
||||
('Describe <bbox>', '<ref-object>'),
|
||||
('<bbox> is', '<ref-object>'),
|
||||
('The bounding box coordinate <bbox> contains', '<ref-object>'),
|
||||
],
|
||||
'zh': [
|
||||
('<bbox>', '<ref-object>'),
|
||||
('<bbox>是什么', '<ref-object>'),
|
||||
('<bbox>的位置包含', '<ref-object>'),
|
||||
('描述<bbox>', '<ref-object>'),
|
||||
('<bbox>中是', '<ref-object>'),
|
||||
('坐标<bbox>描述了什么', '<ref-object>'),
|
||||
('描述<bbox>中的事物', '<ref-object>'),
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
def construct_grounding_prompt(self):
|
||||
# TODO Only support one bbox to one object
|
||||
lang = np.random.choice(['en', 'zh'], p=[0.8, 0.2])
|
||||
prompts = GroundingMixin._grounding_prompts[self.task_type][lang]
|
||||
query, response = prompts[np.random.choice(range(len(prompts)))]
|
||||
return query, response
|
||||
|
||||
|
||||
class TextGenerationPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def __init__(self,
|
||||
*,
|
||||
prompt: str,
|
||||
query_tag: str = '{{QUERY}}',
|
||||
columns: Optional[Dict[str, str]] = None,
|
||||
**kwargs) -> None:
|
||||
self.query_tag = query_tag
|
||||
self.prompt = prompt
|
||||
super().__init__(columns=columns, **kwargs)
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
row['query'] = self.prompt.replace(self.query_tag, row['query'])
|
||||
return super().preprocess(row)
|
||||
|
||||
|
||||
class ClsGenerationPreprocessor(ResponsePreprocessor):
|
||||
|
||||
def __init__(self,
|
||||
labels: List[str],
|
||||
*,
|
||||
task: str,
|
||||
is_pair_seq: bool = False,
|
||||
columns: Optional[Dict[str, str]] = None,
|
||||
**kwargs) -> None:
|
||||
self.labels = labels
|
||||
self.task = task
|
||||
self.is_pair_seq = is_pair_seq
|
||||
|
||||
category = ', '.join(labels)
|
||||
self.sentence2_key = 'sentence2'
|
||||
self.label_key = 'label'
|
||||
if is_pair_seq:
|
||||
self.sentence_key = 'sentence1'
|
||||
inputs = 'Sentence1: {sentence1}\nSentence2: {sentence2}'
|
||||
else:
|
||||
self.sentence_key = 'sentence'
|
||||
inputs = 'Sentence: {sentence}'
|
||||
self.prompt = f"""Task: {task}
|
||||
{inputs}
|
||||
Category: {category}
|
||||
Output:"""
|
||||
super().__init__(columns=columns, **kwargs)
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
label = row.pop(self.label_key, None)
|
||||
if label is None:
|
||||
return
|
||||
|
||||
if self.is_pair_seq:
|
||||
query = self.prompt.format(sentence1=row.pop(self.sentence_key), sentence2=row.pop(self.sentence2_key))
|
||||
else:
|
||||
query = self.prompt.format(sentence=row.pop(self.sentence_key))
|
||||
row['query'] = query
|
||||
row['response'] = self.labels[int(label)]
|
||||
return super().preprocess(row)
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import json
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from swift.utils import get_logger, use_hf_hub
|
||||
from .dataset_meta import DATASET_MAPPING, DatasetMeta, SubsetDataset
|
||||
from .preprocessor import AutoPreprocessor, MessagesPreprocessor
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def get_dataset_list():
|
||||
datasets = []
|
||||
for key in DATASET_MAPPING:
|
||||
if use_hf_hub():
|
||||
if key[1]:
|
||||
datasets.append(key[1])
|
||||
else:
|
||||
if key[0]:
|
||||
datasets.append(key[0])
|
||||
return datasets
|
||||
|
||||
|
||||
def register_dataset(dataset_meta: DatasetMeta, *, exist_ok: bool = False) -> None:
|
||||
"""Register dataset
|
||||
|
||||
Args:
|
||||
dataset_meta: The `DatasetMeta` info of the dataset.
|
||||
exist_ok: If the dataset id exists, raise error or update it.
|
||||
"""
|
||||
if dataset_meta.dataset_name:
|
||||
dataset_name = dataset_meta.dataset_name
|
||||
else:
|
||||
dataset_name = dataset_meta.ms_dataset_id, dataset_meta.hf_dataset_id, dataset_meta.dataset_path
|
||||
if not exist_ok and dataset_name in DATASET_MAPPING:
|
||||
raise ValueError(f'The `{dataset_name}` has already been registered in the DATASET_MAPPING.')
|
||||
|
||||
DATASET_MAPPING[dataset_name] = dataset_meta
|
||||
|
||||
|
||||
def _preprocess_d_info(d_info: Dict[str, Any], *, base_dir: Optional[str] = None) -> Dict[str, Any]:
|
||||
d_info = deepcopy(d_info)
|
||||
|
||||
columns = None
|
||||
if 'columns' in d_info:
|
||||
columns = d_info.pop('columns')
|
||||
|
||||
if 'messages' in d_info:
|
||||
d_info['preprocess_func'] = MessagesPreprocessor(**d_info.pop('messages'), columns=columns)
|
||||
else:
|
||||
d_info['preprocess_func'] = AutoPreprocessor(columns=columns)
|
||||
|
||||
if 'dataset_path' in d_info:
|
||||
dataset_path = d_info.pop('dataset_path')
|
||||
if base_dir is not None and not os.path.isabs(dataset_path):
|
||||
dataset_path = os.path.join(base_dir, dataset_path)
|
||||
dataset_path = os.path.abspath(os.path.expanduser(dataset_path))
|
||||
|
||||
d_info['dataset_path'] = dataset_path
|
||||
|
||||
if 'subsets' in d_info:
|
||||
subsets = d_info.pop('subsets')
|
||||
for i, subset in enumerate(subsets):
|
||||
if isinstance(subset, dict):
|
||||
subsets[i] = SubsetDataset(**_preprocess_d_info(subset))
|
||||
d_info['subsets'] = subsets
|
||||
return d_info
|
||||
|
||||
|
||||
def _register_d_info(d_info: Dict[str, Any], *, base_dir: Optional[str] = None) -> DatasetMeta:
|
||||
"""Register a single dataset to dataset mapping
|
||||
|
||||
Args:
|
||||
d_info: The dataset info
|
||||
"""
|
||||
d_info = _preprocess_d_info(d_info, base_dir=base_dir)
|
||||
dataset_meta = DatasetMeta(**d_info)
|
||||
register_dataset(dataset_meta)
|
||||
return dataset_meta
|
||||
|
||||
|
||||
def register_dataset_info(dataset_info: Union[str, List[str], None] = None) -> List[DatasetMeta]:
|
||||
"""Register dataset from the `dataset_info.json` or a custom dataset info file
|
||||
This is used to deal with the datasets defined in the json info file.
|
||||
|
||||
Args:
|
||||
dataset_info: The dataset info path
|
||||
"""
|
||||
# dataset_info_path: path, json or None
|
||||
if dataset_info is None:
|
||||
dataset_info = os.path.join(os.path.dirname(__file__), 'data', 'dataset_info.json')
|
||||
assert isinstance(dataset_info, (str, list))
|
||||
base_dir = None
|
||||
log_msg = None
|
||||
if isinstance(dataset_info, str):
|
||||
dataset_path = os.path.abspath(os.path.expanduser(dataset_info))
|
||||
if os.path.isfile(dataset_path):
|
||||
log_msg = dataset_path
|
||||
base_dir = os.path.dirname(dataset_path)
|
||||
with open(dataset_path, 'r', encoding='utf-8') as f:
|
||||
dataset_info = json.load(f)
|
||||
else:
|
||||
dataset_info = json.loads(dataset_info) # json
|
||||
if len(dataset_info) == 0:
|
||||
return []
|
||||
res = []
|
||||
for d_info in dataset_info:
|
||||
res.append(_register_d_info(d_info, base_dir=base_dir))
|
||||
|
||||
if log_msg is None:
|
||||
log_msg = dataset_info if len(dataset_info) < 5 else list(dataset_info.keys())
|
||||
logger.info(f'Successfully registered `{log_msg}`.')
|
||||
return res
|
||||
@@ -0,0 +1,153 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import inspect
|
||||
import numpy as np
|
||||
import os
|
||||
import tempfile
|
||||
from datasets import Dataset as HfDataset
|
||||
from modelscope.hub.utils.utils import get_cache_dir
|
||||
from torch.utils.data import Dataset
|
||||
from typing import Any, Callable, Dict, Optional, Union
|
||||
|
||||
from swift.template import MaxLengthError, Template
|
||||
from swift.utils import get_logger
|
||||
from .preprocessor import RowPreprocessor
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def sample_dataset(
|
||||
dataset: HfDataset,
|
||||
dataset_sample: Optional[int],
|
||||
shuffle: bool = True,
|
||||
random_state: Optional[np.random.RandomState] = None,
|
||||
shuffle_all: bool = False, # For compatibility, this defaults to False.
|
||||
) -> HfDataset:
|
||||
"""Sample dataset by a dataset_sample number
|
||||
Args:
|
||||
dataset: The dataset instance, iterable dataset is not supported
|
||||
dataset_sample: The sample number
|
||||
shuffle: Whether to perform random sampling on non-streaming datasets
|
||||
random_state: The random state
|
||||
Returns:
|
||||
The sampled dataset
|
||||
"""
|
||||
if dataset_sample is None:
|
||||
return dataset
|
||||
|
||||
n_repeat_sample = dataset_sample // len(dataset)
|
||||
n_remain_sample = dataset_sample % len(dataset)
|
||||
if n_repeat_sample >= 1 and n_remain_sample >= 1:
|
||||
logger.warning(f'dataset_sample:{dataset_sample} is greater than len(dataset):{len(dataset)}, '
|
||||
'repeated sampling will be performed.')
|
||||
idx = np.tile(range(len(dataset)), n_repeat_sample)
|
||||
if random_state is None:
|
||||
random_state = np.random.RandomState()
|
||||
if n_remain_sample >= 1:
|
||||
if shuffle:
|
||||
idx_remain = random_state.permutation(len(dataset))[:n_remain_sample]
|
||||
else:
|
||||
idx_remain = np.arange(n_remain_sample)
|
||||
idx = np.concatenate([idx, idx_remain])
|
||||
if n_repeat_sample >= 1 and shuffle and shuffle_all:
|
||||
random_state.shuffle(idx)
|
||||
dataset = dataset.select(idx)
|
||||
return dataset
|
||||
|
||||
|
||||
class LazyLLMDataset(Dataset):
|
||||
"""This class if used to lazy tokenize the dataset, and skips bad ones when training"""
|
||||
|
||||
def __init__(self,
|
||||
dataset: HfDataset,
|
||||
encode_func: Callable[[Dict[str, Any]], Dict[str, Any]],
|
||||
*,
|
||||
n_try_fetch: int = 10,
|
||||
strict: bool = False,
|
||||
random_state: Optional[Union[np.random.RandomState, int]] = None,
|
||||
traceback_limit: int = 10) -> None:
|
||||
self.dataset = dataset
|
||||
self.encode_func = encode_func
|
||||
|
||||
n_try_fetch = 1 if strict else min(n_try_fetch, len(self.dataset))
|
||||
assert n_try_fetch >= 1
|
||||
self.strict = strict
|
||||
self.n_try_fetch = n_try_fetch
|
||||
|
||||
if not isinstance(random_state, np.random.RandomState):
|
||||
random_state = np.random.RandomState(random_state)
|
||||
self.random_state = random_state
|
||||
|
||||
self.traceback_limit = traceback_limit
|
||||
self._traceback_counter = 0
|
||||
self._idx = 0
|
||||
self._idx_list = self.random_state.permutation(len(self.dataset)).tolist()
|
||||
|
||||
def __getitem__(self, idx: int) -> Dict[str, Any]:
|
||||
if isinstance(idx, str):
|
||||
return self.dataset[idx]
|
||||
for i in range(self.n_try_fetch):
|
||||
if i > 0:
|
||||
idx = self._idx_list[self._idx]
|
||||
self._idx = (self._idx + 1) % len(self.dataset)
|
||||
data = self.dataset[idx]
|
||||
try:
|
||||
return self.encode_func(data, return_length=True)
|
||||
except Exception as e:
|
||||
if self.strict:
|
||||
logger.warning('To avoid errors, you can pass `strict=False`.')
|
||||
raise
|
||||
if isinstance(e, MaxLengthError):
|
||||
continue
|
||||
if self.traceback_limit is not None and self._traceback_counter < self.traceback_limit:
|
||||
import traceback
|
||||
logger.info(traceback.format_exc())
|
||||
logger.warning('👆👆👆There are errors in the template.encode, '
|
||||
'and another piece of data will be randomly selected.')
|
||||
self._traceback_counter += 1
|
||||
|
||||
raise ValueError('Failed to retrieve the dataset. You can avoid this issue by increasing `max_length` or '
|
||||
'modifying the `truncation_strategy`.')
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.dataset)
|
||||
|
||||
|
||||
class EncodePreprocessor(RowPreprocessor):
|
||||
|
||||
def __init__(self, template: 'Template'):
|
||||
super().__init__()
|
||||
self.template = template
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
return self.template.encode(row, return_length=True)
|
||||
|
||||
|
||||
class AddLengthPreprocessor(EncodePreprocessor):
|
||||
|
||||
def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
encoded = super().preprocess(row)
|
||||
row['lengths'] = encoded['lengths']
|
||||
return row
|
||||
|
||||
|
||||
TEMP_DIR_POOL = {}
|
||||
|
||||
|
||||
def get_temporary_cache_files_directory(prefix=None):
|
||||
if prefix is None:
|
||||
import datasets.config
|
||||
prefix = datasets.config.TEMP_CACHE_DIR_PREFIX
|
||||
if prefix in TEMP_DIR_POOL:
|
||||
TEMP_DIR = TEMP_DIR_POOL[prefix]
|
||||
else:
|
||||
tmp_dir = os.path.join(get_cache_dir(), 'tmp')
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
kwargs = {}
|
||||
parameters = inspect.signature(tempfile.TemporaryDirectory.__init__).parameters
|
||||
if 'ignore_cleanup_errors' in parameters:
|
||||
kwargs['ignore_cleanup_errors'] = True
|
||||
TEMP_DIR = tempfile.TemporaryDirectory(prefix=prefix, dir=tmp_dir, **kwargs)
|
||||
logger.info(f'create tmp_dir: {TEMP_DIR.name}')
|
||||
TEMP_DIR_POOL[prefix] = TEMP_DIR
|
||||
|
||||
return TEMP_DIR.name
|
||||
Reference in New Issue
Block a user