chore: import upstream snapshot with attribution

scripts/utils/plot_loss.py

@@ -0,0 +1,9 @@
+import os
+
+from swift.utils import plot_images
+
+ckpt_dir = 'output/xxx/vx-xxx'
+if __name__ == '__main__':
+    images_dir = os.path.join(ckpt_dir, 'images')
+    tb_dir = os.path.join(ckpt_dir, 'runs')
+    plot_images(images_dir, tb_dir, ['train/loss'], 0.9)

scripts/utils/run_dataset_info.py

@@ -0,0 +1,112 @@
+import numpy as np
+import os
+import re
+
+from swift.dataset import DATASET_MAPPING, EncodePreprocessor, load_dataset
+from swift.model import get_processor
+from swift.template import get_template
+from swift.utils import stat_array
+
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+
+def get_cache_mapping(fpath):
+    with open(fpath, 'r', encoding='utf-8') as f:
+        text = f.read()
+    idx = text.find('| Dataset ID |')
+    text = text[idx:]
+    text_list = text.split('\n')[2:]
+    cache_mapping = {}  # dataset_id -> (dataset_size, stat)
+    for text in text_list:
+        if not text:
+            continue
+        items = text.split('|')
+        key = items[1] if items[1] != '-' else items[6]
+        key = re.search(r'\[(.+?)\]', key).group(1)
+        stat = items[3:5]
+        if stat[0] == '-':
+            stat = ('huge dataset', '-')
+        cache_mapping[key] = stat
+    return cache_mapping
+
+
+def get_dataset_id(key):
+    for dataset_id in key:
+        if dataset_id is not None:
+            break
+    return dataset_id
+
+
+def run_dataset(key, template, cache_mapping):
+    dataset_meta = DATASET_MAPPING[key]
+    ms_id = dataset_meta.ms_dataset_id
+    hf_id = dataset_meta.hf_dataset_id
+    tags = ', '.join(tag for tag in dataset_meta.tags) or '-'
+    dataset_id = ms_id or hf_id
+    use_hf = ms_id is None
+    if ms_id is not None:
+        ms_id = f'[{ms_id}](https://modelscope.cn/datasets/{ms_id})'
+    else:
+        ms_id = '-'
+    if hf_id is not None:
+        hf_id = f'[{hf_id}](https://huggingface.co/datasets/{hf_id})'
+    else:
+        hf_id = '-'
+    subsets = '<br>'.join(subset.name for subset in dataset_meta.subsets)
+
+    if dataset_meta.huge_dataset:
+        dataset_size = 'huge dataset'
+        stat_str = '-'
+    elif dataset_id in cache_mapping:
+        dataset_size, stat_str = cache_mapping[dataset_id]
+    else:
+        num_proc = 4
+        dataset, _ = load_dataset(f'{dataset_id}:all', strict=False, num_proc=num_proc, use_hf=use_hf)
+        dataset_size = len(dataset)
+        random_state = np.random.RandomState(42)
+        idx_list = random_state.choice(dataset_size, size=min(dataset_size, 100000), replace=False)
+        encoded_dataset = EncodePreprocessor(template)(
+            dataset.select(idx_list), num_proc=num_proc, load_from_cache_file=False)
+
+        input_ids = encoded_dataset['input_ids']
+        token_len = [len(tokens) for tokens in input_ids]
+        stat = stat_array(token_len)[0]
+        stat_str = f"{stat['mean']:.1f}±{stat['std']:.1f}, min={stat['min']}, max={stat['max']}"
+
+    return f'|{ms_id}|{subsets}|{dataset_size}|{stat_str}|{tags}|{hf_id}|'
+
+
+def write_dataset_info() -> None:
+    fpaths = [
+        'docs/source/Instruction/Supported-models-and-datasets.md',
+        'docs/source_en/Instruction/Supported-models-and-datasets.md'
+    ]
+    cache_mapping = get_cache_mapping(fpaths[0])
+    res_text_list = []
+    res_text_list.append('| Dataset ID | Subset Name | Dataset Size | Statistic (token) | Tags | HF Dataset ID |')
+    res_text_list.append('| ---------- | ----------- | -------------| ------------------| ---- | ------------- |')
+
+    all_keys = list(DATASET_MAPPING.keys())
+    all_keys = sorted(all_keys, key=lambda x: get_dataset_id(x))
+    tokenizer = get_processor('Qwen/Qwen2.5-7B-Instruct')
+    template = get_template(tokenizer)
+    try:
+        for i, key in enumerate(all_keys):
+            res = run_dataset(key, template, cache_mapping)
+            res_text_list.append(res)
+            print(res)
+    finally:
+        for fpath in fpaths:
+            with open(fpath, 'r', encoding='utf-8') as f:
+                text = f.read()
+            idx = text.find('| Dataset ID |')
+
+            new_text = '\n'.join(res_text_list)
+            text = text[:idx] + new_text + '\n'
+            with open(fpath, 'w', encoding='utf-8') as f:
+                f.write(text)
+    print(f'数据集总数: {len(all_keys)}')
+
+
+if __name__ == '__main__':
+    write_dataset_info()

scripts/utils/run_model_info.py

@@ -0,0 +1,128 @@
+from itertools import chain
+from typing import Any, List
+
+from swift.model import MODEL_MAPPING, ModelType
+from swift.template import TEMPLATE_MAPPING, TemplateType
+from swift.utils import is_megatron_available
+
+
+def get_url_suffix(model_id):
+    if ':' in model_id:
+        return model_id.split(':')[0]
+    return model_id
+
+
+supported_mcore_model_types = None
+
+
+def get_cache_mapping(fpath):
+    with open(fpath, 'r', encoding='utf-8') as f:
+        text = f.read()
+    idx = text.find('| Model ID |')
+    end_idx = text.find('| Dataset ID |')
+    text = text[idx:end_idx]
+    text_list = text.split('\n')[2:]
+    cache_mapping = {}
+    for text in text_list:
+        if not text:
+            continue
+        items = text.split('|')
+        if len(items) < 6:
+            continue
+        cache_mapping[items[1]] = items[5]
+    return cache_mapping
+
+
+def get_model_info_table():
+    global supported_mcore_model_types
+    fpaths = [
+        'docs/source/Instruction/Supported-models-and-datasets.md',
+        'docs/source_en/Instruction/Supported-models-and-datasets.md'
+    ]
+    cache_mapping = get_cache_mapping(fpaths[0])
+    end_words = [['### 多模态大模型', '## 数据集'], ['### Multimodal large models', '## Datasets']]
+    result = [
+        '| Model ID | Model Type | Default Template | Default Agent Template | '
+        'Requires | Support Megatron | Tags | HF Model ID |\n'
+        '| -------- | -----------| ---------------- | ---------------------- | '
+        '-------- | ---------------- | ---- | ----------- |\n'
+    ] * 2
+    res_llm: List[Any] = []
+    res_mllm: List[Any] = []
+    mg_count_llm = 0
+    mg_count_mllm = 0
+    for template in TemplateType.get_template_name_list():
+        assert template in TEMPLATE_MAPPING
+
+    for model_type in ModelType.get_model_name_list():
+        model_meta = MODEL_MAPPING[model_type]
+        for group in model_meta.model_groups:
+            for model in group.models:
+                ms_model_id = model.ms_model_id
+                hf_model_id = model.hf_model_id
+                if ms_model_id:
+                    ms_model_id = f'[{ms_model_id}](https://modelscope.cn/models/{get_url_suffix(ms_model_id)})'
+                else:
+                    ms_model_id = '-'
+                if hf_model_id:
+                    hf_model_id = f'[{hf_model_id}](https://huggingface.co/{get_url_suffix(hf_model_id)})'
+                else:
+                    hf_model_id = '-'
+                tags = ', '.join(group.tags or model_meta.tags) or '-'
+                requires = ', '.join(group.requires or model_meta.requires) or '-'
+                template = group.template or model_meta.template
+                template_meta = TEMPLATE_MAPPING.get(template)
+                agent_template = template_meta.agent_template if template_meta else ''
+                agent_template = agent_template or ''
+                if is_megatron_available():
+                    from mcore_bridge.model import MODEL_MAPPING as MCORE_MODEL_MAPPING
+                    if supported_mcore_model_types is None:
+                        supported_mcore_model_types = set(
+                            list(chain.from_iterable([v.model_types for k, v in MCORE_MODEL_MAPPING.items()])))
+                    if model_meta.mcore_model_type is not None:
+                        support_megatron = True
+                    elif model_meta.model_type in supported_mcore_model_types:
+                        support_megatron = True
+                    else:
+                        support_megatron = False
+                    for word in ['gptq', 'awq', 'bnb', 'aqlm', 'int4', 'int8', 'nf4']:
+                        if word in ms_model_id.lower():
+                            support_megatron = False
+                            break
+                    support_megatron = '&#x2714;' if support_megatron else '&#x2718;'
+                else:
+                    support_megatron = cache_mapping.get(ms_model_id, '&#x2718;')
+                if support_megatron == '&#x2714;':
+                    if model_meta.is_multimodal:
+                        mg_count_mllm += 1
+                    else:
+                        mg_count_llm += 1
+                r = (f'|{ms_model_id}|{model_type}|{template}|{agent_template}|{requires}|'
+                     f'{support_megatron}|{tags}|{hf_model_id}|\n')
+                if model_meta.is_multimodal:
+                    res_mllm.append(r)
+                else:
+                    res_llm.append(r)
+    print(f'LLM总数: {len(res_llm)}, MLLM总数: {len(res_mllm)}')
+    print(f'[Megatron] LLM总数: {mg_count_llm}, MLLM总数: {mg_count_mllm}')
+    text = ['', '']  # llm, mllm
+    for i, res in enumerate([res_llm, res_mllm]):
+        for r in res:
+            text[i] += r
+        result[i] += text[i]
+
+    for i, fpath in enumerate(fpaths):
+        with open(fpath, 'r', encoding='utf-8') as f:
+            text = f.read()
+        llm_start_idx = text.find('| Model ID |')
+        mllm_start_idx = text[llm_start_idx + 1:].find('| Model ID |') + llm_start_idx + 1
+        llm_end_idx = text.find(end_words[i][0])
+        mllm_end_idx = text.find(end_words[i][1])
+        output = text[:llm_start_idx] + result[0] + '\n\n' + text[llm_end_idx:mllm_start_idx] + result[
+            1] + '\n\n' + text[mllm_end_idx:]
+        with open(fpath, 'w', encoding='utf-8') as f:
+            f.write(output)
+
+
+if __name__ == '__main__':
+    get_model_info_table()

scripts/utils/run_template.py

@@ -0,0 +1,8 @@
+from swift.template import TemplateType
+
+if __name__ == '__main__':
+    template_name_list = TemplateType.get_template_name_list()
+    tn_gen = ', '.join([tn for tn in template_name_list if 'generation' in tn])
+    tn_chat = ', '.join([tn for tn in template_name_list if 'generation' not in tn])
+    print(f'Text Generation: {tn_gen}')
+    print(f'Chat: {tn_chat}')

scripts/utils/test_link_valid.py

@@ -0,0 +1,57 @@
+import os
+import re
+import requests
+
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+def check_link(url):
+    try:
+        response = requests.head(url, timeout=5, allow_redirects=True)
+        return response.status_code == 200
+    except requests.RequestException:
+        return False
+
+
+def extract_links_from_md(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        content = file.read()
+        links = re.findall(r'\[.*?\]\((.*?)\)', content)
+        return links
+
+
+def check_links_in_folder(folder_path):
+    for root, _, files in os.walk(folder_path):
+        for file in files:
+            if file.endswith('.md'):
+                if file in ['Supported-models-and-datasets.md', 'Supported-models-and-datasets.md']:
+                    continue
+                file_path = os.path.join(root, file)
+                logger.info(f'Checking links in file: {file_path}')
+                links = extract_links_from_md(file_path)
+                for link in links:
+                    if not link.startswith(('http://', 'https://')):
+                        path = link.rsplit('#', 1)[0]
+                        if path:
+                            path = os.path.abspath(os.path.join(root, path))
+                            if os.path.exists(path):
+                                logger.info(f'✅ Link is valid: {link}')
+                            else:
+                                logger.info(f'❌ Link is broken: {link}')
+                        else:
+                            logger.info(f'Skipping non-HTTP link: {link}')
+                        continue
+                    if check_link(link):
+                        logger.info(f'✅ Link is valid: {link}')
+                    else:
+                        if 'huggingface.co' in link:
+                            logger.info(f'Link is broken: {link}')
+                        else:
+                            logger.info(f'❌ Link is broken: {link}')
+
+
+if __name__ == '__main__':
+    folder_path = './'
+    check_links_in_folder(folder_path)