chore: import upstream snapshot with attribution

tests/general/test_arch.py

@@ -0,0 +1,45 @@
+def test_model_arch():
+    import random
+    from transformers import PretrainedConfig
+
+    from swift.model import MODEL_MAPPING
+    from swift.utils import JsonlWriter, safe_snapshot_download
+    jsonl_writer = JsonlWriter('model_arch.jsonl')
+    for i, (model_type, model_meta) in enumerate(MODEL_MAPPING.items()):
+        if i < 0:
+            continue
+        arch_list = model_meta.architectures
+        for model_group in model_meta.model_groups:
+            model = random.choice(model_group.models).ms_model_id
+            config_dict = None
+            try:
+                model_dir = safe_snapshot_download(model, download_model=False)
+                config_dict = PretrainedConfig.get_config_dict(model_dir)[0]
+            except Exception:
+                pass
+            finally:
+                msg = None
+                if config_dict:
+                    arch = config_dict.get('architectures')
+                    if arch and arch[0] not in arch_list:
+                        msg = {
+                            'model_type': model_type,
+                            'model': model,
+                            'config_arch': arch,
+                            'architectures': arch_list
+                        }
+                    elif not arch and arch_list:
+                        msg = {
+                            'model_type': model_type,
+                            'model': model,
+                            'config_arch': arch,
+                            'architectures': arch_list
+                        }
+                else:
+                    msg = {'msg': 'error', 'model_type': model_type, 'model': model, 'arch_list': arch_list}
+                if msg:
+                    jsonl_writer.append(msg)
+
+
+if __name__ == '__main__':
+    test_model_arch()

tests/general/test_data_preprocess.py

@@ -0,0 +1,207 @@
+import unittest
+
+from swift.dataset import EncodePreprocessor, MessagesPreprocessor, PackingDataset, load_dataset
+from swift.model import get_processor
+from swift.template import get_template
+
+
+class TestDataPreprocess(unittest.TestCase):
+    """Lightweight data preprocessing tests (no model forward/backward).
+
+    These are fast tests suitable for CI. They cover:
+    - SFT dataset encode (input_ids/labels)
+    - Truncation/max_length
+    - Data collator padding (attention_mask)
+    - Multi-turn messages
+    - Tool message
+    - Packing dataset
+
+    Why these tests are needed:
+    - Swift's data preprocessing pipeline is complex (template -> encode -> collate -> pack).
+      NPU training failures often stem from shape/mask/label mismatches before the model
+      even sees the data, not from operator issues.
+    - The original tests/general/test_dataset.py and test_template.py use top-level
+      functions and remote 7B models, so they are never run by unittest discovery
+      and are too heavy for CI.
+    """
+
+    MODEL_PATH = 'Qwen/Qwen2-0.5B'
+
+    @classmethod
+    def setUpClass(cls):
+        cls.processor = get_processor(cls.MODEL_PATH)
+        cls.template = get_template(cls.processor)
+        cls.template.mode = 'train'
+        cls.template.init_processor(cls.processor)
+
+    def _encode_dataset(self, dataset):
+        encode_preprocessor = EncodePreprocessor(self.template)
+        return encode_preprocessor(dataset, num_proc=1, load_from_cache_file=False, strict=False)
+
+    def test_sft_dataset_encode(self):
+        dataset, _ = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#20'], num_proc=1, strict=False)
+        self.assertGreater(len(dataset), 0)
+        encoded_dataset = self._encode_dataset(dataset)
+        first = encoded_dataset[0]
+        self.assertIn('input_ids', first)
+        self.assertIn('labels', first)
+        self.assertEqual(len(first['input_ids']), len(first['labels']))
+
+    def test_truncation_max_length(self):
+        self.template.max_length = 128
+        dataset, _ = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#20'], num_proc=1, strict=False)
+        encoded_dataset = self._encode_dataset(dataset)
+        for row in encoded_dataset:
+            self.assertLessEqual(len(row['input_ids']), self.template.max_length)
+        self.template.max_length = None
+
+    def test_data_collator_padding(self):
+        dataset, _ = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#20'], num_proc=1, strict=False)
+        encoded_dataset = self._encode_dataset(dataset)
+        batch = [encoded_dataset[i] for i in range(4)]
+        collated = self.template.data_collator(batch)
+        self.assertIn('input_ids', collated)
+        self.assertIn('labels', collated)
+        self.assertIn('attention_mask', collated)
+        self.assertEqual(collated['input_ids'].shape[0], 4)
+
+    def test_multi_turn_messages(self):
+        multi_turn_row = {
+            'messages': [
+                {
+                    'role': 'user',
+                    'content': 'What is Python?'
+                },
+                {
+                    'role': 'assistant',
+                    'content': 'Python is a programming language.'
+                },
+                {
+                    'role': 'user',
+                    'content': 'What are its advantages?'
+                },
+                {
+                    'role': 'assistant',
+                    'content': 'Python is easy to learn and use.'
+                },
+            ]
+        }
+        encoded = self.template.encode(multi_turn_row, return_length=True)
+        self.assertIn('input_ids', encoded)
+        self.assertIn('labels', encoded)
+        self.assertGreater(len(encoded['input_ids']), 0)
+        self.assertEqual(len(encoded['input_ids']), len(encoded['labels']))
+
+    def test_tool_message(self):
+        tool_row = {
+            'messages': [
+                {
+                    'role': 'user',
+                    'content': 'What is the weather in Beijing?'
+                },
+                {
+                    'role':
+                    'assistant',
+                    'content':
+                    '',
+                    'tool_calls': [{
+                        'type': 'function',
+                        'function': {
+                            'name': 'get_weather',
+                            'arguments': '{"city": "Beijing"}'
+                        }
+                    }]
+                },
+                {
+                    'role': 'tool',
+                    'content': '{"temperature": 25, "condition": "sunny"}'
+                },
+                {
+                    'role': 'assistant',
+                    'content': 'The weather in Beijing is sunny with a temperature of 25 degrees.'
+                },
+            ]
+        }
+        encoded = self.template.encode(tool_row, return_length=True)
+        self.assertIn('input_ids', encoded)
+        self.assertIn('labels', encoded)
+        self.assertGreater(len(encoded['input_ids']), 0)
+
+    def test_packing_dataset(self):
+        dataset, _ = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#20'], num_proc=1, strict=False)
+        encoded_dataset = self._encode_dataset(dataset)
+        packing_dataset = PackingDataset(
+            self.template,
+            encoded_dataset,
+            num_proc=1,
+            strict=False,
+            load_from_cache_file=False,
+            packing_length=512,
+            packing_num_proc=1,
+        )
+        self.assertGreater(len(packing_dataset), 0)
+        packed = packing_dataset[0]
+        self.assertIsInstance(packed, list)
+        self.assertGreater(len(packed), 0)
+        self.assertIn('input_ids', packed[0])
+        self.assertIn('labels', packed[0])
+
+
+class TestRejectedMessagesPreprocess(unittest.TestCase):
+    """MessagesPreprocessor handling of rejected_messages (no model required)."""
+
+    def test_empty_rejected_messages_does_not_crash(self):
+        """A DPO row whose rejected_messages repair to empty must not crash.
+
+        The recursive preprocess() call returns None when rejected_messages is
+        empty (the same graceful-skip path used for the main messages list), so
+        subscripting it with ['messages'] raised TypeError and aborted the whole
+        dataset map. Downstream already treats rejected_messages is None as
+        'no rejected', so the row should fall back to None instead.
+        """
+        row = {
+            'messages': [
+                {
+                    'role': 'user',
+                    'content': 'Q'
+                },
+                {
+                    'role': 'assistant',
+                    'content': 'good'
+                },
+            ],
+            'rejected_messages': [],
+        }
+        result = MessagesPreprocessor().preprocess(row)
+        self.assertIsNotNone(result)
+        self.assertIsNone(result['rejected_messages'])
+
+    def test_valid_rejected_messages_preserved(self):
+        row = {
+            'messages': [
+                {
+                    'role': 'user',
+                    'content': 'Q'
+                },
+                {
+                    'role': 'assistant',
+                    'content': 'good'
+                },
+            ],
+            'rejected_messages': [
+                {
+                    'role': 'user',
+                    'content': 'Q'
+                },
+                {
+                    'role': 'assistant',
+                    'content': 'bad'
+                },
+            ],
+        }
+        result = MessagesPreprocessor().preprocess(row)
+        self.assertEqual(result['rejected_messages'][-1]['content'], 'bad')
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/general/test_dataset.py

@@ -0,0 +1,90 @@
+from typing import List
+
+from swift.dataset import load_dataset
+
+
+def _test_dataset(datasets: List[str], num_proc: int = 1, strict: bool = False, **kwargs):
+    dataset = load_dataset(datasets, num_proc=num_proc, strict=strict, **kwargs)
+    print(f'dataset[0]: {dataset[0]}')
+    print(f'dataset[1]: {dataset[1]}')
+
+
+def test_sft():
+    # swift/SlimOrca  swift/cosmopedia-100k
+    # _test_dataset(['lvjianjin/AdvertiseGen'])
+    # _test_dataset(['AI-ModelScope/Duet-v0.5'])
+    # _test_dataset(['swift/SlimOrca', 'swift/cosmopedia-100k'])
+    # _test_dataset(['OmniData/Zhihu-KOL-More-Than-100-Upvotes'])
+    # _test_dataset(['OmniData/Zhihu-KOL'])
+    _test_dataset([
+        'AI-ModelScope/alpaca-gpt4-data-zh#1000', 'AI-ModelScope/alpaca-gpt4-data-en#1000',
+        'AI-ModelScope/LongAlpaca-12k#1000'
+    ])
+    # _test_dataset(['swift/Infinity-Instruct:all'])
+    # _test_dataset(['swift/sharegpt:all'])
+    # _test_dataset(['AI-ModelScope/sharegpt_gpt4:all'])
+    # _test_dataset(['iic/ms_bench'])
+    # _test_dataset(['swift/tagengo-gpt4'])
+
+
+def test_mllm():
+    # _test_dataset(['AI-ModelScope/ShareGPT4V:all'])
+    # _test_dataset(['AI-ModelScope/LLaVA-Pretrain'])
+    # _test_dataset(['swift/TextCaps'])
+    # _test_dataset(['swift/RLAIF-V-Dataset:all'])
+    # _test_dataset(['swift/OK-VQA_train'])
+    # _test_dataset(['swift/OCR-VQA'])
+    # _test_dataset(['swift/A-OKVQA'])
+    # _test_dataset(['AI-ModelScope/MovieChat-1K-test'])
+    _test_dataset([
+        'AI-ModelScope/LaTeX_OCR:all', 'modelscope/coco_2014_caption:validation',
+        'speech_asr/speech_asr_aishell1_trainsets:validation'
+    ],
+                  strict=False)
+    # _test_dataset(['swift/VideoChatGPT:all'])
+    # _test_dataset(['speech_asr/speech_asr_aishell1_trainsets:validation'])
+    # _test_dataset(['AI-ModelScope/captcha-images'])
+    # _test_dataset(['swift/gpt4v-dataset:all'])
+    # _test_dataset(['modelscope/coco_2014_caption:validation'])
+    # _test_dataset(['AI-ModelScope/LLaVA-Instruct-150K'], num_proc=16)
+
+
+def test_agent():
+    _test_dataset(['swift/ToolBench'])
+    # _test_dataset(['AI-ModelScope/ms_agent_for_agentfabric:all'])
+
+
+def test_dpo():
+    _test_dataset(['AI-ModelScope/orpo-dpo-mix-40k'])
+    _test_dataset(['AI-ModelScope/hh-rlhf:all'])
+    _test_dataset(['AI-ModelScope/hh_rlhf_cn:all'])
+    _test_dataset(['hjh0119/shareAI-Llama3-DPO-zh-en-emoji:all'])
+
+
+def test_kto():
+    _test_dataset(['AI-ModelScope/ultrafeedback-binarized-preferences-cleaned-kto'])
+
+
+def test_pretrain():
+    _test_dataset(['AI-ModelScope/ruozhiba:all'])
+
+
+def test_dataset_info():
+    _test_dataset(['swift/self-cognition#500'], model_name='xiao huang', model_author='swift')
+    # _test_dataset(['codefuse-ai/CodeExercise-Python-27k'])
+
+
+def test_cls():
+    _test_dataset(['simpleai/HC3-Chinese:baike'])
+    _test_dataset(['simpleai/HC3-Chinese:baike_cls'])
+
+
+if __name__ == '__main__':
+    # test_sft()
+    # test_agent()
+    # test_dpo()
+    # test_kto()
+    test_mllm()
+    # test_pretrain()
+    # test_dataset_info()
+    # test_cls()

tests/general/test_model.py

@@ -0,0 +1,53 @@
+import os
+import torch
+import unittest
+
+from swift.utils import get_device
+
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+
+def test_qwen2():
+    import os
+
+    from swift.model import get_model_processor
+    model, tokenizer = get_model_processor('Qwen/Qwen2-7B-Instruct', load_model=False)
+    print(f'model: {model}, tokenizer: {tokenizer}')
+    # test hf
+    model, tokenizer = get_model_processor('Qwen/Qwen2-7B-Instruct', load_model=False, use_hf=True)
+
+    model, tokenizer = get_model_processor(
+        'Qwen/Qwen2-7B-Instruct', torch_dtype=torch.float32, device_map=get_device(), attn_impl='flash_attn')
+    print(f'model: {model}, tokenizer: {tokenizer}')
+
+
+def test_modelscope_hub():
+    from swift.model import get_model_processor
+    model, tokenizer = get_model_processor('Qwen/Qwen2___5-Math-1___5B-Instruct/', load_model=False)
+
+
+class TestMolmo2Registration(unittest.TestCase):
+
+    def test_registration(self):
+        from swift.model import MODEL_MAPPING, MLLMModelType
+        from swift.template import TEMPLATE_MAPPING, TemplateType
+
+        model_meta = MODEL_MAPPING[MLLMModelType.molmo2]
+        self.assertEqual(model_meta.template, TemplateType.molmo2)
+        self.assertEqual(model_meta.model_arch.arch_name, 'molmo')
+        self.assertIn('Molmo2ForConditionalGeneration', model_meta.architectures)
+
+        hf_model_ids = []
+        for group in model_meta.model_groups:
+            for model in group.models:
+                hf_model_ids.append(model.hf_model_id)
+
+        self.assertIn('allenai/Molmo2-4B', hf_model_ids)
+        self.assertIn('allenai/Molmo2-8B', hf_model_ids)
+        self.assertIn('allenai/Molmo2-O-7B', hf_model_ids)
+        self.assertIn(TemplateType.molmo2, TEMPLATE_MAPPING)
+
+
+if __name__ == '__main__':
+    test_qwen2()
+    # test_modelscope_hub()

tests/general/test_stream.py

@@ -0,0 +1,20 @@
+from swift.dataset import load_dataset
+
+
+def test_local_dataset():
+    # please use git clone
+    from swift.utils import git_clone_github
+    model_dir = git_clone_github('https://www.modelscope.cn/datasets/swift/swift-sft-mixture.git')
+    dataset = load_dataset(datasets=[f'{model_dir}:firefly'], streaming=True)[0]
+    print(next(iter(dataset)))
+
+
+def test_hub_dataset():
+    local_dataset = 'swift/swift-sft-mixture:firefly'
+    dataset = load_dataset(datasets=[local_dataset], streaming=True)[0]
+    print(next(iter(dataset)))
+
+
+if __name__ == '__main__':
+    test_local_dataset()
+    # test_hub_dataset()

tests/general/test_template.py

@@ -0,0 +1,79 @@
+from swift.dataset import EncodePreprocessor, load_dataset
+from swift.model import get_processor
+from swift.template import TemplateInputs, get_template
+
+
+def test_template():
+    tokenizer = get_processor('Qwen/Qwen2-7B-Instruct')
+    template = get_template(tokenizer)
+    template_inputs = TemplateInputs.from_dict({
+        'messages': [{
+            'role': 'system',
+            'content': 'AAA'
+        }, {
+            'role': 'user',
+            'content': 'BBB'
+        }, {
+            'role': 'assistant',
+            'content': 'CCC'
+        }, {
+            'role': 'user',
+            'content': 'DDD'
+        }]
+    })
+    inputs = template.encode(template_inputs)
+    print(f'inputs.keys(): {inputs.keys()}')
+    print(tokenizer.decode(inputs['input_ids']))
+
+
+def test_mllm():
+    processor = get_processor('Qwen/Qwen2-VL-7B-Instruct')
+    template = get_template(processor)
+    template_inputs = TemplateInputs(
+        chosen={
+            'messages': [{
+                'role': 'system',
+                'content': 'AAA'
+            }, {
+                'role': 'user',
+                'content': '<image>BBB'
+            }, {
+                'role': 'assistant',
+                'content': 'CCC'
+            }, {
+                'role': 'user',
+                'content': 'DDD'
+            }],
+            'images': ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png']
+        })
+    inputs = template.encode(template_inputs)
+    print(f'inputs.keys(): {inputs.keys()}')
+    print(template.safe_decode(inputs['input_ids']))
+
+
+def _test_dataset_map(model_id: str, dataset_id: str):
+    tokenizer = get_processor(model_id)
+    template = get_template(tokenizer)
+    dataset = load_dataset([dataset_id], num_proc=2)[0]
+
+    # 1: 1500
+    # 16: 10766.36 examples/s
+    new_dataset = EncodePreprocessor(template)(dataset, num_proc=4)
+    print(f'new_dataset: {new_dataset}')
+    print(template.safe_decode(new_dataset[0]['input_ids']))
+    print(template.safe_decode(new_dataset[1]['input_ids']))
+
+
+def test_llm_dataset_map():
+    _test_dataset_map('Qwen/Qwen2-7B-Instruct', 'AI-ModelScope/alpaca-gpt4-data-zh')
+
+
+def test_mllm_dataset_map():
+    _test_dataset_map('Qwen/Qwen2-VL-7B-Instruct', 'modelscope/coco_2014_caption:validation#100')
+
+
+if __name__ == '__main__':
+    test_template()
+    test_mllm()
+    test_llm_dataset_map()
+    test_mllm_dataset_map()

tests/general/test_template_meta.py

@@ -0,0 +1,22 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.template import TemplateMeta
+
+
+def test_replace_system_preserves_non_string_elements():
+    """_replace_system must not drop list elements like ['bos_token_id'].
+
+    Templates such as ziya, bluelm and emu3_chat use
+    ``prefix=[['bos_token_id'], '{{SYSTEM}}']``.  When no system message is
+    provided the prefix is produced by _replace_system, which should keep every
+    non-string element intact and only strip the placeholder from strings.
+    """
+    meta = TemplateMeta(
+        template_type='_test_replace_system_bug',
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=['{{QUERY}}'],
+        chat_sep=['\n'],
+    )
+    # __post_init__ moves prefix to system_prefix and builds a no-system prefix
+    # via _replace_system.  The list element must survive.
+    assert any(isinstance(p, list) for p in meta.prefix), (f'_replace_system dropped the bos_token_id list; '
+                                                           f'meta.prefix={meta.prefix!r}')