chore: import upstream snapshot with attribution

tests/infer/test_agent.py

@@ -0,0 +1,28 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+kwargs = {
+    'per_device_train_batch_size': 2,
+    'save_steps': 50,
+    'gradient_accumulation_steps': 4,
+    'num_train_epochs': 1,
+}
+
+
+def test_sft():
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1'
+    from swift import SftArguments, sft_main
+    sft_main(SftArguments(model='Qwen/Qwen2-7B-Instruct', dataset=['iic/ms_agent#2000'], loss_scale='react', **kwargs))
+
+
+def test_infer():
+    from swift import InferArguments, infer_main
+    ckpt_dir = 'output/Qwen2-7B-Instruct/vx-xxx/checkpoint-xxx'
+    infer_main(InferArguments(adapters=[ckpt_dir]))
+
+
+if __name__ == '__main__':
+    test_sft()
+    # test_infer()

tests/infer/test_infer.py

@@ -0,0 +1,70 @@
+import os
+import torch
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def _prepare(infer_backend: Literal['vllm', 'transformers', 'lmdeploy']):
+    from swift.infer_engine import InferRequest
+    if infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine('OpenGVLab/InternVL2_5-2B', torch_dtype=torch.float32)
+    elif infer_backend == 'transformers':
+        from swift.infer_engine import TransformersEngine
+        engine = TransformersEngine('Qwen/Qwen2-7B-Instruct', max_batch_size=16)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine('Qwen/Qwen2-7B-Instruct')
+    infer_requests = [
+        # InferRequest([{'role': 'user', 'content': '晚上睡不着觉怎么办'}]) for i in range(100)
+        InferRequest([{
+            'role': 'user',
+            'content': 'hello! who are you'
+        }]) for i in range(100)
+    ]
+    return engine, infer_requests
+
+
+def test_infer(infer_backend):
+    from swift.infer_engine import RequestConfig
+    from swift.metrics import InferStats
+    engine, infer_requests = _prepare(infer_backend=infer_backend)
+    request_config = RequestConfig(temperature=0)
+    infer_stats = InferStats()
+
+    response_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    for response in response_list[:2]:
+        print(response.choices[0].message.content)
+    print(infer_stats.compute())
+
+
+def test_stream(infer_backend):
+    from swift.infer_engine import RequestConfig
+    from swift.metrics import InferStats
+    engine, infer_requests = _prepare(infer_backend=infer_backend)
+    infer_stats = InferStats()
+    request_config = RequestConfig(temperature=0, stream=True, logprobs=True)
+
+    gen_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    for response in gen_list[0]:
+        if response is None:
+            continue
+        print(response.choices[0].delta.content, end='', flush=True)
+    print()
+    print(infer_stats.compute())
+
+    gen_list = engine.infer(infer_requests, request_config=request_config, use_tqdm=True, metrics=[infer_stats])
+
+    for response in gen_list[0]:
+        pass
+
+    print(infer_stats.compute())
+
+
+if __name__ == '__main__':
+    test_infer('transformers')
+    # test_stream('transformers')

tests/infer/test_logprobs.py

@@ -0,0 +1,69 @@
+import os
+import torch
+from typing import Literal
+
+if __name__ == '__main__':
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def _prepare(infer_backend: Literal['vllm', 'transformers', 'lmdeploy']):
+    from swift.infer_engine import InferRequest
+
+    if infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine('Qwen/Qwen2-7B-Instruct', torch_dtype=torch.float32)
+    elif infer_backend == 'transformers':
+        from swift.infer_engine import TransformersEngine
+        engine = TransformersEngine('Qwen/Qwen2-7B-Instruct')
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine('Qwen/Qwen2-7B-Instruct')
+    infer_requests = [
+        InferRequest([{
+            'role': 'user',
+            'content': '晚上睡不着觉怎么办'
+        }]),
+        InferRequest([{
+            'role': 'user',
+            'content': 'hello! who are you'
+        }])
+    ]
+    return engine, infer_requests
+
+
+def test_infer(engine, infer_requests):
+    from swift.infer_engine import RequestConfig
+    from swift.metrics import InferStats
+
+    request_config = RequestConfig(temperature=0, logprobs=True, top_logprobs=2)
+    infer_stats = InferStats()
+
+    response_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    for response in response_list[:2]:
+        print(response.choices[0].message.content)
+    print(infer_stats.compute())
+
+
+def test_stream(engine, infer_requests):
+    from swift.infer_engine import RequestConfig
+    from swift.metrics import InferStats
+
+    infer_stats = InferStats()
+    request_config = RequestConfig(temperature=0, stream=True, logprobs=True, top_logprobs=2)
+
+    gen_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    for response in gen_list[0]:
+        if response is None:
+            continue
+        print(response.choices[0].delta.content, end='', flush=True)
+
+    print(infer_stats.compute())
+
+
+if __name__ == '__main__':
+    engine, infer_requests = _prepare(infer_backend='transformers')
+    test_infer(engine, infer_requests)
+    test_stream(engine, infer_requests)

tests/infer/test_main.py

@@ -0,0 +1,76 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def test_cli(infer_backend):
+    from swift import InferArguments, infer_main
+    args = InferArguments(model='Qwen/Qwen2-VL-7B-Instruct', infer_backend=infer_backend)
+    infer_main(args)
+
+
+def test_cli_jinja(infer_backend):
+    from swift import InferArguments, infer_main
+    args = InferArguments(model='Qwen/Qwen2-VL-7B-Instruct', infer_backend=infer_backend, template_backend='jinja')
+    infer_main(args)
+
+
+def test_dataset(infer_backend):
+    from swift import InferArguments, infer_main
+    args = InferArguments(
+        model='Qwen/Qwen2-7B-Instruct',
+        infer_backend=infer_backend,
+        val_dataset=['AI-ModelScope/alpaca-gpt4-data-zh#10'],
+        stream=True)
+    infer_main(args)
+
+
+def test_mllm_dataset(infer_backend):
+    from swift import InferArguments, infer_main
+    args = InferArguments(
+        model='Qwen/Qwen2-VL-7B-Instruct',
+        infer_backend=infer_backend,
+        val_dataset=['modelscope/coco_2014_caption:validation#1000'],
+        stream=True)
+    infer_main(args)
+
+
+def test_dataset_ddp():
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+    from swift import InferArguments, infer_main
+    args = InferArguments(
+        model='Qwen/Qwen2-7B-Instruct', max_batch_size=64, val_dataset=['AI-ModelScope/alpaca-gpt4-data-zh#1000'])
+    infer_main(args)
+
+
+def test_dataset_mp_ddp():
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3'
+    from swift import InferArguments, infer_main
+    args = InferArguments(
+        model='Qwen/Qwen2-7B-Instruct', max_batch_size=64, val_dataset=['AI-ModelScope/alpaca-gpt4-data-zh#1000'])
+    infer_main(args)
+
+
+def test_emu3_gen(infer_backend):
+    from swift import InferArguments, infer_main
+    args = InferArguments(
+        model='BAAI/Emu3-Gen',
+        infer_backend=infer_backend,
+        stream=False,
+        use_chat_template=False,
+        top_k=2048,
+        max_new_tokens=40960)
+    infer_main(args)
+
+
+if __name__ == '__main__':
+    # test_cli('transformers')
+    # test_cli_jinja('transformers')
+    # test_dataset('transformers')
+    # test_mllm_dataset('transformers')
+    # test_dataset_ddp()
+    # test_dataset_mp_ddp()
+    test_emu3_gen('transformers')

tests/infer/test_max_memory.py

@@ -0,0 +1,10 @@
+from swift import InferArguments, infer_main
+
+
+def test_max_memory():
+    infer_main(
+        InferArguments(model='Qwen/Qwen2.5-7B-Instruct', max_memory='{0: "50GB", 1: "5GB"}', device_map='sequential'))
+
+
+if __name__ == '__main__':
+    test_max_memory()

tests/infer/test_mllm.py

@@ -0,0 +1,76 @@
+import os
+import torch
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def _prepare(infer_backend: Literal['vllm', 'transformers', 'lmdeploy']):
+    from swift.infer_engine import InferRequest
+    if infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine('Qwen/Qwen-VL-Chat', torch_dtype=torch.float32)
+    elif infer_backend == 'transformers':
+        from swift.infer_engine import TransformersEngine
+        engine = TransformersEngine('Qwen/Qwen2-VL-7B-Instruct')
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine('Qwen/Qwen2-VL-7B-Instruct')
+    infer_requests = [
+        InferRequest([{
+            'role': 'user',
+            'content': '晚上睡不着觉怎么办'
+        }]),
+        InferRequest([{
+            'role':
+            'user',
+            'content': [{
+                'type': 'image_url',
+                'image_url': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png'
+            }]
+        }])
+    ]
+    return engine, infer_requests
+
+
+def test_infer(engine, infer_requests):
+    from swift.infer_engine import RequestConfig
+    from swift.metrics import InferStats
+    request_config = RequestConfig(temperature=0)
+    infer_stats = InferStats()
+
+    response_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    for response in response_list[:2]:
+        print(response.choices[0].message.content)
+    print(infer_stats.compute())
+
+
+def test_stream(engine, infer_requests):
+    from swift.infer_engine import RequestConfig
+    from swift.metrics import InferStats
+    infer_stats = InferStats()
+    request_config = RequestConfig(temperature=0, stream=True, logprobs=True)
+
+    gen_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    for response in gen_list[0]:
+        if response is None:
+            continue
+        print(response.choices[0].delta.content, end='', flush=True)
+    print()
+    print(infer_stats.compute())
+
+    gen_list = engine.infer(infer_requests, request_config=request_config, use_tqdm=True, metrics=[infer_stats])
+
+    for response in gen_list[0]:
+        pass
+
+    print(infer_stats.compute())
+
+
+if __name__ == '__main__':
+    engine, infer_requests = _prepare(infer_backend='transformers')
+    test_infer(engine, infer_requests)
+    test_stream(engine, infer_requests)

tests/infer/test_sglang.py

@@ -0,0 +1,55 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+
+
+def test_engine():
+    from swift.dataset import load_dataset
+    from swift.infer_engine import RequestConfig, SglangEngine
+    dataset = load_dataset('AI-ModelScope/alpaca-gpt4-data-zh#20')[0]
+    engine = SglangEngine('Qwen/Qwen2.5-0.5B-Instruct')
+    request_config = RequestConfig(max_tokens=1024)
+    resp_list = engine.infer(list(dataset), request_config=request_config)
+    for resp in resp_list[:5]:
+        print(resp)
+    resp_list = engine.infer(list(dataset), request_config=request_config)
+    for resp in resp_list[:5]:
+        print(resp)
+
+
+def test_engine_stream():
+    from swift.dataset import load_dataset
+    from swift.infer_engine import RequestConfig, SglangEngine
+    dataset = load_dataset('AI-ModelScope/alpaca-gpt4-data-zh#1')[0]
+    engine = SglangEngine('Qwen/Qwen2.5-0.5B-Instruct')
+    request_config = RequestConfig(max_tokens=1024, stream=True)
+    gen_list = engine.infer(list(dataset), request_config=request_config)
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, flush=True, end='')
+
+
+def test_infer():
+    from swift import InferArguments, infer_main
+    infer_main(
+        InferArguments(model='Qwen/Qwen2.5-0.5B-Instruct', stream=True, infer_backend='sglang', max_new_tokens=2048))
+
+
+def test_eval():
+    from swift import EvalArguments, eval_main
+    eval_main(
+        EvalArguments(
+            model='Qwen/Qwen2-7B-Instruct',
+            eval_dataset='arc_c',
+            infer_backend='sglang',
+            eval_backend='OpenCompass',
+        ))
+
+
+if __name__ == '__main__':
+    test_engine()
+    # test_engine_stream()
+    # test_infer()
+    # test_eval()

tests/infer/test_transformers_engine.py

@@ -0,0 +1,73 @@
+import os
+
+from swift import TransformersEngine
+from swift.infer_engine import InferRequest, RequestConfig
+from swift.metrics import InferStats
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+engine = TransformersEngine('Qwen/Qwen2-0.5B', max_batch_size=4)
+
+
+def test_batch_infer():
+    infer_requests = [InferRequest([{'role': 'user', 'content': 'hello, who are you?'}]) for _ in range(4)]
+    request_config = RequestConfig(temperature=0, max_tokens=32)
+    infer_stats = InferStats()
+
+    response_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    assert len(response_list) == len(infer_requests)
+    for response in response_list:
+        assert len(response.choices) > 0
+        assert response.choices[0].message.content is not None
+
+    stats = infer_stats.compute()
+    assert stats['num_samples'] > 0
+    assert stats['num_generated_tokens'] > 0
+
+
+def test_stream_infer():
+    infer_requests = [InferRequest([{'role': 'user', 'content': 'What is 1+1? Answer briefly.'}])]
+    request_config = RequestConfig(temperature=0, max_tokens=32, stream=True)
+    infer_stats = InferStats()
+
+    gen_list = engine.infer(infer_requests, request_config=request_config, metrics=[infer_stats])
+
+    full_content = ''
+    for chunk in gen_list[0]:
+        if chunk is None:
+            continue
+        delta = chunk.choices[0].delta.content
+        if delta:
+            full_content += delta
+
+    assert len(full_content) > 0, 'Stream infer produced no content'
+
+    stats = infer_stats.compute()
+    assert stats['num_samples'] > 0
+    assert stats['num_generated_tokens'] > 0
+
+
+def test_single_infer_with_system():
+    infer_requests = [
+        InferRequest([{
+            'role': 'system',
+            'content': 'You are a helpful assistant.'
+        }, {
+            'role': 'user',
+            'content': 'Say hello in one word.'
+        }])
+    ]
+    request_config = RequestConfig(temperature=0, max_tokens=16)
+
+    response_list = engine.infer(infer_requests, request_config=request_config)
+
+    assert len(response_list) == 1
+    assert len(response_list[0].choices) > 0
+    assert response_list[0].choices[0].message.content is not None
+
+
+if __name__ == '__main__':
+    test_batch_infer()
+    test_stream_infer()
+    test_single_infer_with_system()