chore: import upstream snapshot with attribution

tests/deploy/test_dataset.py

@@ -0,0 +1,63 @@
+def _test_client(port=8000):
+    import time
+
+    from swift.dataset import load_dataset
+    from swift.infer_engine import InferClient, InferRequest, RequestConfig
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], num_proc=4)
+    infer_client = InferClient(port=port)
+    while True:
+        try:
+            infer_client.models
+            break
+        except Exception:
+            time.sleep(1)
+            pass
+    infer_requests = []
+    for data in dataset[0]:
+        infer_requests.append(InferRequest(**data))
+    request_config = RequestConfig(seed=42, max_tokens=256, temperature=0.8)
+
+    resp = infer_client.infer(infer_requests, request_config=request_config, use_tqdm=False)
+    print(len(resp))
+
+
+def _test(infer_backend):
+    import os
+
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+    from swift.arguments import DeployArguments
+    from swift.pipelines import run_deploy
+    args = DeployArguments(model='Qwen/Qwen2-7B-Instruct', infer_backend=infer_backend, verbose=False)
+    with run_deploy(args) as port:
+        _test_client(port)
+
+
+def test_vllm():
+    _test('vllm')
+
+
+def test_lmdeploy():
+    _test('lmdeploy')
+
+
+def test_pt():
+    _test('transformers')
+
+
+def test_vllm_origin():
+    import subprocess
+    import sys
+    from modelscope import snapshot_download
+    model_dir = snapshot_download('Qwen/Qwen2-7B-Instruct')
+    args = [sys.executable, '-m', 'vllm.entrypoints.openai.api_server', '--model', model_dir]
+    process = subprocess.Popen(args)
+    _test_client()
+    process.terminate()
+
+
+if __name__ == '__main__':
+    # test_vllm_origin()
+    # test_vllm()
+    test_lmdeploy()
+    # test_pt()

tests/deploy/test_logprobs.py

@@ -0,0 +1,116 @@
+def _test_client(port: int, print_logprobs: bool = False, test_vlm: bool = False):
+    import aiohttp
+    import time
+    from pprint import pprint
+
+    from swift.infer_engine import InferClient, InferRequest, RequestConfig
+
+    infer_client = InferClient(port=port)
+
+    while True:
+        try:
+            models = infer_client.models
+            print(f'models: {models}')
+        except aiohttp.ClientConnectorError:
+            time.sleep(5)
+            continue
+        break
+
+    if test_vlm:
+        query = '这是什么'
+        # http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png
+        messages = [{
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'text',
+                    'text': '这是什么'
+                },
+                {
+                    'type': 'image_url',
+                    'image_url': {
+                        'url': 'cat.png'
+                    }
+                },
+            ]
+        }]
+    else:
+        query = '123*234=?'
+        messages = [{'role': 'user', 'content': query}]
+
+    infer_request = InferRequest(messages=messages)
+    request_config = RequestConfig(seed=42, max_tokens=256, temperature=0.8, logprobs=True, top_logprobs=5)
+
+    resp = infer_client.infer([infer_request], request_config=request_config)[0]
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    if print_logprobs:
+        pprint(resp.choices[0].logprobs)
+
+    request_config = RequestConfig(
+        stream=True, seed=42, max_tokens=256, temperature=0.8, top_k=20, top_p=0.8, logprobs=True, top_logprobs=5)
+    gen_list = infer_client.infer([infer_request], request_config=request_config)
+    print(f'query: {query}')
+    print('response: ', end='')
+    for chunk in gen_list[0]:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+        if print_logprobs and chunk.choices[0].logprobs is not None:
+            pprint(chunk.choices[0].logprobs)
+    print()
+
+
+def _test(infer_backend, test_vlm: bool = False):
+    import os
+
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+    os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0'
+    import multiprocessing
+
+    from swift import DeployArguments, deploy_main
+    mp = multiprocessing.get_context('spawn')
+    model = 'Qwen/Qwen2-VL-7B-Instruct' if test_vlm else 'Qwen/Qwen2-7B-Instruct'
+    args = DeployArguments(model=model, infer_backend=infer_backend, verbose=False)
+    process = mp.Process(target=deploy_main, args=(args, ))
+    process.start()
+    _test_client(args.port, True, test_vlm)
+    process.terminate()
+
+
+def test_vllm_vlm():
+    _test('vllm', test_vlm=True)
+
+
+def test_vllm():
+    _test('vllm')
+
+
+def test_lmdeploy():
+    _test('lmdeploy')
+
+
+def test_pt():
+    _test('transformers')
+
+
+def test_vllm_origin():
+    import os
+    import subprocess
+    import sys
+    from modelscope import snapshot_download
+    model_dir = snapshot_download('Qwen/Qwen2-7B-Instruct')
+    args = [sys.executable, '-m', 'vllm.entrypoints.openai.api_server', '--model', model_dir]
+    process = subprocess.Popen(args)
+    _test_client(8000)
+    process.terminate()
+
+
+if __name__ == '__main__':
+    # test_vllm_origin()
+    # test_vllm()
+    test_vllm_vlm()
+    # test_lmdeploy()
+    # test_pt()