chore: import upstream snapshot with attribution

examples/deploy/client/llm/base/openai_client.py

@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    query = messages[0]['content']
+    print(f'query: {query}')
+    resp = client.completions.create(model=model, prompt=query, max_tokens=64, temperature=0)
+    response = resp.choices[0].text
+    print(f'response: {response}')
+    # or (The two calling methods are equivalent.)
+    resp = client.chat.completions.create(model=model, messages=messages, max_tokens=64, temperature=0)
+    response = resp.choices[0].message.content
+    print(f'response: {response}')
+    return response
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{'role': 'user', 'content': '浙江 -> 杭州\n安徽 -> 合肥\n四川 ->'}]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen2.5-1.5B',
+                verbose=False,
+                log_interval=-1,
+                infer_backend='transformers',
+                use_chat_template=False)) as port:
+        run_client(port=port)

examples/deploy/client/llm/base/swift_client.py

@@ -0,0 +1,37 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=64, temperature=0)
+
+    resp_list = engine.infer(infer_requests, request_config)
+
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+
+    infer_requests = [InferRequest(messages=[{'role': 'user', 'content': '浙江 -> 杭州\n安徽 -> 合肥\n四川 ->'}])]
+    infer_batch(engine, infer_requests)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, InferClient, InferEngine, InferRequest, RequestConfig, run_deploy
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen2.5-1.5B',
+                verbose=False,
+                log_interval=-1,
+                infer_backend='transformers',
+                use_chat_template=False)) as port:
+        run_client(port=port)

examples/deploy/client/llm/chat/openai_client.py

@@ -0,0 +1,65 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        max_tokens=512,
+        temperature=0,
+        extra_body={
+            'chat_template_kwargs': {
+                'enable_thinking': False
+            },
+        })
+    query = messages[0]['content']
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    return response
+
+
+# streaming
+def infer_stream(client, model: str, messages):
+    gen = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        stream=True,
+        temperature=0,
+        extra_body={
+            'chat_template_kwargs': {
+                'enable_thinking': False
+            },
+        })
+    print(f'messages: {messages}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    query = 'Where is the capital of Zhejiang?'
+    messages = [{'role': 'user', 'content': query}]
+    response = infer(client, model, messages)
+    messages.append({'role': 'assistant', 'content': response})
+    messages.append({'role': 'user', 'content': 'What delicious food is there?'})
+    infer_stream(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(DeployArguments(model='Qwen/Qwen3.5-4B', verbose=False, log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/client/llm/chat/swift_client.py

@@ -0,0 +1,59 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    # # The asynchronous interface below is equivalent to the synchronous interface above.
+    # async def _run():
+    #     tasks = [engine.infer_async(infer_request, request_config) for infer_request in infer_requests]
+    #     return await asyncio.gather(*tasks)
+    # resp_list = asyncio.run(_run())
+
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{'role': 'user', 'content': 'who are you?'}]
+    infer_stream(engine, InferRequest(messages=messages, chat_template_kwargs={'enable_thinking': False}))
+
+
+if __name__ == '__main__':
+    from swift import (DeployArguments, InferClient, InferEngine, InferRequest, InferStats, RequestConfig, load_dataset,
+                       run_deploy)
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(DeployArguments(model='Qwen/Qwen3.5-4B', verbose=False, log_interval=-1,
+                                    infer_backend='vllm')) as port:
+        run_client(port=port)

examples/deploy/client/mllm/openai_client.py

@@ -0,0 +1,98 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages, max_tokens=512, temperature=0)
+    query = messages[0]['content']
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    return response
+
+
+# streaming
+def infer_stream(client, model: str, messages):
+    gen = client.chat.completions.create(model=model, messages=messages, stream=True, temperature=0)
+    print(f'messages: {messages}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'image',
+                'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+            }, {
+                'type': 'text',
+                'text': 'How many sheep are there in the picture?'
+            }]
+        }
+
+    elif mm_type == 'video':
+        # # use base64
+        # import base64
+        # with open('baby.mp4', 'rb') as f:
+        #     vid_base64 = base64.b64encode(f.read()).decode('utf-8')
+        # video = f'data:video/mp4;base64,{vid_base64}'
+
+        # use url
+        video = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+        message = {
+            'role': 'user',
+            'content': [{
+                'type': 'video',
+                'video': video
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    query = 'who are you?'
+    messages = [{'role': 'user', 'content': query}]
+    response = infer(client, model, messages)
+    messages.append({'role': 'assistant', 'content': response})
+    messages.append(get_message(mm_type='video'))
+    infer_stream(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(DeployArguments(model='Qwen/Qwen2.5-VL-3B-Instruct', verbose=False, log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/client/mllm/swift_client.py

@@ -0,0 +1,127 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List, Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'image',
+                    # url or local_path or PIL.Image or base64
+                    'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+                },
+                {
+                    'type': 'text',
+                    'text': 'How many sheep are there in the picture?'
+                }
+            ]
+        }
+
+    elif mm_type == 'video':
+        # # use base64
+        # import base64
+        # with open('baby.mp4', 'rb') as f:
+        #     vid_base64 = base64.b64encode(f.read()).decode('utf-8')
+        # video = f'data:video/mp4;base64,{vid_base64}'
+
+        # use url
+        video = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+        message = {
+            'role': 'user',
+            'content': [{
+                'type': 'video',
+                'video': video
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def get_data(mm_type: Literal['text', 'image', 'video', 'audio']):
+    data = {}
+    if mm_type == 'text':
+        messages = [{'role': 'user', 'content': 'who are you?'}]
+    elif mm_type == 'image':
+        # The number of <image> tags must be the same as len(images).
+        messages = [{'role': 'user', 'content': '<image>How many sheep are there in the picture?'}]
+        # Support URL/Path/base64/PIL.Image
+        data['images'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']
+    elif mm_type == 'video':
+        messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+        data['videos'] = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    elif mm_type == 'audio':
+        messages = [{'role': 'user', 'content': '<audio>What does this audio say?'}]
+        data['audios'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    data['messages'] = messages
+    return data
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/LaTeX_OCR:small#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_stream(engine, InferRequest(messages=[get_message(mm_type='video')]))
+    # This writing is equivalent to the above writing.
+    infer_stream(engine, InferRequest(**get_data(mm_type='video')))
+
+
+if __name__ == '__main__':
+    from swift import (DeployArguments, InferClient, InferEngine, InferRequest, InferStats, RequestConfig, load_dataset,
+                       run_deploy)
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(model='Qwen/Qwen2.5-VL-3B-Instruct', verbose=False, log_interval=-1,
+                            infer_backend='vllm')) as port:
+        run_client(port=port)