chore: import upstream snapshot with attribution

examples/deploy/README.md

@@ -0,0 +1,9 @@
+Please refer to the examples in [examples/infer](../../infer/) and change `swift infer` to `swift deploy` to start the service. (You need to additionally remove `--val_dataset`)
+
+e.g.
+```shell
+CUDA_VISIBLE_DEVICES=0 \
+swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm
+```

examples/deploy/agent/client.py

@@ -0,0 +1,90 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def get_infer_request():
+    messages = [{'role': 'user', 'content': "How's the weather in Beijing today?"}]
+    tools = [{
+        'name': 'get_current_weather',
+        'description': 'Get the current weather in a given location',
+        'parameters': {
+            'type': 'object',
+            'properties': {
+                'location': {
+                    'type': 'string',
+                    'description': 'The city and state, e.g. San Francisco, CA'
+                },
+                'unit': {
+                    'type': 'string',
+                    'enum': ['celsius', 'fahrenheit']
+                }
+            },
+            'required': ['location']
+        }
+    }]
+    return messages, tools
+
+
+def infer(client, model: str, messages, tools):
+    messages = messages.copy()
+    query = messages[0]['content']
+    resp = client.chat.completions.create(model=model, messages=messages, tools=tools, max_tokens=512, temperature=0)
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    print(f'tool_calls: {resp.choices[0].message.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    resp = client.chat.completions.create(model=model, messages=messages, tools=tools, max_tokens=512, temperature=0)
+    response2 = resp.choices[0].message.content
+    print(f'response2: {response2}')
+
+
+# streaming
+def infer_stream(client, model: str, messages, tools):
+    messages = messages.copy()
+    query = messages[0]['content']
+    gen = client.chat.completions.create(
+        model=model, messages=messages, tools=tools, max_tokens=512, temperature=0, stream=True)
+    response = ''
+    print(f'query: {query}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        delta = chunk.choices[0].delta.content
+        response += delta
+        print(delta, end='', flush=True)
+    print()
+    print(f'tool_calls: {chunk.choices[0].delta.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    gen = client.chat.completions.create(
+        model=model, messages=messages, tools=tools, max_tokens=512, temperature=0, stream=True)
+    print(f'query: {query}\nresponse2: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+if __name__ == '__main__':
+    host: str = '127.0.0.1'
+    port: int = 8000
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages, tools = get_infer_request()
+    infer(client, model, messages, tools)
+    infer_stream(client, model, messages, tools)

examples/deploy/agent/server.sh

@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --agent_template hermes \
+    --served_model_name Qwen2.5-7B-Instruct

examples/deploy/bert/client.py

@@ -0,0 +1,29 @@
+from typing import List
+
+from swift.infer_engine import InferClient, InferRequest
+
+
+def infer_batch(engine: InferClient, infer_requests: List[InferRequest]):
+    resp_list = engine.infer(infer_requests)
+    query0 = infer_requests[0].messages[0]['content']
+    query1 = infer_requests[1].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'query1: {query1}')
+    print(f'response1: {resp_list[1].choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    engine = InferClient(host='127.0.0.1', port=8000)
+    models = engine.models
+    print(f'models: {models}')
+    infer_batch(engine, [
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '今天天气真好呀'
+        }]),
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '真倒霉'
+        }])
+    ])

examples/deploy/bert/server.sh

@@ -0,0 +1,10 @@
+# Since `swift/test_lora` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --adapters swift/test_bert \
+    --served_model_name bert-base-chinese \
+    --infer_backend transformers \
+    --truncation_strategy right \
+    --max_length 512

examples/deploy/client/llm/base/openai_client.py

@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    query = messages[0]['content']
+    print(f'query: {query}')
+    resp = client.completions.create(model=model, prompt=query, max_tokens=64, temperature=0)
+    response = resp.choices[0].text
+    print(f'response: {response}')
+    # or (The two calling methods are equivalent.)
+    resp = client.chat.completions.create(model=model, messages=messages, max_tokens=64, temperature=0)
+    response = resp.choices[0].message.content
+    print(f'response: {response}')
+    return response
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{'role': 'user', 'content': '浙江 -> 杭州\n安徽 -> 合肥\n四川 ->'}]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen2.5-1.5B',
+                verbose=False,
+                log_interval=-1,
+                infer_backend='transformers',
+                use_chat_template=False)) as port:
+        run_client(port=port)

examples/deploy/client/llm/base/swift_client.py

@@ -0,0 +1,37 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=64, temperature=0)
+
+    resp_list = engine.infer(infer_requests, request_config)
+
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+
+    infer_requests = [InferRequest(messages=[{'role': 'user', 'content': '浙江 -> 杭州\n安徽 -> 合肥\n四川 ->'}])]
+    infer_batch(engine, infer_requests)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, InferClient, InferEngine, InferRequest, RequestConfig, run_deploy
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen2.5-1.5B',
+                verbose=False,
+                log_interval=-1,
+                infer_backend='transformers',
+                use_chat_template=False)) as port:
+        run_client(port=port)

examples/deploy/client/llm/chat/openai_client.py

@@ -0,0 +1,65 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        max_tokens=512,
+        temperature=0,
+        extra_body={
+            'chat_template_kwargs': {
+                'enable_thinking': False
+            },
+        })
+    query = messages[0]['content']
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    return response
+
+
+# streaming
+def infer_stream(client, model: str, messages):
+    gen = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        stream=True,
+        temperature=0,
+        extra_body={
+            'chat_template_kwargs': {
+                'enable_thinking': False
+            },
+        })
+    print(f'messages: {messages}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    query = 'Where is the capital of Zhejiang?'
+    messages = [{'role': 'user', 'content': query}]
+    response = infer(client, model, messages)
+    messages.append({'role': 'assistant', 'content': response})
+    messages.append({'role': 'user', 'content': 'What delicious food is there?'})
+    infer_stream(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(DeployArguments(model='Qwen/Qwen3.5-4B', verbose=False, log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/client/llm/chat/swift_client.py

@@ -0,0 +1,59 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    # # The asynchronous interface below is equivalent to the synchronous interface above.
+    # async def _run():
+    #     tasks = [engine.infer_async(infer_request, request_config) for infer_request in infer_requests]
+    #     return await asyncio.gather(*tasks)
+    # resp_list = asyncio.run(_run())
+
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{'role': 'user', 'content': 'who are you?'}]
+    infer_stream(engine, InferRequest(messages=messages, chat_template_kwargs={'enable_thinking': False}))
+
+
+if __name__ == '__main__':
+    from swift import (DeployArguments, InferClient, InferEngine, InferRequest, InferStats, RequestConfig, load_dataset,
+                       run_deploy)
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(DeployArguments(model='Qwen/Qwen3.5-4B', verbose=False, log_interval=-1,
+                                    infer_backend='vllm')) as port:
+        run_client(port=port)

examples/deploy/client/mllm/openai_client.py

@@ -0,0 +1,98 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages, max_tokens=512, temperature=0)
+    query = messages[0]['content']
+    response = resp.choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    return response
+
+
+# streaming
+def infer_stream(client, model: str, messages):
+    gen = client.chat.completions.create(model=model, messages=messages, stream=True, temperature=0)
+    print(f'messages: {messages}\nresponse: ', end='')
+    for chunk in gen:
+        if chunk is None:
+            continue
+        print(chunk.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'image',
+                'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+            }, {
+                'type': 'text',
+                'text': 'How many sheep are there in the picture?'
+            }]
+        }
+
+    elif mm_type == 'video':
+        # # use base64
+        # import base64
+        # with open('baby.mp4', 'rb') as f:
+        #     vid_base64 = base64.b64encode(f.read()).decode('utf-8')
+        # video = f'data:video/mp4;base64,{vid_base64}'
+
+        # use url
+        video = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+        message = {
+            'role': 'user',
+            'content': [{
+                'type': 'video',
+                'video': video
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    query = 'who are you?'
+    messages = [{'role': 'user', 'content': query}]
+    response = infer(client, model, messages)
+    messages.append({'role': 'assistant', 'content': response})
+    messages.append(get_message(mm_type='video'))
+    infer_stream(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(DeployArguments(model='Qwen/Qwen2.5-VL-3B-Instruct', verbose=False, log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/client/mllm/swift_client.py

@@ -0,0 +1,127 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List, Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'image',
+                    # url or local_path or PIL.Image or base64
+                    'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+                },
+                {
+                    'type': 'text',
+                    'text': 'How many sheep are there in the picture?'
+                }
+            ]
+        }
+
+    elif mm_type == 'video':
+        # # use base64
+        # import base64
+        # with open('baby.mp4', 'rb') as f:
+        #     vid_base64 = base64.b64encode(f.read()).decode('utf-8')
+        # video = f'data:video/mp4;base64,{vid_base64}'
+
+        # use url
+        video = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+        message = {
+            'role': 'user',
+            'content': [{
+                'type': 'video',
+                'video': video
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def get_data(mm_type: Literal['text', 'image', 'video', 'audio']):
+    data = {}
+    if mm_type == 'text':
+        messages = [{'role': 'user', 'content': 'who are you?'}]
+    elif mm_type == 'image':
+        # The number of <image> tags must be the same as len(images).
+        messages = [{'role': 'user', 'content': '<image>How many sheep are there in the picture?'}]
+        # Support URL/Path/base64/PIL.Image
+        data['images'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']
+    elif mm_type == 'video':
+        messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+        data['videos'] = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    elif mm_type == 'audio':
+        messages = [{'role': 'user', 'content': '<audio>What does this audio say?'}]
+        data['audios'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    data['messages'] = messages
+    return data
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    engine = InferClient(host=host, port=port)
+    print(f'models: {engine.models}')
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/LaTeX_OCR:small#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_stream(engine, InferRequest(messages=[get_message(mm_type='video')]))
+    # This writing is equivalent to the above writing.
+    infer_stream(engine, InferRequest(**get_data(mm_type='video')))
+
+
+if __name__ == '__main__':
+    from swift import (DeployArguments, InferClient, InferEngine, InferRequest, InferStats, RequestConfig, load_dataset,
+                       run_deploy)
+
+    # NOTE: In a real deployment scenario, please comment out the context of run_deploy.
+    with run_deploy(
+            DeployArguments(model='Qwen/Qwen2.5-VL-3B-Instruct', verbose=False, log_interval=-1,
+                            infer_backend='vllm')) as port:
+        run_client(port=port)

examples/deploy/embedding/client.py

@@ -0,0 +1,56 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    # You can also use client.embeddings.create
+    # But this interface does not support multi-modal medias
+    resp = client.chat.completions.create(model=model, messages=messages)
+    emb = resp.data[0]['embedding']
+    shape = len(emb)
+    sample = str(emb)
+    if len(emb) > 6:
+        sample = str(emb[:3])[:-1] + ', ..., ' + str(emb[-3:])[1:]
+    print(f'messages: {messages}')
+    print(f'Embedding(shape: [1, {shape}]): {sample}')
+    return emb
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role':
+        'user',
+        'content': [
+            # {
+            #   'type': 'image',
+            #   'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+            # },
+            {
+                'type': 'text',
+                'text': 'What is the capital of China?'
+            },
+        ]
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen3-Embedding-0.6B',  # GME/GTE models or your checkpoints are also supported
+                task_type='embedding',
+                infer_backend='vllm',
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/embedding/server.sh

@@ -0,0 +1,8 @@
+# GME/GTE models or your checkpoints are also supported
+# transformers/vllm/sglang supported
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --task_type embedding \
+    --model Qwen/Qwen3-Embedding-0.6B \
+    --infer_backend sglang

examples/deploy/lora/client.py

@@ -0,0 +1,27 @@
+from swift.infer_engine import InferClient, InferRequest, RequestConfig
+
+
+def infer_multilora(engine: InferClient, infer_request: InferRequest):
+    # Dynamic LoRA
+    models = engine.models
+    print(f'models: {models}')
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+
+    # use lora1
+    resp_list = engine.infer([infer_request], request_config, model=models[1])
+    response = resp_list[0].choices[0].message.content
+    print(f'lora1-response: {response}')
+    # origin model
+    resp_list = engine.infer([infer_request], request_config, model=models[0])
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+    # use lora2
+    resp_list = engine.infer([infer_request], request_config, model=models[2])
+    response = resp_list[0].choices[0].message.content
+    print(f'lora2-response: {response}')
+
+
+if __name__ == '__main__':
+    engine = InferClient(host='127.0.0.1', port=8000)
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+    infer_multilora(engine, infer_request)

examples/deploy/lora/server.sh

@@ -0,0 +1,7 @@
+# Since `swift/test_lora` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --adapters lora1=swift/test_lora lora2=swift/test_lora2 \
+    --infer_backend vllm

examples/deploy/reranker/client.py

@@ -0,0 +1,46 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages)
+    scores = resp.choices[0].message.content
+    print(f'messages: {messages}')
+    print(f'scores: {scores}')
+    return scores
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role': 'user',
+        'content': 'what is the capital of China?',
+    }, {
+        'role': 'assistant',
+        'content': 'Beijing',
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='BAAI/bge-reranker-v2-m3',
+                task_type='reranker',
+                infer_backend='vllm',
+                gpu_memory_utilization=0.7,
+                vllm_enforce_eager=True,
+                reranker_use_activation=True,
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/reranker/client_generative.py

@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages)
+    scores = resp.choices[0].message.content
+    print(f'messages: {messages}')
+    print(f'scores: {scores}')
+    return scores
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role': 'user',
+        'content': 'what is the capital of China?',
+    }, {
+        'role': 'assistant',
+        'content': 'Beijing.',
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='Qwen/Qwen3-Reranker-0.6B',
+                task_type='generative_reranker',
+                infer_backend='vllm',
+                gpu_memory_utilization=0.7,
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/reranker/server.sh

@@ -0,0 +1,9 @@
+# GME/GTE models or your checkpoints are also supported
+# transformers/vllm/sglang supported
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --model BAAI/bge-reranker-v2-m3 \
+    --infer_backend vllm \
+    --task_type reranker \
+    --vllm_enforce_eager true \

examples/deploy/reward_model/client.py

@@ -0,0 +1,17 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.infer_engine import InferClient, InferRequest
+
+if __name__ == '__main__':
+    engine = InferClient(host='127.0.0.1', port=8000)
+    models = engine.models
+    print(f'models: {models}')
+    messages = [{
+        'role': 'user',
+        'content': "Hello! What's your name?"
+    }, {
+        'role': 'assistant',
+        'content': 'My name is InternLM2! A helpful AI assistant. What can I do for you?'
+    }]
+    resp_list = engine.infer([InferRequest(messages=messages)])
+    print(f'messages: {messages}')
+    print(f'response: {resp_list[0].choices[0].message.content}')

examples/deploy/reward_model/server.sh

@@ -0,0 +1,5 @@
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
+    --infer_backend transformers

examples/deploy/seq_cls/client.py

@@ -0,0 +1,44 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from openai import OpenAI
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer(client, model: str, messages):
+    resp = client.chat.completions.create(model=model, messages=messages)
+    classify = resp.choices[0].message.content
+    print(f'messages: {messages}')
+    print(f'classify: {classify}')
+    return classify
+
+
+def run_client(host: str = '127.0.0.1', port: int = 8000):
+    client = OpenAI(
+        api_key='EMPTY',
+        base_url=f'http://{host}:{port}/v1',
+    )
+    model = client.models.list().data[0].id
+    print(f'model: {model}')
+
+    messages = [{
+        'role': 'user',
+        'content': 'What is the capital of China?',
+    }, {
+        'role': 'assistant',
+        'content': 'Beijing',
+    }]
+    infer(client, model, messages)
+
+
+if __name__ == '__main__':
+    from swift import DeployArguments, run_deploy
+    with run_deploy(
+            DeployArguments(
+                model='/your/seq_cls/checkpoint-xxx',
+                task_type='seq_cls',
+                infer_backend='vllm',
+                num_labels=2,
+                verbose=False,
+                log_interval=-1)) as port:
+        run_client(port=port)

examples/deploy/seq_cls/server.sh

@@ -0,0 +1,9 @@
+# GME/GTE models or your checkpoints are also supported
+# transformers/vllm/sglang supported
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --model /your/seq_cls/checkpoint-xxx \
+    --infer_backend vllm \
+    --task_type seq_cls \
+    --num_labels 2 \

examples/deploy/sglang.sh

@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+swift deploy \
+    --model Qwen/Qwen3-8B \
+    --infer_backend sglang \
+    --max_new_tokens 2048 \
+    --sglang_context_length 8192 \
+    --sglang_tp_size 2 \
+    --served_model_name Qwen3-8B
+
+# After the server-side deployment above is successful, use the command below to perform a client call test.
+
+# curl http://localhost:8000/v1/chat/completions \
+# -H "Content-Type: application/json" \
+# -d '{
+# "model": "Qwen3-8B",
+# "messages": [{"role": "user", "content": "What is your name?"}],
+# "temperature": 0
+# }'

examples/deploy/vllm.sh

@@ -0,0 +1,14 @@
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --served_model_name Qwen2.5-7B-Instruct
+
+# After the server-side deployment above is successful, use the command below to perform a client call test.
+
+# curl http://localhost:8000/v1/chat/completions \
+# -H "Content-Type: application/json" \
+# -d '{
+# "model": "Qwen2.5-7B-Instruct",
+# "messages": [{"role": "user", "content": "What is your name?"}],
+# "temperature": 0
+# }'

examples/deploy/vllm_dp.sh

@@ -0,0 +1,22 @@
+CUDA_VISIBLE_DEVICES=0,1 swift deploy \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --infer_backend vllm \
+    --served_model_name Qwen2.5-VL-7B-Instruct \
+    --vllm_max_model_len 8192 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_data_parallel_size 2
+
+# After the server-side deployment above is successful, use the command below to perform a client call test.
+
+# curl http://localhost:8000/v1/chat/completions \
+# -H "Content-Type: application/json" \
+# -d '{
+# "model": "Qwen2.5-VL-7B-Instruct",
+# "messages": [{"role": "user", "content": [
+#     {"type": "image", "image": "http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png"},
+#     {"type": "image", "image": "http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png"},
+#     {"type": "text", "text": "What is the difference between the two images?"}
+# ]}],
+# "max_tokens": 256,
+# "temperature": 0
+# }'