chore: import upstream snapshot with attribution

examples/infer/cli_demo.sh

@@ -0,0 +1,6 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend transformers \
+    --stream true \
+    --max_new_tokens 2048

examples/infer/demo.py

@@ -0,0 +1,57 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+    # metric.reset()  # reuse
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+if __name__ == '__main__':
+    from swift import InferEngine, InferRequest, InferStats, RequestConfig, TransformersEngine, load_dataset
+    model = 'Qwen/Qwen2.5-1.5B-Instruct'
+    infer_backend = 'transformers'
+
+    if infer_backend == 'transformers':
+        engine = TransformersEngine(model, max_batch_size=64)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine(model, max_model_len=8192)
+    elif infer_backend == 'sglang':
+        from swift.infer_engine import SglangEngine
+        engine = SglangEngine(model)
+    elif infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine(model)
+
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{'role': 'user', 'content': 'who are you?'}]
+    infer_stream(engine, InferRequest(messages=messages))

examples/infer/demo_agent.py

@@ -0,0 +1,114 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+# os.environ['SWIFT_DEBUG'] = '1'
+
+
+def infer(engine: 'InferEngine', infer_request: 'InferRequest'):
+    stop = [engine.template.agent_template.keyword.observation]  # compat react_en
+    request_config = RequestConfig(max_tokens=512, temperature=0, stop=stop)
+    resp_list = engine.infer([infer_request], request_config)
+    query = infer_request.messages[0]['content']
+    response = resp_list[0].choices[0].message.content
+    print(f'query: {query}')
+    print(f'response: {response}')
+    print(f'tool_calls: {resp_list[0].choices[0].message.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}')
+    infer_request.messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    resp_list = engine.infer([infer_request], request_config)
+    response2 = resp_list[0].choices[0].message.content
+    print(f'response2: {response2}')
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    stop = [engine.template.agent_template.keyword.observation]
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True, stop=stop)
+    gen_list = engine.infer([infer_request], request_config)
+    query = infer_request.messages[0]['content']
+    response = ''
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        delta = resp.choices[0].delta.content
+        response += delta
+        print(delta, end='', flush=True)
+    print()
+    print(f'tool_calls: {resp.choices[0].delta.tool_calls}')
+
+    tool = '{"temperature": 32, "condition": "Sunny", "humidity": 50}'
+    print(f'tool_response: {tool}\nresponse2: ', end='')
+    infer_request.messages += [{'role': 'assistant', 'content': response}, {'role': 'tool', 'content': tool}]
+    gen_list = engine.infer([infer_request], request_config)
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+
+
+def get_infer_request():
+    return InferRequest(
+        messages=[{
+            'role': 'user',
+            'content': "How's the weather in Beijing today?"
+        }],
+        tools=[{
+            'name': 'get_current_weather',
+            'description': 'Get the current weather in a given location',
+            'parameters': {
+                'type': 'object',
+                'properties': {
+                    'location': {
+                        'type': 'string',
+                        'description': 'The city and state, e.g. San Francisco, CA'
+                    },
+                    'unit': {
+                        'type': 'string',
+                        'enum': ['celsius', 'fahrenheit']
+                    }
+                },
+                'required': ['location']
+            }
+        }])
+
+
+def infer_continue_generate(engine):
+    # Continue generating after the assistant message.
+    infer_request = InferRequest(messages=[{
+        'role': 'user',
+        'content': 'How is the weather today?'
+    }, {
+        'role': 'assistant',
+        'content': 'It is sunny today, '
+    }])
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    resp_list = engine.infer([infer_request], request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+
+
+if __name__ == '__main__':
+    from swift.agent_template import agent_template_map
+    from swift.infer_engine import InferEngine, InferRequest, RequestConfig, TransformersEngine
+    model = 'Qwen/Qwen2.5-1.5B-Instruct'
+    infer_backend = 'transformers'
+
+    if infer_backend == 'transformers':
+        engine = TransformersEngine(model, max_batch_size=64)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine(model, max_model_len=8192)
+    elif infer_backend == 'lmdeploy':
+        from swift.infer_engine import LmdeployEngine
+        engine = LmdeployEngine(model)
+
+    # engine.template._agent_template = 'hermes'  # react_en/qwen_en/qwen_en_parallel
+
+    infer(engine, get_infer_request())
+    infer_stream(engine, get_infer_request())
+
+    # infer_continue_generate(engine)

examples/infer/demo_bert.py

@@ -0,0 +1,55 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+# demo_seq_cls: https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls/qwen2_5_omni/infer.py
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    resp_list = engine.infer(infer_requests)
+    query0 = infer_requests[0].messages[0]['content']
+    query1 = infer_requests[1].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'query1: {query1}')
+    print(f'response1: {resp_list[1].choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    # This is an example of BERT with LoRA.
+    from peft import PeftModel
+
+    from swift import BaseArguments, InferEngine, InferRequest, TransformersEngine, load_dataset, safe_snapshot_download
+    adapter_path = safe_snapshot_download('swift/test_bert')
+    args = BaseArguments.from_pretrained(adapter_path)
+    args.max_length = 512
+    args.truncation_strategy = 'right'
+    # method1
+    model, processor = args.get_model_processor()
+    model = PeftModel.from_pretrained(model, adapter_path)
+    template = args.get_template(processor)
+    engine = TransformersEngine(model, template=template, max_batch_size=64)
+
+    # method2
+    # engine = TransformersEngine(args.model, adapters=[adapter_path], max_batch_size=64,
+    #                   task_type=args.task_type, num_labels=args.num_labels)
+    # template = args.get_template(engine.processor)
+    # engine.template = template
+
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['DAMO_NLP/jd:cls#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(messages=data['messages']) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_batch(engine, [
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '今天天气真好呀'
+        }]),
+        InferRequest(messages=[{
+            'role': 'user',
+            'content': '真倒霉'
+        }])
+    ])

examples/infer/demo_embedding.py

@@ -0,0 +1,64 @@
+import torch
+
+from swift.infer_engine import InferRequest, TransformersEngine
+
+
+def run_qwen3_emb():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-Embedding-4B', task_type='embedding', torch_dtype=torch.float16, attn_impl='flash_attention_2')
+
+    infer_requests = [
+        InferRequest(messages=[
+            {
+                'role':
+                'user',
+                'content':
+                'Instruct: Given a web search query, retrieve relevant passages that answer the query\n'
+                'Query:What is the capital of China?'
+            },
+        ]),
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'The capital of China is Beijing.'
+            },
+        ])
+    ]
+    resp_list = engine.infer(infer_requests)
+    embedding0 = torch.tensor(resp_list[0].data[0].embedding)
+    embedding1 = torch.tensor(resp_list[1].data[0].embedding)
+    print(f'scores: {(embedding0 * embedding1).sum()}')
+
+
+def run_qwen3_vl_emb():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-VL-Embedding-2B', task_type='embedding', max_batch_size=2, attn_impl='flash_attention_2')
+
+    infer_requests = [
+        InferRequest(messages=[
+            {
+                'role': 'user',
+                'content': 'A woman playing with her dog on a beach at sunset.'
+            },
+        ]),
+        InferRequest(
+            messages=[
+                {
+                    'role':
+                    'user',
+                    'content':
+                    '<image>A woman shares a joyful moment with her golden retriever on a sun-drenched beach at '
+                    'sunset, as the dog offers its paw in a heartwarming display of companionship and trust.'
+                },
+            ],
+            images=['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg'])
+    ]
+    resp_list = engine.infer(infer_requests)
+    embedding0 = torch.tensor(resp_list[0].data[0].embedding)
+    embedding1 = torch.tensor(resp_list[1].data[0].embedding)
+    print(f'scores: {(embedding0 * embedding1).sum()}')
+
+
+if __name__ == '__main__':
+    # run_qwen3_emb()
+    run_qwen3_vl_emb()

examples/infer/demo_grounding.py

@@ -0,0 +1,45 @@
+import os
+import re
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+os.environ['MAX_PIXELS'] = '1003520'
+
+
+def draw_bbox_qwen2_vl(image, response, norm_bbox: Literal['norm1000', 'none']):
+    matches = re.findall(
+        r'<\|object_ref_start\|>(.*?)<\|object_ref_end\|><\|box_start\|>\((\d+),(\d+)\),\((\d+),(\d+)\)<\|box_end\|>',
+        response)
+    ref = []
+    bbox = []
+    for match_ in matches:
+        ref.append(match_[0])
+        bbox.append(list(match_[1:]))
+    draw_bbox(image, ref, bbox, norm_bbox=norm_bbox)
+
+
+def infer_grounding():
+    # use transformers==4.51.3
+    from swift import BaseArguments, InferRequest, RequestConfig, TransformersEngine, safe_snapshot_download
+    output_path = 'bbox.png'
+    image = load_image('http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png')
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'Task: Object Detection'}], images=[image])
+
+    request_config = RequestConfig(max_tokens=512, temperature=0, return_details=True)
+    adapter_path = safe_snapshot_download('swift/test_grounding')
+    args = BaseArguments.from_pretrained(adapter_path)
+
+    engine = TransformersEngine(args.model, adapters=[adapter_path])
+    resp_list = engine.infer([infer_request], request_config)
+    image = image.resize(resp_list[0].images_size[0])
+    response = resp_list[0].choices[0].message.content
+    print(f'lora-response: {response}')
+
+    draw_bbox_qwen2_vl(image, response, norm_bbox=args.norm_bbox)
+    print(f'output_path: {output_path}')
+    image.save(output_path)
+
+
+if __name__ == '__main__':
+    from swift.template import draw_bbox, load_image
+    infer_grounding()

examples/infer/demo_hf.py

@@ -0,0 +1,66 @@
+def infer_hf():
+    from modelscope import snapshot_download
+    from peft import PeftModel
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    model_dir = snapshot_download('Qwen/Qwen2.5-7B-Instruct')
+    adapter_dir = snapshot_download('swift/test_lora')
+    model = AutoModelForCausalLM.from_pretrained(
+        model_dir, torch_dtype='auto', device_map='auto', trust_remote_code=True)
+    model = PeftModel.from_pretrained(model, adapter_dir)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
+
+    messages = [{
+        'role': 'system',
+        'content': 'You are a helpful assistant.'
+    }, {
+        'role': 'user',
+        'content': 'who are you?'
+    }]
+    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    model_inputs = tokenizer([text], return_tensors='pt', add_special_tokens=False).to(model.device)
+
+    generated_ids = model.generate(**model_inputs, max_new_tokens=512, do_sample=False)
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    print(f'response: {response}')
+    return response
+
+
+def infer_swift():
+    from modelscope import snapshot_download
+    from peft import PeftModel
+
+    from swift import get_model_processor, get_template
+    from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine
+    from swift.tuners import Swift
+    model_dir = snapshot_download('Qwen/Qwen2.5-7B-Instruct')
+    adapter_dir = snapshot_download('swift/test_lora')
+    model, tokenizer = get_model_processor(model_dir, device_map='auto')
+    model = Swift.from_pretrained(model, adapter_dir)
+    # You can also write it as:
+    # model = PeftModel.from_pretrained(model, adapter_dir)
+    template = get_template(tokenizer)
+    engine = TransformersEngine(model, template=template)
+
+    messages = [{
+        'role': 'system',
+        'content': 'You are a helpful assistant.'
+    }, {
+        'role': 'user',
+        'content': 'who are you?'
+    }]
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    resp_list = engine.infer([InferRequest(messages=messages)], request_config=request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+    return response
+
+
+if __name__ == '__main__':
+    response = infer_hf()
+    response2 = infer_swift()
+    assert response == response2

examples/infer/demo_lora.py

@@ -0,0 +1,70 @@
+import os
+from typing import Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_multilora(infer_request: 'InferRequest', infer_backend: Literal['vllm', 'transformers']):
+    # Dynamic LoRA
+    adapter_path = safe_snapshot_download('swift/test_lora')
+    adapter_path2 = safe_snapshot_download('swift/test_lora2')
+    args = BaseArguments.from_pretrained(adapter_path)
+    if infer_backend == 'transformers':
+        engine = TransformersEngine(args.model)
+    elif infer_backend == 'vllm':
+        from swift.infer_engine import VllmEngine
+        engine = VllmEngine(args.model, enable_lora=True, max_loras=1, max_lora_rank=16)
+    template = get_template(engine.processor, template_type=args.template, default_system=args.system)
+    engine.template = template
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    adapter_request = AdapterRequest('lora1', adapter_path)
+    adapter_request2 = AdapterRequest('lora2', adapter_path2)
+
+    # use lora
+    resp_list = engine.infer([infer_request], request_config, adapter_request=adapter_request)
+    response = resp_list[0].choices[0].message.content
+    print(f'lora1-response: {response}')
+    # origin model
+    resp_list = engine.infer([infer_request], request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'response: {response}')
+    # use lora
+    resp_list = engine.infer([infer_request], request_config, adapter_request=adapter_request2)
+    response = resp_list[0].choices[0].message.content
+    print(f'lora2-response: {response}')
+
+
+def infer_lora(infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    adapter_path = safe_snapshot_download('swift/test_lora')
+    args = BaseArguments.from_pretrained(adapter_path)
+    # method1
+    # engine = TransformersEngine(args.model, adapters=[adapter_path])
+    # template = get_template(engine.processor, args.system, template_type=args.template)
+    # engine.template = template
+
+    # method2
+    # model, processor = args.get_model_processor()
+    # model = PeftModel.from_pretrained(model, adapter_path)
+    # template = args.get_template(processor)
+    # engine = TransformersEngine(model, template=template)
+
+    # method3
+    model, tokenizer = get_model_processor(args.model)
+    model = PeftModel.from_pretrained(model, adapter_path)
+    template = get_template(tokenizer, args.system, template_type=args.template)
+    engine = TransformersEngine(model, template=template)
+
+    resp_list = engine.infer([infer_request], request_config)
+    response = resp_list[0].choices[0].message.content
+    print(f'lora-response: {response}')
+
+
+if __name__ == '__main__':
+    from peft import PeftModel
+
+    from swift import (AdapterRequest, BaseArguments, InferRequest, RequestConfig, TransformersEngine,
+                       get_model_processor, get_template, safe_snapshot_download)
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+    # infer_lora(infer_request)
+    infer_multilora(infer_request, 'transformers')

examples/infer/demo_mllm.py

@@ -0,0 +1,145 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List, Literal
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    request_config = RequestConfig(max_tokens=512, temperature=0)
+    metric = InferStats()
+    resp_list = engine.infer(infer_requests, request_config, metrics=[metric])
+    query0 = infer_requests[0].messages[0]['content']
+    print(f'query0: {query0}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+    print(f'metric: {metric.compute()}')
+    # metric.reset()  # reuse
+
+
+def infer_stream(engine: 'InferEngine', infer_request: 'InferRequest'):
+    request_config = RequestConfig(max_tokens=512, temperature=0, stream=True)
+    metric = InferStats()
+    gen_list = engine.infer([infer_request], request_config, metrics=[metric])
+    query = infer_request.messages[0]['content']
+    print(f'query: {query}\nresponse: ', end='')
+    for resp in gen_list[0]:
+        if resp is None:
+            continue
+        print(resp.choices[0].delta.content, end='', flush=True)
+    print()
+    print(f'metric: {metric.compute()}')
+
+
+def get_message(mm_type: Literal['text', 'image', 'video', 'audio']):
+    if mm_type == 'text':
+        message = {'role': 'user', 'content': 'who are you?'}
+    elif mm_type == 'image':
+        message = {
+            'role':
+            'user',
+            'content': [
+                {
+                    'type': 'image',
+                    # url or local_path or PIL.Image or base64
+                    'image': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png'
+                },
+                {
+                    'type': 'text',
+                    'text': 'How many sheep are there in the picture?'
+                }
+            ]
+        }
+
+    elif mm_type == 'video':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'video',
+                'video': 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'
+            }, {
+                'type': 'text',
+                'text': 'Describe this video.'
+            }]
+        }
+    elif mm_type == 'audio':
+        message = {
+            'role':
+            'user',
+            'content': [{
+                'type': 'audio',
+                'audio': 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
+            }, {
+                'type': 'text',
+                'text': 'What does this audio say?'
+            }]
+        }
+    return message
+
+
+def get_data(mm_type: Literal['text', 'image', 'video', 'audio']):
+    data = {}
+    if mm_type == 'text':
+        messages = [{'role': 'user', 'content': 'who are you?'}]
+    elif mm_type == 'image':
+        # The number of <image> tags must be the same as len(images).
+        messages = [{'role': 'user', 'content': '<image>How many sheep are there in the picture?'}]
+        # Support URL/Path/base64/PIL.Image
+        data['images'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']
+    elif mm_type == 'video':
+        messages = [{'role': 'user', 'content': '<video>Describe this video.'}]
+        data['videos'] = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
+    elif mm_type == 'audio':
+        messages = [{'role': 'user', 'content': '<audio>What does this audio say?'}]
+        data['audios'] = ['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav']
+    data['messages'] = messages
+    return data
+
+
+if __name__ == '__main__':
+    # The inference of the trained model can be referred to as:
+    # https://github.com/modelscope/ms-swift/tree/main/examples/notebook
+    from swift import InferEngine, InferRequest, InferStats, RequestConfig, TransformersEngine, load_dataset
+    infer_backend = 'transformers'
+
+    if infer_backend == 'transformers':
+        # test env: transformers==4.55.2
+        model = 'Qwen/Qwen2.5-Omni-7B'
+        mm_type = 'audio'
+        engine = TransformersEngine(model, max_batch_size=64, attn_impl='flash_attention_2')
+    elif infer_backend == 'vllm':
+        # test env: vllm==0.8.5.post1, transformers==4.51.3
+        # The meaning of environment variables can be found at:
+        # https://swift.readthedocs.io/zh-cn/latest/Instruction/%E5%91%BD%E4%BB%A4%E8%A1%8C%E5%8F%82%E6%95%B0.html#id17
+        from swift.infer_engine import VllmEngine
+        os.environ['MAX_PIXELS'] = '1003520'
+        os.environ['VIDEO_MAX_PIXELS'] = '50176'
+        os.environ['FPS_MAX_FRAMES'] = '12'
+        model = 'Qwen/Qwen2.5-VL-3B-Instruct'
+        # If you encounter insufficient GPU memory, please reduce `max_model_len` and set `max_num_seqs=5`.
+        engine = VllmEngine(model, max_model_len=8192, limit_mm_per_prompt={'image': 5, 'video': 2})
+        mm_type = 'image'  # or 'video'
+    elif infer_backend == 'lmdeploy':
+        # test env: lmdeploy==0.7.1
+        from swift.infer_engine import LmdeployEngine
+        model = 'OpenGVLab/InternVL2_5-1B'
+        engine = LmdeployEngine(model, vision_batch_size=8)
+        mm_type = 'image'  # or 'video'
+
+    # infer dataset
+    if mm_type == 'audio':
+        dataset = 'speech_asr/speech_asr_aishell1_trainsets:validation#1000'
+    elif mm_type == 'image':
+        dataset = 'AI-ModelScope/LaTeX_OCR:small#1000'
+    elif mm_type == 'video':
+        dataset = 'swift/VideoChatGPT:Generic#100'
+
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset([dataset], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    infer_stream(engine, InferRequest(messages=[get_message(mm_type)]))
+    # This writing is equivalent to the above writing.
+    infer_stream(engine, InferRequest(**get_data(mm_type)))

examples/infer/demo_reranker.py

@@ -0,0 +1,55 @@
+import torch
+
+from swift.infer_engine import InferRequest, TransformersEngine
+
+
+def run_qwen3_reranker():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-Reranker-4B',
+        task_type='generative_reranker',
+        torch_dtype=torch.float16,
+        attn_impl='flash_attention_2')
+
+    infer_request = InferRequest(
+        messages=[{
+            'role': 'system',
+            'content': 'Given a web search query, retrieve relevant passages that answer the query'
+        }, {
+            'role': 'user',
+            'content': 'What is the capital of China?'
+        }, {
+            'role': 'assistant',
+            'content': 'The capital of China is Beijing.'
+        }])
+
+    response = engine.infer([infer_request])[0]
+    print(f'scores: {response.choices[0].message.content}')
+
+
+def run_qwen3_vl_reranker():
+    engine = TransformersEngine(
+        'Qwen/Qwen3-VL-Reranker-2B', task_type='generative_reranker', attn_impl='flash_attention_2')
+
+    infer_request = InferRequest(
+        messages=[{
+            'role': 'system',
+            'content': "Retrieval relevant image or text with user's query"
+        }, {
+            'role': 'user',
+            'content': 'A woman playing with her dog on a beach at sunset.'
+        }, {
+            'role':
+            'assistant',
+            'content':
+            '<image>A woman shares a joyful moment with her golden retriever on a sun-drenched beach '
+            'at sunset, as the dog offers its paw in a heartwarming display of companionship and trust.'
+        }],
+        images=['https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg'])
+
+    response = engine.infer([infer_request])[0]
+    print(f'scores: {response.choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    # run_qwen3_reranker()
+    run_qwen3_vl_reranker()

examples/infer/demo_reward_model.py

@@ -0,0 +1,31 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import os
+from typing import List
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+
+def infer_batch(engine: 'InferEngine', infer_requests: List['InferRequest']):
+    resp_list = engine.infer(infer_requests)
+    print(f'messages0: {infer_requests[0].messages}')
+    print(f'response0: {resp_list[0].choices[0].message.content}')
+
+
+if __name__ == '__main__':
+    from swift import InferEngine, InferRequest, TransformersEngine, load_dataset
+    model = 'Shanghai_AI_Laboratory/internlm2-1_8b-reward'
+    engine = TransformersEngine(model, max_batch_size=64)
+    # Here, `load_dataset` is used for convenience; `infer_batch` does not require creating a dataset.
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], seed=42)[0]
+    print(f'dataset: {dataset}')
+    infer_requests = [InferRequest(**data) for data in dataset]
+    infer_batch(engine, infer_requests)
+
+    messages = [{
+        'role': 'user',
+        'content': "Hello! What's your name?"
+    }, {
+        'role': 'assistant',
+        'content': 'My name is InternLM2! A helpful AI assistant. What can I do for you?'
+    }]
+    infer_batch(engine, [InferRequest(messages=messages)])

examples/infer/demo_vllm_reasoning_parser.py

@@ -0,0 +1,85 @@
+"""
+Example of using reasoning_parser
+
+This example demonstrates how to use reasoning_parser in Swift's VllmEngine to support reasoning models.
+"""
+
+from swift.infer_engine import InferRequest, RequestConfig, VllmEngine
+
+
+def main(engine: VllmEngine):
+    # Create inference request
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': '9.11 and 9.8, which is greater?'}])
+
+    # Configure request parameters
+    request_config = RequestConfig(
+        max_tokens=8192,
+        temperature=0.7,
+        stream=False  # Non-streaming inference
+    )
+
+    # Execute inference
+    responses = engine.infer(infer_requests=[infer_request], request_config=request_config)
+
+    # Process responses
+    for response in responses:
+        if hasattr(response, 'choices') and response.choices:
+            choice = response.choices[0]
+            message = choice.message
+
+            print('=== Reasoning Content ===')
+            if message.reasoning_content:
+                print(f'Reasoning steps: {message.reasoning_content}')
+            else:
+                print('No reasoning content detected')
+
+            print('\n=== Final Answer ===')
+            print(f'Answer: {message.content}')
+
+            print('\n=== Finish Reason ===')
+            print(f'Reason: {choice.finish_reason}')
+
+
+def streaming_example(engine: VllmEngine):
+    """Streaming inference example"""
+    infer_request = InferRequest(messages=[{'role': 'user', 'content': 'Calculate the result of 15 + 27'}])
+
+    request_config = RequestConfig(
+        max_tokens=8192,
+        temperature=0.7,
+        stream=True  # Enable streaming inference
+    )
+
+    # Streaming inference
+    responses = engine.infer(infer_requests=[infer_request], request_config=request_config)
+
+    print('=== Streaming Inference Results ===')
+    for chunk in responses[0]:  # responses[0] is the streaming generator
+        if chunk and chunk.choices:
+            choice = chunk.choices[0]
+            delta = choice.delta
+
+            if delta.reasoning_content:
+                print(f'Reasoning: {delta.reasoning_content}', end='', flush=True)
+
+            if delta.content:
+                print(f'Content: {delta.content}', end='', flush=True)
+
+    print('\n=== Inference Complete ===')
+
+
+if __name__ == '__main__':
+    # Initialize VllmEngine with reasoning_parser enabled
+    engine = VllmEngine(
+        model_id_or_path='Qwen/Qwen3-8B',
+        reasoning_parser='qwen3',  # Specify reasoning parser
+        gpu_memory_utilization=0.9,
+    )
+
+    print('=== Non-streaming Inference Example ===')
+    main(engine)
+
+    print('\n' + '=' * 50 + '\n')
+
+    print('=== Streaming Inference Example ===')
+    streaming_example(engine)

examples/infer/lmdeploy/batch_ddp.sh

@@ -0,0 +1,8 @@
+# test env: lmdeploy 0.9.2.post1
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend lmdeploy \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#1000 \
+    --max_new_tokens 512

examples/infer/lmdeploy/mllm_tp.sh

@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+swift infer \
+    --model OpenGVLab/InternVL2_5-1B \
+    --infer_backend lmdeploy \
+    --val_dataset AI-ModelScope/captcha-images#1000 \
+    --lmdeploy_tp 2 \
+    --lmdeploy_vision_batch_size 8 \
+    --max_new_tokens 2048

examples/infer/sglang/demo.sh

@@ -0,0 +1,7 @@
+# test_env: pip install "sglang[all]==0.4.6.*" -U
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend sglang \
+    --stream true \
+    --max_new_tokens 2048

examples/infer/sglang/distill_qwen3_235b.sh

@@ -0,0 +1,9 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+swift infer \
+    --model Qwen/Qwen3-235B-A22B-Instruct-2507 \
+    --infer_backend sglang \
+    --val_dataset liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT \
+    --sglang_context_length 12000 \
+    --sglang_tp_size 8 \
+    --write_batch_size 10000 \
+    --result_path distill_qwen3_235b.jsonl

examples/infer/sglang/mtp.sh

@@ -0,0 +1,13 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model ZhipuAI/GLM-4.5-Air \
+    --sglang_tp_size 4 \
+    --infer_backend sglang \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#100 \
+    --sglang_context_length 8192 \
+    --max_new_tokens 2048 \
+    --sglang_mem_fraction_static 0.7 \
+    --sglang_speculative_algorithm EAGLE \
+    --sglang_speculative_eagle_topk 1 \
+    --sglang_speculative_num_steps 3 \
+    --sglang_speculative_num_draft_tokens 4

examples/infer/sglang/tp.sh

@@ -0,0 +1,9 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+swift infer \
+    --model Qwen/Qwen3-8B \
+    --infer_backend sglang \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#2000 \
+    --max_new_tokens 2048 \
+    --sglang_context_length 8192 \
+    --sglang_tp_size 2 \
+    --write_batch_size 1000

examples/infer/transformers/batch_ddp.sh

@@ -0,0 +1,9 @@
+# 18GB
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen2.5-1.5B-Instruct \
+    --infer_backend transformers \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#1000 \
+    --max_batch_size 16 \
+    --max_new_tokens 512

examples/infer/transformers/bert.sh

@@ -0,0 +1,8 @@
+# Since `swift/test_lora` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+# To disable this behavior, please set `--load_args false`.
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters swift/test_bert \
+    --truncation_strategy right \
+    --max_length 512

examples/infer/transformers/lora.sh

@@ -0,0 +1,10 @@
+# Since `swift/test_lora` is trained by swift and contains an `args.json` file,
+# there is no need to explicitly set `--model`, `--system`, etc., as they will be automatically read.
+# To disable this behavior, please set `--load_args false`.
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters swift/test_lora \
+    --infer_backend transformers \
+    --stream true \
+    --temperature 0 \
+    --max_new_tokens 2048

examples/infer/transformers/mllm_device_map.sh

@@ -0,0 +1,9 @@
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+MAX_PIXELS=1003520 \
+swift infer \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --infer_backend transformers \
+    --val_dataset AI-ModelScope/LaTeX_OCR#1000 \
+    --max_batch_size 16 \
+    --max_new_tokens 512

examples/infer/transformers/prm.sh

@@ -0,0 +1,4 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model Qwen/Qwen2.5-Math-PRM-7B \
+    --infer_backend transformers

examples/infer/transformers/reward_model.sh

@@ -0,0 +1,5 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#1000 \
+    --max_batch_size 64

examples/infer/vllm/dp_tp.sh

@@ -0,0 +1,11 @@
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#2000 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --vllm_tensor_parallel_size 2 \
+    --max_new_tokens 2048 \
+    --write_batch_size 1000

examples/infer/vllm/mllm_ddp.sh

@@ -0,0 +1,11 @@
+# You need to use flash-attn (manual installation) instead of xformers.
+NPROC_PER_NODE=2 \
+CUDA_VISIBLE_DEVICES=0,1 \
+swift infer \
+    --model Qwen/Qwen2.5-Omni-7B \
+    --infer_backend vllm \
+    --val_dataset speech_asr/speech_asr_aishell1_trainsets:validation#1000 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --max_new_tokens 2048 \
+    --vllm_limit_mm_per_prompt '{"audio": 5}'

examples/infer/vllm/mllm_tp.sh

@@ -0,0 +1,11 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+MAX_PIXELS=1003520 \
+swift infer \
+    --model Qwen/Qwen2.5-VL-3B-Instruct \
+    --infer_backend vllm \
+    --val_dataset AI-ModelScope/LaTeX_OCR#1000 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_tensor_parallel_size 2 \
+    --vllm_max_model_len 32768 \
+    --max_new_tokens 2048 \
+    --vllm_limit_mm_per_prompt '{"image": 5, "video": 2}'

examples/infer/vllm/mtp.sh

@@ -0,0 +1,10 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --vllm_tensor_parallel_size 4 \
+    --infer_backend vllm \
+    --vllm_max_model_len 8192 \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#100 \
+    --vllm_speculative_config '{"method":"qwen3_next_mtp","num_speculative_tokens":2}' \
+    --vllm_gpu_memory_utilization 0.9 \
+    --max_new_tokens 2048