This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
from swift.infer_engine import InferClient, InferRequest
|
||||
|
||||
if __name__ == '__main__':
|
||||
engine = InferClient(host='127.0.0.1', port=8000)
|
||||
models = engine.models
|
||||
print(f'models: {models}')
|
||||
messages = [{
|
||||
'role': 'user',
|
||||
'content': "Hello! What's your name?"
|
||||
}, {
|
||||
'role': 'assistant',
|
||||
'content': 'My name is InternLM2! A helpful AI assistant. What can I do for you?'
|
||||
}]
|
||||
resp_list = engine.infer([InferRequest(messages=messages)])
|
||||
print(f'messages: {messages}')
|
||||
print(f'response: {resp_list[0].choices[0].message.content}')
|
||||
@@ -0,0 +1,5 @@
|
||||
CUDA_VISIBLE_DEVICES=0 swift deploy \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
|
||||
--infer_backend transformers
|
||||
Reference in New Issue
Block a user