wehub-resource-sync
85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
|
|
""" Starting with llmware 0.3.7, we have integrated support for ONNX Runtime Generative models.
|
|
|
|
To get started:
|
|
|
|
`pip install onnxruntime_genai`
|
|
|
|
Please note that onnxruntime_genai is supported on a wide range of Windows, Linux and x86 platforms, but
|
|
does not build for Mac Metal - so it will not work on Macs.
|
|
|
|
"""
|
|
|
|
from llmware.models import ModelCatalog
|
|
|
|
from importlib import util
|
|
if not util.find_spec("onnxruntime_genai"):
|
|
print("\nto run this example, you need to install onnxruntime_genai first, e.g., pip3 install onnxruntime_genai")
|
|
|
|
# we will be adding more ONNX models to the default catalog, but we currently support:
|
|
# -- bling-tiny-llama-onnx
|
|
# -- bling-phi-3-onnx
|
|
# -- phi-3-onnx
|
|
|
|
# please see the example 'adding_openvino_or_onnx_model.py' to add your own ONNX and OpenVino models
|
|
|
|
|
|
def getting_started():
|
|
|
|
""" Simple 'hello world' example. """
|
|
|
|
model = ModelCatalog().load_model("bling-tiny-llama-onnx", temperature=0.0, sample=False,
|
|
max_output=100)
|
|
|
|
query= "What was Microsoft's revenue in the 3rd quarter?"
|
|
|
|
context = ("Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — "
|
|
"Microsoft Corp. today announced the following results for the quarter ended March 31, 2023,"
|
|
" as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion"
|
|
" and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion "
|
|
"and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and "
|
|
"increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 "
|
|
"and increased 10% (up 14% in constant currency).\n")
|
|
|
|
response = model.inference(query,add_context=context)
|
|
|
|
print(f"\ngetting_started example - query - {query}")
|
|
print("getting_started example - response: ", response)
|
|
|
|
return response
|
|
|
|
|
|
def streaming_example():
|
|
|
|
prompt = "What are the benefits of small specialized LLMs?"
|
|
|
|
print(f"\nstreaming_example - prompt: {prompt}")
|
|
|
|
# since model.stream provides a generator, then use as follows to consume the generator
|
|
model = ModelCatalog().load_model("phi-3-onnx", max_output=500)
|
|
text_out = ""
|
|
token_count = 0
|
|
|
|
for streamed_token in model.stream(prompt):
|
|
|
|
text_out += streamed_token
|
|
if text_out.strip():
|
|
print(streamed_token, end="")
|
|
|
|
token_count += 1
|
|
|
|
print("total text: ", text_out)
|
|
print("total tokens: ", token_count)
|
|
|
|
return text_out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
getting_started()
|
|
|
|
streaming_example()
|
|
|
|
|
|
|